From 4ddfaf8bf2ba937005d4c921dbf45ef63a63d112 Mon Sep 17 00:00:00 2001 From: Asim Viladi Oglu Manizada Date: Tue, 6 Jan 2026 18:54:00 +0100 Subject: [PATCH 001/311] smb: client: reject userspace cifs.spnego descriptions cifs.spnego key descriptions contain authority-bearing fields such as pid, uid, creduid, and upcall_target that cifs.upcall treats as kernel-originating inputs. However, userspace can also create keys of this type through request_key(2) or add_key(2), allowing those fields to be supplied without CIFS origin. Only accept cifs.spnego descriptions while CIFS is using its private spnego_cred to request the key. Fixes: f1d662a7d5e5 ("[CIFS] Add upcall files for cifs to use spnego/kerberos") Assisted-by:avom-custom-harness:gpt-5.5-qwen3.6-mod-mix Reviewed-by: David Howells Signed-off-by: Asim Viladi Oglu Manizada Signed-off-by: Steve French CVE-2026-46243 (cherry picked from commit 3da1fdf4efbc490041eb4f836bf596201203f8f2) Signed-off-by: Massimiliano Pellizzer Acked-by: Changwei Zou Acked-by: Ross Porter Signed-off-by: Stefan Bader --- fs/smb/client/cifs_spnego.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/fs/smb/client/cifs_spnego.c b/fs/smb/client/cifs_spnego.c index 3a41bbada04c7..44c4072756804 100644 --- a/fs/smb/client/cifs_spnego.c +++ b/fs/smb/client/cifs_spnego.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -40,12 +41,27 @@ cifs_spnego_key_destroy(struct key *key) kfree(key->payload.data[0]); } +static int +cifs_spnego_key_vet_description(const char *description) +{ + /* + * cifs.spnego descriptions are authority-bearing inputs to cifs.upcall. + * They are only valid when produced by CIFS while using the private + * spnego_cred installed below. Do not let userspace create this type + * of key through request_key(2)/add_key(2), since the helper treats + * pid/uid/creduid/upcall_target as kernel-originating fields. + */ + if (current_cred() != spnego_cred) + return -EPERM; + return 0; +} /* * keytype for CIFS spnego keys */ struct key_type cifs_spnego_key_type = { .name = "cifs.spnego", + .vet_description = cifs_spnego_key_vet_description, .instantiate = cifs_spnego_key_instantiate, .destroy = cifs_spnego_key_destroy, .describe = user_describe, From 55cdc809a73d97a1433cf560cfb479bfd6eecb9d Mon Sep 17 00:00:00 2001 From: Naman Jain Date: Fri, 10 Apr 2026 15:34:13 +0000 Subject: [PATCH 002/311] block: add pgmap check to biovec_phys_mergeable commit 13920e4b7b784b40cf4519ff1f0f3e513476a499 upstream. biovec_phys_mergeable() is used by the request merge, DMA mapping, and integrity merge paths to decide if two physically contiguous bvec segments can be coalesced into one. It currently has no check for whether the segments belong to different dev_pagemaps. When zone device memory is registered in multiple chunks, each chunk gets its own dev_pagemap. A single bio can legitimately contain bvecs from different pgmaps -- iov_iter_extract_bvecs() breaks at pgmap boundaries but the outer loop in bio_iov_iter_get_pages() continues filling the same bio. If such bvecs are physically contiguous, biovec_phys_mergeable() will coalesce them, making it impossible to recover the correct pgmap for the merged segment via page_pgmap(). Add a zone_device_pages_have_same_pgmap() check to prevent merging bvec segments that span different pgmaps. Fixes: 49580e690755 ("block: add check when merging zone device pages") Cc: stable@vger.kernel.org Reviewed-by: Christoph Hellwig Signed-off-by: Naman Jain Link: https://patch.msgid.link/20260410153414.4159050-2-namjain@linux.microsoft.com Signed-off-by: Jens Axboe Signed-off-by: Greg Kroah-Hartman CVE-2026-46115 Signed-off-by: Noah Wager Signed-off-by: Stefan Bader --- block/blk.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/block/blk.h b/block/blk.h index a55e2e4fcda4f..a7abf3be34ef8 100644 --- a/block/blk.h +++ b/block/blk.h @@ -132,6 +132,8 @@ static inline bool biovec_phys_mergeable(struct request_queue *q, if (addr1 + vec1->bv_len != addr2) return false; + if (!zone_device_pages_have_same_pgmap(vec1->bv_page, vec2->bv_page)) + return false; if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2->bv_page)) return false; if ((addr1 | mask) != ((addr2 + vec2->bv_len - 1) | mask)) From 2bb6ef5a8a83260175b6a3e9266c36606047bed0 Mon Sep 17 00:00:00 2001 From: Zisen Ye Date: Wed, 6 May 2026 11:49:08 +0800 Subject: [PATCH 003/311] smb/client: fix out-of-bounds read in smb2_compound_op() commit 8d09328dfda089675e4c049f3f256064a1d1996b upstream. If a server sends a truncated response but a large OutputBufferLength, and terminates the EA list early, check_wsl_eas() returns success without validating that the entire OutputBufferLength fits within iov_len. Then smb2_compound_op() does: memcpy(idata->wsl.eas, data[0], size[0]); Where size[0] is OutputBufferLength. If iov_len is smaller than size[0], memcpy can read beyond the end of the rsp_iov allocation and leak adjacent kernel heap memory. Link: https://lore.kernel.org/linux-cifs/d998240c-aca9-420d-9dbd-f5ba24af19e0@chenxiaosong.com/ Fixes: ea41367b2a60 ("smb: client: introduce SMB2_OP_QUERY_WSL_EA") Cc: stable@vger.kernel.org Signed-off-by: Zisen Ye Reviewed-by: ChenXiaoSong Signed-off-by: Steve French Signed-off-by: Greg Kroah-Hartman CVE-2026-46155 Signed-off-by: Noah Wager Signed-off-by: Stefan Bader --- fs/smb/client/smb2inode.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/fs/smb/client/smb2inode.c b/fs/smb/client/smb2inode.c index fe1c9d7765806..3b09cf8ab0f27 100644 --- a/fs/smb/client/smb2inode.c +++ b/fs/smb/client/smb2inode.c @@ -111,7 +111,7 @@ static int check_wsl_eas(struct kvec *rsp_iov) u32 outlen, next; u16 vlen; u8 nlen; - u8 *end; + u8 *ea_end, *iov_end; outlen = le32_to_cpu(rsp->OutputBufferLength); if (outlen < SMB2_WSL_MIN_QUERY_EA_RESP_SIZE || @@ -120,15 +120,19 @@ static int check_wsl_eas(struct kvec *rsp_iov) ea = (void *)((u8 *)rsp_iov->iov_base + le16_to_cpu(rsp->OutputBufferOffset)); - end = (u8 *)rsp_iov->iov_base + rsp_iov->iov_len; + ea_end = (u8 *)ea + outlen; + iov_end = (u8 *)rsp_iov->iov_base + rsp_iov->iov_len; + if (ea_end > iov_end) + return -EINVAL; + for (;;) { - if ((u8 *)ea > end - sizeof(*ea)) + if ((u8 *)ea > ea_end - sizeof(*ea)) return -EINVAL; nlen = ea->ea_name_length; vlen = le16_to_cpu(ea->ea_value_length); if (nlen != SMB2_WSL_XATTR_NAME_LEN || - (u8 *)ea->ea_data + nlen + 1 + vlen > end) + (u8 *)ea->ea_data + nlen + 1 + vlen > ea_end) return -EINVAL; switch (vlen) { From 74aa44a918fc50bf6c3917d3dc1cac397e5497ab Mon Sep 17 00:00:00 2001 From: Chaitanya Kulkarni Date: Wed, 8 Apr 2026 00:51:31 -0700 Subject: [PATCH 004/311] nvmet-tcp: fix race between ICReq handling and queue teardown commit 5293a8882c549fab4a878bc76b0b6c951f980a61 upstream. nvmet_tcp_handle_icreq() updates queue->state after sending an Initialization Connection Response (ICResp), but it does so without serializing against target-side queue teardown. If an NVMe/TCP host sends an Initialization Connection Request (ICReq) and immediately closes the connection, target-side teardown may start in softirq context before io_work drains the already buffered ICReq. In that case, nvmet_tcp_schedule_release_queue() sets queue->state to NVMET_TCP_Q_DISCONNECTING and drops the queue reference under state_lock. If io_work later processes that ICReq, nvmet_tcp_handle_icreq() can still overwrite the state back to NVMET_TCP_Q_LIVE. That defeats the DISCONNECTING-state guard in nvmet_tcp_schedule_release_queue() and allows a later socket state change to re-enter teardown and issue a second kref_put() on an already released queue. The ICResp send failure path has the same problem. If teardown has already moved the queue to DISCONNECTING, a send error can still overwrite the state with NVMET_TCP_Q_FAILED, again reopening the window for a second teardown path to drop the queue reference. Fix this by serializing both post-send state transitions with state_lock and bailing out if teardown has already started. Use -ESHUTDOWN as an internal sentinel for that bail-out path rather than propagating it as a transport error like -ECONNRESET. Keep nvmet_tcp_socket_error() setting rcv_state to NVMET_TCP_RECV_ERR before honoring that sentinel so receive-side parsing stays quiesced until the existing release path completes. Fixes: c46a6465bac2 ("nvmet-tcp: add NVMe over TCP target driver") Cc: stable@vger.kernel.org Reported-by: Shivam Kumar Tested-by: Shivam Kumar Signed-off-by: Chaitanya Kulkarni Signed-off-by: Keith Busch Signed-off-by: Greg Kroah-Hartman CVE-2026-46135 Signed-off-by: Noah Wager Signed-off-by: Stefan Bader --- drivers/nvme/target/tcp.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index acc71a26733f9..255ebd948dfe1 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -398,6 +398,19 @@ static void nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd *cmd) static void nvmet_tcp_fatal_error(struct nvmet_tcp_queue *queue) { + /* + * Keep rcv_state at RECV_ERR even for the internal -ESHUTDOWN path. + * nvmet_tcp_handle_icreq() can return -ESHUTDOWN after the ICReq has + * already been consumed and queue teardown has started. + * + * If nvmet_tcp_data_ready() or nvmet_tcp_write_space() queues + * nvmet_tcp_io_work() again before nvmet_tcp_release_queue_work() + * cancels it, the queue must not keep that old receive state. + * Otherwise the next nvmet_tcp_io_work() run can reach + * nvmet_tcp_done_recv_pdu() and try to handle the same ICReq again. + * + * That is why queue->rcv_state needs to be updated before we return. + */ queue->rcv_state = NVMET_TCP_RECV_ERR; if (queue->nvme_sq.ctrl) nvmet_ctrl_fatal_error(queue->nvme_sq.ctrl); @@ -922,11 +935,24 @@ static int nvmet_tcp_handle_icreq(struct nvmet_tcp_queue *queue) iov.iov_len = sizeof(*icresp); ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); if (ret < 0) { + spin_lock_bh(&queue->state_lock); + if (queue->state == NVMET_TCP_Q_DISCONNECTING) { + spin_unlock_bh(&queue->state_lock); + return -ESHUTDOWN; + } queue->state = NVMET_TCP_Q_FAILED; + spin_unlock_bh(&queue->state_lock); return ret; /* queue removal will cleanup */ } + spin_lock_bh(&queue->state_lock); + if (queue->state == NVMET_TCP_Q_DISCONNECTING) { + spin_unlock_bh(&queue->state_lock); + /* Tell nvmet_tcp_socket_error() teardown is in progress. */ + return -ESHUTDOWN; + } queue->state = NVMET_TCP_Q_LIVE; + spin_unlock_bh(&queue->state_lock); nvmet_prepare_receive_pdu(queue); return 0; } From 89146f0ba00827dc043a778a64ddbb51ce6ddbb9 Mon Sep 17 00:00:00 2001 From: Raphael Zimmer Date: Tue, 21 Apr 2026 10:27:01 +0200 Subject: [PATCH 005/311] libceph: Fix slab-out-of-bounds access in auth message processing commit 1c439de70b1c3eb3c6bffa8245c16b9fc318f114 upstream. If a (potentially corrupted) message of type CEPH_MSG_AUTH_REPLY contains a positive value in its result field, it is treated as an error code by ceph_handle_auth_reply() and returned to handle_auth_reply(). Thereafter, an attempt is made to send the preallocated message of type CEPH_MSG_AUTH, where the returned value is interpreted as the size of the front segment to send. If the result value in the message is greater than the size of the memory buffer allocated for the front segment, an out-of-bounds access occurs, and the content of the memory region beyond this buffer is sent out. This patch fixes the issue by treating only negative values in the result field as errors. Positive values are therefore treated as success in the same way as a zero value. Additionally, a BUG_ON is added to __send_prepared_auth_request() comparing the len parameter to front_alloc_len to prevent sending the message if it exceeds the bounds of the allocation and to make it easier to catch any logic flaws leading to this. Cc: stable@vger.kernel.org Signed-off-by: Raphael Zimmer Reviewed-by: Ilya Dryomov Signed-off-by: Ilya Dryomov Signed-off-by: Greg Kroah-Hartman CVE-2026-46119 Signed-off-by: Noah Wager Signed-off-by: Stefan Bader --- net/ceph/auth.c | 2 +- net/ceph/mon_client.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/net/ceph/auth.c b/net/ceph/auth.c index 3314705e59146..17660bde896be 100644 --- a/net/ceph/auth.c +++ b/net/ceph/auth.c @@ -257,7 +257,7 @@ int ceph_handle_auth_reply(struct ceph_auth_client *ac, ac->negotiating = false; } - if (result) { + if (result < 0) { pr_err("auth protocol '%s' mauth authentication failed: %d\n", ceph_auth_proto_name(ac->protocol), result); ret = result; diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c index d5080530ce0cc..d2cdc8ee31551 100644 --- a/net/ceph/mon_client.c +++ b/net/ceph/mon_client.c @@ -174,6 +174,8 @@ int ceph_monmap_contains(struct ceph_monmap *m, struct ceph_entity_addr *addr) */ static void __send_prepared_auth_request(struct ceph_mon_client *monc, int len) { + BUG_ON(len > monc->m_auth->front_alloc_len); + monc->pending_auth = 1; monc->m_auth->front.iov_len = len; monc->m_auth->hdr.front_len = cpu_to_le32(len); From 858610b13d81b65bd0dc5033c7ebf826d05c283d Mon Sep 17 00:00:00 2001 From: "Christian A. Ehrhardt" Date: Thu, 26 Mar 2026 22:49:01 +0100 Subject: [PATCH 006/311] lib/scatterlist: fix length calculations in extract_kvec_to_sg commit 07b7d66e65d9cfe6b9c2c34aa22cfcaac37a5c45 upstream. Patch series "Fix bugs in extract_iter_to_sg()", v3. Fix bugs in the kvec and user variants of extract_iter_to_sg. This series is growing due to useful remarks made by sashiko.dev. The main bugs are: - The length for an sglist entry when extracting from a kvec can exceed the number of bytes in the page. This is obviously not intended. - When extracting a user buffer the sglist is temporarily used as a scratch buffer for extracted page pointers. If the sglist already contains some elements this scratch buffer could overlap with existing entries in the sglist. The series adds test cases to the kunit_iov_iter test that demonstrate all of these bugs. Additionally, there is a memory leak fix for the test itself. The bugs were orignally introduced into kernel v6.3 where the function lived in fs/netfs/iterator.c. It was later moved to lib/scatterlist.c in v6.5. Thus the actual fix is only marked for backports to v6.5+. This patch (of 5): When extracting from a kvec to a scatterlist, do not cross page boundaries. The required length was already calculated but not used as intended. Adjust the copied length if the loop runs out of sglist entries without extracting everything. While there, return immediately from extract_iter_to_sg if there are no sglist entries at all. A subsequent commit will add kunit test cases that demonstrate that the patch is necessary. Link: https://lkml.kernel.org/r/20260326214905.818170-1-lk@c--e.de Link: https://lkml.kernel.org/r/20260326214905.818170-2-lk@c--e.de Fixes: 018584697533 ("netfs: Add a function to extract an iterator into a scatterlist") Signed-off-by: Christian A. Ehrhardt Cc: David Gow Cc: David Howells Cc: Kees Cook Cc: Petr Mladek Cc: [v6.5+] Signed-off-by: Andrew Morton Signed-off-by: Greg Kroah-Hartman CVE-2026-46289 Signed-off-by: Noah Wager Signed-off-by: Stefan Bader --- lib/scatterlist.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/scatterlist.c b/lib/scatterlist.c index d773720d11bf2..befdc4b9c11d3 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -1247,7 +1247,7 @@ static ssize_t extract_kvec_to_sg(struct iov_iter *iter, else page = virt_to_page((void *)kaddr); - sg_set_page(sg, page, len, off); + sg_set_page(sg, page, seg, off); sgtable->nents++; sg++; sg_max--; @@ -1256,6 +1256,7 @@ static ssize_t extract_kvec_to_sg(struct iov_iter *iter, kaddr += PAGE_SIZE; off = 0; } while (len > 0 && sg_max > 0); + ret -= len; if (maxsize <= 0 || sg_max == 0) break; @@ -1409,7 +1410,7 @@ ssize_t extract_iter_to_sg(struct iov_iter *iter, size_t maxsize, struct sg_table *sgtable, unsigned int sg_max, iov_iter_extraction_t extraction_flags) { - if (maxsize == 0) + if (maxsize == 0 || sg_max == 0) return 0; switch (iov_iter_type(iter)) { From c911e58e697a5ef859c11cbd30b4689e77d97253 Mon Sep 17 00:00:00 2001 From: Michael Bommarito Date: Mon, 20 Apr 2026 10:47:47 -0400 Subject: [PATCH 007/311] smb: client: validate dacloffset before building DACL pointers commit f98b48151cc502ada59d9778f0112d21f2586ca3 upstream. parse_sec_desc(), build_sec_desc(), and the chown path in id_mode_to_cifs_acl() all add the server-supplied dacloffset to pntsd before proving a DACL header fits inside the returned security descriptor. On 32-bit builds a malicious server can return dacloffset near U32_MAX, wrap the derived DACL pointer below end_of_acl, and then slip past the later pointer-based bounds checks. build_sec_desc() and id_mode_to_cifs_acl() can then dereference DACL fields from the wrapped pointer in the chmod/chown rewrite paths. Validate dacloffset numerically before building any DACL pointer and reuse the same helper at the three DACL entry points. Fixes: bc3e9dd9d104 ("cifs: Change SIDs in ACEs while transferring file ownership.") Cc: stable@vger.kernel.org Assisted-by: Claude:claude-opus-4-6 Signed-off-by: Michael Bommarito Signed-off-by: Steve French Signed-off-by: Greg Kroah-Hartman CVE-2026-46195 Signed-off-by: Noah Wager Signed-off-by: Stefan Bader --- fs/smb/client/cifsacl.c | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c index 4ec204d2c7742..fcf135b6ff808 100644 --- a/fs/smb/client/cifsacl.c +++ b/fs/smb/client/cifsacl.c @@ -1264,6 +1264,17 @@ static int parse_sid(struct smb_sid *psid, char *end_of_acl) return 0; } +static bool dacl_offset_valid(unsigned int acl_len, __u32 dacloffset) +{ + if (acl_len < sizeof(struct smb_acl)) + return false; + + if (dacloffset < sizeof(struct smb_ntsd)) + return false; + + return dacloffset <= acl_len - sizeof(struct smb_acl); +} + /* Convert CIFS ACL to POSIX form */ static int parse_sec_desc(struct cifs_sb_info *cifs_sb, @@ -1284,7 +1295,6 @@ static int parse_sec_desc(struct cifs_sb_info *cifs_sb, group_sid_ptr = (struct smb_sid *)((char *)pntsd + le32_to_cpu(pntsd->gsidoffset)); dacloffset = le32_to_cpu(pntsd->dacloffset); - dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); cifs_dbg(NOISY, "revision %d type 0x%x ooffset 0x%x goffset 0x%x sacloffset 0x%x dacloffset 0x%x\n", pntsd->revision, pntsd->type, le32_to_cpu(pntsd->osidoffset), le32_to_cpu(pntsd->gsidoffset), @@ -1315,11 +1325,18 @@ static int parse_sec_desc(struct cifs_sb_info *cifs_sb, return rc; } - if (dacloffset) + if (dacloffset) { + if (!dacl_offset_valid(acl_len, dacloffset)) { + cifs_dbg(VFS, "Server returned illegal DACL offset\n"); + return -EINVAL; + } + + dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); parse_dacl(dacl_ptr, end_of_acl, owner_sid_ptr, group_sid_ptr, fattr, get_mode_from_special_sid); - else + } else { cifs_dbg(FYI, "no ACL\n"); /* BB grant all or default perms? */ + } return rc; } @@ -1342,6 +1359,11 @@ static int build_sec_desc(struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd, dacloffset = le32_to_cpu(pntsd->dacloffset); if (dacloffset) { + if (!dacl_offset_valid(secdesclen, dacloffset)) { + cifs_dbg(VFS, "Server returned illegal DACL offset\n"); + return -EINVAL; + } + dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); rc = validate_dacl(dacl_ptr, end_of_acl); if (rc) @@ -1710,6 +1732,12 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, nsecdesclen = sizeof(struct smb_ntsd) + (sizeof(struct smb_sid) * 2); dacloffset = le32_to_cpu(pntsd->dacloffset); if (dacloffset) { + if (!dacl_offset_valid(secdesclen, dacloffset)) { + cifs_dbg(VFS, "Server returned illegal DACL offset\n"); + rc = -EINVAL; + goto id_mode_to_cifs_acl_exit; + } + dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); rc = validate_dacl(dacl_ptr, (char *)pntsd + secdesclen); if (rc) { @@ -1752,6 +1780,7 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, rc = ops->set_acl(pnntsd, nsecdesclen, inode, path, aclflag); cifs_dbg(NOISY, "set_cifs_acl rc: %d\n", rc); } +id_mode_to_cifs_acl_exit: cifs_put_tlink(tlink); kfree(pnntsd); From 48bf0b54a2a07461f4a9396b977b3793e14f3172 Mon Sep 17 00:00:00 2001 From: Zisen Ye Date: Sat, 2 May 2026 18:48:36 +0800 Subject: [PATCH 008/311] smb/client: fix out-of-bounds read in symlink_data() commit d62b8d236fab503c6fec1d3e9a38bea71feaca20 upstream. Since smb2_check_message() returns success without length validation for the symlink error response, in symlink_data() it is possible for iov->iov_len to be smaller than sizeof(struct smb2_err_rsp). If the buffer only contains the base SMB2 header (64 bytes), accessing err->ErrorContextCount (at offset 66) or err->ByteCount later in symlink_data() will cause an out-of-bounds read. Link: https://lore.kernel.org/linux-cifs/297d8d9b-adf7-42fd-a1c2-5b1f230032bc@chenxiaosong.com/ Fixes: 76894f3e2f71 ("cifs: improve symlink handling for smb2+") Cc: Stable@vger.kernel.org Signed-off-by: Zisen Ye Reviewed-by: ChenXiaoSong Signed-off-by: Steve French Signed-off-by: Greg Kroah-Hartman CVE-2026-46185 Signed-off-by: Noah Wager Signed-off-by: Stefan Bader --- fs/smb/client/smb2misc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/smb/client/smb2misc.c b/fs/smb/client/smb2misc.c index 973fce3c959c4..2a7355ce1a078 100644 --- a/fs/smb/client/smb2misc.c +++ b/fs/smb/client/smb2misc.c @@ -241,7 +241,8 @@ smb2_check_message(char *buf, unsigned int pdu_len, unsigned int len, if (len != calc_len) { /* create failed on symlink */ if (command == SMB2_CREATE_HE && - shdr->Status == STATUS_STOPPED_ON_SYMLINK) + shdr->Status == STATUS_STOPPED_ON_SYMLINK && + len > calc_len) return 0; /* Windows 7 server returns 24 bytes more */ if (calc_len + 24 == len && command == SMB2_OPLOCK_BREAK_HE) From a4fd0c6900e44ea3816c0b506efe83a487a0da54 Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Tue, 5 May 2026 17:00:50 +0200 Subject: [PATCH 009/311] mptcp: pm: ADD_ADDR rtx: allow ID 0 commit 03f324f3f1f7619a47b9c91282cb12775ab0a2f1 upstream. ADD_ADDR can be sent for the ID 0, which corresponds to the local address and port linked to the initial subflow. Indeed, this address could be removed, and re-added later on, e.g. what is done in the "delete re-add signal" MPTCP Join selftests. So no reason to ignore it. Fixes: 00cfd77b9063 ("mptcp: retransmit ADD_ADDR when timeout") Cc: stable@vger.kernel.org Reviewed-by: Mat Martineau Signed-off-by: Matthieu Baerts (NGI0) Link: https://patch.msgid.link/20260505-net-mptcp-pm-fixes-7-1-rc3-v1-2-fca8091060a4@kernel.org Signed-off-by: Jakub Kicinski Signed-off-by: Greg Kroah-Hartman CVE-2026-46137 Signed-off-by: Noah Wager Signed-off-by: Stefan Bader --- net/mptcp/pm.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c index 57a4566904067..5056eb8db24e0 100644 --- a/net/mptcp/pm.c +++ b/net/mptcp/pm.c @@ -337,9 +337,6 @@ static void mptcp_pm_add_timer(struct timer_list *timer) if (inet_sk_state_load(sk) == TCP_CLOSE) return; - if (!entry->addr.id) - return; - if (mptcp_pm_should_add_signal_addr(msk)) { sk_reset_timer(sk, timer, jiffies + TCP_RTO_MAX / 8); goto out; From 7f42f5f70d29634bf33360e62fcb8a6d5729c41a Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Tue, 5 May 2026 17:00:51 +0200 Subject: [PATCH 010/311] mptcp: pm: ADD_ADDR rtx: fix potential data-race commit 5cd6e0ad79d2615264f63929f8b457ad97ae550d upstream. This mptcp_pm_add_timer() helper is executed as a timer callback in softirq context. To avoid any data races, the socket lock needs to be held with bh_lock_sock(). If the socket is in use, retry again soon after, similar to what is done with the keepalive timer. Fixes: 00cfd77b9063 ("mptcp: retransmit ADD_ADDR when timeout") Cc: stable@vger.kernel.org Reviewed-by: Mat Martineau Signed-off-by: Matthieu Baerts (NGI0) Link: https://patch.msgid.link/20260505-net-mptcp-pm-fixes-7-1-rc3-v1-3-fca8091060a4@kernel.org Signed-off-by: Jakub Kicinski Signed-off-by: Greg Kroah-Hartman CVE-2026-46137 Signed-off-by: Noah Wager Signed-off-by: Stefan Bader --- net/mptcp/pm.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c index 5056eb8db24e0..3912128d9b866 100644 --- a/net/mptcp/pm.c +++ b/net/mptcp/pm.c @@ -337,6 +337,13 @@ static void mptcp_pm_add_timer(struct timer_list *timer) if (inet_sk_state_load(sk) == TCP_CLOSE) return; + bh_lock_sock(sk); + if (sock_owned_by_user(sk)) { + /* Try again later. */ + sk_reset_timer(sk, timer, jiffies + HZ / 20); + goto out; + } + if (mptcp_pm_should_add_signal_addr(msk)) { sk_reset_timer(sk, timer, jiffies + TCP_RTO_MAX / 8); goto out; @@ -365,6 +372,7 @@ static void mptcp_pm_add_timer(struct timer_list *timer) mptcp_pm_subflow_established(msk); out: + bh_unlock_sock(sk); __sock_put(sk); } From ea3c5c702adb31fd4d76f275bf4db6683a9ff802 Mon Sep 17 00:00:00 2001 From: Yizhou Zhao Date: Tue, 12 May 2026 01:30:41 +0800 Subject: [PATCH 011/311] netfilter: nft_inner: Fix IPv6 inner_thoff desync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit b6a91f68ebfed9c38e0e9150f58a9b85da07181c upstream. In nft_inner_parse_l2l3(), when processing inner IPv6 packets, ipv6_find_hdr() correctly computes the transport header offset traversing all extension headers, but the result is immediately overwritten with nhoff + sizeof(_ip6h) (40 bytes), which only accounts for the IPv6 base header. This creates a desync between inner_thoff (wrong — points to extension header start) and l4proto (correct — e.g., IPPROTO_TCP), enabling transport header forgery and potential firewall bypass. This issue affects stable versions from Linux 6.2. For comparison, the normal (non-inner) IPv6 path correctly preserves ipv6_find_hdr()'s result. Removing the incorrect overwrite ensures that ipv6_find_hdr()'s calculated transport header offset is preserved, thereby fixing the desynchronization. Fixes: 3a07327d10a0 ("netfilter: nft_inner: support for inner tunnel header matching") Cc: stable@vger.kernel.org Reported-by: Yizhou Zhao Reported-by: Yuxiang Yang Reported-by: Xuewei Feng Reported-by: Qi Li Reported-by: Ke Xu Assisted-by: GLM:5.1 Z.ai Signed-off-by: Yizhou Zhao Reviewed-by: Fernando Fernandez Mancera Signed-off-by: Pablo Neira Ayuso Signed-off-by: Greg Kroah-Hartman CVE-2026-46244 Signed-off-by: Noah Wager Signed-off-by: Stefan Bader --- net/netfilter/nft_inner.c | 1 - 1 file changed, 1 deletion(-) diff --git a/net/netfilter/nft_inner.c b/net/netfilter/nft_inner.c index c4569d4b92285..1b3e7a976f560 100644 --- a/net/netfilter/nft_inner.c +++ b/net/netfilter/nft_inner.c @@ -163,7 +163,6 @@ static int nft_inner_parse_l2l3(const struct nft_inner *priv, return -1; if (fragoff == 0) { - thoff = nhoff + sizeof(_ip6h); ctx->flags |= NFT_PAYLOAD_CTX_INNER_TH; ctx->inner_thoff = thoff; ctx->l4proto = l4proto; From 589303a2f68bc6940fcdb7bdd954ca07acaeb5ce Mon Sep 17 00:00:00 2001 From: Hyunwoo Kim Date: Sun, 6 Dec 2026 21:13:00 +0100 Subject: [PATCH 012/311] KVM: arm64: vgic-its: Drop the translation cache reference only for the erased entry vgic_its_invalidate_cache() walks the per-ITS translation cache with xa_for_each() and drops the cache's reference on each entry with vgic_put_irq(). It puts the iterated pointer, though, rather than the value returned by xa_erase(). The function is called from contexts that do not exclude one another: the ITS command handlers hold its_lock, the GITS_CTLR write path holds cmd_lock, and the path that clears EnableLPIs in a redistributor's GICR_CTLR holds neither. Two or more of them can drain the same cache concurrently, and if each one observes the same entry, erases it and then puts it, the single reference the cache holds on that entry is dropped more than once. The entry can then be freed while an ITE still maps it. xa_erase() is atomic and returns the previous entry, so put only the entry that this context actually removed. The cache reference is then dropped exactly once per entry even when the invalidations run concurrently, and the behavior is unchanged when only one context runs. Fixes: 8201d1028caa ("KVM: arm64: vgic-its: Maintain a translation cache per ITS") Signed-off-by: Hyunwoo Kim Reviewed-by: Oliver Upton Link:https://patch.msgid.link/ah2c5lu4JbUg7dj-@v4bel Signed-off-by: Marc Zyngier Cc:stable@vger.kernel.org CVE-2026-46316 (cherry picked from commit 13031fb6b8357fbbcded2a7f4cba73e4781ee594) Signed-off-by: Massimiliano Pellizzer Acked-by: Edoardo Canepa Acked-by: Ross Porter Signed-off-by: Stefan Bader --- arch/arm64/kvm/vgic/vgic-its.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 2ea9f1c7ebcd0..963d2699cd9a3 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -597,8 +597,10 @@ static void vgic_its_invalidate_cache(struct vgic_its *its) unsigned long idx; xa_for_each(&its->translation_cache, idx, irq) { - xa_erase(&its->translation_cache, idx); - vgic_put_irq(kvm, irq); + /* Only the context that erases the entry drops its cache ref. */ + irq = xa_erase(&its->translation_cache, idx); + if (irq) + vgic_put_irq(kvm, irq); } } From 769356a86a12c29170e1dc16692d2659ed401119 Mon Sep 17 00:00:00 2001 From: Edoardo Canepa via kernel-team Date: Thu, 18 Jun 2026 17:54:58 +0200 Subject: [PATCH 013/311] UBUNTU: [Config] Disable NOVA_CORE BugLink: https://bugs.launchpad.net/bugs/2150845 Signed-off-by: Edoardo Canepa Acked-by: Alessio Faina via kernel-team Acked-by: Manuel Diewald Signed-off-by: Manuel Diewald --- debian.master/config/annotations | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/debian.master/config/annotations b/debian.master/config/annotations index 0a6dfbb7a3063..dde1e2948bdea 100644 --- a/debian.master/config/annotations +++ b/debian.master/config/annotations @@ -459,6 +459,9 @@ CONFIG_NLS note<'dependancy of boot essenti CONFIG_NOP_USB_XCEIV policy<{'amd64': 'm', 'arm64': 'm', 'armhf': 'y', 'ppc64el': 'm', 'riscv64': 'm', 's390x': '-'}> CONFIG_NOP_USB_XCEIV note<'boot essential on omap/highbank'> +CONFIG_NOVA_CORE policy<{'amd64': 'n', 'arm64': 'n'}> +CONFIG_NOVA_CORE note<'LP: #2150845'> + CONFIG_NO_HZ_FULL policy<{'amd64': 'y', 'arm64': 'y', 'armhf': 'n', 'ppc64el': 'n', 'riscv64': 'n'}> CONFIG_NO_HZ_FULL note<'LP: #2051342'> @@ -9857,7 +9860,6 @@ CONFIG_NOUVEAU_DEBUG_DEFAULT policy<{'amd64': '3', 'arm64': ' CONFIG_NOUVEAU_DEBUG_MMU policy<{'amd64': 'n', 'arm64': 'n', 'armhf': 'n', 'ppc64el': 'n', 'riscv64': 'n', 's390x': '-'}> CONFIG_NOUVEAU_DEBUG_PUSH policy<{'amd64': 'n', 'arm64': 'n', 'armhf': 'n', 'ppc64el': 'n', 'riscv64': 'n', 's390x': '-'}> CONFIG_NOUVEAU_PLATFORM_DRIVER policy<{'arm64': 'y', 'armhf': 'y'}> -CONFIG_NOVA_CORE policy<{'amd64': 'm', 'arm64': 'm'}> CONFIG_NOZOMI policy<{'amd64': 'm', 'arm64': 'm', 'armhf': 'm', 'ppc64el': 'm', 'riscv64': 'm', 's390x': 'n'}> CONFIG_NO_HZ policy<{'amd64': 'y', 'arm64': 'y', 'armhf': 'y', 'ppc64el': 'y', 'riscv64': 'y', 's390x': 'y'}> CONFIG_NO_HZ_COMMON policy<{'amd64': 'y', 'arm64': 'y', 'armhf': 'y', 'ppc64el': 'y', 'riscv64': 'y', 's390x': 'y'}> From ecd9d2855a0c6bb720042d1689e4c5ce9979fd4b Mon Sep 17 00:00:00 2001 From: Manuel Diewald Date: Thu, 18 Jun 2026 18:44:05 +0200 Subject: [PATCH 014/311] UBUNTU: [Packaging] update annotations scripts BugLink: https://bugs.launchpad.net/bugs/1786013 Signed-off-by: Manuel Diewald --- debian/scripts/misc/kconfig/annotations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/scripts/misc/kconfig/annotations.py b/debian/scripts/misc/kconfig/annotations.py index 5f34410d4ef87..ba9db44964708 100644 --- a/debian/scripts/misc/kconfig/annotations.py +++ b/debian/scripts/misc/kconfig/annotations.py @@ -80,7 +80,7 @@ def _parse_body(self, data: str, parent=True): if not line: continue - # Catpure flavors of included files + # Capture flavors of included files if line.startswith("# FLAVOUR: "): self.include_flavour += line.split(" ")[2:] continue @@ -211,7 +211,7 @@ def _json_parse(self, data, is_included=False): self.include = data["attributes"]["include"] self.include_flavour = [] else: - # We are procesing an imported annotations, so merge all the + # We are processing an imported annotations, so merge all the # configs and attributes. try: self.config = data["config"] | self.config From 960837d10f1aaa5de742218384a5fad5d15ae6d6 Mon Sep 17 00:00:00 2001 From: Manuel Diewald Date: Thu, 18 Jun 2026 18:44:09 +0200 Subject: [PATCH 015/311] UBUNTU: Start new release Ignore: yes Signed-off-by: Manuel Diewald --- debian.master/changelog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debian.master/changelog b/debian.master/changelog index a323b50416a91..f65e8342b96ef 100644 --- a/debian.master/changelog +++ b/debian.master/changelog @@ -1,3 +1,11 @@ +linux (7.0.0-27.27) UNRELEASED; urgency=medium + + CHANGELOG: Do not edit directly. Autogenerated at release. + CHANGELOG: Use the printchanges target to see the current changes. + CHANGELOG: Use the insertchanges target to create the final log. + + -- Manuel Diewald Thu, 18 Jun 2026 18:44:08 +0200 + linux (7.0.0-26.26) resolute; urgency=medium * resolute/linux: 7.0.0-26.26 -proposed tracker (LP: #2154530) From 9487e91776770ef04653de0180865cca9ec7a053 Mon Sep 17 00:00:00 2001 From: Manuel Diewald Date: Thu, 18 Jun 2026 18:49:59 +0200 Subject: [PATCH 016/311] UBUNTU: link-to-tracker: update tracking bug BugLink: https://bugs.launchpad.net/bugs/2157114 Properties: no-test-build Signed-off-by: Manuel Diewald --- debian.master/tracking-bug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian.master/tracking-bug b/debian.master/tracking-bug index afdbf0d4c9bc1..57fff0be3022b 100644 --- a/debian.master/tracking-bug +++ b/debian.master/tracking-bug @@ -1 +1 @@ -2154530 2026.05.18-6 +2157114 s2026.05.18-1 From 01543ba213ac86680193399096806a6e265e1cdf Mon Sep 17 00:00:00 2001 From: Manuel Diewald Date: Thu, 18 Jun 2026 18:54:56 +0200 Subject: [PATCH 017/311] UBUNTU: Ubuntu-7.0.0-27.27 Signed-off-by: Manuel Diewald --- debian.master/changelog | 50 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/debian.master/changelog b/debian.master/changelog index f65e8342b96ef..6bc92ed5dac8c 100644 --- a/debian.master/changelog +++ b/debian.master/changelog @@ -1,10 +1,50 @@ -linux (7.0.0-27.27) UNRELEASED; urgency=medium +linux (7.0.0-27.27) resolute; urgency=medium - CHANGELOG: Do not edit directly. Autogenerated at release. - CHANGELOG: Use the printchanges target to see the current changes. - CHANGELOG: Use the insertchanges target to create the final log. + * resolute/linux: 7.0.0-27.27 -proposed tracker (LP: #2157114) - -- Manuel Diewald Thu, 18 Jun 2026 18:44:08 +0200 + * Packaging resync (LP: #1786013) + - [Packaging] update annotations scripts + + * Ubuntu 26.04 linux kernel has non-functional nova-core GPU driver enabled, + conflicting with nouveau (LP: #2150845) + - [Config] Disable NOVA_CORE + + * CVE-2026-46316 + - KVM: arm64: vgic-its: Drop the translation cache reference only for the + erased entry + + * CVE-2026-46244 + - netfilter: nft_inner: Fix IPv6 inner_thoff desync + + * CVE-2026-46137 + - mptcp: pm: ADD_ADDR rtx: allow ID 0 + - mptcp: pm: ADD_ADDR rtx: fix potential data-race + + * CVE-2026-46185 + - smb/client: fix out-of-bounds read in symlink_data() + + * CVE-2026-46195 + - smb: client: validate dacloffset before building DACL pointers + + * CVE-2026-46289 + - lib/scatterlist: fix length calculations in extract_kvec_to_sg + + * CVE-2026-46119 + - libceph: Fix slab-out-of-bounds access in auth message processing + + * CVE-2026-46135 + - nvmet-tcp: fix race between ICReq handling and queue teardown + + * CVE-2026-46155 + - smb/client: fix out-of-bounds read in smb2_compound_op() + + * CVE-2026-46115 + - block: add pgmap check to biovec_phys_mergeable + + * CVE-2026-46243 + - smb: client: reject userspace cifs.spnego descriptions + + -- Manuel Diewald Thu, 18 Jun 2026 18:54:56 +0200 linux (7.0.0-26.26) resolute; urgency=medium From 4338e9a08d21869c6fd291956e820585d8e1a847 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Fri, 27 Feb 2026 14:50:31 -0600 Subject: [PATCH 018/311] UBUNTU: [Packaging] Initialize resolute/linux-nvidia Ignore: yes Signed-off-by: Jacob Martin --- Ubuntu.md | 6 +- debian.nvidia/changelog | 77 ++++++++ debian.nvidia/config/README.rst | 185 ++++++++++++++++++ debian.nvidia/config/annotations | 14 ++ debian.nvidia/control.d/flavour-control.stub | 87 ++++++++ .../control.d/flavour-signed-control.stub | 38 ++++ debian.nvidia/control.d/vars.nvidia | 5 + debian.nvidia/control.d/vars.nvidia-64k | 5 + debian.nvidia/control.stub.in | 99 ++++++++++ debian.nvidia/dkms-versions | 8 + debian.nvidia/etc/update.conf | 7 + debian.nvidia/modprobe.d/common.conf | 3 + debian.nvidia/reconstruct | 34 ++++ debian.nvidia/rules.d/amd64.mk | 20 ++ debian.nvidia/rules.d/arm64.mk | 20 ++ debian.nvidia/tracking-bug | 1 + debian.nvidia/upstream-stable | 3 + debian.nvidia/variants | 4 + debian/debian.env | 2 +- 19 files changed, 614 insertions(+), 4 deletions(-) create mode 100644 debian.nvidia/changelog create mode 100644 debian.nvidia/config/README.rst create mode 100644 debian.nvidia/config/annotations create mode 100644 debian.nvidia/control.d/flavour-control.stub create mode 100644 debian.nvidia/control.d/flavour-signed-control.stub create mode 100644 debian.nvidia/control.d/vars.nvidia create mode 100644 debian.nvidia/control.d/vars.nvidia-64k create mode 100644 debian.nvidia/control.stub.in create mode 100644 debian.nvidia/dkms-versions create mode 100644 debian.nvidia/etc/update.conf create mode 100644 debian.nvidia/modprobe.d/common.conf create mode 100644 debian.nvidia/reconstruct create mode 100644 debian.nvidia/rules.d/amd64.mk create mode 100644 debian.nvidia/rules.d/arm64.mk create mode 100644 debian.nvidia/tracking-bug create mode 100644 debian.nvidia/upstream-stable create mode 100644 debian.nvidia/variants diff --git a/Ubuntu.md b/Ubuntu.md index a49309c1d8c9d..85afdc3463f40 100644 --- a/Ubuntu.md +++ b/Ubuntu.md @@ -1,8 +1,8 @@ -Name: linux +Name: linux-nvidia Version: 7.0.0 Series: 26.04 (resolute) Description: - This is the source code for the Ubuntu linux kernel for the 26.04 series. This - source tree is used to produce the flavours: generic, generic-64k. + This is the source code for the NVIDIA linux kernel for the 26.04 series. This + source tree is used to produce the flavours: nvidia, nvidia-64k. This kernel is configured to support the widest range of desktop, laptop and server configurations. diff --git a/debian.nvidia/changelog b/debian.nvidia/changelog new file mode 100644 index 0000000000000..5f3985aeee68a --- /dev/null +++ b/debian.nvidia/changelog @@ -0,0 +1,77 @@ +linux-nvidia (6.19.0-1000.0) resolute; urgency=medium + + * Initial changelog entry. + + -- Jacob Martin Fri, 27 Feb 2026 14:49:42 -0600 + +linux (6.19.0-6.6) resolute; urgency=medium + + * resolute/linux: 6.19.0-6.6 -proposed tracker (LP: #2142114) + + * Resolute update: v6.19.2 upstream stable release (LP: #2142112) + - Revert "driver core: enforce device_lock for driver_match_device()" + - Linux 6.19.2 + + * Resolute update: v6.19.1 upstream stable release (LP: #2142111) + - io_uring/io-wq: add exit-on-idle state + - io_uring: allow io-wq workers to exit when unused + - smb: client: split cached_fid bitfields to avoid shared-byte RMW races + - ksmbd: fix infinite loop caused by next_smb2_rcv_hdr_off reset in error + paths + - ksmbd: add chann_lock to protect ksmbd_chann_list xarray + - smb: server: fix leak of active_num_conn in ksmbd_tcp_new_connection() + - smb: smbdirect: introduce smbdirect_socket.recv_io.credits.available + - smb: smbdirect: introduce smbdirect_socket.send_io.bcredits.* + - smb: server: make use of smbdirect_socket.recv_io.credits.available + - smb: server: let recv_done() queue a refill when the peer is low on + credits + - smb: server: make use of smbdirect_socket.send_io.bcredits + - smb: server: fix last send credit problem causing disconnects + - smb: server: let send_done handle a completion without IB_SEND_SIGNALED + - smb: client: make use of smbdirect_socket.recv_io.credits.available + - smb: client: let recv_done() queue a refill when the peer is low on + credits + - smb: client: let smbd_post_send() make use of request->wr + - smb: client: remove pointless sc->recv_io.credits.count rollback + - smb: client: remove pointless sc->send_io.pending handling in + smbd_post_send_iter() + - smb: client: port and use the wait_for_credits logic used by server + - smb: client: split out smbd_ib_post_send() + - smb: client: introduce and use smbd_{alloc, free}_send_io() + - smb: client: use smbdirect_send_batch processing + - smb: client: make use of smbdirect_socket.send_io.bcredits + - smb: client: fix last send credit problem causing disconnects + - smb: client: let smbd_post_send_negotiate_req() use smbd_post_send() + - smb: client: let send_done handle a completion without IB_SEND_SIGNALED + - driver core: enforce device_lock for driver_match_device() + - Bluetooth: btusb: Add USB ID 7392:e611 for Edimax EW-7611UXB + - ALSA: hda/conexant: Add quirk for HP ZBook Studio G4 + - crypto: iaa - Fix out-of-bounds index in find_empty_iaa_compression_mode + - crypto: octeontx - Fix length check to avoid truncation in + ucode_load_store + - crypto: omap - Allocate OMAP_CRYPTO_FORCE_COPY scatterlists correctly + - crypto: virtio - Add spinlock protection with virtqueue notification + - crypto: virtio - Remove duplicated virtqueue_kick in + virtio_crypto_skcipher_crypt_req + - nilfs2: Fix potential block overflow that cause system hang + - hfs: ensure sb->s_fs_info is always cleaned up + - wifi: rtw88: Fix alignment fault in rtw_core_enable_beacon() + - scsi: qla2xxx: Validate sp before freeing associated memory + - scsi: qla2xxx: Allow recovery for tape devices + - scsi: qla2xxx: Delay module unload while fabric scan in progress + - scsi: qla2xxx: Free sp in error path to fix system crash + - scsi: qla2xxx: Query FW again before proceeding with login + - sched/mmcid: Don't assume CID is CPU owned on mode switch + - bus: fsl-mc: fix use-after-free in driver_override_show() + - erofs: fix UAF issue for file-backed mounts w/ directio option + - xfs: fix UAF in xchk_btree_check_block_owner + - drm/exynos: vidi: use ctx->lock to protect struct vidi_context member + variables related to memory alloc/free + - PCI: endpoint: Avoid creating sub-groups asynchronously + - wifi: rtl8xxxu: fix slab-out-of-bounds in rtl8xxxu_sta_add + - Linux 6.19.1 + + * AppArmor blocks write(2) to network sockets with Linux 6.19 (LP: #2141298) + - SAUCE: apparmor: fix aa_label_sk_perm to check for RULE_MEDIATES_NET + + -- Timo Aaltonen Wed, 18 Feb 2026 14:31:48 +0200 diff --git a/debian.nvidia/config/README.rst b/debian.nvidia/config/README.rst new file mode 100644 index 0000000000000..751ce7f3b284d --- /dev/null +++ b/debian.nvidia/config/README.rst @@ -0,0 +1,185 @@ +================== +Config Annotations +================== + +:Author: Andrea Righi + +Overview +======== + +Each Ubuntu kernel needs to maintain its own .config for each supported +architecture and each flavour. + +Every time a new patch is applied or a kernel is rebased on top of a new +one, we need to update the .config's accordingly (config options can be +added, removed and also renamed). + +So, we need to make sure that some critical config options are always +matching the desired value in order to have a functional kernel. + +State of the art +================ + +At the moment configs are maintained as a set of Kconfig chunks (inside +`debian./config/`): a global one, plus per-arch / per-flavour +chunks. + +In addition to that, we need to maintain also a file called +'annotations'; the purpose of this file is to make sure that some +critical config options are not silently removed or changed when the +real .config is re-generated (for example after a rebase or after +applying a new set of patches). + +The main problem with this approach is that, often, we have duplicate +information that is stored both in the Kconfig chunks *and* in the +annotations files and, at the same time, the whole .config's information +is distributed between Kconfig chunks and annotations, making it hard to +maintain, review and manage in general. + +Proposed solution +================= + +The proposed solution is to store all the config information into the +"annotations" format and get rid of the config chunks (basically the +real .config's can be produced "compiling" annotations). + +Implementation +============== + +To help the management of the annotations an helper script is provided +(`debian/scripts/misc/annotations`): + +``` +usage: annotations [-h] [--version] [--file FILE] [--arch ARCH] [--flavour FLAVOUR] [--config CONFIG] + (--query | --export | --import FILE | --update FILE | --check FILE) + +Manage Ubuntu kernel .config and annotations + +options: + -h, --help show this help message and exit + --version, -v show program's version number and exit + --file FILE, -f FILE Pass annotations or .config file to be parsed + --arch ARCH, -a ARCH Select architecture + --flavour FLAVOUR, -l FLAVOUR + Select flavour (default is "generic") + --config CONFIG, -c CONFIG + Select a specific config option + +Action: + --query, -q Query annotations + --export, -e Convert annotations to .config format + --import FILE, -i FILE + Import a full .config for a specific arch and flavour into annotations + --update FILE, -u FILE + Import a partial .config into annotations (only resync configs specified in FILE) + --check FILE, -k FILE + Validate kernel .config with annotations +``` + +This script allows to query config settings (per arch/flavour/config), +export them into the Kconfig format (generating the real .config files) +and check if the final .config matches the rules defined in the +annotations. + +Examples (annotations is defined as an alias to `debian/scripts/annotations`): + + - Show settings for `CONFIG_DEBUG_INFO_BTF` for master kernel across all the + supported architectures and flavours: + +``` +$ annotations --query --config CONFIG_DEBUG_INFO_BTF +{ + "policy": { + "amd64": "y", + "arm64": "y", + "armhf": "n", + "ppc64el": "y", + "riscv64": "y", + "s390x": "y" + }, + "note": "'Needs newer pahole for armhf'" +} +``` + + - Dump kernel .config for arm64 and flavour generic-64k: + +``` +$ annotations --arch arm64 --flavour generic-64k --export +CONFIG_DEBUG_FS=y +CONFIG_DEBUG_KERNEL=y +CONFIG_COMPAT=y +... +``` + + - Update annotations file with a new kernel .config for amd64 flavour + generic: + +``` +$ annotations --arch amd64 --flavour generic --import build/.config +``` + +Moreover, an additional kernelconfig commands are provided +(via debian/rules targets): + - `migrateconfigs`: automatically merge all the previous configs into + annotations (local changes still need to be committed) + +Annotations headers +=================== + +The main annotations file should contain a header to define the architectures +and flavours that are supported. + +Here is the format of the header for the generic kernel: +``` +# Menu: HEADER +# FORMAT: 4 +# ARCH: amd64 arm64 armhf ppc64el riscv64 s390x +# FLAVOUR: amd64-generic arm64-generic arm64-generic-64k armhf-generic armhf-generic-lpae ppc64el-generic riscv64-generic s390x-generic + +``` + +Example header of a derivative (linux-aws): +``` +# Menu: HEADER +# FORMAT: 4 +# ARCH: amd64 arm64 +# FLAVOUR: amd64-aws arm64-aws +# FLAVOUR_DEP: {'amd64-aws': 'amd64-generic', 'arm64-aws': 'arm64-generic'} + +include "../../debian.master/config/annotations" + +# Below you can define only the specific linux-aws configs that differ from linux generic + +``` + +Pros and Cons +============= + + Pros: + - avoid duplicate information in .config's and annotations + - allow to easily define groups of config settings (for a specific + environment or feature, such as annotations.clouds, annotations.ubuntu, + annotations.snapd, etc.) + - config options are more accessible, easy to change and review + - we can easily document how config options are managed (and external + contributors won't be discouraged anymore when they need to to change a + config option) + + Cons: + - potential regressions: the new tool/scripts can have potential bugs, + so we could experience regressions due to some missed config changes + - kernel team need to understand the new process (even if everything + is transparent, kernel cranking process is the same, there might be + corner cases that need to be addressed and resolved manually) + +TODO +==== + + - Migrate all flavour and arch definitions into annotations (rather + than having this information defined in multiple places inside + debian/scripts); right now this information is "partially" migrated, + meaning that we need to define arches and flavours in the headers + section of annotations (so that the annotations tool can figure out + the list of supported arches and flavours), but arches and flavours + are still defined elsewhere, ideally we would like to have arches and + flavours defined only in one place: annotations. diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations new file mode 100644 index 0000000000000..66f95c1d5d6bd --- /dev/null +++ b/debian.nvidia/config/annotations @@ -0,0 +1,14 @@ +# Menu: HEADER +# FORMAT: 4 +# ARCH: amd64 arm64 +# FLAVOUR: amd64-nvidia arm64-nvidia arm64-nvidia-64k +# FLAVOUR_DEP: {'amd64-nvidia': 'amd64-generic', 'arm64-nvidia': 'arm64-generic', 'arm64-nvidia-64k': 'arm64-generic-64k'} + +include "../../debian.master/config/annotations" + + +# ---- Annotations without notes ---- + +CONFIG_AS_VERSION policy<{'amd64': '24600', 'arm64': '24600'}> +CONFIG_CC_VERSION_TEXT policy<{'amd64': '"x86_64-linux-gnu-gcc (Ubuntu 15.2.0-14ubuntu1) 15.2.0"', 'arm64': '"aarch64-linux-gnu-gcc (Ubuntu 15.2.0-13ubuntu3) 15.2.0"'}> +CONFIG_LD_VERSION policy<{'amd64': '24600', 'arm64': '24600'}> diff --git a/debian.nvidia/control.d/flavour-control.stub b/debian.nvidia/control.d/flavour-control.stub new file mode 100644 index 0000000000000..4055c769adb64 --- /dev/null +++ b/debian.nvidia/control.d/flavour-control.stub @@ -0,0 +1,87 @@ +# Items that get replaced: +# FLAVOUR +# ARCH +# SUPPORTED +# TARGET +# BOOTLOADER +# =PROVIDES= +# +# Items marked with =FOO= are optional +# +# This file describes the template for packages that are created for each flavour +# in debian/control.d/vars.* +# +# This file gets edited in a couple of places. See the debian/control.stub rule in +# debian/rules. PGGVER, ABINUM, and SRCPKGNAME are all converted in the +# process of creating debian/control. +# +# The flavour specific strings (ARCH, etc) are converted using values from the various +# flavour files in debian/control.d/vars.* +# +# XXX: Leave the blank line before the first package!! + +Package: linux-modules-PKGVER-ABINUM-FLAVOUR +Build-Profiles: +Architecture: ARCH +Section: kernel +Priority: optional +Depends: ${misc:Depends}, ${shlibs:Depends}, wireless-regdb +Built-Using: ${linux:BuiltUsing} +Description: Linux kernel modules for version PKGVER + Contains the corresponding System.map file, the modules built by the + packager, and scripts that try to ensure that the system is not left in an + unbootable state after an update. + . + Supports SUPPORTED processors. + . + TARGET + . + You likely do not want to install this package directly. Instead, install + the linux-FLAVOUR meta-package, which will ensure that upgrades work + correctly, and that supporting packages are also installed. + +Package: linux-headers-PKGVER-ABINUM-FLAVOUR +Build-Profiles: +Architecture: ARCH +Section: devel +Priority: optional +Depends: ${misc:Depends}, SRCPKGNAME-headers-PKGVER-ABINUM, ${shlibs:Depends} +Provides: linux-headers, linux-headers-3.0 +Description: Linux kernel headers for version PKGVER + This package provides kernel header files for version PKGVER. + . + This is for sites that want the latest kernel headers. Please read + /usr/share/doc/linux-headers-PKGVER-ABINUM/debian.README.gz for details. + +Package: linux-lib-rust-PKGVER-ABINUM-FLAVOUR +Build-Profiles: +Architecture: amd64 +Multi-Arch: foreign +Section: devel +Priority: optional +Depends: ${misc:Depends}, coreutils +Description: Rust library files related to Linux kernel version PKGVER + This package provides kernel library files for version PKGVER, that allow to + compile out-of-tree kernel modules written in Rust. + +Package: linux-tools-PKGVER-ABINUM-FLAVOUR +Build-Profiles: +Architecture: ARCH +Section: devel +Priority: optional +Depends: ${misc:Depends}, SRCPKGNAME-tools-PKGVER-ABINUM +Description: Linux kernel version specific tools for version PKGVER-ABINUM + This package provides the architecture dependant parts for kernel + version locked tools (such as x86_energy_perf_policy) for + version PKGVER-ABINUM. + +Package: linux-cloud-tools-PKGVER-ABINUM-FLAVOUR +Build-Profiles: +Architecture: ARCH +Section: devel +Priority: optional +Depends: ${misc:Depends}, SRCPKGNAME-cloud-tools-PKGVER-ABINUM +Description: Linux kernel version specific cloud tools for version PKGVER-ABINUM + This package provides the architecture dependant parts for kernel + version locked tools for cloud for version PKGVER-ABINUM. + diff --git a/debian.nvidia/control.d/flavour-signed-control.stub b/debian.nvidia/control.d/flavour-signed-control.stub new file mode 100644 index 0000000000000..b8551a52e743f --- /dev/null +++ b/debian.nvidia/control.d/flavour-signed-control.stub @@ -0,0 +1,38 @@ +Package: linux-image=SIGN-ME-PKG=-PKGVER-ABINUM-FLAVOUR +Build-Profiles: +Architecture: ARCH +Section: kernel +Priority: optional +Provides: linux-image, fuse-module, =PROVIDES=${linux:rprovides} +Depends: ${misc:Depends}, ${shlibs:Depends}, kmod, linux-base (>= 4.5ubuntu1~16.04.1), linux-modules-PKGVER-ABINUM-FLAVOUR +Recommends: BOOTLOADER, initramfs-tools | linux-initramfs-tool +Breaks: flash-kernel (<< 3.90ubuntu2) [arm64 armhf], s390-tools (<< 2.3.0-0ubuntu3) [s390x] +Conflicts: linux-image=SIGN-PEER-PKG=-PKGVER-ABINUM-FLAVOUR +Suggests: bpftool, linux-perf, SRCPKGNAME-tools, linux-headers-PKGVER-ABINUM-FLAVOUR +Description: Linux kernel image for version PKGVER + This package contains the=SIGN-ME-TXT= Linux kernel image for version PKGVER. + . + Supports SUPPORTED processors. + . + TARGET + . + You likely do not want to install this package directly. Instead, install + the linux-FLAVOUR meta-package, which will ensure that upgrades work + correctly, and that supporting packages are also installed. + +Package: linux-image=SIGN-ME-PKG=-PKGVER-ABINUM-FLAVOUR-dbgsym +Build-Profiles: +Architecture: ARCH +Section: devel +Priority: optional +Depends: ${misc:Depends} +Provides: linux-debug +Description: Linux kernel debug image for version PKGVER + This package provides the=SIGN-ME-TXT= kernel debug image for version PKGVER. + . + This is for sites that wish to debug the kernel. + . + The kernel image contained in this package is NOT meant to boot from. It + is uncompressed, and unstripped. This package also includes the + unstripped modules. + diff --git a/debian.nvidia/control.d/vars.nvidia b/debian.nvidia/control.d/vars.nvidia new file mode 100644 index 0000000000000..1444699fc2bf2 --- /dev/null +++ b/debian.nvidia/control.d/vars.nvidia @@ -0,0 +1,5 @@ +arch="amd64 arm64" +supported="NVIDIA" +target="Intended for NVIDIA platforms" +bootloader="grub-pc [amd64] | grub-efi-amd64 [amd64] | grub-efi-ia32 [amd64] | grub [amd64] | lilo [amd64] | flash-kernel [armhf arm64] | grub-efi-arm64 [arm64] | grub-efi-arm [armhf] | grub-ieee1275 [ppc64el]" +provides="kvm-api-4, redhat-cluster-modules, ivtv-modules, virtualbox-guest-modules [amd64]" diff --git a/debian.nvidia/control.d/vars.nvidia-64k b/debian.nvidia/control.d/vars.nvidia-64k new file mode 100644 index 0000000000000..072ec63f21074 --- /dev/null +++ b/debian.nvidia/control.d/vars.nvidia-64k @@ -0,0 +1,5 @@ +arch="arm64" +supported="NVIDIA 64K pages" +target="Intended for NVIDIA systems" +bootloader="grub-efi-arm64 [arm64] | flash-kernel [arm64]" +provides="kvm-api-4, redhat-cluster-modules, ivtv-modules" diff --git a/debian.nvidia/control.stub.in b/debian.nvidia/control.stub.in new file mode 100644 index 0000000000000..2528e3ae03ab7 --- /dev/null +++ b/debian.nvidia/control.stub.in @@ -0,0 +1,99 @@ +Source: SRCPKGNAME +Section: devel +Priority: optional +Maintainer: Ubuntu Kernel Team +Rules-Requires-Root: no +Standards-Version: 3.9.4.0 +Build-Depends: + autoconf , + automake , + bc , + bindgen:native [amd64 arm64], + bison , + clang-21:native [amd64 arm64], + cpio, + curl , + debhelper-compat (= 10), + default-jdk-headless:native , + dkms , + flex , + gawk , + java-common , + kmod , + libaudit-dev , + libcap-dev , + libdebuginfod-dev [amd64 arm64] , + libdw-dev , + libelf-dev , + libiberty-dev , + liblzma-dev , + libnewt-dev , + libnuma-dev [amd64 arm64] , + libpci-dev , + libssl-dev , + libstdc++-dev, + libtool , + libtraceevent-dev [amd64 arm64] , + libtracefs-dev [amd64 arm64] , + libudev-dev , + libunwind8-dev [amd64 arm64] , + llvm-21-dev, + makedumpfile:native [amd64] , + openssl , + pahole (>= 1.29-2ubuntu2) [amd64 arm64] | dwarves (>= 1.21) [amd64 arm64] , + pkg-config , + python3:native , + python3-dev:native , + libpython3-dev , + python3-setuptools, + rsync [!i386] , + rust-src:native [amd64 arm64], + rustc:native (>= 1.82) [amd64 arm64], + rustfmt:native [amd64 arm64], + uuid-dev , + zstd , + bpftool:native [amd64 arm64] , +Build-Depends-Indep: + asciidoc , + bzip2 , + python3-docutils , + sharutils , + xmlto , +Vcs-Git: git://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-nvidia/+git/=SERIES= +XS-Testsuite: autopkgtest +#XS-Testsuite-Depends: gcc-4.7 binutils + +Package: SRCPKGNAME-headers-PKGVER-ABINUM +Build-Profiles: +Architecture: all +Multi-Arch: foreign +Section: devel +Priority: optional +Depends: ${misc:Depends}, coreutils +Description: Header files related to Linux kernel version PKGVER + This package provides kernel header files for version PKGVER, for sites + that want the latest kernel headers. Please read + /usr/share/doc/SRCPKGNAME-headers-PKGVER-ABINUM/debian.README.gz for details + +Package: SRCPKGNAME-tools-PKGVER-ABINUM +Build-Profiles: +Architecture: amd64 arm64 +Section: devel +Priority: optional +Depends: ${misc:Depends}, ${shlibs:Depends}, linux-tools-common +Description: Linux kernel version specific tools for version PKGVER-ABINUM + This package provides the architecture dependant parts for kernel + version locked tools (such as perf and x86_energy_perf_policy) for + version PKGVER-ABINUM. + You probably want to install linux-tools-PKGVER-ABINUM-. + +Package: SRCPKGNAME-cloud-tools-PKGVER-ABINUM +Build-Profiles: +Architecture: amd64 +Section: devel +Priority: optional +Depends: ${misc:Depends}, ${shlibs:Depends}, linux-cloud-tools-common +Description: Linux kernel version specific cloud tools for version PKGVER-ABINUM + This package provides the architecture dependant parts for kernel + version locked tools for cloud tools for version PKGVER-ABINUM. + You probably want to install linux-cloud-tools-PKGVER-ABINUM-. diff --git a/debian.nvidia/dkms-versions b/debian.nvidia/dkms-versions new file mode 100644 index 0000000000000..5ccbdb3d3e79c --- /dev/null +++ b/debian.nvidia/dkms-versions @@ -0,0 +1,8 @@ +zfs-linux 2.4.0-1ubuntu3 modulename=zfs debpath=pool/universe/z/%package%/zfs-dkms_%version%_all.deb arch=amd64 arch=arm64 arch=ppc64el arch=s390x rprovides=spl-modules rprovides=spl-dkms rprovides=zfs-modules rprovides=zfs-dkms +evdi 1.14.12+dfsg-1ubuntu1 modulename=evdi debpath=pool/universe/e/%package%/evdi-dkms_%version%_all.deb rprovides=evdi-modules rprovides=evdi-dkms type=standalone +ipu6-drivers 0~git202511120800.9766e218-0ubuntu2 modulename=ipu6 debpath=pool/universe/i/%package%/intel-ipu6-dkms_%version%_amd64.deb arch=amd64 rprovides=ipu6-modules rprovides=intel-ipu6-dkms type=standalone +ipu7-drivers 0~git202511120800.fc335577-0ubuntu1 modulename=ipu7 debpath=pool/universe/i/%package%/intel-ipu7-dkms_%version%_amd64.deb arch=amd64 rprovides=ipu7-modules rprovides=intel-ipu7-dkms type=standalone +backport-iwlwifi-dkms 1:0~96.13623-gitd16e74cc-0ubuntu2 modulename=iwlwifi debpath=pool/universe/b/%package%/backport-iwlwifi-dkms_%version%_all.deb arch=amd64 rprovides=iwlwifi-modules rprovides=backport-iwlwifi-dkms type=standalone +v4l2loopback 0.15.3-1ubuntu2 modulename=v4l2loopback debpath=pool/universe/v/%package%/v4l2loopback-dkms_%version%_all.deb arch=amd64 rprovides=v4l2loopback-modules rprovides=v4l2loopback-dkms +usbio-drivers 0~git202510282139.ee221eca-0ubuntu1 modulename=usbio debpath=pool/universe/u/%package%/intel-usbio-dkms_%version%_amd64.deb arch=amd64 rprovides=usbio-modules rprovides=intel-usbio-dkms type=standalone +vision-drivers 0~git202511121832.a8d772f2-0ubuntu1 modulename=vision debpath=pool/universe/v/%package%/intel-vision-dkms_%version%_amd64.deb arch=amd64 rprovides=vision-modules rprovides=intel-vision-dkms type=standalone diff --git a/debian.nvidia/etc/update.conf b/debian.nvidia/etc/update.conf new file mode 100644 index 0000000000000..3917a390a39f5 --- /dev/null +++ b/debian.nvidia/etc/update.conf @@ -0,0 +1,7 @@ +# WARNING: we do not create update.conf when we are not a +# derivative. Various cranky components make use of this. +# If we start unconditionally creating update.conf we need +# to fix at least cranky close and cranky rebase. +RELEASE_REPO=git://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/resolute +SOURCE_RELEASE_BRANCH=master-next +DEBIAN_MASTER=debian.master diff --git a/debian.nvidia/modprobe.d/common.conf b/debian.nvidia/modprobe.d/common.conf new file mode 100644 index 0000000000000..e0fbbd6e060d4 --- /dev/null +++ b/debian.nvidia/modprobe.d/common.conf @@ -0,0 +1,3 @@ +# LP:1434842 -- disable OSS drivers by default to allow pulseaudio to emulate +blacklist snd-mixer-oss +blacklist snd-pcm-oss diff --git a/debian.nvidia/reconstruct b/debian.nvidia/reconstruct new file mode 100644 index 0000000000000..16e52ee71b8a0 --- /dev/null +++ b/debian.nvidia/reconstruct @@ -0,0 +1,34 @@ +# Recreate any symlinks created since the orig. +[ ! -L 'ubuntu/igh-ecat/master/rtdm-ioctl.c' ] && ln -sf 'ioctl.c' 'ubuntu/igh-ecat/master/rtdm-ioctl.c' +chmod +x 'debian/cloud-tools/hv_get_dhcp_info' +chmod +x 'debian/cloud-tools/hv_get_dns_info' +chmod +x 'debian/cloud-tools/hv_set_ifconfig' +chmod +x 'debian/rules' +chmod +x 'debian/scripts/checks/final-checks' +chmod +x 'debian/scripts/checks/module-signature-check' +chmod +x 'debian/scripts/control-create' +chmod +x 'debian/scripts/dkms-build' +chmod +x 'debian/scripts/dkms-build--nvidia-N' +chmod +x 'debian/scripts/dkms-build-configure--zfs' +chmod +x 'debian/scripts/file-downloader' +chmod +x 'debian/scripts/link-headers' +chmod +x 'debian/scripts/link-lib-rust' +chmod +x 'debian/scripts/misc/annotations' +chmod +x 'debian/scripts/misc/find-missing-sauce.sh' +chmod +x 'debian/scripts/misc/gen-auto-reconstruct' +chmod +x 'debian/scripts/misc/git-ubuntu-log' +chmod +x 'debian/scripts/misc/insert-changes' +chmod +x 'debian/scripts/misc/insert-ubuntu-changes' +chmod +x 'debian/scripts/misc/kernelconfig' +chmod +x 'debian/scripts/sign-module' +chmod +x 'debian/templates/extra.postinst.in' +chmod +x 'debian/templates/extra.postrm.in' +chmod +x 'debian/templates/headers.postinst.in' +chmod +x 'debian/templates/image.postinst.in' +chmod +x 'debian/templates/image.postrm.in' +chmod +x 'debian/templates/image.preinst.in' +chmod +x 'debian/templates/image.prerm.in' +chmod +x 'debian/tests/rebuild' +chmod +x 'debian/tests/ubuntu-regression-suite' +# Remove any files deleted from the orig. +exit 0 diff --git a/debian.nvidia/rules.d/amd64.mk b/debian.nvidia/rules.d/amd64.mk new file mode 100644 index 0000000000000..8aa96b3e758cc --- /dev/null +++ b/debian.nvidia/rules.d/amd64.mk @@ -0,0 +1,20 @@ +build_arch = x86 +defconfig = defconfig +flavours = nvidia +build_image = bzImage +kernel_file = arch/$(build_arch)/boot/bzImage +install_file = vmlinuz +vdso = vdso_install +no_dumpfile = true +uefi_signed = true +do_tools_usbip = true +do_tools_cpupower = true +do_tools_perf = true +do_tools_perf_jvmti = true +do_tools_perf_python = true +do_tools_bpftool = true +do_tools_x86 = true +do_tools_hyperv = false +do_tools_rtla = true +do_tools_acpidbg = true +do_lib_rust = false diff --git a/debian.nvidia/rules.d/arm64.mk b/debian.nvidia/rules.d/arm64.mk new file mode 100644 index 0000000000000..f086214eb37ad --- /dev/null +++ b/debian.nvidia/rules.d/arm64.mk @@ -0,0 +1,20 @@ +build_arch = arm64 +defconfig = defconfig +flavours = nvidia nvidia-64k +build_image = vmlinuz.efi +kernel_file = arch/$(build_arch)/boot/vmlinuz.efi +install_file = vmlinuz +no_dumpfile = true +uefi_signed = true + +vdso = vdso_install + +do_tools_usbip = true +do_tools_cpupower = true +do_tools_perf = true +do_tools_perf_jvmti = true +do_tools_perf_python = true +do_tools_bpftool = true +do_tools_rtla = true + +do_dtbs = true diff --git a/debian.nvidia/tracking-bug b/debian.nvidia/tracking-bug new file mode 100644 index 0000000000000..eaf24103d1343 --- /dev/null +++ b/debian.nvidia/tracking-bug @@ -0,0 +1 @@ +2142114 d2026.02.16-1 diff --git a/debian.nvidia/upstream-stable b/debian.nvidia/upstream-stable new file mode 100644 index 0000000000000..30e10948e88f9 --- /dev/null +++ b/debian.nvidia/upstream-stable @@ -0,0 +1,3 @@ +# The following upstream stable releases have been ported: +[upstream-stable] + linux-6.17.y = v6.17.1 diff --git a/debian.nvidia/variants b/debian.nvidia/variants new file mode 100644 index 0000000000000..6606318691bcd --- /dev/null +++ b/debian.nvidia/variants @@ -0,0 +1,4 @@ +-6.19 +-- +-hwe-24.04 +-hwe-24.04-edge diff --git a/debian/debian.env b/debian/debian.env index be31a0c270197..2a9c07b235b89 100644 --- a/debian/debian.env +++ b/debian/debian.env @@ -1 +1 @@ -DEBIAN=debian.master +DEBIAN=debian.nvidia From 2321f850cf98bc39f3d26546a6e75de26127df88 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Tue, 3 Mar 2026 10:54:50 -0600 Subject: [PATCH 019/311] UBUNTU: Start new release Ignore: yes Signed-off-by: Jacob Martin --- debian.nvidia/changelog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debian.nvidia/changelog b/debian.nvidia/changelog index 5f3985aeee68a..1195a150e741c 100644 --- a/debian.nvidia/changelog +++ b/debian.nvidia/changelog @@ -1,3 +1,11 @@ +linux-nvidia (6.19.0-1001.1) UNRELEASED; urgency=medium + + CHANGELOG: Do not edit directly. Autogenerated at release. + CHANGELOG: Use the printchanges target to see the current changes. + CHANGELOG: Use the insertchanges target to create the final log. + + -- Jacob Martin Tue, 03 Mar 2026 10:54:50 -0600 + linux-nvidia (6.19.0-1000.0) resolute; urgency=medium * Initial changelog entry. From abbc2e231b9197ac23ef3ef2e1373a3712e02c40 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Tue, 3 Mar 2026 11:01:18 -0600 Subject: [PATCH 020/311] UBUNTU: Ubuntu-nvidia-6.19.0-1001.1 Signed-off-by: Jacob Martin --- debian.nvidia/changelog | 76 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/debian.nvidia/changelog b/debian.nvidia/changelog index 1195a150e741c..bfbe572a8b282 100644 --- a/debian.nvidia/changelog +++ b/debian.nvidia/changelog @@ -1,10 +1,76 @@ -linux-nvidia (6.19.0-1001.1) UNRELEASED; urgency=medium +linux-nvidia (6.19.0-1001.1) resolute; urgency=medium - CHANGELOG: Do not edit directly. Autogenerated at release. - CHANGELOG: Use the printchanges target to see the current changes. - CHANGELOG: Use the insertchanges target to create the final log. + * linux-tools: consider linking perf against LLVM (LP: #2138328) + - [Packaging] Add llvm-21-dev to build-depends for perf - -- Jacob Martin Tue, 03 Mar 2026 10:54:50 -0600 + [ Ubuntu: 6.19.0-6.6 ] + + * resolute/linux: 6.19.0-6.6 -proposed tracker (LP: #2142114) + * Resolute update: v6.19.2 upstream stable release (LP: #2142112) + - Revert "driver core: enforce device_lock for driver_match_device()" + - Linux 6.19.2 + * Resolute update: v6.19.1 upstream stable release (LP: #2142111) + - io_uring/io-wq: add exit-on-idle state + - io_uring: allow io-wq workers to exit when unused + - smb: client: split cached_fid bitfields to avoid shared-byte RMW races + - ksmbd: fix infinite loop caused by next_smb2_rcv_hdr_off reset in error + paths + - ksmbd: add chann_lock to protect ksmbd_chann_list xarray + - smb: server: fix leak of active_num_conn in ksmbd_tcp_new_connection() + - smb: smbdirect: introduce smbdirect_socket.recv_io.credits.available + - smb: smbdirect: introduce smbdirect_socket.send_io.bcredits.* + - smb: server: make use of smbdirect_socket.recv_io.credits.available + - smb: server: let recv_done() queue a refill when the peer is low on + credits + - smb: server: make use of smbdirect_socket.send_io.bcredits + - smb: server: fix last send credit problem causing disconnects + - smb: server: let send_done handle a completion without IB_SEND_SIGNALED + - smb: client: make use of smbdirect_socket.recv_io.credits.available + - smb: client: let recv_done() queue a refill when the peer is low on + credits + - smb: client: let smbd_post_send() make use of request->wr + - smb: client: remove pointless sc->recv_io.credits.count rollback + - smb: client: remove pointless sc->send_io.pending handling in + smbd_post_send_iter() + - smb: client: port and use the wait_for_credits logic used by server + - smb: client: split out smbd_ib_post_send() + - smb: client: introduce and use smbd_{alloc, free}_send_io() + - smb: client: use smbdirect_send_batch processing + - smb: client: make use of smbdirect_socket.send_io.bcredits + - smb: client: fix last send credit problem causing disconnects + - smb: client: let smbd_post_send_negotiate_req() use smbd_post_send() + - smb: client: let send_done handle a completion without IB_SEND_SIGNALED + - driver core: enforce device_lock for driver_match_device() + - Bluetooth: btusb: Add USB ID 7392:e611 for Edimax EW-7611UXB + - ALSA: hda/conexant: Add quirk for HP ZBook Studio G4 + - crypto: iaa - Fix out-of-bounds index in find_empty_iaa_compression_mode + - crypto: octeontx - Fix length check to avoid truncation in + ucode_load_store + - crypto: omap - Allocate OMAP_CRYPTO_FORCE_COPY scatterlists correctly + - crypto: virtio - Add spinlock protection with virtqueue notification + - crypto: virtio - Remove duplicated virtqueue_kick in + virtio_crypto_skcipher_crypt_req + - nilfs2: Fix potential block overflow that cause system hang + - hfs: ensure sb->s_fs_info is always cleaned up + - wifi: rtw88: Fix alignment fault in rtw_core_enable_beacon() + - scsi: qla2xxx: Validate sp before freeing associated memory + - scsi: qla2xxx: Allow recovery for tape devices + - scsi: qla2xxx: Delay module unload while fabric scan in progress + - scsi: qla2xxx: Free sp in error path to fix system crash + - scsi: qla2xxx: Query FW again before proceeding with login + - sched/mmcid: Don't assume CID is CPU owned on mode switch + - bus: fsl-mc: fix use-after-free in driver_override_show() + - erofs: fix UAF issue for file-backed mounts w/ directio option + - xfs: fix UAF in xchk_btree_check_block_owner + - drm/exynos: vidi: use ctx->lock to protect struct vidi_context member + variables related to memory alloc/free + - PCI: endpoint: Avoid creating sub-groups asynchronously + - wifi: rtl8xxxu: fix slab-out-of-bounds in rtl8xxxu_sta_add + - Linux 6.19.1 + * AppArmor blocks write(2) to network sockets with Linux 6.19 (LP: #2141298) + - SAUCE: apparmor: fix aa_label_sk_perm to check for RULE_MEDIATES_NET + + -- Jacob Martin Tue, 03 Mar 2026 11:01:18 -0600 linux-nvidia (6.19.0-1000.0) resolute; urgency=medium From 5425b06bdf944fd441ca1dea9342f1fa9fd21c72 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Tue, 10 Mar 2026 10:56:27 -0500 Subject: [PATCH 021/311] UBUNTU: Start new release Ignore: yes Signed-off-by: Jacob Martin --- debian.nvidia/changelog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debian.nvidia/changelog b/debian.nvidia/changelog index bfbe572a8b282..ddea5556ec148 100644 --- a/debian.nvidia/changelog +++ b/debian.nvidia/changelog @@ -1,3 +1,11 @@ +linux-nvidia (7.0.0-1003.3) UNRELEASED; urgency=medium + + CHANGELOG: Do not edit directly. Autogenerated at release. + CHANGELOG: Use the printchanges target to see the current changes. + CHANGELOG: Use the insertchanges target to create the final log. + + -- Jacob Martin Tue, 10 Mar 2026 10:56:27 -0500 + linux-nvidia (6.19.0-1001.1) resolute; urgency=medium * linux-tools: consider linking perf against LLVM (LP: #2138328) From ad5d410f0e584498347b448cfa07e338b21d1cf6 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Tue, 8 Apr 2025 09:00:03 -0500 Subject: [PATCH 022/311] UBUNTU: [Config] nvidia-6.14: import misc configs from noble:linux-nvidia Ignore: yes Signed-off-by: Jacob Martin (cherry picked from commit 1a32c7f18ea7bf2799f3413ecdae10e9de21da74 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 0d1a2deae660bfb5061640b96fe7a67402e63750 noble:linux-nvidia-6.17) [jacobmartin: dropped uses of CONFIG_PREEMPT_NONE / CONFIG_PREEMPT_VOLUNTARY, these have been disabled upstream for arm64 and amd64 arches by commit 7dadeaa6e851 ("sched: Further restrict the preemption modes") in favor of CONFIG_PREEMPT_LAZY, which is default in the parent kernel.] Signed-off-by: Jacob Martin --- debian.nvidia/config/annotations | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index 66f95c1d5d6bd..2fb4abe2cc9af 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -6,9 +6,34 @@ include "../../debian.master/config/annotations" +CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND policy<{'arm64': 'n'}> +CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND note<'required for NVIDIA workloads'> + +CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE policy<{'amd64': 'n', 'arm64': 'y'}> +CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE note<'required for NVIDIA workloads'> + +CONFIG_MTD policy<{'amd64': 'm', 'arm64': 'y'}> +CONFIG_MTD note<'Essential for boot on ARM64'> + +CONFIG_NR_CPUS policy<{'amd64': '8192', 'arm64': '512'}> +CONFIG_NR_CPUS note<'LP: #1864198'> + +CONFIG_SPI_TEGRA210_QUAD policy<{'arm64': 'y'}> +CONFIG_SPI_TEGRA210_QUAD note<'Ensures the TPM is available before the IMA driver initializes'> + +CONFIG_TCG_TIS_SPI policy<{'amd64': 'm', 'arm64': 'y'}> +CONFIG_TCG_TIS_SPI note<'Ensures the TPM is available before the IMA driver initializes'> + +CONFIG_UBUNTU_ODM_DRIVERS policy<{'amd64': 'n', 'arm64': 'n'}> +CONFIG_UBUNTU_ODM_DRIVERS note<'Disable all Ubuntu ODM drivers'> + # ---- Annotations without notes ---- -CONFIG_AS_VERSION policy<{'amd64': '24600', 'arm64': '24600'}> -CONFIG_CC_VERSION_TEXT policy<{'amd64': '"x86_64-linux-gnu-gcc (Ubuntu 15.2.0-14ubuntu1) 15.2.0"', 'arm64': '"aarch64-linux-gnu-gcc (Ubuntu 15.2.0-13ubuntu3) 15.2.0"'}> -CONFIG_LD_VERSION policy<{'amd64': '24600', 'arm64': '24600'}> +CONFIG_BCH policy<{'amd64': 'm', 'arm64': 'y'}> +CONFIG_CC_VERSION_TEXT policy<{'amd64': '"x86_64-linux-gnu-gcc (Ubuntu 15.2.0-15ubuntu1) 15.2.0"', 'arm64': '"aarch64-linux-gnu-gcc (Ubuntu 15.2.0-15ubuntu1) 15.2.0"'}> +CONFIG_GPIO_AAEON policy<{'amd64': '-'}> +CONFIG_LEDS_AAEON policy<{'amd64': '-'}> +CONFIG_MFD_AAEON policy<{'amd64': '-'}> +CONFIG_MTD_NAND_CORE policy<{'amd64': 'm', 'arm64': 'y'}> +CONFIG_SENSORS_AAEON policy<{'amd64': '-'}> From 3e4cf1bce6bee45547336f814f529eddaebe77ea Mon Sep 17 00:00:00 2001 From: Brad Figg Date: Fri, 5 Jan 2024 08:18:39 -0800 Subject: [PATCH 023/311] UBUNTU: [Packaging] dkms-versions standalone provides support Add support for exposing rprovides data for standalone modules too. Switch to exposing provides as a shared debian/substvar file and use that in the templates. Ignore: yes Signed-off-by: Brad Figg Signed-off-by: Ian May (cherry picked from commit afacdda832a97ab283d8e88d64fae1d7ce5b7060 noble:linux-nvidia/main-next) Signed-off-by: Jacob Martin (cherry picked from commit 52ba185348888cbe7f723260f751a552dfb3a78a) (cherry picked from commit 52ba18534888 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 8f0710a888eff2b25f67a6599377625f3fc54d7a noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- debian/control.d/flavour-module.stub | 1 + debian/rules | 4 ++++ debian/rules.d/2-binary-arch.mk | 3 +-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/debian/control.d/flavour-module.stub b/debian/control.d/flavour-module.stub index 2810f83bb361f..4aa9ddbe76b95 100644 --- a/debian/control.d/flavour-module.stub +++ b/debian/control.d/flavour-module.stub @@ -4,6 +4,7 @@ Build-Profiles: Architecture: ARCH Section: kernel Priority: optional +Provides: ${MODULE:rprovides} Depends: ${misc:Depends}, linux-image-PKGVER-ABINUM-FLAVOUR | linux-image-unsigned-PKGVER-ABINUM-FLAVOUR, diff --git a/debian/rules b/debian/rules index 4f87ba2dcf796..7ab8e690e038a 100755 --- a/debian/rules +++ b/debian/rules @@ -147,6 +147,10 @@ clean: debian/control debian/canonical-certs.pem debian/canonical-revoked-certs. rm -f debian/scripts/fix-filenames + # SUBSTVARS: rprovides for all DKMS packages + echo "linux:rprovides=$(foreach dkms,$(all_built-in_dkms_modules),$(foreach provides,$(dkms_$(dkms)_rprovides),$(provides)$(comma)))" >"debian/substvars" + echo "$(foreach dkms,$(all_standalone_dkms_modules),$(dkms):rprovides=$(foreach provides,$(dkms_$(dkms)_rprovides),$(provides)$(comma))=NL=)" | sed -e "s/=NL= */\n/g" >>"debian/substvars" + .PHONY: distclean distclean: clean rm -rf debian/control debian/changelog diff --git a/debian/rules.d/2-binary-arch.mk b/debian/rules.d/2-binary-arch.mk index a249f782bca8e..7a8bb73afd78b 100644 --- a/debian/rules.d/2-binary-arch.mk +++ b/debian/rules.d/2-binary-arch.mk @@ -499,7 +499,7 @@ define dh_all dh_shlibdeps -p$(1) $(shlibdeps_opts) dh_installdeb -p$(1) dh_installdebconf -p$(1) - $(lockme) dh_gencontrol -p$(1) -- -Vlinux:rprovides='$(rprovides)' $(2) + $(lockme) dh_gencontrol -p$(1) -- -Tdebian/substvars $(2) dh_md5sums -p$(1) dh_builddeb -p$(1) endef @@ -531,7 +531,6 @@ binary-%: pkgcloud = $(cloud_flavour_pkg_name)-$* $(foreach _m,$(all_dkms_modules), \ $(eval binary-%: enable_$(_m) = $$(filter true,$$(call custom_override,do_$(_m),$$*))) \ ) -binary-%: rprovides = $(foreach _m,$(all_built-in_dkms_modules),$(if $(enable_$(_m)),$(foreach _r,$(dkms_$(_m)_rprovides),$(_r)$(comma) ))) binary-%: $(stampdir)/stamp-install-% @echo Debug: $@ dh_testdir From 4fcb7ec5ad5da825e9e87989084e6f23da5bc3bb Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Wed, 11 Mar 2026 11:38:23 -0500 Subject: [PATCH 024/311] Revert "UBUNTU: [Packaging] remove stale debian/dkms-versions scripting" This reverts commit 7a51fffb97c621f80da06bd0c8442359f78d735d. This stale debian/dkms-versions scripting is still used for derivatives of linux without a linux-main-modules package to parse the main package's dkms-versions file for out-of-tree module builds. Ignore: yes Signed-off-by: Jacob Martin --- debian/rules.d/0-common-vars.mk | 47 +++++++++++++++++++++++++++++++++ debian/scripts/control-create | 27 +++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/debian/rules.d/0-common-vars.mk b/debian/rules.d/0-common-vars.mk index 1ea787d668dcc..e68d6eb170198 100644 --- a/debian/rules.d/0-common-vars.mk +++ b/debian/rules.d/0-common-vars.mk @@ -219,3 +219,50 @@ custom_override = $(or $($(1)_$(2)),$($(1))) # selftests that Ubuntu cares about ubuntu_selftests = breakpoints cpu-hotplug efivarfs memfd memory-hotplug mount net ptrace seccomp timers powerpc user ftrace + +# DKMS +all_dkms_modules = + +subst_paired = $(subst $(firstword $(subst =, ,$(1))),$(lastword $(subst =, ,$(1))),$(2)) +recursive_call = $(if $(2),$(call recursive_call,$(1),$(wordlist 2,$(words $(2)),$(2)),$(call $(1),$(firstword $(2)),$(3))),$(3)) + +$(foreach _line,$(shell gawk '{ OFS = "!"; $$1 = $$1; print }' $(DEBIAN)/dkms-versions), \ + $(eval _params = $(subst !, ,$(_line))) \ + $(eval _deb_pkgname = $(firstword $(_params))) \ + $(eval _deb_version = $(word 2,$(_params))) \ + $(if $(filter modulename=%,$(_params)), \ + $(eval _m = $(word 2,$(subst =, ,$(filter modulename=%,$(_params))))) \ + , \ + $(info modulename for $(_deb_pkgname) not specified in dkms-versions. Assume $(_deb_pkgname).) \ + $(eval _m = $(_deb_pkgname)) \ + ) \ + $(eval all_dkms_modules += $(_m)) \ + $(eval dkms_$(_m)_version = $(_deb_version)) \ + $(foreach _p,$(patsubst debpath=%,%,$(filter debpath=%,$(_params))), \ + $(eval dkms_$(_m)_debpath += $(strip \ + $(call recursive_call,subst_paired, \ + %module%=$(_m) \ + %package%=$(_deb_pkgname) \ + %version%=$(lastword $(subst :, ,$(_deb_version))) \ + , \ + $(_p) \ + ) \ + )) \ + ) \ + $(if $(dkms_$(_m)_debpath),,$(error debpath for $(_deb_pkgname) not specified.)) \ + $(if $(filter arch=%,$(_params)), \ + $(eval dkms_$(_m)_archs = $(patsubst arch=%,%,$(filter arch=%,$(_params)))) \ + , \ + $(eval dkms_$(_m)_archs = any) \ + ) \ + $(eval dkms_$(_m)_rprovides = $(patsubst rprovides=%,%,$(filter rprovides=%,$(_params)))) \ + $(eval dkms_$(_m)_type = $(word 1,$(patsubst type=%,%,$(filter type=%,$(_params))) built-in)) \ + $(eval all_$(dkms_$(_m)_type)_dkms_modules += $(_m)) \ + $(if $(filter standalone,$(dkms_$(_m)_type)), \ + $(eval dkms_$(_m)_pkg_name = linux-modules-$(_m)-$(abi_release)) \ + $(eval dkms_$(_m)_subdir = ubuntu) \ + , \ + $(eval dkms_$(_m)_pkg_name = $(mods_pkg_name)) \ + $(eval dkms_$(_m)_subdir = kernel) \ + ) \ +) diff --git a/debian/scripts/control-create b/debian/scripts/control-create index f96da1ec1168c..acdaa36ec5144 100755 --- a/debian/scripts/control-create +++ b/debian/scripts/control-create @@ -100,6 +100,33 @@ gen_per_flavour () { sed "${sed_common_patterns[@]}" \ -e "s/ARCH/${arch}/g" \ "debian/control.d/flavour-buildinfo.stub" + + while read -r package version extras + do + module="$package" + module_type= + + # Module arch parameters are skipped here, so a package section will + # be generated for each flavour, and its Architecture will be set to + # all architectures with that flavour. Even that is being generated, + # it doesn't follow all of them will be built. That's to work-around + # dkms_exclude/dkms_include that manipulates supported architectures + # in $(DEBIAN)/rules.d/$(arch).mk. + for param in $extras; do + case "$param" in + modulename=*) module="${param#modulename=}" ;; + type=*) module_type="${param#type=}" ;; + *) continue ;; + esac + done + + [ "$module_type" = "standalone" ] || continue + + sed "${sed_common_patterns[@]}" \ + -e "s/ARCH/${arch}/g" \ + -e "s/MODULE/${module}/g" \ + debian/control.d/flavour-module.stub + done < "${DEBIAN}/dkms-versions" } gen_common From 08630e55658a453bb7f7495c7d66fa3e151ea927 Mon Sep 17 00:00:00 2001 From: Ian May Date: Thu, 21 Mar 2024 17:12:26 -0500 Subject: [PATCH 025/311] UBUNTU: [Packaging] add versioning to dkms standalone rprovides When nvidia-fs-dkms is available as a dkms package, we want to default to using the signed modules if possible. Adding a version number for the nvidia-fs modules package enables the inbox modules to be selected over an equivalent dkms version. Ignore: yes Signed-off-by: Ian May (cherry picked from commit 607379d81d95894ef1f2575008242c50ba7c5d72 noble:linux-nvidia/main-next) Signed-off-by: Jacob Martin (cherry picked from commit f6927df081a74a0f84b3ca49b9e699c7bef361a6) (cherry picked from commit f6927df081a7 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 750ba56ba02182b042ce337385880f8e33e0fa11 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- debian/rules | 2 +- debian/rules.d/0-common-vars.mk | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/debian/rules b/debian/rules index 7ab8e690e038a..ec843ccde5ef5 100755 --- a/debian/rules +++ b/debian/rules @@ -149,7 +149,7 @@ clean: debian/control debian/canonical-certs.pem debian/canonical-revoked-certs. # SUBSTVARS: rprovides for all DKMS packages echo "linux:rprovides=$(foreach dkms,$(all_built-in_dkms_modules),$(foreach provides,$(dkms_$(dkms)_rprovides),$(provides)$(comma)))" >"debian/substvars" - echo "$(foreach dkms,$(all_standalone_dkms_modules),$(dkms):rprovides=$(foreach provides,$(dkms_$(dkms)_rprovides),$(provides)$(comma))=NL=)" | sed -e "s/=NL= */\n/g" >>"debian/substvars" + echo "$(foreach dkms,$(all_standalone_dkms_modules),$(dkms):rprovides=$(strip $(foreach provides,$(dkms_$(dkms)_rprovides),$(provides)$(comma)))=NL=)" | sed -e 's/~(/ (/g' -e 's/, (/ (/g' -e 's/=NL= */\n/g' >>"debian/substvars" .PHONY: distclean distclean: clean diff --git a/debian/rules.d/0-common-vars.mk b/debian/rules.d/0-common-vars.mk index e68d6eb170198..17af44bc5239f 100644 --- a/debian/rules.d/0-common-vars.mk +++ b/debian/rules.d/0-common-vars.mk @@ -255,7 +255,8 @@ $(foreach _line,$(shell gawk '{ OFS = "!"; $$1 = $$1; print }' $(DEBIAN)/dkms-ve , \ $(eval dkms_$(_m)_archs = any) \ ) \ - $(eval dkms_$(_m)_rprovides = $(patsubst rprovides=%,%,$(filter rprovides=%,$(_params)))) \ + $(eval _rprovides_raw = $(filter rprovides=%,$(_params))) \ + $(eval dkms_$(_m)_rprovides = $(patsubst rprovides=%,%,$(_rprovides_raw))) \ $(eval dkms_$(_m)_type = $(word 1,$(patsubst type=%,%,$(filter type=%,$(_params))) built-in)) \ $(eval all_$(dkms_$(_m)_type)_dkms_modules += $(_m)) \ $(if $(filter standalone,$(dkms_$(_m)_type)), \ From 29300a7426b9ec33c9a54d1977dfcd049196d2f6 Mon Sep 17 00:00:00 2001 From: Brad Figg Date: Thu, 4 Apr 2024 11:22:16 -0700 Subject: [PATCH 026/311] NVIDIA: [Config]: Disable the NOUVEAU driver which is not used with -nvidia kernels BugLink: https://bugs.launchpad.net/bugs/2060327 Signed-off-by: Brad Figg Acked-by: Brad Figg Signed-off-by: Ian May [jacobmartin: Add note to changed configs] Signed-off-by: Jacob Martin (cherry picked from commit 9b2615a63f73 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit e3b5061c441ebca291c15b99260894d7eeb9f034 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- debian.nvidia/config/annotations | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index 2fb4abe2cc9af..743a202d78fdd 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -12,9 +12,39 @@ CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND note<'required for NVIDIA worklo CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE policy<{'amd64': 'n', 'arm64': 'y'}> CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE note<'required for NVIDIA workloads'> +CONFIG_DRM_NOUVEAU policy<{'amd64': 'n', 'arm64': 'n'}> +CONFIG_DRM_NOUVEAU note<'Disable nouveau for NVIDIA kernels'> + +CONFIG_DRM_NOUVEAU_BACKLIGHT policy<{'amd64': '-', 'arm64': '-'}> +CONFIG_DRM_NOUVEAU_BACKLIGHT note<'Disable nouveau for NVIDIA kernels'> + +CONFIG_DRM_NOUVEAU_CH7006 policy<{'amd64': '-', 'arm64': '-'}> +CONFIG_DRM_NOUVEAU_CH7006 note<'Disable nouveau for NVIDIA kernels'> + +CONFIG_DRM_NOUVEAU_SIL164 policy<{'amd64': '-', 'arm64': '-'}> +CONFIG_DRM_NOUVEAU_SIL164 note<'Disable nouveau for NVIDIA kernels'> + +CONFIG_DRM_NOUVEAU_SVM policy<{'amd64': '-', 'arm64': '-'}> +CONFIG_DRM_NOUVEAU_SVM note<'Disable nouveau for NVIDIA kernels'> + CONFIG_MTD policy<{'amd64': 'm', 'arm64': 'y'}> CONFIG_MTD note<'Essential for boot on ARM64'> +CONFIG_NOUVEAU_DEBUG policy<{'amd64': '-', 'arm64': '-'}> +CONFIG_NOUVEAU_DEBUG note<'Disable nouveau for NVIDIA kernels'> + +CONFIG_NOUVEAU_DEBUG_DEFAULT policy<{'amd64': '-', 'arm64': '-'}> +CONFIG_NOUVEAU_DEBUG_DEFAULT note<'Disable nouveau for NVIDIA kernels'> + +CONFIG_NOUVEAU_DEBUG_MMU policy<{'amd64': '-', 'arm64': '-'}> +CONFIG_NOUVEAU_DEBUG_MMU note<'Disable nouveau for NVIDIA kernels'> + +CONFIG_NOUVEAU_DEBUG_PUSH policy<{'amd64': '-', 'arm64': '-'}> +CONFIG_NOUVEAU_DEBUG_PUSH note<'Disable nouveau for NVIDIA kernels'> + +CONFIG_NOUVEAU_PLATFORM_DRIVER policy<{'arm64': '-'}> +CONFIG_NOUVEAU_PLATFORM_DRIVER note<'Disable nouveau for NVIDIA kernels'> + CONFIG_NR_CPUS policy<{'amd64': '8192', 'arm64': '512'}> CONFIG_NR_CPUS note<'LP: #1864198'> From 93bd7b10f046fd9be6726ab2241a57187dab42fa Mon Sep 17 00:00:00 2001 From: Brad Figg Date: Fri, 5 Apr 2024 11:57:09 -0700 Subject: [PATCH 027/311] NVIDIA: [Config]: Adding CORESIGHT and ARM64_ERRATUM configs to annotations BugLink: https://bugs.launchpad.net/bugs/2060327 Signed-off-by: Brad Figg Acked-by: Brad Figg Signed-off-by: Ian May [jacobmartin: Add annotations note for changed configs] Signed-off-by: Jacob Martin (cherry picked from commit 3d31ea05380f noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 6de4075392d6b80387919934a10ac91c7fdfebb4 noble:linux-nvidia-6.17) [jacobmartin: set new CoreSight configs: - CONFIG_CORESIGHT_TNOC=m - CONFIG_CORESIGHT_CTCU=m] Signed-off-by: Jacob Martin --- debian.nvidia/config/annotations | 93 ++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index 743a202d78fdd..07524eed867db 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -6,6 +6,87 @@ include "../../debian.master/config/annotations" +CONFIG_ARM64_ERRATUM_1902691 policy<{'arm64': 'y'}> +CONFIG_ARM64_ERRATUM_1902691 note<'Required for Grace enablement'> + +CONFIG_ARM64_ERRATUM_2038923 policy<{'arm64': 'y'}> +CONFIG_ARM64_ERRATUM_2038923 note<'Required for Grace enablement'> + +CONFIG_ARM64_ERRATUM_2064142 policy<{'arm64': 'y'}> +CONFIG_ARM64_ERRATUM_2064142 note<'Required for Grace enablement'> + +CONFIG_ARM64_ERRATUM_2119858 policy<{'arm64': 'y'}> +CONFIG_ARM64_ERRATUM_2119858 note<'Required for Grace enablement'> + +CONFIG_ARM64_ERRATUM_2139208 policy<{'arm64': 'y'}> +CONFIG_ARM64_ERRATUM_2139208 note<'Required for Grace enablement'> + +CONFIG_ARM64_ERRATUM_2224489 policy<{'arm64': 'y'}> +CONFIG_ARM64_ERRATUM_2224489 note<'Required for Grace enablement'> + +CONFIG_ARM64_ERRATUM_2253138 policy<{'arm64': 'y'}> +CONFIG_ARM64_ERRATUM_2253138 note<'Required for Grace enablement'> + +CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE policy<{'arm64': 'y'}> +CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE note<'Required for Grace enablement'> + +CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE policy<{'arm64': 'y'}> +CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE note<'Required for Grace enablement'> + +CONFIG_CORESIGHT policy<{'arm64': 'm'}> +CONFIG_CORESIGHT note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_CATU policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_CATU note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_CPU_DEBUG policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_CPU_DEBUG note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_CPU_DEBUG_DEFAULT_ON policy<{'arm64': 'n'}> +CONFIG_CORESIGHT_CPU_DEBUG_DEFAULT_ON note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_CTCU policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_CTCU note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_CTI policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_CTI note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_CTI_INTEGRATION_REGS policy<{'arm64': 'n'}> +CONFIG_CORESIGHT_CTI_INTEGRATION_REGS note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_DUMMY policy<{'arm64': 'n'}> +CONFIG_CORESIGHT_DUMMY note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_LINKS_AND_SINKS policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_LINKS_AND_SINKS note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_LINK_AND_SINK_TMC policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_LINK_AND_SINK_TMC note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_SINK_ETBV10 policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_SINK_ETBV10 note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_SINK_TPIU policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_SINK_TPIU note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_SOURCE_ETM4X policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_SOURCE_ETM4X note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_STM policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_STM note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_TNOC policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_TNOC note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_TPDA policy<{'arm64': 'n'}> +CONFIG_CORESIGHT_TPDA note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_TPDM policy<{'arm64': 'n'}> +CONFIG_CORESIGHT_TPDM note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_TRBE policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_TRBE note<'Required for Grace enablement'> + CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND policy<{'arm64': 'n'}> CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND note<'required for NVIDIA workloads'> @@ -27,6 +108,9 @@ CONFIG_DRM_NOUVEAU_SIL164 note<'Disable nouveau for NVIDIA CONFIG_DRM_NOUVEAU_SVM policy<{'amd64': '-', 'arm64': '-'}> CONFIG_DRM_NOUVEAU_SVM note<'Disable nouveau for NVIDIA kernels'> +CONFIG_ETM4X_IMPDEF_FEATURE policy<{'arm64': 'n'}> +CONFIG_ETM4X_IMPDEF_FEATURE note<'Required for Grace enablement'> + CONFIG_MTD policy<{'amd64': 'm', 'arm64': 'y'}> CONFIG_MTD note<'Essential for boot on ARM64'> @@ -48,6 +132,12 @@ CONFIG_NOUVEAU_PLATFORM_DRIVER note<'Disable nouveau for NVIDIA CONFIG_NR_CPUS policy<{'amd64': '8192', 'arm64': '512'}> CONFIG_NR_CPUS note<'LP: #1864198'> +CONFIG_PID_IN_CONTEXTIDR policy<{'arm64': 'y'}> +CONFIG_PID_IN_CONTEXTIDR note<'Required for Grace enablement'> + +CONFIG_SAMPLE_CORESIGHT_SYSCFG policy<{'arm64': 'n'}> +CONFIG_SAMPLE_CORESIGHT_SYSCFG note<'Required for Grace enablement'> + CONFIG_SPI_TEGRA210_QUAD policy<{'arm64': 'y'}> CONFIG_SPI_TEGRA210_QUAD note<'Ensures the TPM is available before the IMA driver initializes'> @@ -57,6 +147,9 @@ CONFIG_TCG_TIS_SPI note<'Ensures the TPM is availab CONFIG_UBUNTU_ODM_DRIVERS policy<{'amd64': 'n', 'arm64': 'n'}> CONFIG_UBUNTU_ODM_DRIVERS note<'Disable all Ubuntu ODM drivers'> +CONFIG_ULTRASOC_SMB policy<{'arm64': 'n'}> +CONFIG_ULTRASOC_SMB note<'Required for Grace enablement'> + # ---- Annotations without notes ---- From 4b4462ad1eb21f7ad28fd4a30628d2b4beec87ca Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Thu, 17 Oct 2024 15:01:53 -0500 Subject: [PATCH 028/311] UBUNTU: [Config] Disable Ubuntu ODM drivers for NVIDIA kernels Ignore: yes Signed-off-by: Jacob Martin (cherry picked from commit 448ddcb3bec6206e2e86f653574c0e7a1ce30fac) (cherry picked from commit 448ddcb3bec6 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit b4e9b917c8c779f30c4873f9a0bc90f2f66a5e1e noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- debian.nvidia/config/annotations | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index 07524eed867db..2b9471014acf1 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -111,6 +111,15 @@ CONFIG_DRM_NOUVEAU_SVM note<'Disable nouveau for NVIDIA CONFIG_ETM4X_IMPDEF_FEATURE policy<{'arm64': 'n'}> CONFIG_ETM4X_IMPDEF_FEATURE note<'Required for Grace enablement'> +CONFIG_GPIO_AAEON policy<{'amd64': '-'}> +CONFIG_GPIO_AAEON note<'Disable all Ubuntu ODM drivers'> + +CONFIG_LEDS_AAEON policy<{'amd64': '-'}> +CONFIG_LEDS_AAEON note<'Disable all Ubuntu ODM drivers'> + +CONFIG_MFD_AAEON policy<{'amd64': '-'}> +CONFIG_MFD_AAEON note<'Disable all Ubuntu ODM drivers'> + CONFIG_MTD policy<{'amd64': 'm', 'arm64': 'y'}> CONFIG_MTD note<'Essential for boot on ARM64'> @@ -138,6 +147,9 @@ CONFIG_PID_IN_CONTEXTIDR note<'Required for Grace enablem CONFIG_SAMPLE_CORESIGHT_SYSCFG policy<{'arm64': 'n'}> CONFIG_SAMPLE_CORESIGHT_SYSCFG note<'Required for Grace enablement'> +CONFIG_SENSORS_AAEON policy<{'amd64': '-'}> +CONFIG_SENSORS_AAEON note<'Disable all Ubuntu ODM drivers'> + CONFIG_SPI_TEGRA210_QUAD policy<{'arm64': 'y'}> CONFIG_SPI_TEGRA210_QUAD note<'Ensures the TPM is available before the IMA driver initializes'> @@ -155,8 +167,4 @@ CONFIG_ULTRASOC_SMB note<'Required for Grace enablem CONFIG_BCH policy<{'amd64': 'm', 'arm64': 'y'}> CONFIG_CC_VERSION_TEXT policy<{'amd64': '"x86_64-linux-gnu-gcc (Ubuntu 15.2.0-15ubuntu1) 15.2.0"', 'arm64': '"aarch64-linux-gnu-gcc (Ubuntu 15.2.0-15ubuntu1) 15.2.0"'}> -CONFIG_GPIO_AAEON policy<{'amd64': '-'}> -CONFIG_LEDS_AAEON policy<{'amd64': '-'}> -CONFIG_MFD_AAEON policy<{'amd64': '-'}> CONFIG_MTD_NAND_CORE policy<{'amd64': 'm', 'arm64': 'y'}> -CONFIG_SENSORS_AAEON policy<{'amd64': '-'}> From 8427aa123967d5930e2217b8b5d9c095c3713d06 Mon Sep 17 00:00:00 2001 From: Ian May Date: Wed, 24 Apr 2024 22:45:17 -0500 Subject: [PATCH 029/311] UBUNTU: [Packaging] blacklist coresight_etm4x BugLink: https://bugs.launchpad.net/bugs/2061930 BugLink: https://bugs.launchpad.net/bugs/2067106 There are systems in production that don't have firmware that supports coresight_etm4x. Instead of removing completely, blacklist coresight_etm4x so systems with the correct firmware can use the module. Signed-off-by: Ian May Signed-off-by: Jamie Nguyen Acked-by: Brad Figg Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off-by: Brad Figg Signed-off-by: Jacob Martin (backported from commit 217d1ae2aad8b33ff247bdab358f9134b90f6d4e noble:linux-nvidia-6.14) [maskedarray: adjusted context] Signed-off-by: Abdur Rahman (cherry picked from commit 3f7d9007f3b47889a2ce7eecf773bf99bf78f1d5 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- debian.nvidia/modprobe.d/common.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/debian.nvidia/modprobe.d/common.conf b/debian.nvidia/modprobe.d/common.conf index e0fbbd6e060d4..619c9a23fe210 100644 --- a/debian.nvidia/modprobe.d/common.conf +++ b/debian.nvidia/modprobe.d/common.conf @@ -1,3 +1,4 @@ # LP:1434842 -- disable OSS drivers by default to allow pulseaudio to emulate blacklist snd-mixer-oss blacklist snd-pcm-oss +blacklist coresight_etm4x From 19d91d3c04a0e8724fb3d61201e7501714451e4c Mon Sep 17 00:00:00 2001 From: Brad Figg Date: Wed, 7 Aug 2024 11:13:22 -0700 Subject: [PATCH 030/311] NVIDIA: [Config] EFI: set CAPSULE_LOADER=y for arm64 BugLink: https://bugs.launchpad.net/bugs/2067111 Nvidia provide a way to flash the UEFI via capsule loader in arm64. CAPSULE_LOADER is also built-in in L4T kernel so for the easy use, need to make CAPSULE_LOADER as built-in in arm64. Nvidia-BugLink: https://nvbugspro.nvidia.com/bug/4601764 Signed-off-by: Brad Figg Acked-by: Jacob Martin Acked-by: Noah Wager (cherry picked from commit efbc80a791437c6bd3f477c656d6d84970f826d8 noble:linux-nvidia-6.11) Signed-off-by: Jacob Martin (cherry picked from commit 58d6077a21f6914b3b274dd1c29757343a29e32b) (cherry picked from commit 58d6077a21f6 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 812ae1ef0031de4d63257648e6d2a4c6708174df noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- debian.nvidia/config/annotations | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index 2b9471014acf1..bbc3a8faf0d49 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -108,6 +108,9 @@ CONFIG_DRM_NOUVEAU_SIL164 note<'Disable nouveau for NVIDIA CONFIG_DRM_NOUVEAU_SVM policy<{'amd64': '-', 'arm64': '-'}> CONFIG_DRM_NOUVEAU_SVM note<'Disable nouveau for NVIDIA kernels'> +CONFIG_EFI_CAPSULE_LOADER policy<{'amd64': 'm', 'arm64': 'y'}> +CONFIG_EFI_CAPSULE_LOADER note<'LP: #2067111'> + CONFIG_ETM4X_IMPDEF_FEATURE policy<{'arm64': 'n'}> CONFIG_ETM4X_IMPDEF_FEATURE note<'Required for Grace enablement'> From 39069497463bac43ac3034bc330f80177d8ff92e Mon Sep 17 00:00:00 2001 From: Ankit Agrawal Date: Wed, 15 Nov 2023 10:27:43 +0000 Subject: [PATCH 031/311] NVIDIA: SAUCE: WAR: iommufd/pages: Bypass PFNMAP BugLink: https://bugs.launchpad.net/bugs/2095028 This is used for GPU memory mapping. The solution is a WAR while waiting for the upstream solution that would use dmabuf to map the entire range in a single sequence. Related topics: https://lore.kernel.org/kvm/20240624065552.1572580-1-vivek.kasireddy@intel.com/ https://lore.kernel.org/kvm/cover.1719909395.git.leon@kernel.org/ Signed-off-by: Ankit Agrawal (cherry picked from commit d3d7b64f1a3274e5df04dee1a8062f54a3fa1116 nvidia/kstable/dev/nic/iommufd_vsmmu-12122024) Signed-off-by: Koba Ko Acked-by: Matt Ochs Acked-by: Brad Figg Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off-by: Brad Figg (cherry picked from commit 15e066a3cc7484e59f1e1c26d651947c98cf42dd noble:linux-nvidia-6.11) Signed-off-by: Jacob Martin (cherry picked from commit 8fcaed8d5824 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit ef306c832155a7f6555dcc428e995f54b5bab4a1 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/iommu/iommufd/pages.c | 81 +++++++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/iommufd/pages.c b/drivers/iommu/iommufd/pages.c index 9b49f0c5b4599..27a1628e92a08 100644 --- a/drivers/iommu/iommufd/pages.c +++ b/drivers/iommu/iommufd/pages.c @@ -711,9 +711,10 @@ static void batch_unpin(struct pfn_batch *batch, struct iopt_pages *pages, size_t to_unpin = min_t(size_t, npages, batch->npfns[cur] - first_page_off); - unpin_user_page_range_dirty_lock( - pfn_to_page(batch->pfns[cur] + first_page_off), - to_unpin, pages->writable); + if (pfn_valid(batch->pfns[cur] + first_page_off)) + unpin_user_page_range_dirty_lock( + pfn_to_page(batch->pfns[cur] + first_page_off), + to_unpin, pages->writable); iopt_pages_sub_npinned(pages, to_unpin); cur++; first_page_off = 0; @@ -873,6 +874,41 @@ static long pin_memfd_pages(struct pfn_reader_user *user, unsigned long start, return npages_out; } +static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, + unsigned long vaddr, unsigned long *pfn, + bool write_fault) +{ + struct follow_pfnmap_args args = { .vma = vma, .address = vaddr }; + int ret; + + ret = follow_pfnmap_start(&args); + if (ret) { + bool unlocked = false; + + ret = fixup_user_fault(mm, vaddr, + FAULT_FLAG_REMOTE | + (write_fault ? FAULT_FLAG_WRITE : 0), + &unlocked); + if (unlocked) + return -EAGAIN; + + if (ret) + return ret; + + ret = follow_pfnmap_start(&args); + if (ret) + return ret; + } + + if (write_fault && !args.writable) + ret = -EFAULT; + else + *pfn = args.pfn; + + follow_pfnmap_end(&args); + return ret; +} + static int pfn_reader_user_pin(struct pfn_reader_user *user, struct iopt_pages *pages, unsigned long start_index, @@ -941,6 +977,42 @@ static int pfn_reader_user_pin(struct pfn_reader_user *user, user->gup_flags, user->upages, &user->locked); } + + if (rc < 0) { + struct vm_area_struct *vma; + unsigned long vaddr; + unsigned long pfn; + int pinned = 0; + + /* fast path above doesn't hold the lock */ + if (!user->locked) + mmap_read_lock(pages->source_mm); + vaddr = untagged_addr_remote(pages->source_mm, uptr); +retry: + vma = vma_lookup(pages->source_mm, vaddr); + if (vma && vma->vm_flags & VM_PFNMAP) { + do { + rc = follow_fault_pfn(vma, pages->source_mm, vaddr, + &pfn, pages->writable); + if (rc == -EAGAIN) + goto retry; + if (!rc) { + if (!pfn_valid(pfn)) { + user->upages[pinned] = pfn_to_page(pfn); + pinned += 1; + vaddr += PAGE_SIZE; + } else { + rc = -EFAULT; + } + } + } while (pinned < npages && vaddr < vma->vm_end && !rc); + } + if (pinned) + rc = pinned; + if (!user->locked) + mmap_read_unlock(pages->source_mm); + } + if (rc <= 0) { if (WARN_ON(!rc)) return -EFAULT; @@ -1313,7 +1385,8 @@ static void pfn_reader_release_pins(struct pfn_reader *pfns) user->upages_start; if (!user->file) { - unpin_user_pages(user->upages + start_index, npages); + if (pfn_valid(page_to_pfn(user->upages[0]))) + unpin_user_pages(user->upages + start_index, npages); } else { long n = user->ufolios_len / sizeof(*user->ufolios); From 002da9cb16c99e8a37495a3fe2f70879823a6761 Mon Sep 17 00:00:00 2001 From: "Matthew R. Ochs" Date: Tue, 16 Jul 2024 01:47:44 +0000 Subject: [PATCH 032/311] NVIDIA: SAUCE: [Config] nvidia: Update annotations for Grace I/O virtualization BugLink: https://bugs.launchpad.net/bugs/2095028 This adds the following config options to annotations: CONFIG_ARM_SMMU_V3_IOMMUFD=y CONFIG_IOMMUFD_DRIVER_CORE=y CONFIG_IOMMUFD_VFIO_CONTAINER=y CONFIG_NVGRACE_GPU_VFIO_PCI=m CONFIG_VFIO_CONTAINER=n CONFIG_VFIO_IOMMU_TYPE1=- For CMA size requirements, the 64K kernel configuration needs 640MB in the worst-case scenario, while the 4K kernel configuration requires 40MB. Due to the current CMA alignment requirement of 512MB on 64k kernel and 128MB on 4k kernel, use each as default For 64k kernel, CONFIG_CMA_SIZE_MBYTES=1024 For 4k kernel, CONFIG_CMA_SIZE_MBYTES=128 These config options has been defined in debian.master CONFIG_IOMMUFD=m CONFIG_IOMMU_IOPF=y Signed-off-by: Matthew R. Ochs Acked-by: Kai-Heng Feng Acked-by: Koba Ko Signed-off-by: Matthew R. Ochs (backported from commit 35a55f343e80627c03640759886aa5d1c732acdf 24.04_linux-nvidia-adv-6.8-next) Signed-off-by: Koba Ko Acked-by: Matt Ochs Acked-by: Brad Figg Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off-by: Brad Figg (backported from commit 1314cf03bfb0510f83cb861c7345a65c8c2e25a9 noble:linux-nvidia-6.11) Signed-off-by: Jacob Martin (cherry picked from commit d09b7e27c860 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (backported from commit 3660ee50e5984e96c615133e96ea90b11651193c noble:linux-nvidia-6.17) [mochs: Removed CONFIG_TEGRA241_CMDQV=n; we want it =y from debian.master] Signed-off-by: Matthew R. Ochs --- debian.nvidia/config/annotations | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index bbc3a8faf0d49..7bfa5bcee00ba 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -33,6 +33,12 @@ CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE note<'Required for Grace enable CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE policy<{'arm64': 'y'}> CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE note<'Required for Grace enablement'> +CONFIG_ARM_SMMU_V3_IOMMUFD policy<{'arm64': 'y'}> +CONFIG_ARM_SMMU_V3_IOMMUFD note<'LP: #2095028'> + +CONFIG_CMA_SIZE_MBYTES policy<{'amd64': '0', 'arm64': '32', 'arm64-nvidia': '128', 'arm64-nvidia-64k': '1024'}> +CONFIG_CMA_SIZE_MBYTES note<'LP: #2095028'> + CONFIG_CORESIGHT policy<{'arm64': 'm'}> CONFIG_CORESIGHT note<'Required for Grace enablement'> @@ -117,6 +123,9 @@ CONFIG_ETM4X_IMPDEF_FEATURE note<'Required for Grace enablem CONFIG_GPIO_AAEON policy<{'amd64': '-'}> CONFIG_GPIO_AAEON note<'Disable all Ubuntu ODM drivers'> +CONFIG_IOMMUFD_VFIO_CONTAINER policy<{'arm64': 'y'}> +CONFIG_IOMMUFD_VFIO_CONTAINER note<'LP: #2095028'> + CONFIG_LEDS_AAEON policy<{'amd64': '-'}> CONFIG_LEDS_AAEON note<'Disable all Ubuntu ODM drivers'> @@ -165,6 +174,12 @@ CONFIG_UBUNTU_ODM_DRIVERS note<'Disable all Ubuntu ODM dri CONFIG_ULTRASOC_SMB policy<{'arm64': 'n'}> CONFIG_ULTRASOC_SMB note<'Required for Grace enablement'> +CONFIG_VFIO_CONTAINER policy<{'amd64': 'y', 'arm64': 'n'}> +CONFIG_VFIO_CONTAINER note<'LP: #2095028'> + +CONFIG_VFIO_IOMMU_TYPE1 policy<{'amd64': 'm', 'arm64': '-'}> +CONFIG_VFIO_IOMMU_TYPE1 note<'LP: #2095028'> + # ---- Annotations without notes ---- From 6ec98a5dee1f21ae2d1f7a39434df5e86869d9f2 Mon Sep 17 00:00:00 2001 From: Yenchia Chen Date: Tue, 11 Feb 2025 10:36:31 +0800 Subject: [PATCH 033/311] NVIDIA: SAUCE: serial: 8250_mtk: Add ACPI support BugLink: https://bugs.launchpad.net/bugs/2096888 Add ACPI support to 8250_mtk driver. This makes it possible to use UART on ARM-based desktops with EDK2 UEFI firmware. Acked-by: Brad Figg Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off-by: Brad Figg (cherry picked from commit 4647186b002bde9bf50ec26db8a776c3f22b6196 noble:linux-nvidia-6.11) Signed-off-by: Jacob Martin (cherry picked from commit d73760e5ac8d noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 072848cc6957a7dea5078840c0b84473b2b14861 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/tty/serial/8250/8250_mtk.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/drivers/tty/serial/8250/8250_mtk.c b/drivers/tty/serial/8250/8250_mtk.c index 5875a7b9b4b10..39e8268cd4b9a 100644 --- a/drivers/tty/serial/8250/8250_mtk.c +++ b/drivers/tty/serial/8250/8250_mtk.c @@ -19,6 +19,7 @@ #include #include #include +#include #include "8250.h" @@ -521,6 +522,7 @@ static int mtk8250_probe(struct platform_device *pdev) struct mtk8250_data *data; struct resource *regs; int irq, err; + struct fwnode_handle *fwnode = dev_fwnode(&pdev->dev); irq = platform_get_irq(pdev, 0); if (irq < 0) @@ -543,12 +545,13 @@ static int mtk8250_probe(struct platform_device *pdev) data->clk_count = 0; - if (pdev->dev.of_node) { + if (is_of_node(fwnode)) { err = mtk8250_probe_of(pdev, &uart.port, data); if (err) return err; - } else + } else if (!fwnode) { return -ENODEV; + } spin_lock_init(&uart.port.lock); uart.port.mapbase = regs->start; @@ -564,14 +567,18 @@ static int mtk8250_probe(struct platform_device *pdev) uart.port.startup = mtk8250_startup; uart.port.set_termios = mtk8250_set_termios; uart.port.uartclk = clk_get_rate(data->uart_clk); + if (!uart.port.uartclk) + uart.port.uartclk = 26 * HZ_PER_MHZ; #ifdef CONFIG_SERIAL_8250_DMA if (data->dma) uart.dma = data->dma; #endif - /* Disable Rate Fix function */ - writel(0x0, uart.port.membase + + if (is_of_node(fwnode)) { + /* Disable Rate Fix function */ + writel(0x0, uart.port.membase + (MTK_UART_RATE_FIX << uart.port.regshift)); + } platform_set_drvdata(pdev, data); @@ -649,11 +656,18 @@ static const struct of_device_id mtk8250_of_match[] = { }; MODULE_DEVICE_TABLE(of, mtk8250_of_match); +static const struct acpi_device_id mtk8250_acpi_match[] = { + { "MTKI0511" }, + {} +}; +MODULE_DEVICE_TABLE(acpi, mtk8250_acpi_match); + static struct platform_driver mtk8250_platform_driver = { .driver = { .name = "mt6577-uart", .pm = &mtk8250_pm_ops, .of_match_table = mtk8250_of_match, + .acpi_match_table = mtk8250_acpi_match, }, .probe = mtk8250_probe, .remove = mtk8250_remove, From b566a8248c379a96d3c69d622273a03affc2e5c2 Mon Sep 17 00:00:00 2001 From: Brad Figg Date: Tue, 11 Feb 2025 11:49:29 -0800 Subject: [PATCH 034/311] NVIDIA: SAUCE: Adds MT7925 BT devices BugLink: https://bugs.launchpad.net/bugs/2096882 Acked-by: Brad Figg Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off-by: Brad Figg (backported from commit a99eb0fc56550d106e276c1847db19996af18336 noble:linux-nvidia-6.11) [jacobmartin: Drop addition of 13d3:3604 already added by upstream commit f9685f315fd ("Bluetooth: btusb: Add MediaTek MT7925-B22M support ID 0x13d3:0x3604"). Drop driver_info flag "BTUSB_VALID_LE_STATES" as it was inverted by upstream commit 0fec656d08a ("Bluetooth: btusb: Invert LE State flag to set invalid rather then valid")] Signed-off-by: Jacob Martin (backported from commit a1d77cd8297c2bff09fd739a1ac3c0ed95e9e2d4 noble:linux-nvidia-6.14) [maskedarray: adjusted context] Signed-off-by: Abdur Rahman (cherry picked from commit f79eaa905f72fe2eff8774d787c1a84946c7a7fe noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/bluetooth/btusb.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index bb0c65a81da59..be32dafd129f5 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -754,6 +754,8 @@ static const struct usb_device_id quirks_table[] = { BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3608), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x13d3, 0x3609), .driver_info = BTUSB_MEDIATEK | + BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3613), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3627), .driver_info = BTUSB_MEDIATEK | From ad61ec5c36717b80bcc461fd2b8466bb8399392f Mon Sep 17 00:00:00 2001 From: Us Chien Date: Sat, 12 Apr 2025 15:25:35 +0800 Subject: [PATCH 035/311] NVIDIA: SAUCE: MEDIATEK: usb: host: xhci-plat: support usb3 bulks stream low power BugLink: https://bugs.launchpad.net/bugs/2107509 Add a quirk to avoid U1 and U2 low power state operations during bulk stream transfers. Change-Id: Iaff484625eca6708713d0c2acaeddfc1103ac7d2 Signed-off-by: Us Chien Signed-off-by: Yenchia Chen Signed-off-by: Terje Bergstrom Acked-by: Brad Figg Acked-by: Matt Ochs Acked-by: Jamie Nguyen Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off-by: Brad Figg (cherry picked from commit 07399e87163635f78a35627866097a5cf6b6494a noble:linux-nvidia-6.11) Signed-off-by: Jacob Martin (backported from commit e521e80f789101cbf5041ee8d5606b9262b8250f) [maskedarray: changed the XHCI_NVIDIA_MT8901_HOST quirk bit value to 51] Signed-off-by: Abdur Rahman (cherry picked from commit 08ca4af323903e5341d555fc8c947491263eec85 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/usb/host/xhci-plat.c | 3 +++ drivers/usb/host/xhci.c | 5 +++++ drivers/usb/host/xhci.h | 1 + 3 files changed, 9 insertions(+) diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index 074d9c731639f..771836ab20ef0 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -277,6 +277,9 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s if (device_property_read_bool(tmpdev, "xhci-skip-phy-init-quirk")) xhci->quirks |= XHCI_SKIP_PHY_INIT; + if (device_property_read_bool(tmpdev, "xhci-nvidia-mediatek-host")) + xhci->quirks |= XHCI_NVIDIA_MT8901_HOST; + device_property_read_u32(tmpdev, "imod-interval-ns", &xhci->imod_interval); device_property_read_u16(tmpdev, "num-hc-interrupters", diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index 8d8f0865fc121..35bd19073b685 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -3731,6 +3731,11 @@ static int xhci_alloc_streams(struct usb_hcd *hcd, struct usb_device *udev, if (ret < 0) goto cleanup; + if (xhci->quirks & XHCI_NVIDIA_MT8901_HOST) { + xhci_hub_control(hcd, SetPortFeature, USB_PORT_FEAT_U1_TIMEOUT, 0, NULL, 0); + xhci_hub_control(hcd, SetPortFeature, USB_PORT_FEAT_U2_TIMEOUT, 0, NULL, 0); + } + spin_lock_irqsave(&xhci->lock, flags); for (i = 0; i < num_eps; i++) { ep_index = xhci_get_endpoint_index(&eps[i]->desc); diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index 2b0796f6d00ea..7421a806a6eae 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1644,6 +1644,7 @@ struct xhci_hcd { #define XHCI_CDNS_SCTX_QUIRK BIT_ULL(48) #define XHCI_ETRON_HOST BIT_ULL(49) #define XHCI_LIMIT_ENDPOINT_INTERVAL_9 BIT_ULL(50) +#define XHCI_NVIDIA_MT8901_HOST BIT_ULL(51) unsigned int num_active_eps; unsigned int limit_active_eps; From 07ae56cc134515cd737dc88ea994ed70f68b0348 Mon Sep 17 00:00:00 2001 From: Abhishek Sahu Date: Mon, 28 Apr 2025 15:41:03 +0000 Subject: [PATCH 036/311] NVIDIA: SAUCE: r8127: Add Realtek r8127 ethernet driver BugLink: https://bugs.launchpad.net/bugs/2109730 Realtek R8127 driver can be downloaded from https://www.realtek.com/Download/List?cate_id=584 Where it is maintained as out of tree module. This patch adds the extracted content of r8127-11.014.00.tar.bz2 in the folder drivers/net/ethernet/realtek/r8127. 4bd62fc87de32760fb1f3b9cd3ec14e933035623 r8127-11.014.00.tar.bz2 All the clean-up, makefile and Kconfig related changes will be done in the subsequent commits. The source code contains a GPL2 compatible license. All the license information and Realtek copyright notice will be maintained in each file and newly added files. Signed-off-by: Abhishek Sahu Acked-by: Matt Ochs Acked-by: Carol L Soto Acked-by: Ian May Acked-by: Jacob Martin Acked-by: Noah Wager Signed-off-by: Ian May (cherry picked from commit 7faf7ac3fffa292c696fa762de3863ca1f969f59 noble:linux-nvidia-6.11) Signed-off-by: Jacob Martin (cherry picked from commit e45f1b764ad7 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 24068b28f3b4b0ac746d520ffeb8864a844ef72f noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/net/ethernet/realtek/r8127/Makefile | 59 + drivers/net/ethernet/realtek/r8127/README | 147 + drivers/net/ethernet/realtek/r8127/autorun.sh | 101 + .../net/ethernet/realtek/r8127/src/Makefile | 209 + .../realtek/r8127/src/Makefile_linux24x | 75 + .../net/ethernet/realtek/r8127/src/r8127.h | 3068 +++ .../ethernet/realtek/r8127/src/r8127_dash.h | 261 + .../realtek/r8127/src/r8127_firmware.c | 264 + .../realtek/r8127/src/r8127_firmware.h | 68 + .../net/ethernet/realtek/r8127/src/r8127_n.c | 17824 ++++++++++++++++ .../ethernet/realtek/r8127/src/r8127_ptp.c | 944 + .../ethernet/realtek/r8127/src/r8127_ptp.h | 202 + .../realtek/r8127/src/r8127_realwow.h | 118 + .../ethernet/realtek/r8127/src/r8127_rss.c | 583 + .../ethernet/realtek/r8127/src/r8127_rss.h | 76 + .../ethernet/realtek/r8127/src/rtl_eeprom.c | 285 + .../ethernet/realtek/r8127/src/rtl_eeprom.h | 58 + .../net/ethernet/realtek/r8127/src/rtltool.c | 270 + .../net/ethernet/realtek/r8127/src/rtltool.h | 86 + 19 files changed, 24698 insertions(+) create mode 100755 drivers/net/ethernet/realtek/r8127/Makefile create mode 100755 drivers/net/ethernet/realtek/r8127/README create mode 100755 drivers/net/ethernet/realtek/r8127/autorun.sh create mode 100755 drivers/net/ethernet/realtek/r8127/src/Makefile create mode 100755 drivers/net/ethernet/realtek/r8127/src/Makefile_linux24x create mode 100755 drivers/net/ethernet/realtek/r8127/src/r8127.h create mode 100755 drivers/net/ethernet/realtek/r8127/src/r8127_dash.h create mode 100755 drivers/net/ethernet/realtek/r8127/src/r8127_firmware.c create mode 100755 drivers/net/ethernet/realtek/r8127/src/r8127_firmware.h create mode 100755 drivers/net/ethernet/realtek/r8127/src/r8127_n.c create mode 100755 drivers/net/ethernet/realtek/r8127/src/r8127_ptp.c create mode 100755 drivers/net/ethernet/realtek/r8127/src/r8127_ptp.h create mode 100755 drivers/net/ethernet/realtek/r8127/src/r8127_realwow.h create mode 100755 drivers/net/ethernet/realtek/r8127/src/r8127_rss.c create mode 100755 drivers/net/ethernet/realtek/r8127/src/r8127_rss.h create mode 100755 drivers/net/ethernet/realtek/r8127/src/rtl_eeprom.c create mode 100755 drivers/net/ethernet/realtek/r8127/src/rtl_eeprom.h create mode 100755 drivers/net/ethernet/realtek/r8127/src/rtltool.c create mode 100755 drivers/net/ethernet/realtek/r8127/src/rtltool.h diff --git a/drivers/net/ethernet/realtek/r8127/Makefile b/drivers/net/ethernet/realtek/r8127/Makefile new file mode 100755 index 0000000000000..39e846ad3fc9f --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/Makefile @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: GPL-2.0-only +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ + +################################################################################ +# This product is covered by one or more of the following patents: +# US6,570,884, US6,115,776, and US6,327,625. +################################################################################ + +KFLAG := 2$(shell uname -r | sed -ne 's/^2\.[4]\..*/4/p')x + +all: clean modules install + +modules: +ifeq ($(KFLAG),24x) + $(MAKE) -C src/ -f Makefile_linux24x modules +else + $(MAKE) -C src/ modules +endif + +clean: +ifeq ($(KFLAG),24x) + $(MAKE) -C src/ -f Makefile_linux24x clean +else + $(MAKE) -C src/ clean +endif + +install: +ifeq ($(KFLAG),24x) + $(MAKE) -C src/ -f Makefile_linux24x install +else + $(MAKE) -C src/ install +endif + + + diff --git a/drivers/net/ethernet/realtek/r8127/README b/drivers/net/ethernet/realtek/r8127/README new file mode 100755 index 0000000000000..a2d451d938cab --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/README @@ -0,0 +1,147 @@ + + + This is the Linux device driver released for Realtek 5 Gigabit Ethernet controllers with PCI-Express interface. + + + + - Kernel source tree (supported Linux kernel 2.6.x and 2.4.x) + - For linux kernel 2.4.x, this driver supports 2.4.20 and latter. + - Compiler/binutils for kernel compilation + + + Unpack the tarball : + # tar vjxf r8127-11.aaa.bb.tar.bz2 + + Change to the directory: + # cd r8127-11.aaa.bb + + If you are running the target kernel, then you should be able to do : + + # ./autorun.sh (as root or with sudo) + + You can check whether the driver is loaded by using following commands. + + # lsmod | grep r8127 + # ifconfig -a + + If there is a device name, ethX, shown on the monitor, the linux + driver is loaded. Then, you can use the following command to activate + the ethX. + + # ifconfig ethX up + + ,where X=0,1,2,... + + + 1. Set manually + a. Set the IP address of your machine. + + # ifconfig ethX "the IP address of your machine" + + b. Set the IP address of DNS. + + Insert the following configuration in /etc/resolv.conf. + + nameserver "the IP address of DNS" + + c. Set the IP address of gateway. + + # route add default gw "the IP address of gateway" + + 2. Set by doing configurations in /etc/sysconfig/network-scripts + /ifcfg-ethX for Redhat and Fedora, or /etc/sysconfig/network + /ifcfg-ethX for SuSE. There are two examples to set network + configurations. + + a. Fixed IP address: + DEVICE=eth0 + BOOTPROTO=static + ONBOOT=yes + TYPE=ethernet + NETMASK=255.255.255.0 + IPADDR=192.168.1.1 + GATEWAY=192.168.1.254 + BROADCAST=192.168.1.255 + + b. DHCP: + DEVICE=eth0 + BOOTPROTO=dhcp + ONBOOT=yes + + + There are two ways to modify the MAC address of the NIC. + 1. Use ifconfig: + + # ifconfig ethX hw ether YY:YY:YY:YY:YY:YY + + ,where X is the device number assigned by Linux kernel, and + YY:YY:YY:YY:YY:YY is the MAC address assigned by the user. + + 2. Use ip: + + # ip link set ethX address YY:YY:YY:YY:YY:YY + + ,where X is the device number assigned by Linux kernel, and + YY:YY:YY:YY:YY:YY is the MAC address assigned by the user. + + + + 1. Force the link status when insert the driver. + + If the user is in the path ~/r8127, the link status can be forced + to one of the 5 modes as following command. + + # insmod ./src/r8127.ko speed=SPEED_MODE duplex=DUPLEX_MODE autoneg=NWAY_OPTION + + ,where + SPEED_MODE = 1000 for 1000Mbps + = 100 for 100Mbps + = 10 for 10Mbps + DUPLEX_MODE = 0 for half-duplex + = 1 for full-duplex + NWAY_OPTION = 0 for auto-negotiation off (true force) + = 1 for auto-negotiation on (nway force) + For example: + + # insmod ./src/r8127.ko speed=100 duplex=0 autoneg=1 + + will force PHY to operate in 100Mpbs Half-duplex(nway force). + + 2. Force the link status by using ethtool. + a. Insert the driver first. + b. Make sure that ethtool exists in /sbin. + c. Force the link status as the following command. + + 2.5G before kernel v4.10 + # ethtool -s eth0 autoneg on advertise 0x802f + + 2.5G for kernel v4.10 and later + # ethtool -s eth0 autoneg on advertise 0x80000000002f + + 5G for kernel v4.10 and later (Couldn't be supported before kernel v4.10) + # ethtool -s eth0 autoneg on advertise 0x180000000002f + + # ethtool -s eth0 autoneg on advertise 0x1000 (10G) + # ethtool -s eth0 autoneg on advertise 0x002f (1G) + # ethtool -s eth0 autoneg on advertise 0x000f (100M full) + # ethtool -s eth0 autoneg on advertise 0x0003 (10M full) + + + Transmitting Jumbo Frames, whose packet size is bigger than 1500 bytes, please change mtu by the following command. + + # ifconfig ethX mtu MTU + + , where X=0,1,2,..., and MTU is configured by user. + + RTL8127 supports Jumbo Frame size up to 9 kBytes. + + + Get/Set device EEE status + + Get EEE device status + # ethtool --show-eee enp1s0 + + Set EEE device status + # ethtool --set-eee enp1s0 eee on tx-lpi on tx-timer 1546 advertise 0x0008 (100M full) + # ethtool --set-eee enp1s0 eee on tx-lpi on tx-timer 1546 advertise 0x0020 (1G) + # ethtool --set-eee enp1s0 eee on tx-lpi on tx-timer 1546 advertise 0x8000 (2.5G) diff --git a/drivers/net/ethernet/realtek/r8127/autorun.sh b/drivers/net/ethernet/realtek/r8127/autorun.sh new file mode 100755 index 0000000000000..fd87bced11583 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/autorun.sh @@ -0,0 +1,101 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only + +# invoke insmod with all arguments we got +# and use a pathname, as insmod doesn't look in . by default + +TARGET_PATH=$(find /lib/modules/$(uname -r)/kernel/drivers/net/ethernet -name realtek -type d) +if [ "$TARGET_PATH" = "" ]; then + TARGET_PATH=$(find /lib/modules/$(uname -r)/kernel/drivers/net -name realtek -type d) +fi +if [ "$TARGET_PATH" = "" ]; then + TARGET_PATH=/lib/modules/$(uname -r)/kernel/drivers/net +fi +echo +echo "Check old driver and unload it." +check=`lsmod | grep r8169` +if [ "$check" != "" ]; then + echo "rmmod r8169" + /sbin/rmmod r8169 +fi + +check=`lsmod | grep r8127` +if [ "$check" != "" ]; then + echo "rmmod r8127" + /sbin/rmmod r8127 +fi + +echo "Build the module and install" +echo "-------------------------------" >> log.txt +date 1>>log.txt +make $@ all 1>>log.txt || exit 1 +module=`ls src/*.ko` +module=${module#src/} +module=${module%.ko} + +if [ "$module" = "" ]; then + echo "No driver exists!!!" + exit 1 +elif [ "$module" != "r8169" ]; then + if test -e $TARGET_PATH/r8169.ko ; then + echo "Backup r8169.ko" + if test -e $TARGET_PATH/r8169.bak ; then + i=0 + while test -e $TARGET_PATH/r8169.bak$i + do + i=$(($i+1)) + done + echo "rename r8169.ko to r8169.bak$i" + mv $TARGET_PATH/r8169.ko $TARGET_PATH/r8169.bak$i + else + echo "rename r8169.ko to r8169.bak" + mv $TARGET_PATH/r8169.ko $TARGET_PATH/r8169.bak + fi + fi + if test -e $TARGET_PATH/r8169.ko.zst ; then + echo "Backup r8169.ko.zst" + if test -e $TARGET_PATH/r8169.zst.bak ; then + i=0 + while test -e $TARGET_PATH/r8169.zst.bak$i + do + i=$(($i+1)) + done + echo "rename r8169.ko.zst to r8169.zst.bak$i" + mv $TARGET_PATH/r8169.ko.zst $TARGET_PATH/r8169.zst.bak$i + else + echo "rename r8169.ko.zst to r8169.zst.bak" + mv $TARGET_PATH/r8169.ko.zst $TARGET_PATH/r8169.zst.bak + fi + fi +fi + +echo "DEPMOD $(uname -r)" +depmod `uname -r` +echo "load module $module" +modprobe $module + +is_update_initramfs=n +distrib_list="ubuntu debian" + +if [ -r /etc/debian_version ]; then + is_update_initramfs=y +elif [ -r /etc/lsb-release ]; then + for distrib in $distrib_list + do + /bin/grep -i "$distrib" /etc/lsb-release 2>&1 /dev/null && \ + is_update_initramfs=y && break + done +fi + +if [ "$is_update_initramfs" = "y" ]; then + if which update-initramfs >/dev/null ; then + echo "Updating initramfs. Please wait." + update-initramfs -u -k $(uname -r) + else + echo "update-initramfs: command not found" + exit 1 + fi +fi + +echo "Completed." +exit 0 diff --git a/drivers/net/ethernet/realtek/r8127/src/Makefile b/drivers/net/ethernet/realtek/r8127/src/Makefile new file mode 100755 index 0000000000000..d270904691bf3 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/src/Makefile @@ -0,0 +1,209 @@ +# SPDX-License-Identifier: GPL-2.0-only +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ + +################################################################################ +# This product is covered by one or more of the following patents: +# US6,570,884, US6,115,776, and US6,327,625. +################################################################################ + +CONFIG_SOC_LAN = y +ENABLE_REALWOW_SUPPORT = n +ENABLE_DASH_SUPPORT = n +ENABLE_DASH_PRINTER_SUPPORT = n +CONFIG_DOWN_SPEED_100 = n +CONFIG_ASPM = y +ENABLE_S5WOL = y +ENABLE_S5_KEEP_CURR_MAC = n +ENABLE_EEE = y +ENABLE_S0_MAGIC_PACKET = n +ENABLE_TX_NO_CLOSE = y +ENABLE_MULTIPLE_TX_QUEUE = n +ENABLE_PTP_SUPPORT = n +ENABLE_RSS_SUPPORT = n +ENABLE_LIB_SUPPORT = n +ENABLE_USE_FIRMWARE_FILE = n +DISABLE_WOL_SUPPORT = n +DISABLE_MULTI_MSIX_VECTOR = n +ENABLE_DOUBLE_VLAN = n +ENABLE_PAGE_REUSE = n +ENABLE_RX_PACKET_FRAGMENT = n +ENABLE_GIGA_LITE = y + +ifneq ($(KERNELRELEASE),) + obj-m := r8127.o + r8127-objs := r8127_n.o rtl_eeprom.o rtltool.o + ifeq ($(CONFIG_SOC_LAN), y) + EXTRA_CFLAGS += -DCONFIG_SOC_LAN + endif + ifeq ($(ENABLE_REALWOW_SUPPORT), y) + r8127-objs += r8127_realwow.o + EXTRA_CFLAGS += -DENABLE_REALWOW_SUPPORT + endif + ifeq ($(ENABLE_DASH_SUPPORT), y) + r8127-objs += r8127_dash.o + EXTRA_CFLAGS += -DENABLE_DASH_SUPPORT + endif + ifeq ($(ENABLE_DASH_PRINTER_SUPPORT), y) + r8127-objs += r8127_dash.o + EXTRA_CFLAGS += -DENABLE_DASH_SUPPORT -DENABLE_DASH_PRINTER_SUPPORT + endif + EXTRA_CFLAGS += -DCONFIG_R8127_NAPI + EXTRA_CFLAGS += -DCONFIG_R8127_VLAN + ifeq ($(CONFIG_DOWN_SPEED_100), y) + EXTRA_CFLAGS += -DCONFIG_DOWN_SPEED_100 + endif + ifeq ($(CONFIG_ASPM), y) + EXTRA_CFLAGS += -DCONFIG_ASPM + endif + ifeq ($(ENABLE_S5WOL), y) + EXTRA_CFLAGS += -DENABLE_S5WOL + endif + ifeq ($(ENABLE_S5_KEEP_CURR_MAC), y) + EXTRA_CFLAGS += -DENABLE_S5_KEEP_CURR_MAC + endif + ifeq ($(ENABLE_EEE), y) + EXTRA_CFLAGS += -DENABLE_EEE + endif + ifeq ($(ENABLE_S0_MAGIC_PACKET), y) + EXTRA_CFLAGS += -DENABLE_S0_MAGIC_PACKET + endif + ifeq ($(ENABLE_TX_NO_CLOSE), y) + EXTRA_CFLAGS += -DENABLE_TX_NO_CLOSE + endif + ifeq ($(ENABLE_MULTIPLE_TX_QUEUE), y) + EXTRA_CFLAGS += -DENABLE_MULTIPLE_TX_QUEUE + endif + ifeq ($(ENABLE_PTP_SUPPORT), y) + r8127-objs += r8127_ptp.o + EXTRA_CFLAGS += -DENABLE_PTP_SUPPORT + endif + ifeq ($(ENABLE_RSS_SUPPORT), y) + r8127-objs += r8127_rss.o + EXTRA_CFLAGS += -DENABLE_RSS_SUPPORT + endif + ifeq ($(ENABLE_LIB_SUPPORT), y) + r8127-objs += r8127_lib.o + EXTRA_CFLAGS += -DENABLE_LIB_SUPPORT + endif + ifeq ($(ENABLE_USE_FIRMWARE_FILE), y) + r8127-objs += r8127_firmware.o + EXTRA_CFLAGS += -DENABLE_USE_FIRMWARE_FILE + endif + ifeq ($(DISABLE_WOL_SUPPORT), y) + EXTRA_CFLAGS += -DDISABLE_WOL_SUPPORT + endif + ifeq ($(DISABLE_MULTI_MSIX_VECTOR), y) + EXTRA_CFLAGS += -DDISABLE_MULTI_MSIX_VECTOR + endif + ifeq ($(ENABLE_DOUBLE_VLAN), y) + EXTRA_CFLAGS += -DENABLE_DOUBLE_VLAN + endif + ifeq ($(ENABLE_PAGE_REUSE), y) + EXTRA_CFLAGS += -DENABLE_PAGE_REUSE + endif + ifeq ($(ENABLE_RX_PACKET_FRAGMENT), y) + EXTRA_CFLAGS += -DENABLE_RX_PACKET_FRAGMENT + endif + ifeq ($(ENABLE_GIGA_LITE), y) + EXTRA_CFLAGS += -DENABLE_GIGA_LITE + endif +else + BASEDIR := /lib/modules/$(shell uname -r) + KERNELDIR ?= $(BASEDIR)/build + PWD :=$(shell pwd) + DRIVERDIR := $(shell find $(BASEDIR)/kernel/drivers/net/ethernet -name realtek -type d) + ifeq ($(DRIVERDIR),) + DRIVERDIR := $(shell find $(BASEDIR)/kernel/drivers/net -name realtek -type d) + endif + ifeq ($(DRIVERDIR),) + DRIVERDIR := $(BASEDIR)/kernel/drivers/net + endif + RTKDIR := $(subst $(BASEDIR)/,,$(DRIVERDIR)) + + KERNEL_GCC_VERSION := $(shell cat /proc/version | sed -n 's/.*gcc version \([[:digit:]]\.[[:digit:]]\.[[:digit:]]\).*/\1/p') + CCVERSION = $(shell $(CC) -dumpversion) + + KVER = $(shell uname -r) + KMAJ = $(shell echo $(KVER) | \ + sed -e 's/^\([0-9][0-9]*\)\.[0-9][0-9]*\.[0-9][0-9]*.*/\1/') + KMIN = $(shell echo $(KVER) | \ + sed -e 's/^[0-9][0-9]*\.\([0-9][0-9]*\)\.[0-9][0-9]*.*/\1/') + KREV = $(shell echo $(KVER) | \ + sed -e 's/^[0-9][0-9]*\.[0-9][0-9]*\.\([0-9][0-9]*\).*/\1/') + + kver_ge = $(shell \ + echo test | awk '{if($(KMAJ) < $(1)) {print 0} else { \ + if($(KMAJ) > $(1)) {print 1} else { \ + if($(KMIN) < $(2)) {print 0} else { \ + if($(KMIN) > $(2)) {print 1} else { \ + if($(KREV) < $(3)) {print 0} else { print 1 } \ + }}}}}' \ + ) + +.PHONY: all +all: print_vars clean modules install + +print_vars: + @echo + @echo "CC: " $(CC) + @echo "CCVERSION: " $(CCVERSION) + @echo "KERNEL_GCC_VERSION: " $(KERNEL_GCC_VERSION) + @echo "KVER: " $(KVER) + @echo "KMAJ: " $(KMAJ) + @echo "KMIN: " $(KMIN) + @echo "KREV: " $(KREV) + @echo "BASEDIR: " $(BASEDIR) + @echo "DRIVERDIR: " $(DRIVERDIR) + @echo "PWD: " $(PWD) + @echo "RTKDIR: " $(RTKDIR) + @echo + +.PHONY:modules +modules: +#ifeq ($(call kver_ge,5,0,0),1) + $(MAKE) -C $(KERNELDIR) M=$(PWD) modules +#else +# $(MAKE) -C $(KERNELDIR) SUBDIRS=$(PWD) modules +#endif + +.PHONY:clean +clean: +#ifeq ($(call kver_ge,5,0,0),1) + $(MAKE) -C $(KERNELDIR) M=$(PWD) clean +#else +# $(MAKE) -C $(KERNELDIR) SUBDIRS=$(PWD) clean +#endif + +.PHONY:install +install: +#ifeq ($(call kver_ge,5,0,0),1) + $(MAKE) -C $(KERNELDIR) M=$(PWD) INSTALL_MOD_DIR=$(RTKDIR) modules_install +#else +# $(MAKE) -C $(KERNELDIR) SUBDIRS=$(PWD) INSTALL_MOD_DIR=$(RTKDIR) modules_install +#endif + +endif diff --git a/drivers/net/ethernet/realtek/r8127/src/Makefile_linux24x b/drivers/net/ethernet/realtek/r8127/src/Makefile_linux24x new file mode 100755 index 0000000000000..7cb3d91a85a64 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/src/Makefile_linux24x @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: GPL-2.0-only +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ + +################################################################################ +# This product is covered by one or more of the following patents: +# US6,570,884, US6,115,776, and US6,327,625. +################################################################################ + +CC := gcc +LD := ld +ARCH := $(shell uname -m | sed 's/i.86/i386/') +KSRC := /lib/modules/$(shell uname -r)/build +CONFIG_FILE := $(KSRC)/include/linux/autoconf.h +KMISC := /lib/modules/$(shell uname -r)/kernel/drivers/net/ + + +ifeq ($(ARCH),x86_64) + MODCFLAGS += -mcmodel=kernel -mno-red-zone +endif + +#standard flags for module builds +MODCFLAGS += -DLINUX -D__KERNEL__ -DMODULE -O2 -pipe -Wall +MODCFLAGS += -I$(KSRC)/include -I. +MODCFLAGS += -DMODVERSIONS -DEXPORT_SYMTAB -include $(KSRC)/include/linux/modversions.h +SOURCE := r8127_n.c rtl_eeprom.c rtltool.c +OBJS := $(SOURCE:.c=.o) + + +SMP := $(shell $(CC) $(MODCFLAGS) -E -dM $(CONFIG_FILE) | \ + grep CONFIG_SMP | awk '{print $$3}') + +ifneq ($(SMP),1) + SMP := 0 +endif + +ifeq ($(SMP),1) + MODCFLAGS += -D__SMP__ +endif + +modules: $(OBJS) + $(LD) -r $^ -o r8127.o + strip --strip-debug r8127.o + +%.o: %.c + $(CC) $(MODCFLAGS) -c $< -o $@ + +clean: + rm *.o -f + +install: + install -m 744 -c r8127.o $(KMISC) diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127.h b/drivers/net/ethernet/realtek/r8127/src/r8127.h new file mode 100755 index 0000000000000..fccb974bc08c4 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/src/r8127.h @@ -0,0 +1,3068 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#ifndef __R8127_H +#define __R8127_H + +//#include +#include +#include +#include +#include "r8127_dash.h" +#include "r8127_realwow.h" +#ifdef ENABLE_PTP_SUPPORT +#include "r8127_ptp.h" +#endif +#include "r8127_rss.h" +#ifdef ENABLE_LIB_SUPPORT +#include "r8127_lib.h" +#endif + +#ifndef fallthrough +#define fallthrough +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) +#define netif_xmit_stopped netif_tx_queue_stopped +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,7,0) +#ifndef MDIO_AN_EEE_ADV_100TX +#define MDIO_AN_EEE_ADV_100TX 0x0002 /* Advertise 100TX EEE cap */ +#endif +#ifndef MDIO_AN_EEE_ADV_1000T +#define MDIO_AN_EEE_ADV_1000T 0x0004 /* Advertise 1000T EEE cap */ +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0) +#define MDIO_EEE_100TX MDIO_AN_EEE_ADV_100TX /* 100TX EEE cap */ +#define MDIO_EEE_1000T MDIO_AN_EEE_ADV_1000T /* 1000T EEE cap */ +#define MDIO_EEE_10GT 0x0008 /* 10GT EEE cap */ +#define MDIO_EEE_1000KX 0x0010 /* 1000KX EEE cap */ +#define MDIO_EEE_10GKX4 0x0020 /* 10G KX4 EEE cap */ +#define MDIO_EEE_10GKR 0x0040 /* 10G KR EEE cap */ +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0) */ + +static inline u32 mmd_eee_adv_to_ethtool_adv_t(u16 eee_adv) +{ + u32 adv = 0; + + if (eee_adv & MDIO_EEE_100TX) + adv |= ADVERTISED_100baseT_Full; + if (eee_adv & MDIO_EEE_1000T) + adv |= ADVERTISED_1000baseT_Full; + if (eee_adv & MDIO_EEE_10GT) + adv |= ADVERTISED_10000baseT_Full; + if (eee_adv & MDIO_EEE_1000KX) + adv |= ADVERTISED_1000baseKX_Full; + if (eee_adv & MDIO_EEE_10GKX4) + adv |= ADVERTISED_10000baseKX4_Full; + if (eee_adv & MDIO_EEE_10GKR) + adv |= ADVERTISED_10000baseKR_Full; + + return adv; +} + +static inline u16 ethtool_adv_to_mmd_eee_adv_t(u32 adv) +{ + u16 reg = 0; + + if (adv & ADVERTISED_100baseT_Full) + reg |= MDIO_EEE_100TX; + if (adv & ADVERTISED_1000baseT_Full) + reg |= MDIO_EEE_1000T; + if (adv & ADVERTISED_10000baseT_Full) + reg |= MDIO_EEE_10GT; + if (adv & ADVERTISED_1000baseKX_Full) + reg |= MDIO_EEE_1000KX; + if (adv & ADVERTISED_10000baseKX4_Full) + reg |= MDIO_EEE_10GKX4; + if (adv & ADVERTISED_10000baseKR_Full) + reg |= MDIO_EEE_10GKR; + + return reg; +} +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,7,0) */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,9,0) +static inline bool skb_transport_header_was_set(const struct sk_buff *skb) +{ + return skb->transport_header != ~0U; +} +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,9,0) */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,20,0) +static inline void linkmode_set_bit(int nr, volatile unsigned long *addr) +{ + __set_bit(nr, addr); +} + +static inline void linkmode_clear_bit(int nr, volatile unsigned long *addr) +{ + __clear_bit(nr, addr); +} + +static inline int linkmode_test_bit(int nr, volatile unsigned long *addr) +{ + return test_bit(nr, addr); +} +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4,20,0) */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,0,0) +static inline void linkmode_mod_bit(int nr, volatile unsigned long *addr, + int set) +{ + if (set) + linkmode_set_bit(nr, addr); + else + linkmode_clear_bit(nr, addr); +} +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5,0,0) */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,3,0) +static inline +ssize_t strscpy(char *dest, const char *src, size_t count) +{ + long res = 0; + + if (count == 0) + return -E2BIG; + + while (count) { + char c; + + c = src[res]; + dest[res] = c; + if (!c) + return res; + res++; + count--; + } + + /* Hit buffer length without finding a NUL; force NUL-termination. */ + if (res) + dest[res-1] = '\0'; + + return -E2BIG; +} +#endif + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0)) +static inline unsigned char *skb_checksum_start(const struct sk_buff *skb) +{ +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,22)) + return skb->head + skb->csum_start; +#else /* < 2.6.22 */ + return skb_transport_header(skb); +#endif +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) +static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue, + unsigned int bytes) +{} +static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue, + unsigned int pkts, + unsigned int bytes) +{} +static inline void netdev_tx_reset_queue(struct netdev_queue *q) {} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,8,0) +static inline void fsleep(unsigned long usecs) +{ + if (usecs <= 10) + udelay(usecs); + else if (usecs <= 20000) + usleep_range(usecs, 2 * usecs); + else + msleep(DIV_ROUND_UP(usecs, 1000)); +} +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5,8,0) */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,2,0) +#define netdev_xmit_more() (0) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,8,0) +#define netif_testing_on(dev) +#define netif_testing_off(dev) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,2,0) +#define netdev_sw_irq_coalesce_default_on(dev) +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(6,2,0) */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,32) +typedef int netdev_tx_t; +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,12,0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,1,9) +static inline bool page_is_pfmemalloc(struct page *page) +{ + /* + * Page index cannot be this large so this must be + * a pfmemalloc page. + */ + return page->index == -1UL; +} +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4,1,9) */ +static inline bool dev_page_is_reusable(struct page *page) +{ + return likely(page_to_nid(page) == numa_mem_id() && + !page_is_pfmemalloc(page)); +} +#endif + +/* +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0)&& !defined(ENABLE_LIB_SUPPORT) +#define RTL_USE_NEW_INTR_API +#endif +*/ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,10,0) +#define dma_map_page_attrs(dev, page, offset, size, dir, attrs) \ + dma_map_page(dev, page, offset, size, dir) +#define dma_unmap_page_attrs(dev, page, size, dir, attrs) \ + dma_unmap_page(dev, page, size, dir) +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,10,0) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) +#define page_ref_inc(page) atomic_inc(&page->_count) +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,4,216) +#define page_ref_count(page) atomic_read(&page->_count) +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,4,216) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) +#define skb_transport_offset(skb) (skb->h.raw - skb->data) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,26) +#define device_set_wakeup_enable(dev, val) do {} while (0) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,14,0) +static inline void ether_addr_copy(u8 *dst, const u8 *src) +{ + u16 *a = (u16 *)dst; + const u16 *b = (const u16 *)src; + + a[0] = b[0]; + a[1] = b[1]; + a[2] = b[2]; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,15,0) +#define IS_ERR_OR_NULL(ptr) (!ptr) +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,13,0) +#define reinit_completion(x) ((x)->done = 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,39) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,32) +#define pm_runtime_mark_last_busy(x) +#define pm_runtime_put_autosuspend(x) pm_runtime_put(x) +#define pm_runtime_put_sync_autosuspend(x) pm_runtime_put_sync(x) + +static inline bool pm_runtime_suspended(struct device *dev) +{ + return dev->power.runtime_status == RPM_SUSPENDED + && !dev->power.disable_depth; +} + +static inline bool pm_runtime_active(struct device *dev) +{ + return dev->power.runtime_status == RPM_ACTIVE + || dev->power.disable_depth; +} +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,36) +#define queue_delayed_work(long_wq, work, delay) schedule_delayed_work(work, delay) +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,34) +#define netif_printk(priv, type, level, netdev, fmt, args...) \ + do { \ + if (netif_msg_##type(priv)) \ + printk(level "%s: " fmt,(netdev)->name , ##args); \ + } while (0) + +#define netif_emerg(priv, type, netdev, fmt, args...) \ + netif_printk(priv, type, KERN_EMERG, netdev, fmt, ##args) +#define netif_alert(priv, type, netdev, fmt, args...) \ + netif_printk(priv, type, KERN_ALERT, netdev, fmt, ##args) +#define netif_crit(priv, type, netdev, fmt, args...) \ + netif_printk(priv, type, KERN_CRIT, netdev, fmt, ##args) +#define netif_err(priv, type, netdev, fmt, args...) \ + netif_printk(priv, type, KERN_ERR, netdev, fmt, ##args) +#define netif_warn(priv, type, netdev, fmt, args...) \ + netif_printk(priv, type, KERN_WARNING, netdev, fmt, ##args) +#define netif_notice(priv, type, netdev, fmt, args...) \ + netif_printk(priv, type, KERN_NOTICE, netdev, fmt, ##args) +#define netif_info(priv, type, netdev, fmt, args...) \ + netif_printk(priv, type, KERN_INFO, (netdev), fmt, ##args) +#endif +#endif +#endif +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15) +#define setup_timer(_timer, _function, _data) \ +do { \ + (_timer)->function = _function; \ + (_timer)->data = _data; \ + init_timer(_timer); \ +} while (0) +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,0,0) +#if defined(skb_vlan_tag_present) && !defined(vlan_tx_tag_present) +#define vlan_tx_tag_present skb_vlan_tag_present +#endif +#if defined(skb_vlan_tag_get) && !defined(vlan_tx_tag_get) +#define vlan_tx_tag_get skb_vlan_tag_get +#endif +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,0,0) + +#define RTL_ALLOC_SKB_INTR(napi, length) dev_alloc_skb(length) +#define R8127_USE_NAPI_ALLOC_SKB 0 +#ifdef CONFIG_R8127_NAPI +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0) +#undef RTL_ALLOC_SKB_INTR +#define RTL_ALLOC_SKB_INTR(napi, length) napi_alloc_skb(napi, length) +#undef R8127_USE_NAPI_ALLOC_SKB +#define R8127_USE_NAPI_ALLOC_SKB 1 +#endif +#endif + +#define RTL_BUILD_SKB_INTR(data, frag_size) build_skb(data, frag_size) +#ifdef CONFIG_R8127_NAPI +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,12,0) +#undef RTL_BUILD_SKB_INTR +#define RTL_BUILD_SKB_INTR(data, frag_size) napi_build_skb(data, frag_size) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0) +#define eth_random_addr(addr) random_ether_addr(addr) +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0) +#define netdev_features_t u32 +#endif +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,5,0) +#define NETIF_F_ALL_CSUM NETIF_F_CSUM_MASK +#else +#ifndef NETIF_F_ALL_CSUM +#define NETIF_F_ALL_CSUM NETIF_F_CSUM_MASK +#endif +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,37) +#define ENABLE_R8127_PROCFS +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,11,0) +#define ENABLE_R8127_SYSFS +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) +#define NETIF_F_HW_VLAN_RX NETIF_F_HW_VLAN_CTAG_RX +#define NETIF_F_HW_VLAN_TX NETIF_F_HW_VLAN_CTAG_TX +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,8,0) +#define __devinit +#define __devexit +#define __devexit_p(func) func +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) +#define CHECKSUM_PARTIAL CHECKSUM_HW +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) +#define irqreturn_t void +#define IRQ_HANDLED 1 +#define IRQ_NONE 0 +#define IRQ_RETVAL(x) +#endif + +#ifndef NETIF_F_RXALL +#define NETIF_F_RXALL 0 +#endif + +#ifndef NETIF_F_RXFCS +#define NETIF_F_RXFCS 0 +#endif + +#if !defined(HAVE_FREE_NETDEV) && (LINUX_VERSION_CODE < KERNEL_VERSION(3,1,0)) +#define free_netdev(x) kfree(x) +#endif + +#ifndef SET_NETDEV_DEV +#define SET_NETDEV_DEV(net, pdev) +#endif + +#ifndef SET_MODULE_OWNER +#define SET_MODULE_OWNER(dev) +#endif + +#ifndef SA_SHIRQ +#define SA_SHIRQ IRQF_SHARED +#endif + +#ifndef NETIF_F_GSO +#define gso_size tso_size +#define gso_segs tso_segs +#endif + +#ifndef PCI_VENDOR_ID_DLINK +#define PCI_VENDOR_ID_DLINK 0x1186 +#endif + +#ifndef dma_mapping_error +#define dma_mapping_error(a,b) 0 +#endif + +#ifndef netif_err +#define netif_err(a,b,c,d) +#endif + +#ifndef AUTONEG_DISABLE +#define AUTONEG_DISABLE 0x00 +#endif + +#ifndef AUTONEG_ENABLE +#define AUTONEG_ENABLE 0x01 +#endif + +#ifndef BMCR_SPEED1000 +#define BMCR_SPEED1000 0x0040 +#endif + +#ifndef BMCR_SPEED100 +#define BMCR_SPEED100 0x2000 +#endif + +#ifndef BMCR_SPEED10 +#define BMCR_SPEED10 0x0000 +#endif + +#ifndef SPEED_UNKNOWN +#define SPEED_UNKNOWN -1 +#endif + +#ifndef DUPLEX_UNKNOWN +#define DUPLEX_UNKNOWN 0xff +#endif + +#ifndef SUPPORTED_Pause +#define SUPPORTED_Pause (1 << 13) +#endif + +#ifndef SUPPORTED_Asym_Pause +#define SUPPORTED_Asym_Pause (1 << 14) +#endif + +#ifndef MDIO_EEE_100TX +#define MDIO_EEE_100TX 0x0002 +#endif + +#ifndef MDIO_EEE_1000T +#define MDIO_EEE_1000T 0x0004 +#endif + +#ifndef MDIO_EEE_2_5GT +#define MDIO_EEE_2_5GT 0x0001 +#endif + +#ifndef MDIO_EEE_5GT +#define MDIO_EEE_5GT 0x0002 +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0) +#define ethtool_keee ethtool_eee +#define rtl8127_ethtool_adv_to_mmd_eee_adv_cap1_t ethtool_adv_to_mmd_eee_adv_t +static inline u32 rtl8127_ethtool_adv_to_mmd_eee_adv_cap2_t(u32 adv) +{ + u32 result = 0; + + if (adv & SUPPORTED_2500baseX_Full) + result |= MDIO_EEE_2_5GT; + + return result; +} +#else +#define rtl8127_ethtool_adv_to_mmd_eee_adv_cap1_t linkmode_to_mii_eee_cap1_t +#define rtl8127_ethtool_adv_to_mmd_eee_adv_cap2_t linkmode_to_mii_eee_cap2_t +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0) */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29) +#ifdef CONFIG_NET_POLL_CONTROLLER +#define RTL_NET_POLL_CONTROLLER dev->poll_controller=rtl8127_netpoll +#else +#define RTL_NET_POLL_CONTROLLER +#endif + +#ifdef CONFIG_R8127_VLAN +#define RTL_SET_VLAN dev->vlan_rx_register=rtl8127_vlan_rx_register +#else +#define RTL_SET_VLAN +#endif + +#define RTL_NET_DEVICE_OPS(ops) dev->open=rtl8127_open; \ + dev->hard_start_xmit=rtl8127_start_xmit; \ + dev->get_stats=rtl8127_get_stats; \ + dev->stop=rtl8127_close; \ + dev->tx_timeout=rtl8127_tx_timeout; \ + dev->set_multicast_list=rtl8127_set_rx_mode; \ + dev->change_mtu=rtl8127_change_mtu; \ + dev->set_mac_address=rtl8127_set_mac_address; \ + dev->do_ioctl=rtl8127_do_ioctl; \ + RTL_NET_POLL_CONTROLLER; \ + RTL_SET_VLAN; +#else +#define RTL_NET_DEVICE_OPS(ops) dev->netdev_ops=&ops +#endif + +#ifndef FALSE +#define FALSE 0 +#endif + +#ifndef TRUE +#define TRUE 1 +#endif + +#ifndef false +#define false 0 +#endif + +#ifndef true +#define true 1 +#endif + +//Hardware will continue interrupt 10 times after interrupt finished. +#define RTK_KEEP_INTERRUPT_COUNT (10) + +//the low 32 bit address of receive buffer must be 8-byte alignment. +#ifndef NET_IP_ALIGN +#define NET_IP_ALIGN 2 +#endif +#define R8127_RX_ALIGN NET_IP_ALIGN + +#ifdef CONFIG_R8127_NAPI +#define NAPI_SUFFIX "-NAPI" +#else +#define NAPI_SUFFIX "" +#endif +#if defined(ENABLE_DASH_PRINTER_SUPPORT) +#define DASH_SUFFIX "-PRINTER" +#elif defined(ENABLE_DASH_SUPPORT) +#define DASH_SUFFIX "-DASH" +#else +#define DASH_SUFFIX "" +#endif + +#if defined(ENABLE_REALWOW_SUPPORT) +#define REALWOW_SUFFIX "-REALWOW" +#else +#define REALWOW_SUFFIX "" +#endif + +#if defined(ENABLE_PTP_SUPPORT) +#define PTP_SUFFIX "-PTP" +#else +#define PTP_SUFFIX "" +#endif + +#if defined(ENABLE_RSS_SUPPORT) +#define RSS_SUFFIX "-RSS" +#else +#define RSS_SUFFIX "" +#endif + +#define RTL8127_VERSION "11.014.00" NAPI_SUFFIX DASH_SUFFIX REALWOW_SUFFIX PTP_SUFFIX RSS_SUFFIX +#define MODULENAME "r8127" +#define PFX MODULENAME ": " + +#define GPL_CLAIM "\ +r8127 Copyright (C) 2025 Realtek NIC software team \n \ +This program comes with ABSOLUTELY NO WARRANTY; for details, please see . \n \ +This is free software, and you are welcome to redistribute it under certain conditions; see . \n" + +#ifdef RTL8127_DEBUG +#define assert(expr) \ + if(!(expr)) { \ + printk("Assertion failed! %s,%s,%s,line=%d\n", \ + #expr,__FILE__,__FUNCTION__,__LINE__); \ + } +#define dprintk(fmt, args...) do { printk(PFX fmt, ## args); } while (0) +#else +#define assert(expr) do {} while (0) +#define dprintk(fmt, args...) do {} while (0) +#endif /* RTL8127_DEBUG */ + +#define R8127_MSG_DEFAULT \ + (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_IFUP | NETIF_MSG_IFDOWN) + +#ifdef CONFIG_R8127_NAPI +#define rtl8127_rx_hwaccel_skb vlan_hwaccel_receive_skb +#define rtl8127_rx_quota(count, quota) min(count, quota) +#else +#define rtl8127_rx_hwaccel_skb vlan_hwaccel_rx +#define rtl8127_rx_quota(count, quota) count +#endif + +/* MAC address length */ +#ifndef MAC_ADDR_LEN +#define MAC_ADDR_LEN 6 +#endif + +#ifndef MAC_PROTOCOL_LEN +#define MAC_PROTOCOL_LEN 2 +#endif + +#ifndef ETH_FCS_LEN +#define ETH_FCS_LEN 4 +#endif + +#ifndef NETIF_F_TSO6 +#define NETIF_F_TSO6 0 +#endif + +#define Reserved2_data 7 +#define RX_DMA_BURST_unlimited 7 /* Maximum PCI burst, '7' is unlimited */ +#define RX_DMA_BURST_512 5 +#define RX_DMA_BURST_256 4 +#define TX_DMA_BURST_unlimited 7 +#define TX_DMA_BURST_1024 6 +#define TX_DMA_BURST_512 5 +#define TX_DMA_BURST_256 4 +#define TX_DMA_BURST_128 3 +#define TX_DMA_BURST_64 2 +#define TX_DMA_BURST_32 1 +#define TX_DMA_BURST_16 0 +#define Reserved1_data 0x3F +#define RxPacketMaxSize 0x3FE8 /* 16K - 1 - ETH_HLEN - VLAN - CRC... */ +#define Jumbo_Frame_1k ETH_DATA_LEN +#define Jumbo_Frame_2k (2*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN) +#define Jumbo_Frame_3k (3*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN) +#define Jumbo_Frame_4k (4*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN) +#define Jumbo_Frame_5k (5*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN) +#define Jumbo_Frame_6k (6*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN) +#define Jumbo_Frame_7k (7*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN) +#define Jumbo_Frame_8k (8*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN) +#define Jumbo_Frame_9k (9*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN) +#define InterFrameGap 0x03 /* 3 means InterFrameGap = the shortest one */ +#define RxEarly_off_V1 (0x07 << 11) +#define RxEarly_off_V2 (1 << 11) +#define Rx_Single_fetch_V2 (1 << 14) +#define Rx_Close_Multiple (1 << 21) +#define Rx_Fetch_Number_8 (1 << 30) + +#define R8127_REGS_SIZE (256) +#define R8127_MAC_REGS_SIZE (256) +#define R8127_PHY_REGS_SIZE (16*2) +#define R8127_EPHY_REGS_SIZE (31*2) +#define R8127_ERI_REGS_SIZE (0x100) +#define R8127_REGS_DUMP_SIZE (0x400) +#define R8127_PCI_REGS_SIZE (0x100) +#define R8127_NAPI_WEIGHT 64 + +#define R8127_MAX_MSIX_VEC_8125A 4 +#define R8127_MAX_MSIX_VEC_8125B 32 +#define R8127_MAX_MSIX_VEC_8125D 32 +#define R8127_MIN_MSIX_VEC_8125B 22 +#define R8127_MIN_MSIX_VEC_8125BP 31 +#define R8127_MIN_MSIX_VEC_8125D 20 +#define R8127_MIN_MSIX_VEC_8127 30 +#define R8127_MAX_MSIX_VEC 32 +#define R8127_MAX_RX_QUEUES_VEC_V3 (16) +#define R8127_MAX_RX_QUEUES_VEC_V4 (8) + +#define RTL8127_TX_TIMEOUT (6 * HZ) +#define RTL8127_LINK_TIMEOUT (1 * HZ) +#define RTL8127_ESD_TIMEOUT (2 * HZ) + +#define rtl8127_rx_page_size(order) (PAGE_SIZE << order) + +#define MAX_NUM_TX_DESC 1024 /* Maximum number of Tx descriptor registers */ +#define MAX_NUM_RX_DESC 1024 /* Maximum number of Rx descriptor registers */ + +#define MIN_NUM_TX_DESC 256 /* Minimum number of Tx descriptor registers */ +#define MIN_NUM_RX_DESC 256 /* Minimum number of Rx descriptor registers */ + +#define NUM_TX_DESC MAX_NUM_TX_DESC /* Number of Tx descriptor registers */ +#define NUM_RX_DESC MAX_NUM_RX_DESC /* Number of Rx descriptor registers */ + +#ifdef ENABLE_DOUBLE_VLAN +#define RX_BUF_SIZE 0x05F6 /* 0x05F6(1526) = 1514 + 8(double vlan) + 4(crc) bytes */ +#define RT_VALN_HLEN 8 /* 8(double vlan) bytes */ +#else +#define RX_BUF_SIZE 0x05F2 /* 0x05F2(1522) = 1514 + 4(single vlan) + 4(crc) bytes */ +#define RT_VALN_HLEN 4 /* 4(single vlan) bytes */ +#endif + +#define R8127_MAX_TX_QUEUES (2) +#define R8127_MAX_RX_QUEUES_V2 (4) +#define R8127_MAX_RX_QUEUES_V3 (16) +#define R8127_MAX_RX_QUEUES R8127_MAX_RX_QUEUES_V3 +#define R8127_MAX_QUEUES R8127_MAX_RX_QUEUES + +#define OCP_STD_PHY_BASE 0xa400 + +//Channel Wait Count +#define R8127_CHANNEL_WAIT_COUNT (20000) +#define R8127_CHANNEL_WAIT_TIME (1) // 1us +#define R8127_CHANNEL_EXIT_DELAY_TIME (20) //20us + +#ifdef ENABLE_LIB_SUPPORT +#define R8127_MULTI_RX_Q(tp) 0 +#else +#define R8127_MULTI_RX_Q(tp) (tp->num_rx_rings > 1) +#endif + +#define NODE_ADDRESS_SIZE 6 + +#define SHORT_PACKET_PADDING_BUF_SIZE 256 + +#define RTK_MAGIC_DEBUG_VALUE 0x0badbeef + +/* write/read MMIO register */ +#define RTL_W8(tp, reg, val8) writeb((val8), tp->mmio_addr + (reg)) +#define RTL_W16(tp, reg, val16) writew((val16), tp->mmio_addr + (reg)) +#define RTL_W32(tp, reg, val32) writel((val32), tp->mmio_addr + (reg)) +#define RTL_R8(tp, reg) readb(tp->mmio_addr + (reg)) +#define RTL_R16(tp, reg) readw(tp->mmio_addr + (reg)) +#define RTL_R32(tp, reg) ((unsigned long) readl(tp->mmio_addr + (reg))) + +#ifndef DMA_64BIT_MASK +#define DMA_64BIT_MASK 0xffffffffffffffffULL +#endif + +#ifndef DMA_32BIT_MASK +#define DMA_32BIT_MASK 0x00000000ffffffffULL +#endif + +#ifndef NETDEV_TX_OK +#define NETDEV_TX_OK 0 /* driver took care of packet */ +#endif + +#ifndef NETDEV_TX_BUSY +#define NETDEV_TX_BUSY 1 /* driver tx path was busy*/ +#endif + +#ifndef NETDEV_TX_LOCKED +#define NETDEV_TX_LOCKED -1t /* driver tx lock was already taken */ +#endif + +#ifndef ADVERTISED_Pause +#define ADVERTISED_Pause (1 << 13) +#endif + +#ifndef ADVERTISED_Asym_Pause +#define ADVERTISED_Asym_Pause (1 << 14) +#endif + +#ifndef ADVERTISE_PAUSE_CAP +#define ADVERTISE_PAUSE_CAP 0x400 +#endif + +#ifndef ADVERTISE_PAUSE_ASYM +#define ADVERTISE_PAUSE_ASYM 0x800 +#endif + +#ifndef MII_CTRL1000 +#define MII_CTRL1000 0x09 +#endif + +#ifndef ADVERTISE_1000FULL +#define ADVERTISE_1000FULL 0x200 +#endif + +#ifndef ADVERTISE_1000HALF +#define ADVERTISE_1000HALF 0x100 +#endif + +#ifndef BIT_ULL +#define BIT_ULL(nr) (1ULL << (nr)) +#endif + +#ifndef ADVERTISED_2500baseX_Full +#define ADVERTISED_2500baseX_Full 0x8000 +#endif +#define RTK_ADVERTISED_5000baseX_Full BIT_ULL(48) +#define RTK_SUPPORTED_5000baseX_Full BIT_ULL(48) + +#define RTK_ADVERTISE_2500FULL 0x80 +#define RTK_ADVERTISE_5000FULL 0x100 +#define RTK_ADVERTISE_10000FULL 0x1000 +#define RTK_LPA_ADVERTISE_2500FULL 0x20 +#define RTK_LPA_ADVERTISE_5000FULL 0x40 +#define RTK_LPA_ADVERTISE_10000FULL 0x800 + +#define RTK_EEE_ADVERTISE_2500FULL BIT(0) +#define RTK_EEE_ADVERTISE_5000FULL BIT(1) +#define RTK_LPA_EEE_ADVERTISE_2500FULL BIT(0) +#define RTK_LPA_EEE_ADVERTISE_5000FULL BIT(1) + +/* Tx NO CLOSE */ +#define MAX_TX_NO_CLOSE_DESC_PTR_V2 0x10000 +#define MAX_TX_NO_CLOSE_DESC_PTR_MASK_V2 0xFFFF +#define MAX_TX_NO_CLOSE_DESC_PTR_V3 0x100000000 +#define MAX_TX_NO_CLOSE_DESC_PTR_MASK_V3 0xFFFFFFFF +#define MAX_TX_NO_CLOSE_DESC_PTR_V4 0x80000000 +#define MAX_TX_NO_CLOSE_DESC_PTR_MASK_V4 0x7FFFFFFF +#define TX_NO_CLOSE_SW_PTR_MASK_V2 0x1FFFF + +#ifndef ETH_MIN_MTU +#define ETH_MIN_MTU 68 +#endif + +#define D0_SPEED_UP_SPEED_DISABLE 0 +#define D0_SPEED_UP_SPEED_1000 1 +#define D0_SPEED_UP_SPEED_2500 2 +#define D0_SPEED_UP_SPEED_5000 3 +#define D0_SPEED_UP_SPEED_10000 4 + +#define RTL8127_MAC_MCU_PAGE_SIZE 256 //256 words + +#ifndef WRITE_ONCE +#define WRITE_ONCE(var, val) (*((volatile typeof(val) *)(&(var))) = (val)) +#endif +#ifndef READ_ONCE +#define READ_ONCE(var) (*((volatile typeof(var) *)(&(var)))) +#endif + +#ifndef SPEED_5000 +#define SPEED_5000 5000 +#endif + +#ifndef SPEED_10000 +#define SPEED_10000 10000 +#endif + +#define R8127_LINK_STATE_OFF 0 +#define R8127_LINK_STATE_ON 1 +#define R8127_LINK_STATE_UNKNOWN 2 + +/*****************************************************************************/ + +//#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,3) +#if ((LINUX_VERSION_CODE < KERNEL_VERSION(2,4,27)) || \ + ((LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)) && \ + (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,3)))) +/* copied from linux kernel 2.6.20 include/linux/netdev.h */ +#define NETDEV_ALIGN 32 +#define NETDEV_ALIGN_CONST (NETDEV_ALIGN - 1) + +static inline void *netdev_priv(struct net_device *dev) +{ + return (char *)dev + ((sizeof(struct net_device) + + NETDEV_ALIGN_CONST) + & ~NETDEV_ALIGN_CONST); +} +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,3) + +/*****************************************************************************/ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) +#define RTLDEV tp +#else +#define RTLDEV dev +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) +/*****************************************************************************/ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) +typedef struct net_device *napi_ptr; +typedef int *napi_budget; + +#define napi dev +#define RTL_NAPI_CONFIG(ndev, priv, function, weig) ndev->poll=function; \ + ndev->weight=weig; +#define RTL_NAPI_QUOTA(budget, ndev) min(*budget, ndev->quota) +#define RTL_GET_PRIV(stuct_ptr, priv_struct) netdev_priv(stuct_ptr) +#define RTL_GET_NETDEV(priv_ptr) +#define RTL_RX_QUOTA(budget) *budget +#define RTL_NAPI_QUOTA_UPDATE(ndev, work_done, budget) *budget -= work_done; \ + ndev->quota -= work_done; +#define RTL_NETIF_RX_COMPLETE(dev, napi, work_done) netif_rx_complete(dev) +#define RTL_NETIF_RX_SCHEDULE_PREP(dev, napi) netif_rx_schedule_prep(dev) +#define __RTL_NETIF_RX_SCHEDULE(dev, napi) __netif_rx_schedule(dev) +#define RTL_NAPI_RETURN_VALUE work_done >= work_to_do +#define RTL_NAPI_ENABLE(dev, napi) netif_poll_enable(dev) +#define RTL_NAPI_DISABLE(dev, napi) netif_poll_disable(dev) +#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1)) +#else +typedef struct napi_struct *napi_ptr; +typedef int napi_budget; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,1,0) +#define RTL_NAPI_CONFIG(ndev, priv, function, weight) netif_napi_add_weight(ndev, &priv->napi, function, weight) +#else +#define RTL_NAPI_CONFIG(ndev, priv, function, weight) netif_napi_add(ndev, &priv->napi, function, weight) +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(6,1,0) +#define RTL_NAPI_QUOTA(budget, ndev) min(budget, budget) +#define RTL_GET_PRIV(stuct_ptr, priv_struct) container_of(stuct_ptr, priv_struct, stuct_ptr) +#define RTL_GET_NETDEV(priv_ptr) struct net_device *dev = priv_ptr->dev; +#define RTL_RX_QUOTA(budget) budget +#define RTL_NAPI_QUOTA_UPDATE(ndev, work_done, budget) +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29) +#define RTL_NETIF_RX_COMPLETE(dev, napi, work_done) netif_rx_complete(dev, napi) +#define RTL_NETIF_RX_SCHEDULE_PREP(dev, napi) netif_rx_schedule_prep(dev, napi) +#define __RTL_NETIF_RX_SCHEDULE(dev, napi) __netif_rx_schedule(dev, napi) +#endif +#if LINUX_VERSION_CODE == KERNEL_VERSION(2,6,29) +#define RTL_NETIF_RX_COMPLETE(dev, napi, work_done) netif_rx_complete(napi) +#define RTL_NETIF_RX_SCHEDULE_PREP(dev, napi) netif_rx_schedule_prep(napi) +#define __RTL_NETIF_RX_SCHEDULE(dev, napi) __netif_rx_schedule(napi) +#endif +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,29) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0) +#define RTL_NETIF_RX_COMPLETE(dev, napi, work_done) napi_complete_done(napi, work_done) +#else +#define RTL_NETIF_RX_COMPLETE(dev, napi, work_done) napi_complete(napi) +#endif +#define RTL_NETIF_RX_SCHEDULE_PREP(dev, napi) napi_schedule_prep(napi) +#define __RTL_NETIF_RX_SCHEDULE(dev, napi) __napi_schedule(napi) +#endif +#define RTL_NAPI_RETURN_VALUE work_done +#define RTL_NAPI_ENABLE(dev, napi) napi_enable(napi) +#define RTL_NAPI_DISABLE(dev, napi) napi_disable(napi) +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,27) +#define RTL_NAPI_DEL(priv) +#else +#define RTL_NAPI_DEL(priv) netif_napi_del(&priv->napi) +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,27) + +/*****************************************************************************/ +#ifdef CONFIG_R8127_NAPI +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0) +#define RTL_NAPI_CONSUME_SKB_ANY(skb, budget) napi_consume_skb(skb, budget) +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0) +#define RTL_NAPI_CONSUME_SKB_ANY(skb, budget) dev_consume_skb_any(skb); +#else +#define RTL_NAPI_CONSUME_SKB_ANY(skb, budget) dev_kfree_skb_any(skb); +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0) +#else //CONFIG_R8127_NAPI +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0) +#define RTL_NAPI_CONSUME_SKB_ANY(skb, budget) dev_consume_skb_any(skb); +#else +#define RTL_NAPI_CONSUME_SKB_ANY(skb, budget) dev_kfree_skb_any(skb); +#endif +#endif //CONFIG_R8127_NAPI + +/*****************************************************************************/ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,9) +#ifdef __CHECKER__ +#define __iomem __attribute__((noderef, address_space(2))) +extern void __chk_io_ptr(void __iomem *); +#define __bitwise __attribute__((bitwise)) +#else +#define __iomem +#define __chk_io_ptr(x) (void)0 +#define __bitwise +#endif +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,9) + +/*****************************************************************************/ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,8) +#ifdef __CHECKER__ +#define __force __attribute__((force)) +#else +#define __force +#endif +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,8) + +#ifndef module_param +#define module_param(v,t,p) MODULE_PARM(v, "i"); +#endif + +#ifndef PCI_DEVICE +#define PCI_DEVICE(vend,dev) \ + .vendor = (vend), .device = (dev), \ + .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID +#endif + +/*****************************************************************************/ +/* 2.5.28 => 2.4.23 */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,28)) + +static inline void _kc_synchronize_irq(void) +{ + synchronize_irq(); +} +#undef synchronize_irq +#define synchronize_irq(X) _kc_synchronize_irq() + +#include +#define work_struct tq_struct +#undef INIT_WORK +#define INIT_WORK(a,b,c) INIT_TQUEUE(a,(void (*)(void *))b,c) +#undef container_of +#define container_of list_entry +#define schedule_work schedule_task +#define flush_scheduled_work flush_scheduled_tasks +#endif /* 2.5.28 => 2.4.17 */ + +/*****************************************************************************/ +/* 2.6.4 => 2.6.0 */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,4)) +#define MODULE_VERSION(_version) MODULE_INFO(version, _version) +#endif /* 2.6.4 => 2.6.0 */ +/*****************************************************************************/ +/* 2.6.0 => 2.5.28 */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)) +#define MODULE_INFO(version, _version) +#ifndef CONFIG_E1000_DISABLE_PACKET_SPLIT +#define CONFIG_E1000_DISABLE_PACKET_SPLIT 1 +#endif + +#define pci_set_consistent_dma_mask(dev,mask) 1 + +#undef dev_put +#define dev_put(dev) __dev_put(dev) + +#ifndef skb_fill_page_desc +#define skb_fill_page_desc _kc_skb_fill_page_desc +extern void _kc_skb_fill_page_desc(struct sk_buff *skb, int i, struct page *page, int off, int size); +#endif + +#ifndef pci_dma_mapping_error +#define pci_dma_mapping_error _kc_pci_dma_mapping_error +static inline int _kc_pci_dma_mapping_error(dma_addr_t dma_addr) +{ + return dma_addr == 0; +} +#endif + +#undef ALIGN +#define ALIGN(x,a) (((x)+(a)-1)&~((a)-1)) + +#endif /* 2.6.0 => 2.5.28 */ + +/*****************************************************************************/ +/* 2.4.22 => 2.4.17 */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,4,22)) +#define pci_name(x) ((x)->slot_name) +#endif /* 2.4.22 => 2.4.17 */ + +/*****************************************************************************/ +/* 2.6.5 => 2.6.0 */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,5)) +#define pci_dma_sync_single_for_cpu pci_dma_sync_single +#define pci_dma_sync_single_for_device pci_dma_sync_single_for_cpu +#endif /* 2.6.5 => 2.6.0 */ + +/*****************************************************************************/ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) +/* + * initialize a work-struct's func and data pointers: + */ +#define PREPARE_WORK(_work, _func, _data) \ + do { \ + (_work)->func = _func; \ + (_work)->data = _data; \ + } while (0) + +#endif +/*****************************************************************************/ +/* 2.6.4 => 2.6.0 */ +#if ((LINUX_VERSION_CODE < KERNEL_VERSION(2,4,25) && \ + LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22)) || \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) && \ + LINUX_VERSION_CODE < KERNEL_VERSION(2,6,4))) +#define ETHTOOL_OPS_COMPAT +#endif /* 2.6.4 => 2.6.0 */ + +/*****************************************************************************/ +/* Installations with ethtool version without eeprom, adapter id, or statistics + * support */ + +#ifndef ETH_GSTRING_LEN +#define ETH_GSTRING_LEN 32 +#endif + +#ifndef ETHTOOL_GSTATS +#define ETHTOOL_GSTATS 0x1d +#undef ethtool_drvinfo +#define ethtool_drvinfo k_ethtool_drvinfo +struct k_ethtool_drvinfo { + u32 cmd; + char driver[32]; + char version[32]; + char fw_version[32]; + char bus_info[32]; + char reserved1[32]; + char reserved2[16]; + u32 n_stats; + u32 testinfo_len; + u32 eedump_len; + u32 regdump_len; +}; + +struct ethtool_stats { + u32 cmd; + u32 n_stats; + u64 data[0]; +}; +#endif /* ETHTOOL_GSTATS */ + +#ifndef ETHTOOL_PHYS_ID +#define ETHTOOL_PHYS_ID 0x1c +#endif /* ETHTOOL_PHYS_ID */ + +#ifndef ETHTOOL_GSTRINGS +#define ETHTOOL_GSTRINGS 0x1b +enum ethtool_stringset { + ETH_SS_TEST = 0, + ETH_SS_STATS, +}; +struct ethtool_gstrings { + u32 cmd; /* ETHTOOL_GSTRINGS */ + u32 string_set; /* string set id e.c. ETH_SS_TEST, etc*/ + u32 len; /* number of strings in the string set */ + u8 data[0]; +}; +#endif /* ETHTOOL_GSTRINGS */ + +#ifndef ETHTOOL_TEST +#define ETHTOOL_TEST 0x1a +enum ethtool_test_flags { + ETH_TEST_FL_OFFLINE = (1 << 0), + ETH_TEST_FL_FAILED = (1 << 1), +}; +struct ethtool_test { + u32 cmd; + u32 flags; + u32 reserved; + u32 len; + u64 data[0]; +}; +#endif /* ETHTOOL_TEST */ + +#ifndef ETHTOOL_GEEPROM +#define ETHTOOL_GEEPROM 0xb +#undef ETHTOOL_GREGS +struct ethtool_eeprom { + u32 cmd; + u32 magic; + u32 offset; + u32 len; + u8 data[0]; +}; + +struct ethtool_value { + u32 cmd; + u32 data; +}; +#endif /* ETHTOOL_GEEPROM */ + +#ifndef ETHTOOL_GLINK +#define ETHTOOL_GLINK 0xa +#endif /* ETHTOOL_GLINK */ + +#ifndef ETHTOOL_GREGS +#define ETHTOOL_GREGS 0x00000004 /* Get NIC registers */ +#define ethtool_regs _kc_ethtool_regs +/* for passing big chunks of data */ +struct _kc_ethtool_regs { + u32 cmd; + u32 version; /* driver-specific, indicates different chips/revs */ + u32 len; /* bytes */ + u8 data[0]; +}; +#endif /* ETHTOOL_GREGS */ + +#ifndef ETHTOOL_GMSGLVL +#define ETHTOOL_GMSGLVL 0x00000007 /* Get driver message level */ +#endif +#ifndef ETHTOOL_SMSGLVL +#define ETHTOOL_SMSGLVL 0x00000008 /* Set driver msg level, priv. */ +#endif +#ifndef ETHTOOL_NWAY_RST +#define ETHTOOL_NWAY_RST 0x00000009 /* Restart autonegotiation, priv */ +#endif +#ifndef ETHTOOL_GLINK +#define ETHTOOL_GLINK 0x0000000a /* Get link status */ +#endif +#ifndef ETHTOOL_GEEPROM +#define ETHTOOL_GEEPROM 0x0000000b /* Get EEPROM data */ +#endif +#ifndef ETHTOOL_SEEPROM +#define ETHTOOL_SEEPROM 0x0000000c /* Set EEPROM data */ +#endif +#ifndef ETHTOOL_GCOALESCE +#define ETHTOOL_GCOALESCE 0x0000000e /* Get coalesce config */ +/* for configuring coalescing parameters of chip */ +#define ethtool_coalesce _kc_ethtool_coalesce +struct _kc_ethtool_coalesce { + u32 cmd; /* ETHTOOL_{G,S}COALESCE */ + + /* How many usecs to delay an RX interrupt after + * a packet arrives. If 0, only rx_max_coalesced_frames + * is used. + */ + u32 rx_coalesce_usecs; + + /* How many packets to delay an RX interrupt after + * a packet arrives. If 0, only rx_coalesce_usecs is + * used. It is illegal to set both usecs and max frames + * to zero as this would cause RX interrupts to never be + * generated. + */ + u32 rx_max_coalesced_frames; + + /* Same as above two parameters, except that these values + * apply while an IRQ is being serviced by the host. Not + * all cards support this feature and the values are ignored + * in that case. + */ + u32 rx_coalesce_usecs_irq; + u32 rx_max_coalesced_frames_irq; + + /* How many usecs to delay a TX interrupt after + * a packet is sent. If 0, only tx_max_coalesced_frames + * is used. + */ + u32 tx_coalesce_usecs; + + /* How many packets to delay a TX interrupt after + * a packet is sent. If 0, only tx_coalesce_usecs is + * used. It is illegal to set both usecs and max frames + * to zero as this would cause TX interrupts to never be + * generated. + */ + u32 tx_max_coalesced_frames; + + /* Same as above two parameters, except that these values + * apply while an IRQ is being serviced by the host. Not + * all cards support this feature and the values are ignored + * in that case. + */ + u32 tx_coalesce_usecs_irq; + u32 tx_max_coalesced_frames_irq; + + /* How many usecs to delay in-memory statistics + * block updates. Some drivers do not have an in-memory + * statistic block, and in such cases this value is ignored. + * This value must not be zero. + */ + u32 stats_block_coalesce_usecs; + + /* Adaptive RX/TX coalescing is an algorithm implemented by + * some drivers to improve latency under low packet rates and + * improve throughput under high packet rates. Some drivers + * only implement one of RX or TX adaptive coalescing. Anything + * not implemented by the driver causes these values to be + * silently ignored. + */ + u32 use_adaptive_rx_coalesce; + u32 use_adaptive_tx_coalesce; + + /* When the packet rate (measured in packets per second) + * is below pkt_rate_low, the {rx,tx}_*_low parameters are + * used. + */ + u32 pkt_rate_low; + u32 rx_coalesce_usecs_low; + u32 rx_max_coalesced_frames_low; + u32 tx_coalesce_usecs_low; + u32 tx_max_coalesced_frames_low; + + /* When the packet rate is below pkt_rate_high but above + * pkt_rate_low (both measured in packets per second) the + * normal {rx,tx}_* coalescing parameters are used. + */ + + /* When the packet rate is (measured in packets per second) + * is above pkt_rate_high, the {rx,tx}_*_high parameters are + * used. + */ + u32 pkt_rate_high; + u32 rx_coalesce_usecs_high; + u32 rx_max_coalesced_frames_high; + u32 tx_coalesce_usecs_high; + u32 tx_max_coalesced_frames_high; + + /* How often to do adaptive coalescing packet rate sampling, + * measured in seconds. Must not be zero. + */ + u32 rate_sample_interval; +}; +#endif /* ETHTOOL_GCOALESCE */ + +#ifndef ETHTOOL_SCOALESCE +#define ETHTOOL_SCOALESCE 0x0000000f /* Set coalesce config. */ +#endif +#ifndef ETHTOOL_GRINGPARAM +#define ETHTOOL_GRINGPARAM 0x00000010 /* Get ring parameters */ +/* for configuring RX/TX ring parameters */ +#define ethtool_ringparam _kc_ethtool_ringparam +struct _kc_ethtool_ringparam { + u32 cmd; /* ETHTOOL_{G,S}RINGPARAM */ + + /* Read only attributes. These indicate the maximum number + * of pending RX/TX ring entries the driver will allow the + * user to set. + */ + u32 rx_max_pending; + u32 rx_mini_max_pending; + u32 rx_jumbo_max_pending; + u32 tx_max_pending; + + /* Values changeable by the user. The valid values are + * in the range 1 to the "*_max_pending" counterpart above. + */ + u32 rx_pending; + u32 rx_mini_pending; + u32 rx_jumbo_pending; + u32 tx_pending; +}; +#endif /* ETHTOOL_GRINGPARAM */ + +#ifndef ETHTOOL_SRINGPARAM +#define ETHTOOL_SRINGPARAM 0x00000011 /* Set ring parameters, priv. */ +#endif +#ifndef ETHTOOL_GPAUSEPARAM +#define ETHTOOL_GPAUSEPARAM 0x00000012 /* Get pause parameters */ +/* for configuring link flow control parameters */ +#define ethtool_pauseparam _kc_ethtool_pauseparam +struct _kc_ethtool_pauseparam { + u32 cmd; /* ETHTOOL_{G,S}PAUSEPARAM */ + + /* If the link is being auto-negotiated (via ethtool_cmd.autoneg + * being true) the user may set 'autonet' here non-zero to have the + * pause parameters be auto-negotiated too. In such a case, the + * {rx,tx}_pause values below determine what capabilities are + * advertised. + * + * If 'autoneg' is zero or the link is not being auto-negotiated, + * then {rx,tx}_pause force the driver to use/not-use pause + * flow control. + */ + u32 autoneg; + u32 rx_pause; + u32 tx_pause; +}; +#endif /* ETHTOOL_GPAUSEPARAM */ + +#ifndef ETHTOOL_SPAUSEPARAM +#define ETHTOOL_SPAUSEPARAM 0x00000013 /* Set pause parameters. */ +#endif +#ifndef ETHTOOL_GRXCSUM +#define ETHTOOL_GRXCSUM 0x00000014 /* Get RX hw csum enable (ethtool_value) */ +#endif +#ifndef ETHTOOL_SRXCSUM +#define ETHTOOL_SRXCSUM 0x00000015 /* Set RX hw csum enable (ethtool_value) */ +#endif +#ifndef ETHTOOL_GTXCSUM +#define ETHTOOL_GTXCSUM 0x00000016 /* Get TX hw csum enable (ethtool_value) */ +#endif +#ifndef ETHTOOL_STXCSUM +#define ETHTOOL_STXCSUM 0x00000017 /* Set TX hw csum enable (ethtool_value) */ +#endif +#ifndef ETHTOOL_GSG +#define ETHTOOL_GSG 0x00000018 /* Get scatter-gather enable +* (ethtool_value) */ +#endif +#ifndef ETHTOOL_SSG +#define ETHTOOL_SSG 0x00000019 /* Set scatter-gather enable +* (ethtool_value). */ +#endif +#ifndef ETHTOOL_TEST +#define ETHTOOL_TEST 0x0000001a /* execute NIC self-test, priv. */ +#endif +#ifndef ETHTOOL_GSTRINGS +#define ETHTOOL_GSTRINGS 0x0000001b /* get specified string set */ +#endif +#ifndef ETHTOOL_PHYS_ID +#define ETHTOOL_PHYS_ID 0x0000001c /* identify the NIC */ +#endif +#ifndef ETHTOOL_GSTATS +#define ETHTOOL_GSTATS 0x0000001d /* get NIC-specific statistics */ +#endif +#ifndef ETHTOOL_GTSO +#define ETHTOOL_GTSO 0x0000001e /* Get TSO enable (ethtool_value) */ +#endif +#ifndef ETHTOOL_STSO +#define ETHTOOL_STSO 0x0000001f /* Set TSO enable (ethtool_value) */ +#endif + +#ifndef ETHTOOL_BUSINFO_LEN +#define ETHTOOL_BUSINFO_LEN 32 +#endif + +/*****************************************************************************/ + +enum RTL8127_registers { + MAC0 = 0x00, /* Ethernet hardware address. */ + MAC4 = 0x04, + MAR0 = 0x08, /* Multicast filter. */ + CounterAddrLow = 0x10, + CounterAddrHigh = 0x14, + CustomLED = 0x18, + TxDescStartAddrLow = 0x20, + TxDescStartAddrHigh = 0x24, + TxHDescStartAddrLow = 0x28, + TxHDescStartAddrHigh = 0x2c, + FLASH = 0x30, + INT_CFG0_8125 = 0x34, + ERSR = 0x36, + ChipCmd = 0x37, + TxPoll = 0x38, + IntrMask = 0x3C, + IntrStatus = 0x3E, + TxConfig = 0x40, + RxConfig = 0x44, + TCTR = 0x48, + Cfg9346 = 0x50, + Config0 = 0x51, + Config1 = 0x52, + Config2 = 0x53, + Config3 = 0x54, + Config4 = 0x55, + Config5 = 0x56, + TDFNR = 0x57, + TimeInt0 = 0x58, + TimeInt1 = 0x5C, + PHYAR = 0x60, + CSIDR = 0x64, + CSIAR = 0x68, + PHYstatus = 0x6C, + MACDBG = 0x6D, + GPIO = 0x6E, + PMCH = 0x6F, + ERIDR = 0x70, + ERIAR = 0x74, + INT_CFG1_8125 = 0x7A, + EPHY_RXER_NUM = 0x7C, + EPHYAR = 0x80, + TimeInt2 = 0x8C, + OCPDR = 0xB0, + MACOCP = 0xB0, + OCPAR = 0xB4, + SecMAC0 = 0xB4, + SecMAC4 = 0xB8, + PHYOCP = 0xB8, + DBG_reg = 0xD1, + TwiCmdReg = 0xD2, + MCUCmd_reg = 0xD3, + RxMaxSize = 0xDA, + EFUSEAR = 0xDC, + CPlusCmd = 0xE0, + IntrMitigate = 0xE2, + RxDescAddrLow = 0xE4, + RxDescAddrHigh = 0xE8, + MTPS = 0xEC, + FuncEvent = 0xF0, + PPSW = 0xF2, + FuncEventMask = 0xF4, + TimeInt3 = 0xF4, + FuncPresetState = 0xF8, + CMAC_IBCR0 = 0xF8, + CMAC_IBCR2 = 0xF9, + CMAC_IBIMR0 = 0xFA, + CMAC_IBISR0 = 0xFB, + FuncForceEvent = 0xFC, + //8125 + IMR0_8125 = 0x38, + ISR0_8125 = 0x3C, + TPPOLL_8125 = 0x90, + IMR1_8125 = 0x800, + ISR1_8125 = 0x802, + IMR2_8125 = 0x804, + ISR2_8125 = 0x806, + IMR3_8125 = 0x808, + ISR3_8125 = 0x80A, + BACKUP_ADDR0_8125 = 0x19E0, + BACKUP_ADDR1_8125 = 0X19E4, + TCTR0_8125 = 0x0048, + TCTR1_8125 = 0x004C, + TCTR2_8125 = 0x0088, + TCTR3_8125 = 0x001C, + TIMER_INT0_8125 = 0x0058, + TIMER_INT1_8125 = 0x005C, + TIMER_INT2_8125 = 0x008C, + TIMER_INT3_8125 = 0x00F4, + INT_MITI_V2_0_RX = 0x0A00, + INT_MITI_V2_0_TX = 0x0A02, + INT_MITI_V2_1_RX = 0x0A08, + INT_MITI_V2_1_TX = 0x0A0A, + IMR_V2_CLEAR_REG_8125 = 0x0D00, + ISR_V2_8125 = 0x0D04, + IMR_V2_SET_REG_8125 = 0x0D0C, + TDU_STA_8125 = 0x0D08, + RDU_STA_8125 = 0x0D0A, + IMR_V4_L2_CLEAR_REG_8125 = 0x0D10, + IMR_V4_L2_SET_REG_8125 = 0x0D18, + ISR_V4_L2_8125 = 0x0D14, + SW_TAIL_PTR0_8125BP = 0x0D30, + SW_TAIL_PTR1_8125BP = 0x0D38, + HW_CLO_PTR0_8125BP = 0x0D34, + HW_CLO_PTR1_8125BP = 0x0D3C, + DOUBLE_VLAN_CONFIG = 0x1000, + TX_NEW_CTRL = 0x203E, + TNPDS_Q1_LOW_8125 = 0x2100, + PLA_TXQ0_IDLE_CREDIT = 0x2500, + PLA_TXQ1_IDLE_CREDIT = 0x2504, + SW_TAIL_PTR0_8125 = 0x2800, + HW_CLO_PTR0_8125 = 0x2802, + SW_TAIL_PTR0_8126 = 0x2800, + HW_CLO_PTR0_8126 = 0x2800, + RDSAR_Q1_LOW_8125 = 0x4000, + RSS_CTRL_8125 = 0x4500, + Q_NUM_CTRL_8125 = 0x4800, + RSS_KEY_8125 = 0x4600, + RSS_INDIRECTION_TBL_8125_V2 = 0x4700, + EEE_TXIDLE_TIMER_8125 = 0x6048, + PTP_CTRL_8125 = 0x6800, + PTP_STATUS_8125 = 0x6802, + PTP_ISR_8125 = 0x6804, + PTP_IMR_8125 = 0x6805, + PTP_TIME_CORRECT_CMD_8125 = 0x6806, + PTP_SOFT_CONFIG_Time_NS_8125 = 0x6808, + PTP_SOFT_CONFIG_Time_S_8125 = 0x680C, + PTP_SOFT_CONFIG_Time_Sign = 0x6812, + PTP_LOCAL_Time_SUB_NS_8125 = 0x6814, + PTP_LOCAL_Time_NS_8125 = 0x6818, + PTP_LOCAL_Time_S_8125 = 0x681C, + PTP_Time_SHIFTER_S_8125 = 0x6856, + PPS_RISE_TIME_NS_8125 = 0x68A0, + PPS_RISE_TIME_S_8125 = 0x68A4, + PTP_EGRESS_TIME_BASE_NS_8125 = 0XCF20, + PTP_EGRESS_TIME_BASE_S_8125 = 0XCF24, + PTP_CTL = 0xE400, + PTP_INER = 0xE402, + PTP_INSR = 0xE404, + PTP_SYNCE_CTL = 0xE406, + PTP_GEN_CFG = 0xE408, + PTP_CLK_CFG_8126 = 0xE410, + PTP_CFG_NS_LO_8126 = 0xE412, + PTP_CFG_NS_HI_8126 = 0xE414, + PTP_CFG_S_LO_8126 = 0xE416, + PTP_CFG_S_MI_8126 = 0xE418, + PTP_CFG_S_HI_8126 = 0xE41A, + PTP_TAI_CFG = 0xE420, + PTP_TAI_TS_S_LO = 0xE42A, + PTP_TAI_TS_S_HI = 0xE42C, + PTP_TRX_TS_STA = 0xE430, + PTP_TRX_TS_NS_LO = 0xE446, + PTP_TRX_TS_NS_HI = 0xE448, + PTP_TRX_TS_S_LO = 0xE44A, + PTP_TRX_TS_S_MI = 0xE44C, + PTP_TRX_TS_S_HI = 0xE44E, + + //TCAM + TCAM_NOTVALID_ADDR = 0xA000, + TCAM_VALID_ADDR = 0xA800, + TCAM_MAC_ADDR = 448, + TCAM_VLAN_TAG = 496, + //TCAM V2 + TCAM_NOTVALID_ADDR_V2 = 0xA000, + TCAM_VALID_ADDR_V2 = 0xB000, + TCAM_MAC_ADDR_V2 = 0x00, + TCAM_VLAN_TAG_V2 = 0x03, +}; + +enum RTL8127_register_content { + /* InterruptStatusBits */ + SYSErr = 0x8000, + PCSTimeout = 0x4000, + SWInt = 0x0100, + TxDescUnavail = 0x0080, + RxFIFOOver = 0x0040, + LinkChg = 0x0020, + RxDescUnavail = 0x0010, + TxErr = 0x0008, + TxOK = 0x0004, + RxErr = 0x0002, + RxOK = 0x0001, + RxDU1 = 0x0002, + RxOK1 = 0x0001, + + /* RxStatusDesc */ + RxRWT = (1 << 22), + RxRES = (1 << 21), + RxRUNT = (1 << 20), + RxCRC = (1 << 19), + + RxRWT_V3 = (1 << 18), + RxRES_V3 = (1 << 20), + RxRUNT_V3 = (1 << 19), + RxCRC_V3 = (1 << 17), + + RxRES_V4 = (1 << 22), + RxRUNT_V4 = (1 << 21), + RxCRC_V4 = (1 << 20), + + /* ChipCmdBits */ + StopReq = 0x80, + CmdReset = 0x10, + CmdRxEnb = 0x08, + CmdTxEnb = 0x04, + RxBufEmpty = 0x01, + + /* Cfg9346Bits */ + Cfg9346_EEM_MASK = 0xC0, + Cfg9346_Lock = 0x00, + Cfg9346_Unlock = 0xC0, + Cfg9346_EEDO = (1 << 0), + Cfg9346_EEDI = (1 << 1), + Cfg9346_EESK = (1 << 2), + Cfg9346_EECS = (1 << 3), + Cfg9346_EEM0 = (1 << 6), + Cfg9346_EEM1 = (1 << 7), + + /* rx_mode_bits */ + AcceptErr = 0x20, + AcceptRunt = 0x10, + AcceptBroadcast = 0x08, + AcceptMulticast = 0x04, + AcceptMyPhys = 0x02, + AcceptAllPhys = 0x01, + AcceppVlanPhys = 0x8000, + + /* Transmit Priority Polling*/ + HPQ = 0x80, + NPQ = 0x40, + FSWInt = 0x01, + + /* RxConfigBits */ + Reserved2_shift = 13, + RxCfgDMAShift = 8, + EnableRxDescV3 = (1 << 24), + EnableRxDescV4_1 = (1 << 24), + EnableOuterVlan = (1 << 23), + EnableInnerVlan = (1 << 22), + RxCfg_128_int_en = (1 << 15), + RxCfg_fet_multi_en = (1 << 14), + RxCfg_half_refetch = (1 << 13), + RxCfg_pause_slot_en = (1 << 11), + RxCfg_9356SEL = (1 << 6), + EnableRxDescV4_0 = (1 << 1), //not in rcr + + /* TxConfigBits */ + TxInterFrameGapShift = 24, + TxDMAShift = 8, /* DMA burst value (0-7) is shift this many bits */ + TxMACLoopBack = (1 << 17), /* MAC loopback */ + + /* Config1 register */ + LEDS1 = (1 << 7), + LEDS0 = (1 << 6), + Speed_down = (1 << 4), + MEMMAP = (1 << 3), + IOMAP = (1 << 2), + VPD = (1 << 1), + PMEnable = (1 << 0), /* Power Management Enable */ + + /* Config2 register */ + PMSTS_En = (1 << 5), + + /* Config3 register */ + Isolate_en = (1 << 12), /* Isolate enable */ + MagicPacket = (1 << 5), /* Wake up when receives a Magic Packet */ + LinkUp = (1 << 4), /* This bit is reserved in RTL8125B.*/ + /* Wake up when the cable connection is re-established */ + ECRCEN = (1 << 3), /* This bit is reserved in RTL8125B*/ + Jumbo_En0 = (1 << 2), /* This bit is reserved in RTL8125B*/ + RDY_TO_L23 = (1 << 1), /* This bit is reserved in RTL8125B*/ + Beacon_en = (1 << 0), /* This bit is reserved in RTL8125B*/ + + /* Config4 register */ + Jumbo_En1 = (1 << 1), /* This bit is reserved in RTL8125B*/ + + /* Config5 register */ + BWF = (1 << 6), /* Accept Broadcast wakeup frame */ + MWF = (1 << 5), /* Accept Multicast wakeup frame */ + UWF = (1 << 4), /* Accept Unicast wakeup frame */ + LanWake = (1 << 1), /* LanWake enable/disable */ + PMEStatus = (1 << 0), /* PME status can be reset by PCI RST# */ + + /* CPlusCmd */ + EnableBist = (1 << 15), + Macdbgo_oe = (1 << 14), + Normal_mode = (1 << 13), + Force_halfdup = (1 << 12), + Force_rxflow_en = (1 << 11), + Force_txflow_en = (1 << 10), + Cxpl_dbg_sel = (1 << 9),//This bit is reserved in RTL8125B + ASF = (1 << 8),//This bit is reserved in RTL8125C + PktCntrDisable = (1 << 7), + RxVlan = (1 << 6), + RxChkSum = (1 << 5), + Macdbgo_sel = 0x001C, + INTT_0 = 0x0000, + INTT_1 = 0x0001, + INTT_2 = 0x0002, + INTT_3 = 0x0003, + + /* rtl8127_PHYstatus */ + PowerSaveStatus = 0x80, + _1000bpsL = 0x80000, + _10000bpsF = 0x4000, + _10000bpsL = 0x2000, + _5000bpsF = 0x1000, + _5000bpsL = 0x800, + _2500bpsF = 0x400, + _2500bpsL = 0x200, + TxFlowCtrl = 0x40, + RxFlowCtrl = 0x20, + _1000bpsF = 0x10, + _100bps = 0x08, + _10bps = 0x04, + LinkStatus = 0x02, + FullDup = 0x01, + + /* DBG_reg */ + Fix_Nak_1 = (1 << 4), + Fix_Nak_2 = (1 << 3), + DBGPIN_E2 = (1 << 0), + + /* ResetCounterCommand */ + CounterReset = 0x1, + /* DumpCounterCommand */ + CounterDump = 0x8, + + /* PHY access */ + PHYAR_Flag = 0x80000000, + PHYAR_Write = 0x80000000, + PHYAR_Read = 0x00000000, + PHYAR_Reg_Mask = 0x1f, + PHYAR_Reg_shift = 16, + PHYAR_Data_Mask = 0xffff, + + /* EPHY access */ + EPHYAR_Flag = 0x80000000, + EPHYAR_Write = 0x80000000, + EPHYAR_Read = 0x00000000, + EPHYAR_Reg_Mask = 0x3f, + EPHYAR_Reg_Mask_v2 = 0x7f, + EPHYAR_Reg_shift = 16, + EPHYAR_Data_Mask = 0xffff, + EPHYAR_EXT_ADDR = 0x0ffe, + + /* CSI access */ + CSIAR_Flag = 0x80000000, + CSIAR_Write = 0x80000000, + CSIAR_Read = 0x00000000, + CSIAR_ByteEn = 0x0f, + CSIAR_ByteEn_shift = 12, + CSIAR_Addr_Mask = 0x0fff, + + /* ERI access */ + ERIAR_Flag = 0x80000000, + ERIAR_Write = 0x80000000, + ERIAR_Read = 0x00000000, + ERIAR_Addr_Align = 4, /* ERI access register address must be 4 byte alignment */ + ERIAR_ExGMAC = 0, + ERIAR_MSIX = 1, + ERIAR_ASF = 2, + ERIAR_OOB = 2, + ERIAR_Type_shift = 16, + ERIAR_ByteEn = 0x0f, + ERIAR_ByteEn_shift = 12, + + /* OCP GPHY access */ + OCPDR_Write = 0x80000000, + OCPDR_Read = 0x00000000, + OCPDR_Reg_Mask = 0xFF, + OCPDR_Data_Mask = 0xFFFF, + OCPDR_GPHY_Reg_shift = 16, + OCPAR_Flag = 0x80000000, + OCPAR_GPHY_Write = 0x8000F060, + OCPAR_GPHY_Read = 0x0000F060, + OCPR_Write = 0x80000000, + OCPR_Read = 0x00000000, + OCPR_Addr_Reg_shift = 16, + OCPR_Flag = 0x80000000, + OCP_STD_PHY_BASE_PAGE = 0x0A40, + + /* MCU Command */ + Now_is_oob = (1 << 7), + Txfifo_empty = (1 << 5), + Rxfifo_empty = (1 << 4), + + /* E-FUSE access */ + EFUSE_WRITE = 0x80000000, + EFUSE_WRITE_OK = 0x00000000, + EFUSE_READ = 0x00000000, + EFUSE_READ_OK = 0x80000000, + EFUSE_WRITE_V3 = 0x40000000, + EFUSE_WRITE_OK_V3 = 0x00000000, + EFUSE_READ_V3 = 0x80000000, + EFUSE_READ_OK_V3 = 0x00000000, + EFUSE_Reg_Mask = 0x03FF, + EFUSE_Reg_Shift = 8, + EFUSE_Check_Cnt = 300, + EFUSE_READ_FAIL = 0xFF, + EFUSE_Data_Mask = 0x000000FF, + + /* GPIO */ + GPIO_en = (1 << 0), + + /* PTP */ + PTP_ISR_TOK = (1 << 1), + PTP_ISR_TER = (1 << 2), + PTP_EXEC_CMD = (1 << 7), + PTP_ADJUST_TIME_NS_NEGATIVE = (1 << 30), + PTP_ADJUST_TIME_S_NEGATIVE = (1ULL << 48), + PTP_SOFT_CONFIG_TIME_NS_NEGATIVE = (1 << 30), + PTP_SOFT_CONFIG_TIME_S_NEGATIVE = (1ULL << 48), + + /* New Interrupt Bits */ + INT_CFG0_ENABLE_8125 = (1 << 0), + INT_CFG0_TIMEOUT0_BYPASS_8125 = (1 << 1), + INT_CFG0_MITIGATION_BYPASS_8125 = (1 << 2), + INT_CFG0_RDU_BYPASS_8126 = (1 << 4), + INT_CFG0_MSIX_ENTRY_NUM_MODE = (1 << 5), + INT_CFG0_AUTO_CLEAR_IMR = (1 << 5), + INT_CFG0_AVOID_MISS_INTR = (1 << 6), + ISRIMR_V2_ROK_Q0 = (1 << 0), + ISRIMR_TOK_Q0 = (1 << 16), + ISRIMR_TOK_Q1 = (1 << 18), + ISRIMR_V2_LINKCHG = (1 << 21), + + ISRIMR_V4_ROK_Q0 = (1 << 0), + ISRIMR_V4_LINKCHG = (1 << 29), + + ISRIMR_V5_ROK_Q0 = (1 << 0), + ISRIMR_V5_TOK_Q0 = (1 << 16), + ISRIMR_V5_TOK_Q1 = (1 << 17), + ISRIMR_V5_LINKCHG = (1 << 18), + + ISRIMR_V6_ROK_Q0 = (1 << 0), + ISRIMR_V6_TOK_Q0 = (1 << 8), + ISRIMR_V6_TOK_Q1 = (1 << 9), + ISRIMR_V6_LINKCHG = (1 << 29), + + /* Magic Number */ + RTL8127_MAGIC_NUMBER = 0x0badbadbadbadbadull, +}; + +enum _DescStatusBit { + DescOwn = (1 << 31), /* Descriptor is owned by NIC */ + RingEnd = (1 << 30), /* End of descriptor ring */ + FirstFrag = (1 << 29), /* First segment of a packet */ + LastFrag = (1 << 28), /* Final segment of a packet */ + + DescOwn_V3 = (DescOwn), /* Descriptor is owned by NIC */ + RingEnd_V3 = (RingEnd), /* End of descriptor ring */ + FirstFrag_V3 = (1 << 25), /* First segment of a packet */ + LastFrag_V3 = (1 << 24), /* Final segment of a packet */ + + DescOwn_V4 = (DescOwn), /* Descriptor is owned by NIC */ + RingEnd_V4 = (RingEnd), /* End of descriptor ring */ + FirstFrag_V4 = (FirstFrag), /* First segment of a packet */ + LastFrag_V4 = (LastFrag), /* Final segment of a packet */ + + /* Tx private */ + /*------ offset 0 of tx descriptor ------*/ + LargeSend = (1 << 27), /* TCP Large Send Offload (TSO) */ + GiantSendv4 = (1 << 26), /* TCP Giant Send Offload V4 (GSOv4) */ + GiantSendv6 = (1 << 25), /* TCP Giant Send Offload V6 (GSOv6) */ + LargeSend_DP = (1 << 16), /* TCP Large Send Offload (TSO) */ + MSSShift = 16, /* MSS value position */ + MSSMask = 0x7FFU, /* MSS value 11 bits */ + TxIPCS = (1 << 18), /* Calculate IP checksum */ + TxUDPCS = (1 << 17), /* Calculate UDP/IP checksum */ + TxTCPCS = (1 << 16), /* Calculate TCP/IP checksum */ + TxVlanTag = (1 << 17), /* Add VLAN tag */ + + /*@@@@@@ offset 4 of tx descriptor => bits for RTL8125 only begin @@@@@@*/ + TxUDPCS_C = (1 << 31), /* Calculate UDP/IP checksum */ + TxTCPCS_C = (1 << 30), /* Calculate TCP/IP checksum */ + TxIPCS_C = (1 << 29), /* Calculate IP checksum */ + TxIPV6F_C = (1 << 28), /* Indicate it is an IPv6 packet */ + /*@@@@@@ offset 4 of tx descriptor => bits for RTL8125 only end @@@@@@*/ + + + /* Rx private */ + /*------ offset 0 of rx descriptor ------*/ + PID1 = (1 << 18), /* Protocol ID bit 1/2 */ + PID0 = (1 << 17), /* Protocol ID bit 2/2 */ + +#define RxProtoUDP (PID1) +#define RxProtoTCP (PID0) +#define RxProtoIP (PID1 | PID0) +#define RxProtoMask RxProtoIP + + RxIPF = (1 << 16), /* IP checksum failed */ + RxUDPF = (1 << 15), /* UDP/IP checksum failed */ + RxTCPF = (1 << 14), /* TCP/IP checksum failed */ + RxVlanTag = (1 << 16), /* VLAN tag available */ + + /*@@@@@@ offset 0 of rx descriptor => bits for RTL8125 only begin @@@@@@*/ + RxUDPT = (1 << 18), + RxTCPT = (1 << 17), + /*@@@@@@ offset 0 of rx descriptor => bits for RTL8125 only end @@@@@@*/ + + /*@@@@@@ offset 4 of rx descriptor => bits for RTL8125 only begin @@@@@@*/ + RxV6F = (1 << 31), + RxV4F = (1 << 30), + /*@@@@@@ offset 4 of rx descriptor => bits for RTL8125 only end @@@@@@*/ + + + PID1_v3 = (1 << 29), /* Protocol ID bit 1/2 */ + PID0_v3 = (1 << 28), /* Protocol ID bit 2/2 */ + +#define RxProtoUDP_v3 (PID1_v3) +#define RxProtoTCP_v3 (PID0_v3) +#define RxProtoIP_v3 (PID1_v3 | PID0_v3) +#define RxProtoMask_v3 RxProtoIP_v3 + + RxIPF_v3 = (1 << 26), /* IP checksum failed */ + RxUDPF_v3 = (1 << 25), /* UDP/IP checksum failed */ + RxTCPF_v3 = (1 << 24), /* TCP/IP checksum failed */ + RxSCTPF_v3 = (1 << 23), /* SCTP checksum failed */ + RxVlanTag_v3 = (RxVlanTag), /* VLAN tag available */ + + /*@@@@@@ offset 0 of rx descriptor => bits for RTL8125 only begin @@@@@@*/ + RxUDPT_v3 = (1 << 29), + RxTCPT_v3 = (1 << 28), + RxSCTP_v3 = (1 << 27), + /*@@@@@@ offset 0 of rx descriptor => bits for RTL8125 only end @@@@@@*/ + + /*@@@@@@ offset 4 of rx descriptor => bits for RTL8125 only begin @@@@@@*/ + RxV6F_v3 = (RxV6F), + RxV4F_v3 = (RxV4F), + /*@@@@@@ offset 4 of rx descriptor => bits for RTL8125 only end @@@@@@*/ + + RxIPF_v4 = (1 << 17), /* IP checksum failed */ + RxUDPF_v4 = (1 << 16), /* UDP/IP checksum failed */ + RxTCPF_v4 = (1 << 15), /* TCP/IP checksum failed */ + RxSCTPF_v4 = (1 << 19), /* SCTP checksum failed */ + RxVlanTag_v4 = (RxVlanTag), /* VLAN tag available */ + + /*@@@@@@ offset 0 of rx descriptor => bits for RTL8125 only begin @@@@@@*/ + RxUDPT_v4 = (1 << 19), + RxTCPT_v4 = (1 << 18), + RxSCTP_v4 = (1 << 19), + /*@@@@@@ offset 0 of rx descriptor => bits for RTL8125 only end @@@@@@*/ + + /*@@@@@@ offset 4 of rx descriptor => bits for RTL8125 only begin @@@@@@*/ + RxV6F_v4 = (RxV6F), + RxV4F_v4 = (RxV4F), + /*@@@@@@ offset 4 of rx descriptor => bits for RTL8125 only end @@@@@@*/ +}; + +enum features { +// RTL_FEATURE_WOL = (1 << 0), + RTL_FEATURE_MSI = (1 << 1), + RTL_FEATURE_MSIX = (1 << 2), +}; + +enum wol_capability { + WOL_DISABLED = 0, + WOL_ENABLED = 1 +}; + +enum bits { + BIT_0 = (1 << 0), + BIT_1 = (1 << 1), + BIT_2 = (1 << 2), + BIT_3 = (1 << 3), + BIT_4 = (1 << 4), + BIT_5 = (1 << 5), + BIT_6 = (1 << 6), + BIT_7 = (1 << 7), + BIT_8 = (1 << 8), + BIT_9 = (1 << 9), + BIT_10 = (1 << 10), + BIT_11 = (1 << 11), + BIT_12 = (1 << 12), + BIT_13 = (1 << 13), + BIT_14 = (1 << 14), + BIT_15 = (1 << 15), + BIT_16 = (1 << 16), + BIT_17 = (1 << 17), + BIT_18 = (1 << 18), + BIT_19 = (1 << 19), + BIT_20 = (1 << 20), + BIT_21 = (1 << 21), + BIT_22 = (1 << 22), + BIT_23 = (1 << 23), + BIT_24 = (1 << 24), + BIT_25 = (1 << 25), + BIT_26 = (1 << 26), + BIT_27 = (1 << 27), + BIT_28 = (1 << 28), + BIT_29 = (1 << 29), + BIT_30 = (1 << 30), + BIT_31 = (1 << 31) +}; + +#define RTL8127_CP_NUM 4 +#define RTL8127_MAX_SUPPORT_CP_LEN 110 + +enum rtl8127_cp_status { + rtl8127_cp_normal = 0, + rtl8127_cp_short, + rtl8127_cp_open, + rtl8127_cp_mismatch, + rtl8127_cp_unknown +}; + +enum efuse { + EFUSE_NOT_SUPPORT = 0, + EFUSE_SUPPORT_V1, + EFUSE_SUPPORT_V2, + EFUSE_SUPPORT_V3, + EFUSE_SUPPORT_V4, +}; +#define RsvdMask 0x3fffc000 +#define RsvdMaskV3 0x3fff8000 +#define RsvdMaskV4 RsvdMaskV3 + +struct TxDesc { + u32 opts1; + u32 opts2; + u64 addr; + u32 reserved0; + u32 reserved1; + u32 reserved2; + u32 reserved3; +}; + +struct RxDesc { + u32 opts1; + u32 opts2; + u64 addr; +}; + +struct RxDescV3 { + union { + struct { + u32 rsv1; + u32 rsv2; + } RxDescDDWord1; + }; + + union { + struct { + u32 RSSResult; + u16 HeaderBufferLen; + u16 HeaderInfo; + } RxDescNormalDDWord2; + + struct { + u32 rsv5; + u32 rsv6; + } RxDescDDWord2; + }; + + union { + u64 addr; + + struct { + u32 TimeStampLow; + u32 TimeStampHigh; + } RxDescTimeStamp; + + struct { + u32 rsv8; + u32 rsv9; + } RxDescDDWord3; + }; + + union { + struct { + u32 opts2; + u32 opts1; + } RxDescNormalDDWord4; + + struct { + u16 TimeStampHHigh; + u16 rsv11; + u32 opts1; + } RxDescPTPDDWord4; + }; +}; + +struct RxDescV4 { + union { + u64 addr; + + struct { + u32 RSSInfo; + u32 RSSResult; + } RxDescNormalDDWord1; + }; + + struct { + u32 opts2; + u32 opts1; + } RxDescNormalDDWord2; +}; + +enum rxdesc_type { + RXDESC_TYPE_NORMAL=0, + RXDESC_TYPE_NEXT, + RXDESC_TYPE_PTP, + RXDESC_TYPE_MAX +}; + +//Rx Desc Type +enum rx_desc_ring_type { + RX_DESC_RING_TYPE_UNKNOWN=0, + RX_DESC_RING_TYPE_1, + RX_DESC_RING_TYPE_2, + RX_DESC_RING_TYPE_3, + RX_DESC_RING_TYPE_4, + RX_DESC_RING_TYPE_MAX +}; + +enum rx_desc_len { + RX_DESC_LEN_TYPE_1 = (sizeof(struct RxDesc)), + RX_DESC_LEN_TYPE_3 = (sizeof(struct RxDescV3)), + RX_DESC_LEN_TYPE_4 = (sizeof(struct RxDescV4)) +}; + +struct ring_info { + struct sk_buff *skb; + u32 len; + unsigned int bytecount; + unsigned short gso_segs; + u8 __pad[sizeof(void *) - sizeof(u32)]; +}; + +struct pci_resource { + u8 cmd; + u8 cls; + u16 io_base_h; + u16 io_base_l; + u16 mem_base_h; + u16 mem_base_l; + u8 ilr; + u16 resv_0x1c_h; + u16 resv_0x1c_l; + u16 resv_0x20_h; + u16 resv_0x20_l; + u16 resv_0x24_h; + u16 resv_0x24_l; + u16 resv_0x2c_h; + u16 resv_0x2c_l; + u32 pci_sn_l; + u32 pci_sn_h; +}; + +enum r8127_flag { + R8127_FLAG_DOWN = 0, + R8127_FLAG_TASK_RESET_PENDING, + R8127_FLAG_TASK_ESD_CHECK_PENDING, + R8127_FLAG_TASK_LINKCHG_CHECK_PENDING, + R8127_FLAG_MAX +}; + +enum r8127_sysfs_flag { + R8127_SYSFS_RTL_ADV = 0, + R8127_SYSFS_FLAG_MAX +}; + +struct rtl8127_tx_ring { + void* priv; + struct net_device *netdev; + u32 index; + u32 cur_tx; /* Index into the Tx descriptor buffer of next Rx pkt. */ + u32 dirty_tx; + u32 num_tx_desc; /* Number of Tx descriptor registers */ + struct TxDesc *TxDescArray; /* 256-aligned Tx descriptor ring */ + dma_addr_t TxPhyAddr; + u32 TxDescAllocSize; + struct ring_info tx_skb[MAX_NUM_TX_DESC]; /* Tx data buffers */ + + u32 NextHwDesCloPtr; + u32 BeginHwDesCloPtr; + + u16 hw_clo_ptr_reg; + u16 sw_tail_ptr_reg; + + u16 tdsar_reg; /* Transmit Descriptor Start Address */ +}; + +struct rtl8127_rx_buffer { + struct page *page; + u32 page_offset; + dma_addr_t dma; + void* data; + struct sk_buff *skb; +}; + +struct rtl8127_rx_ring { + void* priv; + struct net_device *netdev; + u32 index; + u32 cur_rx; /* Index into the Rx descriptor buffer of next Rx pkt. */ + u32 dirty_rx; + u32 num_rx_desc; /* Number of Rx descriptor registers */ + struct RxDesc *RxDescArray; /* 256-aligned Rx descriptor ring */ + u32 RxDescAllocSize; + u64 RxDescPhyAddr[MAX_NUM_RX_DESC]; /* Rx desc physical address*/ + dma_addr_t RxPhyAddr; +#ifdef ENABLE_PAGE_REUSE + struct rtl8127_rx_buffer rx_buffer[MAX_NUM_RX_DESC]; + u16 rx_offset; +#else + struct sk_buff *Rx_skbuff[MAX_NUM_RX_DESC]; /* Rx data buffers */ +#endif //ENABLE_PAGE_REUSE + + u16 rdsar_reg; /* Receive Descriptor Start Address */ +}; + +struct r8127_napi { +#ifdef CONFIG_R8127_NAPI +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,24) + struct napi_struct napi; +#endif +#endif + void* priv; + int index; +}; + +struct r8127_irq { + irq_handler_t handler; + unsigned int vector; + u8 requested; + char name[IFNAMSIZ + 10]; +}; + +#pragma pack(1) +struct rtl8127_regs { + //00 + u8 mac_id[6]; + u16 reg_06; + u8 mar[8]; + //10 + u64 dtccr; + u16 ledsel0; + u16 legreg; + u32 tctr3; + //20 + u32 txq0_dsc_st_addr_0; + u32 txq0_dsc_st_addr_2; + u64 reg_28; + //30 + u16 rit; + u16 ritc; + u16 reg_34; + u8 reg_36; + u8 command; + u32 imr0; + u32 isr0; + //40 + u32 tcr; + u32 rcr; + u32 tctr0; + u32 tctr1; + //50 + u8 cr93c46; + u8 config0; + u8 config1; + u8 config2; + u8 config3; + u8 config4; + u8 config5; + u8 tdfnr; + u32 timer_int0; + u32 timer_int1; + //60 + u32 gphy_mdcmdio; + u32 csidr; + u32 csiar; + u16 phy_status; + u8 config6; + u8 pmch; + //70 + u32 eridr; + u32 eriar; + u16 config7; + u16 reg_7a; + u32 ephy_rxerr_cnt; + //80 + u32 ephy_mdcmdio; + u16 ledsel2; + u16 ledsel1; + u32 tctr2; + u32 timer_int2; + //90 + u8 tppoll0; + u8 reg_91; + u16 reg_92; + u16 led_feature; + u16 ledsel3; + u16 eee_led_config; + u16 reg_9a; + u32 reg_9c; + //a0 + u32 reg_a0; + u32 reg_a4; + u32 reg_a8; + u32 reg_ac; + //b0 + u32 patch_dbg; + u32 reg_b4; + u32 gphy_ocp; + u32 reg_bc; + //c0 + u32 reg_c0; + u32 reg_c4; + u32 reg_c8; + u16 otp_cmd; + u16 otp_pg_config; + //d0 + u16 phy_pwr; + u8 twsi_ctrl; + u8 oob_ctrl; + u16 mac_dbgo; + u16 mac_dbg; + u16 reg_d8; + u16 rms; + u32 efuse_data; + //e0 + u16 cplus_cmd; + u16 reg_e2; + u32 rxq0_dsc_st_addr_0; + u32 rxq0_dsc_st_addr_2; + u16 reg_ec; + u16 tx10midle_cnt; + //f0 + u16 misc0; + u16 misc1; + u32 timer_int3; + u32 cmac_ib; + u16 reg_fc; + u16 sw_rst; +}; +#pragma pack() + +struct rtl8127_regs_save { + union { + u8 mac_io[R8127_MAC_REGS_SIZE]; + + struct rtl8127_regs mac_reg; + }; + u16 pcie_phy[R8127_EPHY_REGS_SIZE/2]; + u16 eth_phy[R8127_PHY_REGS_SIZE/2]; + u32 eri_reg[R8127_ERI_REGS_SIZE/4]; + u32 pci_reg[R8127_PCI_REGS_SIZE/4]; + u16 sw_tail_ptr_reg[R8127_MAX_TX_QUEUES]; + u16 hw_clo_ptr_reg[R8127_MAX_TX_QUEUES]; + + //ktime_t begin_ktime; + //ktime_t end_ktime; + //u64 duration_ns; + + u16 sw0_tail_ptr; + u16 next_hwq0_clo_ptr; + u16 sw1_tail_ptr; + u16 next_hwq1_clo_ptr; + + u16 int_miti_rxq0; + u16 int_miti_txq0; + u16 int_miti_rxq1; + u16 int_miti_txq1; + u8 int_config; + u32 imr_new; + u32 isr_new; + + u8 tdu_status; + u16 rdu_status; + + u16 tc_mode; + + u32 txq1_dsc_st_addr_0; + u32 txq1_dsc_st_addr_2; + + u32 pla_tx_q0_idle_credit; + u32 pla_tx_q1_idle_credit; + + u32 rxq1_dsc_st_addr_0; + u32 rxq1_dsc_st_addr_2; + + u32 rss_ctrl; + u8 rss_key[RTL8127_RSS_KEY_SIZE]; + u8 rss_i_table[RTL8127_MAX_INDIRECTION_TABLE_ENTRIES]; + u16 rss_queue_num_sel_r; +}; + +struct rtl8127_counters { + /* legacy */ + u64 tx_packets; + u64 rx_packets; + u64 tx_errors; + u32 rx_errors; + u16 rx_missed; + u16 align_errors; + u32 tx_one_collision; + u32 tx_multi_collision; + u64 rx_unicast; + u64 rx_broadcast; + u32 rx_multicast; + u16 tx_aborted; + u16 tx_underrun; + + /* extended */ + u64 tx_octets; + u64 rx_octets; + u64 rx_multicast64; + u64 tx_unicast64; + u64 tx_broadcast64; + u64 tx_multicast64; + u32 tx_pause_on; + u32 tx_pause_off; + u32 tx_pause_all; + u32 tx_deferred; + u32 tx_late_collision; + u32 tx_all_collision; + u32 tx_aborted32; + u32 align_errors32; + u32 rx_frame_too_long; + u32 rx_runt; + u32 rx_pause_on; + u32 rx_pause_off; + u32 rx_pause_all; + u32 rx_unknown_opcode; + u32 rx_mac_error; + u32 tx_underrun32; + u32 rx_mac_missed; + u32 rx_tcam_dropped; + u32 tdu; + u32 rdu; +}; + +/* Flow Control Settings */ +enum rtl8127_fc_mode { + rtl8127_fc_none = 0, + rtl8127_fc_rx_pause, + rtl8127_fc_tx_pause, + rtl8127_fc_full, + rtl8127_fc_default +}; + +enum rtl8127_state_t { + __RTL8127_TESTING = 0, + __RTL8127_RESETTING, + __RTL8127_DOWN, + __RTL8127_PTP_TX_IN_PROGRESS, +}; + +#define RTL_FLAG_RX_HWTSTAMP_ENABLED BIT_0 + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0) +struct ethtool_eee { + __u32 cmd; + __u32 supported; + __u32 advertised; + __u32 lp_advertised; + __u32 eee_active; + __u32 eee_enabled; + __u32 tx_lpi_enabled; + __u32 tx_lpi_timer; + __u32 reserved[2]; +}; +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0) */ + +struct rtl8127_private { + void __iomem *mmio_addr; /* memory map physical address */ + struct pci_dev *pci_dev; /* Index of PCI device */ + struct net_device *dev; + struct r8127_napi r8127napi[R8127_MAX_MSIX_VEC]; + struct r8127_irq irq_tbl[R8127_MAX_MSIX_VEC]; + unsigned int irq_nvecs; + unsigned int max_irq_nvecs; + unsigned int min_irq_nvecs; + unsigned int hw_supp_irq_nvecs; + //struct msix_entry msix_entries[R8127_MAX_MSIX_VEC]; + struct net_device_stats stats; /* statistics of net device */ + unsigned long state; + u32 flags; + + u32 msg_enable; + u32 tx_tcp_csum_cmd; + u32 tx_udp_csum_cmd; + u32 tx_ip_csum_cmd; + u32 tx_ipv6_csum_cmd; + int max_jumbo_frame_size; + int chipset; + u32 mcfg; + //u32 cur_rx; /* Index into the Rx descriptor buffer of next Rx pkt. */ + //u32 cur_tx; /* Index into the Tx descriptor buffer of next Rx pkt. */ + //u32 dirty_rx; + //u32 dirty_tx; + //struct TxDesc *TxDescArray; /* 256-aligned Tx descriptor ring */ + //struct RxDesc *RxDescArray; /* 256-aligned Rx descriptor ring */ + //dma_addr_t TxPhyAddr; + //dma_addr_t RxPhyAddr; + //struct sk_buff *Rx_skbuff[MAX_NUM_RX_DESC]; /* Rx data buffers */ + //struct ring_info tx_skb[MAX_NUM_TX_DESC]; /* Tx data buffers */ + unsigned rx_buf_sz; +#ifdef ENABLE_PAGE_REUSE + unsigned rx_buf_page_order; + unsigned rx_buf_page_size; + u32 page_reuse_fail_cnt; +#endif //ENABLE_PAGE_REUSE + u16 HwSuppNumTxQueues; + u16 HwSuppNumRxQueues; + unsigned int num_tx_rings; + unsigned int num_rx_rings; + struct rtl8127_tx_ring tx_ring[R8127_MAX_TX_QUEUES]; + struct rtl8127_rx_ring rx_ring[R8127_MAX_RX_QUEUES]; +#ifdef ENABLE_LIB_SUPPORT + struct blocking_notifier_head lib_nh; + struct rtl8127_ring lib_tx_ring[R8127_MAX_TX_QUEUES]; + struct rtl8127_ring lib_rx_ring[R8127_MAX_RX_QUEUES]; +#endif + //struct timer_list esd_timer; + //struct timer_list link_timer; + struct pci_resource pci_cfg_space; + unsigned int esd_flag; + unsigned int pci_cfg_is_read; + unsigned int rtl8127_rx_config; + u16 rms; + u16 cp_cmd; + u32 intr_mask; + u32 timer_intr_mask; + u16 isr_reg[R8127_MAX_MSIX_VEC]; + u16 imr_reg[R8127_MAX_MSIX_VEC]; + int phy_auto_nego_reg; + int phy_1000_ctrl_reg; + int phy_2500_ctrl_reg; + u8 org_mac_addr[NODE_ADDRESS_SIZE]; + struct rtl8127_counters *tally_vaddr; + dma_addr_t tally_paddr; + +#ifdef CONFIG_R8127_VLAN + struct vlan_group *vlgrp; +#endif + u8 wol_enabled; + u32 wol_opts; + u8 efuse_ver; + u8 eeprom_type; + u8 autoneg; + u8 duplex; + u32 speed; + u64 advertising; + enum rtl8127_fc_mode fcpause; + u32 HwSuppMaxPhyLinkSpeed; + u16 eeprom_len; + u16 cur_page; + u32 bios_setting; + + int (*set_speed)(struct net_device *, u8 autoneg, u32 speed, u8 duplex, u64 adv); +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) + void (*get_settings)(struct net_device *, struct ethtool_cmd *); +#else + void (*get_settings)(struct net_device *, struct ethtool_link_ksettings *); +#endif + void (*phy_reset_enable)(struct net_device *); + unsigned int (*phy_reset_pending)(struct net_device *); + unsigned int (*link_ok)(struct net_device *); +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) + struct work_struct reset_task; + struct work_struct esd_task; + struct work_struct linkchg_task; +#else + struct delayed_work reset_task; + struct delayed_work esd_task; + struct delayed_work linkchg_task; +#endif + DECLARE_BITMAP(task_flags, R8127_FLAG_MAX); + unsigned features; + + u8 org_pci_offset_99; + u8 org_pci_offset_180; + u8 issue_offset_99_event; + + u8 org_pci_offset_80; + u8 org_pci_offset_81; + u8 use_timer_interrupt; + + u32 keep_intr_cnt; + + u8 HwIcVerUnknown; + u8 NotWrRamCodeToMicroP; + u8 NotWrMcuPatchCode; + u8 HwHasWrRamCodeToMicroP; + + u16 sw_ram_code_ver; + u16 hw_ram_code_ver; + + u8 rtk_enable_diag; + + u8 ShortPacketSwChecksum; + + u8 UseSwPaddingShortPkt; + + u8 RequireAdcBiasPatch; + u16 AdcBiasPatchIoffset; + + u8 RequireAdjustUpsTxLinkPulseTiming; + u16 SwrCnt1msIni; + + u8 HwSuppNowIsOobVer; + + u8 RequiredSecLanDonglePatch; + + u8 RequirePhyMdiSwapPatch; + + u32 HwFiberModeVer; + u32 HwFiberStat; + u8 HwSwitchMdiToFiber; + + u16 NicCustLedValue; + + u8 HwSuppMagicPktVer; + + u8 HwSuppLinkChgWakeUpVer; + + u8 HwSuppCheckPhyDisableModeVer; + + u8 random_mac; + + u16 phy_reg_aner; + u16 phy_reg_anlpar; + u16 phy_reg_gbsr; + u16 phy_reg_status_2500; + + u32 HwPcieSNOffset; + + u32 MaxTxDescPtrMask; + u8 HwSuppTxNoCloseVer; + u8 EnableTxNoClose; + + u8 HwSuppIsrVer; + u8 HwCurrIsrVer; + + u8 HwSuppIntMitiVer; + + u8 HwSuppExtendTallyCounterVer; + + u8 check_keep_link_speed; + u8 resume_not_chg_speed; + + u8 HwSuppD0SpeedUpVer; + u8 D0SpeedUpSpeed; + + u8 ring_lib_enabled; + + const char *fw_name; + struct rtl8127_fw *rtl_fw; + u32 ocp_base; + + //Dash+++++++++++++++++ + u8 HwSuppDashVer; + u8 DASH; + u8 dash_printer_enabled; + u8 HwPkgDet; + u8 AllowAccessDashOcp; + void __iomem *mapped_cmac_ioaddr; /* mapped cmac memory map physical address */ + void __iomem *cmac_ioaddr; /* cmac memory map physical address */ + +#ifdef ENABLE_DASH_SUPPORT + u16 AfterRecvFromFwBufLen; + u8 AfterRecvFromFwBuf[RECV_FROM_FW_BUF_SIZE]; + u16 AfterSendToFwBufLen; + u8 AfterSendToFwBuf[SEND_TO_FW_BUF_SIZE]; + u16 SendToFwBufferLen; + u32 SizeOfSendToFwBuffer; + u32 SizeOfSendToFwBufferMemAlloc; + u32 NumOfSendToFwBuffer; + + u8 OobReq; + u8 OobAck; + u32 OobReqComplete; + u32 OobAckComplete; + + u8 RcvFwReqSysOkEvt; + u8 RcvFwDashOkEvt; + u8 SendFwHostOkEvt; + + u8 DashFwDisableRx; + + void *UnalignedSendToFwBufferVa; + void *SendToFwBuffer; + u64 SendToFwBufferPhy; + u8 SendingToFw; + dma_addr_t UnalignedSendToFwBufferPa; + PTX_DASH_SEND_FW_DESC TxDashSendFwDesc; + u64 TxDashSendFwDescPhy; + u8 *UnalignedTxDashSendFwDescVa; + u32 SizeOfTxDashSendFwDescMemAlloc; + u32 SizeOfTxDashSendFwDesc; + u32 NumTxDashSendFwDesc; + u32 CurrNumTxDashSendFwDesc; + u32 LastSendNumTxDashSendFwDesc; + dma_addr_t UnalignedTxDashSendFwDescPa; + + u32 NumRecvFromFwBuffer; + u32 SizeOfRecvFromFwBuffer; + u32 SizeOfRecvFromFwBufferMemAlloc; + void *RecvFromFwBuffer; + u64 RecvFromFwBufferPhy; + + void *UnalignedRecvFromFwBufferVa; + dma_addr_t UnalignedRecvFromFwBufferPa; + PRX_DASH_FROM_FW_DESC RxDashRecvFwDesc; + u64 RxDashRecvFwDescPhy; + u8 *UnalignedRxDashRecvFwDescVa; + u32 SizeOfRxDashRecvFwDescMemAlloc; + u32 SizeOfRxDashRecvFwDesc; + u32 NumRxDashRecvFwDesc; + u32 CurrNumRxDashRecvFwDesc; + dma_addr_t UnalignedRxDashRecvFwDescPa; + u8 DashReqRegValue; + u16 HostReqValue; + + u32 CmacResetIsrCounter; + u8 CmacResetIntr; + u8 CmacResetting; + u8 CmacOobIssueCmacReset; + u32 CmacResetbyFwCnt; + +#if defined(ENABLE_DASH_PRINTER_SUPPORT) + struct completion fw_ack; + struct completion fw_req; + struct completion fw_host_ok; +#endif + //Dash----------------- +#endif //ENABLE_DASH_SUPPORT + + //Realwow++++++++++++++ + u8 HwSuppKCPOffloadVer; + + u8 EnableDhcpTimeoutWake; + u8 EnableTeredoOffload; + u8 EnableKCPOffload; +#ifdef ENABLE_REALWOW_SUPPORT + u32 DhcpTimeout; + MP_KCP_INFO MpKCPInfo; + //Realwow-------------- +#endif //ENABLE_REALWOW_SUPPORT + + struct ethtool_keee eee; + +#ifdef ENABLE_R8127_PROCFS + //Procfs support + struct proc_dir_entry *proc_dir; + struct proc_dir_entry *proc_dir_debug; + struct proc_dir_entry *proc_dir_test; +#endif +#ifdef ENABLE_R8127_SYSFS + //sysfs support + DECLARE_BITMAP(sysfs_flag, R8127_SYSFS_FLAG_MAX); + u32 testmode; +#endif + u8 HwSuppRxDescType; + u8 InitRxDescType; + u16 RxDescLength; //V1 16 Byte V2 32 Bytes + + spinlock_t phy_lock; + + u8 HwSuppPtpVer; + u8 EnablePtp; +#ifdef ENABLE_PTP_SUPPORT + u32 tx_hwtstamp_timeouts; + u32 tx_hwtstamp_skipped; + struct work_struct ptp_tx_work; + struct sk_buff *ptp_tx_skb; + struct hwtstamp_config hwtstamp_config; + unsigned long ptp_tx_start; + struct ptp_clock_info ptp_clock_info; + struct ptp_clock *ptp_clock; + u8 syncE_en; + u8 pps_enable; + struct hrtimer pps_timer; +#endif + + u8 HwSuppRssVer; + u8 EnableRss; + u16 HwSuppIndirTblEntries; +#ifdef ENABLE_RSS_SUPPORT + u32 rss_flags; + /* Receive Side Scaling settings */ + u8 rss_key[RTL8127_RSS_KEY_SIZE]; + u8 rss_indir_tbl[RTL8127_MAX_INDIRECTION_TABLE_ENTRIES]; + u32 rss_options; +#endif + + u8 HwSuppMacMcuVer; + u16 MacMcuPageSize; + u64 hw_mcu_patch_code_ver; + u64 bin_mcu_patch_code_ver; + + u8 HwSuppTcamVer; + + u16 TcamNotValidReg; + u16 TcamValidReg; + u16 TcamMaAddrcOffset; + u16 TcamVlanTagOffset; +}; + +#ifdef ENABLE_LIB_SUPPORT +static inline unsigned int +rtl8127_num_lib_tx_rings(struct rtl8127_private *tp) +{ + int count, i; + + for (count = 0, i = tp->num_tx_rings; i < tp->HwSuppNumTxQueues; i++) + if(tp->lib_tx_ring[i].enabled) + count++; + + return count; +} + +static inline unsigned int +rtl8127_num_lib_rx_rings(struct rtl8127_private *tp) +{ + int count, i; + + for (count = 0, i = tp->num_rx_rings; i < tp->HwSuppNumRxQueues; i++) + if(tp->lib_rx_ring[i].enabled) + count++; + + return count; +} + +#else +static inline unsigned int +rtl8127_num_lib_tx_rings(struct rtl8127_private *tp) +{ + return 0; +} + +static inline unsigned int +rtl8127_num_lib_rx_rings(struct rtl8127_private *tp) +{ + return 0; +} +#endif + +static inline unsigned int +rtl8127_tot_tx_rings(struct rtl8127_private *tp) +{ + return tp->num_tx_rings + rtl8127_num_lib_tx_rings(tp); +} + +static inline unsigned int +rtl8127_tot_rx_rings(struct rtl8127_private *tp) +{ + return tp->num_rx_rings + rtl8127_num_lib_rx_rings(tp); +} + +static inline struct netdev_queue *txring_txq(const struct rtl8127_tx_ring *ring) +{ + return netdev_get_tx_queue(ring->netdev, ring->index); +} + +enum eetype { + EEPROM_TYPE_NONE=0, + EEPROM_TYPE_93C46, + EEPROM_TYPE_93C56, + EEPROM_TWSI +}; + +enum mcfg { + CFG_METHOD_1=1, + CFG_METHOD_2, + CFG_METHOD_DEFAULT, + CFG_METHOD_MAX +}; + +#define LSO_32K 32000 +#define LSO_64K 64000 + +#define NIC_MIN_PHYS_BUF_COUNT (2) +#define NIC_MAX_PHYS_BUF_COUNT_LSO_64K (24) +#define NIC_MAX_PHYS_BUF_COUNT_LSO2 (16*4) + +#define GTTCPHO_SHIFT 18 +#define GTTCPHO_MAX 0x70U +#define GTPKTSIZE_MAX 0x3ffffU +#define TCPHO_SHIFT 18 +#define TCPHO_MAX 0x3ffU +#define LSOPKTSIZE_MAX 0xffffU +#define MSS_MAX 0x07ffu /* MSS value */ + +#define OOB_CMD_RESET 0x00 +#define OOB_CMD_DRIVER_START 0x05 +#define OOB_CMD_DRIVER_STOP 0x06 +#define OOB_CMD_SET_IPMAC 0x41 + +#define WAKEUP_MAGIC_PACKET_NOT_SUPPORT (0) +#define WAKEUP_MAGIC_PACKET_V1 (1) +#define WAKEUP_MAGIC_PACKET_V2 (2) +#define WAKEUP_MAGIC_PACKET_V3 (3) + +//Ram Code Version +#define NIC_RAMCODE_VERSION_CFG_METHOD_1 (0x0015) +#define NIC_RAMCODE_VERSION_CFG_METHOD_2 (0x0015) + +//hwoptimize +#define HW_PATCH_SOC_LAN (BIT_0) +#define HW_PATCH_SAMSUNG_LAN_DONGLE (BIT_2) + +static const u16 other_q_intr_mask = (RxOK1 | RxDU1); + +void rtl8127_mdio_write(struct rtl8127_private *tp, u16 RegAddr, u16 value); +void rtl8127_mdio_prot_write(struct rtl8127_private *tp, u32 RegAddr, u32 value); +void rtl8127_mdio_prot_direct_write_phy_ocp(struct rtl8127_private *tp, u32 RegAddr, u32 value); +u32 rtl8127_mdio_read(struct rtl8127_private *tp, u16 RegAddr); +u32 rtl8127_mdio_prot_read(struct rtl8127_private *tp, u32 RegAddr); +u32 rtl8127_mdio_prot_direct_read_phy_ocp(struct rtl8127_private *tp, u32 RegAddr); +void rtl8127_ephy_write(struct rtl8127_private *tp, int RegAddr, int value); +void rtl8127_mac_ocp_write(struct rtl8127_private *tp, u16 reg_addr, u16 value); +u16 rtl8127_mac_ocp_read(struct rtl8127_private *tp, u16 reg_addr); +void rtl8127_clear_eth_phy_bit(struct rtl8127_private *tp, u8 addr, u16 mask); +void rtl8127_set_eth_phy_bit(struct rtl8127_private *tp, u8 addr, u16 mask); +void rtl8127_ocp_write(struct rtl8127_private *tp, u16 addr, u8 len, u32 data); +void rtl8127_oob_notify(struct rtl8127_private *tp, u8 cmd); +void rtl8127_init_ring_indexes(struct rtl8127_private *tp); +void rtl8127_oob_mutex_lock(struct rtl8127_private *tp); +u32 rtl8127_ocp_read(struct rtl8127_private *tp, u16 addr, u8 len); +u32 rtl8127_ocp_read_with_oob_base_address(struct rtl8127_private *tp, u16 addr, u8 len, u32 base_address); +u32 rtl8127_ocp_write_with_oob_base_address(struct rtl8127_private *tp, u16 addr, u8 len, u32 value, u32 base_address); +u32 rtl8127_eri_read(struct rtl8127_private *tp, int addr, int len, int type); +u32 rtl8127_eri_read_with_oob_base_address(struct rtl8127_private *tp, int addr, int len, int type, u32 base_address); +int rtl8127_eri_write(struct rtl8127_private *tp, int addr, int len, u32 value, int type); +int rtl8127_eri_write_with_oob_base_address(struct rtl8127_private *tp, int addr, int len, u32 value, int type, u32 base_address); +u16 rtl8127_ephy_read(struct rtl8127_private *tp, int RegAddr); +void rtl8127_wait_txrx_fifo_empty(struct net_device *dev); +void rtl8127_enable_now_is_oob(struct rtl8127_private *tp); +void rtl8127_disable_now_is_oob(struct rtl8127_private *tp); +void rtl8127_oob_mutex_unlock(struct rtl8127_private *tp); +void rtl8127_dash2_disable_tx(struct rtl8127_private *tp); +void rtl8127_dash2_enable_tx(struct rtl8127_private *tp); +void rtl8127_dash2_disable_rx(struct rtl8127_private *tp); +void rtl8127_dash2_enable_rx(struct rtl8127_private *tp); +void rtl8127_hw_disable_mac_mcu_bps(struct net_device *dev); +void rtl8127_mark_to_asic(struct rtl8127_private *tp, struct RxDesc *desc, u32 rx_buf_sz); +void rtl8127_mark_as_last_descriptor(struct rtl8127_private *tp, struct RxDesc *desc); + +static inline void +rtl8127_make_unusable_by_asic(struct rtl8127_private *tp, + struct RxDesc *desc) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + ((struct RxDescV3 *)desc)->addr = RTL8127_MAGIC_NUMBER; + ((struct RxDescV3 *)desc)->RxDescNormalDDWord4.opts1 &= ~cpu_to_le32(DescOwn | RsvdMaskV3); + break; + case RX_DESC_RING_TYPE_4: + ((struct RxDescV4 *)desc)->addr = RTL8127_MAGIC_NUMBER; + ((struct RxDescV4 *)desc)->RxDescNormalDDWord2.opts1 &= ~cpu_to_le32(DescOwn | RsvdMaskV4); + break; + default: + desc->addr = RTL8127_MAGIC_NUMBER; + desc->opts1 &= ~cpu_to_le32(DescOwn | RsvdMask); + break; + } +} + +static inline struct RxDesc* +rtl8127_get_rxdesc(struct rtl8127_private *tp, struct RxDesc *RxDescBase, u32 const cur_rx) +{ + return (struct RxDesc*)((u8*)RxDescBase + (cur_rx * tp->RxDescLength)); +} + +static inline void +rtl8127_disable_hw_interrupt_v2(struct rtl8127_private *tp, + u32 message_id) +{ + RTL_W32(tp, IMR_V2_CLEAR_REG_8125, BIT(message_id)); +} + +static inline void +rtl8127_enable_hw_interrupt_v2(struct rtl8127_private *tp, u32 message_id) +{ + RTL_W32(tp, IMR_V2_SET_REG_8125, BIT(message_id)); +} + +int rtl8127_open(struct net_device *dev); +int rtl8127_close(struct net_device *dev); +void rtl8127_hw_config(struct net_device *dev); +void rtl8127_hw_set_timer_int(struct rtl8127_private *tp, u32 message_id, u8 timer_intmiti_val); +void rtl8127_set_rx_q_num(struct rtl8127_private *tp, unsigned int num_rx_queues); +void rtl8127_set_tx_q_num(struct rtl8127_private *tp, unsigned int num_tx_queues); +void rtl8127_enable_mcu(struct rtl8127_private *tp, bool enable); +void rtl8127_hw_start(struct net_device *dev); +void rtl8127_hw_reset(struct net_device *dev); +void rtl8127_tx_clear(struct rtl8127_private *tp); +void rtl8127_rx_clear(struct rtl8127_private *tp); +int rtl8127_init_ring(struct net_device *dev); +void rtl8127_hw_set_rx_packet_filter(struct net_device *dev); +void rtl8127_enable_hw_linkchg_interrupt(struct rtl8127_private *tp); +int rtl8127_dump_tally_counter(struct rtl8127_private *tp, dma_addr_t paddr); +void rtl8127_enable_napi(struct rtl8127_private *tp); +void _rtl8127_wait_for_quiescence(struct net_device *dev); + +void rtl8127_mdio_direct_write_phy_ocp(struct rtl8127_private *tp, u16 RegAddr,u16 value); +u32 rtl8127_mdio_direct_read_phy_ocp(struct rtl8127_private *tp, u16 RegAddr); +void rtl8127_clear_and_set_eth_phy_ocp_bit(struct rtl8127_private *tp, u16 addr, u16 clearmask, u16 setmask); +void rtl8127_clear_eth_phy_ocp_bit(struct rtl8127_private *tp, u16 addr, u16 mask); +void rtl8127_set_eth_phy_ocp_bit(struct rtl8127_private *tp, u16 addr, u16 mask); + +void rtl8127_clear_mac_ocp_bit(struct rtl8127_private *tp, u16 addr, u16 mask); + +#ifndef ENABLE_LIB_SUPPORT +static inline void rtl8127_lib_reset_prepare(struct rtl8127_private *tp) { } +static inline void rtl8127_lib_reset_complete(struct rtl8127_private *tp) { } +#endif + +#define HW_SUPPORT_CHECK_PHY_DISABLE_MODE(_M) ((_M)->HwSuppCheckPhyDisableModeVer > 0) +#define HW_HAS_WRITE_PHY_MCU_RAM_CODE(_M) (((_M)->HwHasWrRamCodeToMicroP == TRUE) ? 1 : 0) +#define HW_SUPPORT_D0_SPEED_UP(_M) ((_M)->HwSuppD0SpeedUpVer > 0) +#define HW_SUPPORT_MAC_MCU(_M) ((_M)->HwSuppMacMcuVer > 0) +#define HW_SUPPORT_TCAM(_M) ((_M)->HwSuppTcamVer > 0) + +#define HW_SUPP_PHY_LINK_SPEED_GIGA(_M) ((_M)->HwSuppMaxPhyLinkSpeed >= 1000) +#define HW_SUPP_PHY_LINK_SPEED_2500M(_M) ((_M)->HwSuppMaxPhyLinkSpeed >= 2500) +#define HW_SUPP_PHY_LINK_SPEED_5000M(_M) ((_M)->HwSuppMaxPhyLinkSpeed >= 5000) +#define HW_SUPP_PHY_LINK_SPEED_10000M(_M) ((_M)->HwSuppMaxPhyLinkSpeed >= 10000) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,34) +#define netdev_mc_count(dev) ((dev)->mc_count) +#define netdev_mc_empty(dev) (netdev_mc_count(dev) == 0) +#define netdev_for_each_mc_addr(mclist, dev) \ + for (mclist = dev->mc_list; mclist; mclist = mclist->next) +#endif + +#endif /* __R8127_H */ diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127_dash.h b/drivers/net/ethernet/realtek/r8127/src/r8127_dash.h new file mode 100755 index 0000000000000..0f6a3d150a005 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/src/r8127_dash.h @@ -0,0 +1,261 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#ifndef _LINUX_R8127_DASH_H +#define _LINUX_R8127_DASH_H + +#include + +#define SIOCDEVPRIVATE_RTLDASH SIOCDEVPRIVATE+2 + +enum rtl_dash_cmd { + RTL_DASH_ARP_NS_OFFLOAD = 0, + RTL_DASH_SET_OOB_IPMAC, + RTL_DASH_NOTIFY_OOB, + + RTL_DASH_SEND_BUFFER_DATA_TO_DASH_FW, + RTL_DASH_CHECK_SEND_BUFFER_TO_DASH_FW_COMPLETE, + RTL_DASH_GET_RCV_FROM_FW_BUFFER_DATA, + RTL_DASH_OOB_REQ, + RTL_DASH_OOB_ACK, + RTL_DASH_DETACH_OOB_REQ, + RTL_DASH_DETACH_OOB_ACK, + + RTL_FW_SET_IPV4 = 0x10, + RTL_FW_GET_IPV4, + RTL_FW_SET_IPV6, + RTL_FW_GET_IPV6, + RTL_FW_SET_EXT_SNMP, + RTL_FW_GET_EXT_SNMP, + RTL_FW_SET_WAKEUP_PATTERN, + RTL_FW_GET_WAKEUP_PATTERN, + RTL_FW_DEL_WAKEUP_PATTERN, + + RTLT_DASH_COMMAND_INVALID, +}; + +struct rtl_dash_ip_mac { + struct sockaddr ifru_addr; + struct sockaddr ifru_netmask; + struct sockaddr ifru_hwaddr; +}; + +struct rtl_dash_ioctl_struct { + __u32 cmd; + __u32 offset; + __u32 len; + union { + __u32 data; + void *data_buffer; + }; +}; + +struct settings_ipv4 { + __u32 IPv4addr; + __u32 IPv4mask; + __u32 IPv4Gateway; +}; + +struct settings_ipv6 { + __u32 reserved; + __u32 prefixLen; + __u16 IPv6addr[8]; + __u16 IPv6Gateway[8]; +}; + +struct settings_ext_snmp { + __u16 index; + __u16 oid_get_len; + __u8 oid_for_get[24]; + __u8 reserved0[26]; + __u16 value_len; + __u8 value[256]; + __u8 supported; + __u8 reserved1[27]; +}; + +struct wakeup_pattern { + __u8 index; + __u8 valid; + __u8 start; + __u8 length; + __u8 name[36]; + __u8 mask[16]; + __u8 pattern[128]; + __u32 reserved[2]; +}; + +typedef struct _RX_DASH_FROM_FW_DESC { + u16 length; + u8 statusLowByte; + u8 statusHighByte; + u32 resv; + u64 BufferAddress; +} +RX_DASH_FROM_FW_DESC, *PRX_DASH_FROM_FW_DESC; + +typedef struct _TX_DASH_SEND_FW_DESC { + u16 length; + u8 statusLowByte; + u8 statusHighByte; + u32 resv; + u64 BufferAddress; +} +TX_DASH_SEND_FW_DESC, *PTX_DASH_SEND_FW_DESC; + +typedef struct _OSOOBHdr { + u32 len; + u8 type; + u8 flag; + u8 hostReqV; + u8 res; +} +OSOOBHdr, *POSOOBHdr; + +typedef struct _RX_DASH_BUFFER_TYPE_2 { + OSOOBHdr oobhdr; + u8 RxDataBuffer[0]; +} +RX_DASH_BUFFER_TYPE_2, *PRX_DASH_BUFFER_TYPE_2; + +#define ALIGN_8 (0x7) +#define ALIGN_16 (0xf) +#define ALIGN_32 (0x1f) +#define ALIGN_64 (0x3f) +#define ALIGN_256 (0xff) +#define ALIGN_4096 (0xfff) + +#define OCP_REG_CONFIG0 (0x10) +#define OCP_REG_CONFIG0_REV_F (0xB8) +#define OCP_REG_DASH_POLL (0x30) +#define OCP_REG_HOST_REQ (0x34) +#define OCP_REG_DASH_REQ (0x35) +#define OCP_REG_CR (0x36) +#define OCP_REG_DMEMSTA (0x38) +#define OCP_REG_GPHYAR (0x60) + + +#define OCP_REG_CONFIG0_DASHEN BIT_15 +#define OCP_REG_CONFIG0_OOBRESET BIT_14 +#define OCP_REG_CONFIG0_APRDY BIT_13 +#define OCP_REG_CONFIG0_FIRMWARERDY BIT_12 +#define OCP_REG_CONFIG0_DRIVERRDY BIT_11 +#define OCP_REG_CONFIG0_OOB_WDT BIT_9 +#define OCP_REG_CONFIG0_DRV_WAIT_OOB BIT_8 +#define OCP_REG_CONFIG0_TLSEN BIT_7 + +#define HW_DASH_SUPPORT_DASH(_M) ((_M)->HwSuppDashVer > 0) +#define HW_DASH_SUPPORT_TYPE_1(_M) ((_M)->HwSuppDashVer == 1) +#define HW_DASH_SUPPORT_TYPE_2(_M) ((_M)->HwSuppDashVer == 2) +#define HW_DASH_SUPPORT_TYPE_3(_M) ((_M)->HwSuppDashVer == 3) + +#define RECV_FROM_FW_BUF_SIZE (1520) +#define SEND_TO_FW_BUF_SIZE (1520) + +#define RX_DASH_FROM_FW_OWN BIT_15 +#define TX_DASH_SEND_FW_OWN BIT_15 +#define TX_DASH_SEND_FW_OWN_HIGHBYTE BIT_7 + +#define TXS_CC3_0 (BIT_0|BIT_1|BIT_2|BIT_3) +#define TXS_EXC BIT_4 +#define TXS_LNKF BIT_5 +#define TXS_OWC BIT_6 +#define TXS_TES BIT_7 +#define TXS_UNF BIT_9 +#define TXS_LGSEN BIT_11 +#define TXS_LS BIT_12 +#define TXS_FS BIT_13 +#define TXS_EOR BIT_14 +#define TXS_OWN BIT_15 + +#define TPPool_HRDY 0x20 + +#define HostReqReg (0xC0) +#define SystemMasterDescStartAddrLow (0xF0) +#define SystemMasterDescStartAddrHigh (0xF4) +#define SystemSlaveDescStartAddrLow (0xF8) +#define SystemSlaveDescStartAddrHigh (0xFC) + +//DASH Request Type +#define WSMANREG 0x01 +#define OSPUSHDATA 0x02 + +#define RXS_OWN BIT_15 +#define RXS_EOR BIT_14 +#define RXS_FS BIT_13 +#define RXS_LS BIT_12 + +#define ISRIMR_DP_DASH_OK BIT_15 +#define ISRIMR_DP_HOST_OK BIT_13 +#define ISRIMR_DP_REQSYS_OK BIT_11 + +#define ISRIMR_DASH_INTR_EN BIT_12 +#define ISRIMR_DASH_INTR_CMAC_RESET BIT_15 + +#define ISRIMR_DASH_TYPE2_ROK BIT_0 +#define ISRIMR_DASH_TYPE2_RDU BIT_1 +#define ISRIMR_DASH_TYPE2_TOK BIT_2 +#define ISRIMR_DASH_TYPE2_TDU BIT_3 +#define ISRIMR_DASH_TYPE2_TX_FIFO_FULL BIT_4 +#define ISRIMR_DASH_TYPE2_TX_DISABLE_IDLE BIT_5 +#define ISRIMR_DASH_TYPE2_RX_DISABLE_IDLE BIT_6 + +#define CMAC_OOB_STOP 0x25 +#define CMAC_OOB_INIT 0x26 +#define CMAC_OOB_RESET 0x2a + +#define NO_BASE_ADDRESS 0x00000000 +#define RTL8168FP_OOBMAC_BASE 0xBAF70000 +#define RTL8168FP_CMAC_IOBASE 0xBAF20000 +#define RTL8168FP_KVM_BASE 0xBAF80400 +#define CMAC_SYNC_REG 0x20 +#define CMAC_RXDESC_OFFSET 0x90 //RX: 0x90 - 0x98 +#define CMAC_TXDESC_OFFSET 0x98 //TX: 0x98 - 0x9F + +/* cmac write/read MMIO register */ +#define RTL_CMAC_W8(tp, reg, val8) writeb ((val8), tp->cmac_ioaddr + (reg)) +#define RTL_CMAC_W16(tp, reg, val16) writew ((val16), tp->cmac_ioaddr + (reg)) +#define RTL_CMAC_W32(tp, reg, val32) writel ((val32), tp->cmac_ioaddr + (reg)) +#define RTL_CMAC_R8(tp, reg) readb (tp->cmac_ioaddr + (reg)) +#define RTL_CMAC_R16(tp, reg) readw (tp->cmac_ioaddr + (reg)) +#define RTL_CMAC_R32(tp, reg) ((unsigned long) readl (tp->cmac_ioaddr + (reg))) + +int rtl8127_dash_ioctl(struct net_device *dev, struct ifreq *ifr); +void HandleDashInterrupt(struct net_device *dev); +int AllocateDashShareMemory(struct net_device *dev); +void FreeAllocatedDashShareMemory(struct net_device *dev); +void DashHwInit(struct net_device *dev); + + +#endif /* _LINUX_R8127_DASH_H */ diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127_firmware.c b/drivers/net/ethernet/realtek/r8127/src/r8127_firmware.c new file mode 100755 index 0000000000000..7ab59f641e77a --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/src/r8127_firmware.c @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#include +#include +#include + +#include "r8127_firmware.h" + +enum rtl_fw_opcode { + PHY_READ = 0x0, + PHY_DATA_OR = 0x1, + PHY_DATA_AND = 0x2, + PHY_BJMPN = 0x3, + PHY_MDIO_CHG = 0x4, + PHY_CLEAR_READCOUNT = 0x7, + PHY_WRITE = 0x8, + PHY_READCOUNT_EQ_SKIP = 0x9, + PHY_COMP_EQ_SKIPN = 0xa, + PHY_COMP_NEQ_SKIPN = 0xb, + PHY_WRITE_PREVIOUS = 0xc, + PHY_SKIPN = 0xd, + PHY_DELAY_MS = 0xe, +}; + +struct fw_info { + u32 magic; + char version[RTL8127_VER_SIZE]; + __le32 fw_start; + __le32 fw_len; + u8 chksum; +} __packed; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,16,0) +#define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER)) +#endif +#define FW_OPCODE_SIZE sizeof_field(struct rtl8127_fw_phy_action, code[0]) + +static bool rtl8127_fw_format_ok(struct rtl8127_fw *rtl_fw) +{ + const struct firmware *fw = rtl_fw->fw; + struct fw_info *fw_info = (struct fw_info *)fw->data; + struct rtl8127_fw_phy_action *pa = &rtl_fw->phy_action; + + if (fw->size < FW_OPCODE_SIZE) + return false; + + if (!fw_info->magic) { + size_t i, size, start; + u8 checksum = 0; + + if (fw->size < sizeof(*fw_info)) + return false; + + for (i = 0; i < fw->size; i++) + checksum += fw->data[i]; + if (checksum != 0) + return false; + + start = le32_to_cpu(fw_info->fw_start); + if (start > fw->size) + return false; + + size = le32_to_cpu(fw_info->fw_len); + if (size > (fw->size - start) / FW_OPCODE_SIZE) + return false; + + strscpy(rtl_fw->version, fw_info->version, RTL8127_VER_SIZE); + + pa->code = (__le32 *)(fw->data + start); + pa->size = size; + } else { + if (fw->size % FW_OPCODE_SIZE) + return false; + + strscpy(rtl_fw->version, rtl_fw->fw_name, RTL8127_VER_SIZE); + + pa->code = (__le32 *)fw->data; + pa->size = fw->size / FW_OPCODE_SIZE; + } + + return true; +} + +static bool rtl8127_fw_data_ok(struct rtl8127_fw *rtl_fw) +{ + struct rtl8127_fw_phy_action *pa = &rtl_fw->phy_action; + size_t index; + + for (index = 0; index < pa->size; index++) { + u32 action = le32_to_cpu(pa->code[index]); + u32 val = action & 0x0000ffff; + u32 regno = (action & 0x0fff0000) >> 16; + + switch (action >> 28) { + case PHY_READ: + case PHY_DATA_OR: + case PHY_DATA_AND: + case PHY_CLEAR_READCOUNT: + case PHY_WRITE: + case PHY_WRITE_PREVIOUS: + case PHY_DELAY_MS: + break; + + case PHY_MDIO_CHG: + if (val > 1) + goto out; + break; + + case PHY_BJMPN: + if (regno > index) + goto out; + break; + case PHY_READCOUNT_EQ_SKIP: + if (index + 2 >= pa->size) + goto out; + break; + case PHY_COMP_EQ_SKIPN: + case PHY_COMP_NEQ_SKIPN: + case PHY_SKIPN: + if (index + 1 + regno >= pa->size) + goto out; + break; + + default: + dev_err(rtl_fw->dev, "Invalid action 0x%08x\n", action); + return false; + } + } + + return true; +out: + dev_err(rtl_fw->dev, "Out of range of firmware\n"); + return false; +} + +void rtl8127_fw_write_firmware(struct rtl8127_private *tp, struct rtl8127_fw *rtl_fw) +{ + struct rtl8127_fw_phy_action *pa = &rtl_fw->phy_action; + rtl8127_fw_write_t fw_write = rtl_fw->phy_write; + rtl8127_fw_read_t fw_read = rtl_fw->phy_read; + int predata = 0, count = 0; + size_t index; + + for (index = 0; index < pa->size; index++) { + u32 action = le32_to_cpu(pa->code[index]); + u32 data = action & 0x0000ffff; + u32 regno = (action & 0x0fff0000) >> 16; + enum rtl_fw_opcode opcode = action >> 28; + + if (!action) + break; + + switch (opcode) { + case PHY_READ: + predata = fw_read(tp, regno); + count++; + break; + case PHY_DATA_OR: + predata |= data; + break; + case PHY_DATA_AND: + predata &= data; + break; + case PHY_BJMPN: + index -= (regno + 1); + break; + case PHY_MDIO_CHG: + if (data) { + fw_write = rtl_fw->mac_mcu_write; + fw_read = rtl_fw->mac_mcu_read; + } else { + fw_write = rtl_fw->phy_write; + fw_read = rtl_fw->phy_read; + } + + break; + case PHY_CLEAR_READCOUNT: + count = 0; + break; + case PHY_WRITE: + fw_write(tp, regno, data); + break; + case PHY_READCOUNT_EQ_SKIP: + if (count == data) + index++; + break; + case PHY_COMP_EQ_SKIPN: + if (predata == data) + index += regno; + break; + case PHY_COMP_NEQ_SKIPN: + if (predata != data) + index += regno; + break; + case PHY_WRITE_PREVIOUS: + fw_write(tp, regno, predata); + break; + case PHY_SKIPN: + index += regno; + break; + case PHY_DELAY_MS: + mdelay(data); + break; + } + } +} + +void rtl8127_fw_release_firmware(struct rtl8127_fw *rtl_fw) +{ + release_firmware(rtl_fw->fw); +} + +int rtl8127_fw_request_firmware(struct rtl8127_fw *rtl_fw) +{ + int rc; + + rc = request_firmware(&rtl_fw->fw, rtl_fw->fw_name, rtl_fw->dev); + if (rc < 0) + goto out; + + if (!rtl8127_fw_format_ok(rtl_fw) || !rtl8127_fw_data_ok(rtl_fw)) { + release_firmware(rtl_fw->fw); + rc = -EINVAL; + goto out; + } + + return 0; +out: + dev_err(rtl_fw->dev, "Unable to load firmware %s (%d)\n", + rtl_fw->fw_name, rc); + return rc; +} diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127_firmware.h b/drivers/net/ethernet/realtek/r8127/src/r8127_firmware.h new file mode 100755 index 0000000000000..6b1acea98a3f8 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/src/r8127_firmware.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#ifndef _LINUX_R8127_FIRMWARE_H +#define _LINUX_R8127_FIRMWARE_H + +#include +#include + +struct rtl8127_private; +typedef void (*rtl8127_fw_write_t)(struct rtl8127_private *tp, u16 reg, u16 val); +typedef u32 (*rtl8127_fw_read_t)(struct rtl8127_private *tp, u16 reg); + +#define RTL8127_VER_SIZE 32 + +struct rtl8127_fw { + rtl8127_fw_write_t phy_write; + rtl8127_fw_read_t phy_read; + rtl8127_fw_write_t mac_mcu_write; + rtl8127_fw_read_t mac_mcu_read; + const struct firmware *fw; + const char *fw_name; + struct device *dev; + + char version[RTL8127_VER_SIZE]; + + struct rtl8127_fw_phy_action { + __le32 *code; + size_t size; + } phy_action; +}; + +int rtl8127_fw_request_firmware(struct rtl8127_fw *rtl_fw); +void rtl8127_fw_release_firmware(struct rtl8127_fw *rtl_fw); +void rtl8127_fw_write_firmware(struct rtl8127_private *tp, struct rtl8127_fw *rtl_fw); + +#endif /* _LINUX_R8127_FIRMWARE_H */ diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127_n.c b/drivers/net/ethernet/realtek/r8127/src/r8127_n.c new file mode 100755 index 0000000000000..4f83e44869deb --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/src/r8127_n.c @@ -0,0 +1,17824 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +/* + * This driver is modified from r8169.c in Linux kernel 2.6.18 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,22) +#include +#include +#endif +#include +#include +#include +#include + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0) +#include +#endif +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,37) +#include +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) +#define dev_printk(A,B,fmt,args...) printk(A fmt,##args) +#else +#include +#include +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,31) +#include +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,4,10) +#include +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6,4,10) */ + +#include +#include + +#include "r8127.h" +#include "rtl_eeprom.h" +#include "rtltool.h" +#include "r8127_firmware.h" + +#ifdef ENABLE_R8127_PROCFS +#include +#include +#endif + +#define FIRMWARE_8127_1 "rtl_nic/rtl8127-1.fw" +#define FIRMWARE_8127_2 "rtl_nic/rtl8127-2.fw" + +static const struct { + const char *name; + const char *fw_name; +} rtl_chip_fw_infos[] = { + /* PCI-E devices. */ + [CFG_METHOD_1] = {"RTL8127", FIRMWARE_8127_1}, + [CFG_METHOD_2] = {"RTL8127", FIRMWARE_8127_2}, + [CFG_METHOD_DEFAULT] = {"Unknown", }, +}; + +#define _R(NAME,MAC,RCR,MASK,JumFrameSz) \ + { .name = NAME, .mcfg = MAC, .RCR_Cfg = RCR, .RxConfigMask = MASK, .jumbo_frame_sz = JumFrameSz } + +static const struct { + const char *name; + u8 mcfg; + u32 RCR_Cfg; + u32 RxConfigMask; /* Clears the bits supported by this chip */ + u32 jumbo_frame_sz; +} rtl_chip_info[] = { + _R("RTL8127", + CFG_METHOD_1, + Rx_Fetch_Number_8 | Rx_Close_Multiple | RxCfg_pause_slot_en | EnableInnerVlan | EnableOuterVlan | (RX_DMA_BURST_512 << RxCfgDMAShift), + 0xff7e5880, + Jumbo_Frame_9k), + + _R("RTL8127", + CFG_METHOD_2, + Rx_Fetch_Number_8 | Rx_Close_Multiple | RxCfg_pause_slot_en | EnableInnerVlan | EnableOuterVlan | (RX_DMA_BURST_512 << RxCfgDMAShift), + 0xff7e5880, + Jumbo_Frame_9k), + + _R("Unknown", + CFG_METHOD_DEFAULT, + (RX_DMA_BURST_512 << RxCfgDMAShift), + 0xff7e5880, + Jumbo_Frame_1k) +}; +#undef _R + + +static struct pci_device_id rtl8127_pci_tbl[] = { + { PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0x8127), }, + { PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0x0E10), }, + {0,}, +}; + +MODULE_DEVICE_TABLE(pci, rtl8127_pci_tbl); + +static int use_dac = 1; +static int timer_count = 0x2600; +static int timer_count_v2 = (0x2600 / 0x200); + +static struct { + u32 msg_enable; +} debug = { -1 }; + +static unsigned int speed_mode = SPEED_10000; +static unsigned int duplex_mode = DUPLEX_FULL; +static unsigned int autoneg_mode = AUTONEG_ENABLE; +#ifdef CONFIG_ASPM +static int aspm = 1; +#else +static int aspm = 0; +#endif +#ifdef ENABLE_S5WOL +static int s5wol = 1; +#else +static int s5wol = 0; +#endif +#ifdef ENABLE_S5_KEEP_CURR_MAC +static int s5_keep_curr_mac = 1; +#else +static int s5_keep_curr_mac = 0; +#endif +#ifdef ENABLE_EEE +static int eee_enable = 1; +#else +static int eee_enable = 0; +#endif +#ifdef CONFIG_SOC_LAN +static ulong hwoptimize = HW_PATCH_SOC_LAN; +#else +static ulong hwoptimize = 0; +#endif +#ifdef ENABLE_S0_MAGIC_PACKET +static int s0_magic_packet = 1; +#else +static int s0_magic_packet = 0; +#endif +#ifdef ENABLE_TX_NO_CLOSE +static int tx_no_close_enable = 1; +#else +static int tx_no_close_enable = 0; +#endif +#ifdef DISABLE_WOL_SUPPORT +static int disable_wol_support = 1; +#else +static int disable_wol_support = 0; +#endif +#ifdef ENABLE_DOUBLE_VLAN +static int enable_double_vlan = 1; +#else +static int enable_double_vlan = 0; +#endif +#ifdef ENABLE_GIGA_LITE +static int eee_giga_lite = 1; +#else +static int eee_giga_lite = 0; +#endif + +MODULE_AUTHOR("Realtek and the Linux r8127 crew "); +MODULE_DESCRIPTION("Realtek r8127 Ethernet controller driver"); + +module_param(speed_mode, uint, 0); +MODULE_PARM_DESC(speed_mode, "force phy operation. Deprecated by ethtool (8)."); + +module_param(duplex_mode, uint, 0); +MODULE_PARM_DESC(duplex_mode, "force phy operation. Deprecated by ethtool (8)."); + +module_param(autoneg_mode, uint, 0); +MODULE_PARM_DESC(autoneg_mode, "force phy operation. Deprecated by ethtool (8)."); + +module_param(aspm, int, 0); +MODULE_PARM_DESC(aspm, "Enable ASPM."); + +module_param(s5wol, int, 0); +MODULE_PARM_DESC(s5wol, "Enable Shutdown Wake On Lan."); + +module_param(s5_keep_curr_mac, int, 0); +MODULE_PARM_DESC(s5_keep_curr_mac, "Enable Shutdown Keep Current MAC Address."); + +module_param(use_dac, int, 0); +MODULE_PARM_DESC(use_dac, "Enable PCI DAC. Unsafe on 32 bit PCI slot."); + +module_param(timer_count, int, 0); +MODULE_PARM_DESC(timer_count, "Timer Interrupt Interval."); + +module_param(eee_enable, int, 0); +MODULE_PARM_DESC(eee_enable, "Enable Energy Efficient Ethernet."); + +module_param(hwoptimize, ulong, 0); +MODULE_PARM_DESC(hwoptimize, "Enable HW optimization function."); + +module_param(s0_magic_packet, int, 0); +MODULE_PARM_DESC(s0_magic_packet, "Enable S0 Magic Packet."); + +module_param(tx_no_close_enable, int, 0); +MODULE_PARM_DESC(tx_no_close_enable, "Enable TX No Close."); + +module_param(disable_wol_support, int, 0); +MODULE_PARM_DESC(disable_wol_support, "Disable PM support."); + +module_param(enable_double_vlan, int, 0); +MODULE_PARM_DESC(enable_double_vlan, "Enable Double VLAN."); + +module_param(eee_giga_lite, int, 0); +MODULE_PARM_DESC(eee_giga_lite, "Enable Giga Lite."); + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) +module_param_named(debug, debug.msg_enable, int, 0); +MODULE_PARM_DESC(debug, "Debug verbosity level (0=none, ..., 16=all)"); +#endif//LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + +MODULE_LICENSE("GPL"); +#ifdef ENABLE_USE_FIRMWARE_FILE +MODULE_FIRMWARE(FIRMWARE_8127_1); +MODULE_FIRMWARE(FIRMWARE_8127_2); +#endif + +MODULE_VERSION(RTL8127_VERSION); + +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) +static void rtl8127_esd_timer(unsigned long __opaque); +#else +static void rtl8127_esd_timer(struct timer_list *t); +#endif +*/ +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) +static void rtl8127_link_timer(unsigned long __opaque); +#else +static void rtl8127_link_timer(struct timer_list *t); +#endif +*/ + +static netdev_tx_t rtl8127_start_xmit(struct sk_buff *skb, struct net_device *dev); +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) +static irqreturn_t rtl8127_interrupt(int irq, void *dev_instance, struct pt_regs *regs); +#else +static irqreturn_t rtl8127_interrupt(int irq, void *dev_instance); +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) +static irqreturn_t rtl8127_interrupt_msix(int irq, void *dev_instance, struct pt_regs *regs); +#else +static irqreturn_t rtl8127_interrupt_msix(int irq, void *dev_instance); +#endif +static void rtl8127_set_rx_mode(struct net_device *dev); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0) +static void rtl8127_tx_timeout(struct net_device *dev, unsigned int txqueue); +#else +static void rtl8127_tx_timeout(struct net_device *dev); +#endif +static int rtl8127_rx_interrupt(struct net_device *, struct rtl8127_private *, struct rtl8127_rx_ring *, napi_budget); +static int rtl8127_tx_interrupt(struct rtl8127_tx_ring *ring, int budget); +static int rtl8127_tx_interrupt_with_vector(struct rtl8127_private *tp, const int message_id, int budget); +static void rtl8127_wait_for_quiescence(struct net_device *dev); +static int rtl8127_change_mtu(struct net_device *dev, int new_mtu); +static void rtl8127_down(struct net_device *dev); + +static int rtl8127_set_mac_address(struct net_device *dev, void *p); +static void rtl8127_rar_set(struct rtl8127_private *tp, const u8 *addr); +static void rtl8127_desc_addr_fill(struct rtl8127_private *); +static void rtl8127_tx_desc_init(struct rtl8127_private *tp); +static void rtl8127_rx_desc_init(struct rtl8127_private *tp); + +static u16 rtl8127_get_hw_phy_mcu_code_ver(struct rtl8127_private *tp); +static void rtl8127_phy_power_up(struct net_device *dev); +static void rtl8127_phy_power_down(struct net_device *dev); +static int rtl8127_set_speed(struct net_device *dev, u8 autoneg, u32 speed, u8 duplex, u64 adv); +static bool rtl8127_set_phy_mcu_patch_request(struct rtl8127_private *tp); +static bool rtl8127_clear_phy_mcu_patch_request(struct rtl8127_private *tp); + +#ifdef CONFIG_R8127_NAPI +static int rtl8127_poll(napi_ptr napi, napi_budget budget); +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) +static void rtl8127_reset_task(void *_data); +static void rtl8127_esd_task(void *_data); +static void rtl8127_linkchg_task(void *_data); +#else +static void rtl8127_reset_task(struct work_struct *work); +static void rtl8127_esd_task(struct work_struct *work); +static void rtl8127_linkchg_task(struct work_struct *work); +#endif +static void rtl8127_schedule_reset_work(struct rtl8127_private *tp); +static void rtl8127_schedule_esd_work(struct rtl8127_private *tp); +static void rtl8127_schedule_linkchg_work(struct rtl8127_private *tp); +static void rtl8127_init_all_schedule_work(struct rtl8127_private *tp); +static void rtl8127_cancel_all_schedule_work(struct rtl8127_private *tp); + +static inline struct device *tp_to_dev(struct rtl8127_private *tp) +{ + return &tp->pci_dev->dev; +} + +#if ((LINUX_VERSION_CODE < KERNEL_VERSION(4,7,0) && \ + LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,00))) +void ethtool_convert_legacy_u32_to_link_mode(unsigned long *dst, + u32 legacy_u32) +{ + bitmap_zero(dst, __ETHTOOL_LINK_MODE_MASK_NBITS); + dst[0] = legacy_u32; +} + +bool ethtool_convert_link_mode_to_legacy_u32(u32 *legacy_u32, + const unsigned long *src) +{ + bool retval = true; + + /* TODO: following test will soon always be true */ + if (__ETHTOOL_LINK_MODE_MASK_NBITS > 32) { + __ETHTOOL_DECLARE_LINK_MODE_MASK(ext); + + bitmap_zero(ext, __ETHTOOL_LINK_MODE_MASK_NBITS); + bitmap_fill(ext, 32); + bitmap_complement(ext, ext, __ETHTOOL_LINK_MODE_MASK_NBITS); + if (bitmap_intersects(ext, src, + __ETHTOOL_LINK_MODE_MASK_NBITS)) { + /* src mask goes beyond bit 31 */ + retval = false; + } + } + *legacy_u32 = src[0]; + return retval; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) + +#ifndef LPA_1000FULL +#define LPA_1000FULL 0x0800 +#endif + +#ifndef LPA_1000HALF +#define LPA_1000HALF 0x0400 +#endif + +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,4,0) +static inline void eth_hw_addr_random(struct net_device *dev) +{ + random_ether_addr(dev->dev_addr); +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) +#undef ethtool_ops +#define ethtool_ops _kc_ethtool_ops + +struct _kc_ethtool_ops { + int (*get_settings)(struct net_device *, struct ethtool_cmd *); + int (*set_settings)(struct net_device *, struct ethtool_cmd *); + void (*get_drvinfo)(struct net_device *, struct ethtool_drvinfo *); + int (*get_regs_len)(struct net_device *); + void (*get_regs)(struct net_device *, struct ethtool_regs *, void *); + void (*get_wol)(struct net_device *, struct ethtool_wolinfo *); + int (*set_wol)(struct net_device *, struct ethtool_wolinfo *); + u32 (*get_msglevel)(struct net_device *); + void (*set_msglevel)(struct net_device *, u32); + int (*nway_reset)(struct net_device *); + u32 (*get_link)(struct net_device *); + int (*get_eeprom_len)(struct net_device *); + int (*get_eeprom)(struct net_device *, struct ethtool_eeprom *, u8 *); + int (*set_eeprom)(struct net_device *, struct ethtool_eeprom *, u8 *); + int (*get_coalesce)(struct net_device *, struct ethtool_coalesce *); + int (*set_coalesce)(struct net_device *, struct ethtool_coalesce *); + void (*get_ringparam)(struct net_device *, struct ethtool_ringparam *); + int (*set_ringparam)(struct net_device *, struct ethtool_ringparam *); + void (*get_pauseparam)(struct net_device *, + struct ethtool_pauseparam*); + int (*set_pauseparam)(struct net_device *, + struct ethtool_pauseparam*); + u32 (*get_rx_csum)(struct net_device *); + int (*set_rx_csum)(struct net_device *, u32); + u32 (*get_tx_csum)(struct net_device *); + int (*set_tx_csum)(struct net_device *, u32); + u32 (*get_sg)(struct net_device *); + int (*set_sg)(struct net_device *, u32); + u32 (*get_tso)(struct net_device *); + int (*set_tso)(struct net_device *, u32); + int (*self_test_count)(struct net_device *); + void (*self_test)(struct net_device *, struct ethtool_test *, u64 *); + void (*get_strings)(struct net_device *, u32 stringset, u8 *); + int (*phys_id)(struct net_device *, u32); + int (*get_stats_count)(struct net_device *); + void (*get_ethtool_stats)(struct net_device *, struct ethtool_stats *, + u64 *); +} *ethtool_ops = NULL; + +#undef SET_ETHTOOL_OPS +#define SET_ETHTOOL_OPS(netdev, ops) (ethtool_ops = (ops)) + +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) +#ifndef SET_ETHTOOL_OPS +#define SET_ETHTOOL_OPS(netdev,ops) \ + ((netdev)->ethtool_ops = (ops)) +#endif //SET_ETHTOOL_OPS +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) + +//#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,5) +#ifndef netif_msg_init +#define netif_msg_init _kc_netif_msg_init +/* copied from linux kernel 2.6.20 include/linux/netdevice.h */ +static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits) +{ + /* use default */ + if (debug_value < 0 || debug_value >= (sizeof(u32) * 8)) + return default_msg_enable_bits; + if (debug_value == 0) /* no output */ + return 0; + /* set low N bits */ + return (1 << debug_value) - 1; +} + +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,5) + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,22) +static inline void eth_copy_and_sum (struct sk_buff *dest, + const unsigned char *src, + int len, int base) +{ + skb_copy_to_linear_data(dest, src, len); +} +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,22) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,7) +/* copied from linux kernel 2.6.20 /include/linux/time.h */ +/* Parameters used to convert the timespec values: */ +#define MSEC_PER_SEC 1000L + +/* copied from linux kernel 2.6.20 /include/linux/jiffies.h */ +/* + * Change timeval to jiffies, trying to avoid the + * most obvious overflows.. + * + * And some not so obvious. + * + * Note that we don't want to return MAX_LONG, because + * for various timeout reasons we often end up having + * to wait "jiffies+1" in order to guarantee that we wait + * at _least_ "jiffies" - so "jiffies+1" had better still + * be positive. + */ +#define MAX_JIFFY_OFFSET ((~0UL >> 1)-1) + +/* + * Convert jiffies to milliseconds and back. + * + * Avoid unnecessary multiplications/divisions in the + * two most common HZ cases: + */ +static inline unsigned int _kc_jiffies_to_msecs(const unsigned long j) +{ +#if HZ <= MSEC_PER_SEC && !(MSEC_PER_SEC % HZ) + return (MSEC_PER_SEC / HZ) * j; +#elif HZ > MSEC_PER_SEC && !(HZ % MSEC_PER_SEC) + return (j + (HZ / MSEC_PER_SEC) - 1)/(HZ / MSEC_PER_SEC); +#else + return (j * MSEC_PER_SEC) / HZ; +#endif +} + +static inline unsigned long _kc_msecs_to_jiffies(const unsigned int m) +{ + if (m > _kc_jiffies_to_msecs(MAX_JIFFY_OFFSET)) + return MAX_JIFFY_OFFSET; +#if HZ <= MSEC_PER_SEC && !(MSEC_PER_SEC % HZ) + return (m + (MSEC_PER_SEC / HZ) - 1) / (MSEC_PER_SEC / HZ); +#elif HZ > MSEC_PER_SEC && !(HZ % MSEC_PER_SEC) + return m * (HZ / MSEC_PER_SEC); +#else + return (m * HZ + MSEC_PER_SEC - 1) / MSEC_PER_SEC; +#endif +} +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,7) + + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,11) + +/* copied from linux kernel 2.6.12.6 /include/linux/pm.h */ +typedef int __bitwise pci_power_t; + +/* copied from linux kernel 2.6.12.6 /include/linux/pci.h */ +typedef u32 __bitwise pm_message_t; + +#define PCI_D0 ((pci_power_t __force) 0) +#define PCI_D1 ((pci_power_t __force) 1) +#define PCI_D2 ((pci_power_t __force) 2) +#define PCI_D3hot ((pci_power_t __force) 3) +#define PCI_D3cold ((pci_power_t __force) 4) +#define PCI_POWER_ERROR ((pci_power_t __force) -1) + +/* copied from linux kernel 2.6.12.6 /drivers/pci/pci.c */ +/** + * pci_choose_state - Choose the power state of a PCI device + * @dev: PCI device to be suspended + * @state: target sleep state for the whole system. This is the value + * that is passed to suspend() function. + * + * Returns PCI power state suitable for given device and given system + * message. + */ + +pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state) +{ + if (!pci_find_capability(dev, PCI_CAP_ID_PM)) + return PCI_D0; + + switch (state) { + case 0: + return PCI_D0; + case 3: + return PCI_D3hot; + default: + printk("They asked me for state %d\n", state); +// BUG(); + } + return PCI_D0; +} +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,11) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,9) +/** + * msleep_interruptible - sleep waiting for waitqueue interruptions + * @msecs: Time in milliseconds to sleep for + */ +#define msleep_interruptible _kc_msleep_interruptible +unsigned long _kc_msleep_interruptible(unsigned int msecs) +{ + unsigned long timeout = _kc_msecs_to_jiffies(msecs); + + while (timeout && !signal_pending(current)) { + set_current_state(TASK_INTERRUPTIBLE); + timeout = schedule_timeout(timeout); + } + return _kc_jiffies_to_msecs(timeout); +} +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,9) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,7) +/* copied from linux kernel 2.6.20 include/linux/sched.h */ +#ifndef __sched +#define __sched __attribute__((__section__(".sched.text"))) +#endif + +/* copied from linux kernel 2.6.20 kernel/timer.c */ +signed long __sched schedule_timeout_uninterruptible(signed long timeout) +{ + __set_current_state(TASK_UNINTERRUPTIBLE); + return schedule_timeout(timeout); +} + +/* copied from linux kernel 2.6.20 include/linux/mii.h */ +#undef if_mii +#define if_mii _kc_if_mii +static inline struct mii_ioctl_data *if_mii(struct ifreq *rq) +{ + return (struct mii_ioctl_data *) &rq->ifr_ifru; +} +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,7) + +static u32 rtl8127_read_thermal_sensor(struct rtl8127_private *tp) +{ + u16 ts_digout; + + ts_digout = rtl8127_mdio_direct_read_phy_ocp(tp, 0xBD84); + ts_digout &= 0x3ff; + + return ts_digout; +} + +int rtl8127_dump_tally_counter(struct rtl8127_private *tp, dma_addr_t paddr) +{ + u32 cmd; + u32 WaitCnt; + int retval = -1; + + RTL_W32(tp, CounterAddrHigh, (u64)paddr >> 32); + cmd = (u64)paddr & DMA_BIT_MASK(32); + RTL_W32(tp, CounterAddrLow, cmd); + RTL_W32(tp, CounterAddrLow, cmd | CounterDump); + + WaitCnt = 0; + while (RTL_R32(tp, CounterAddrLow) & CounterDump) { + udelay(10); + + WaitCnt++; + if (WaitCnt > 20) + break; + } + + if (WaitCnt <= 20) + retval = 0; + + return retval; +} + +static u32 +rtl8127_get_hw_clo_ptr(struct rtl8127_tx_ring *ring) +{ + struct rtl8127_private *tp = ring->priv; + + switch (tp->HwSuppTxNoCloseVer) { + case 3: + return RTL_R16(tp, ring->hw_clo_ptr_reg); + case 4: + case 5: + case 6: + return RTL_R32(tp, ring->hw_clo_ptr_reg); + default: +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + WARN_ON(1); +#endif + return 0; + } +} + +static u32 +rtl8127_get_sw_tail_ptr(struct rtl8127_tx_ring *ring) +{ + struct rtl8127_private *tp = ring->priv; + + switch (tp->HwSuppTxNoCloseVer) { + case 3: + return RTL_R16(tp, ring->sw_tail_ptr_reg); + case 4: + case 5: + case 6: + return RTL_R32(tp, ring->sw_tail_ptr_reg); + default: +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + WARN_ON(1); +#endif + return 0; + } +} + +static bool +rtl8127_sysfs_testmode_on(struct rtl8127_private *tp) +{ +#ifdef ENABLE_R8127_SYSFS + return !!tp->testmode; +#else + return 1; +#endif +} + +static u32 rtl8127_convert_link_speed(u32 status) +{ + u32 speed = SPEED_UNKNOWN; + + if (status & LinkStatus) { + if (status & _10000bpsF) + speed = SPEED_10000; + else if (status & (_5000bpsF | _10000bpsL)) + speed = SPEED_5000; + else if (status & (_2500bpsF | _5000bpsL)) + speed = SPEED_2500; + else if (status & (_1000bpsF | _2500bpsL | _1000bpsL)) + speed = SPEED_1000; + else if (status & _100bps) + speed = SPEED_100; + else if (status & _10bps) + speed = SPEED_10; + } + + return speed; +} + +static void rtl8127_mdi_swap(struct rtl8127_private *tp) +{ + int i; + u16 reg, val, mdi_reverse; + u16 tps_p0, tps_p1, tps_p2, tps_p3, tps_p3_p0; + + switch (tp->mcfg) { + default: + return; + }; + + tps_p3_p0 = rtl8127_mac_ocp_read(tp, 0xD440) & 0xF000; + tps_p3 = !!(tps_p3_p0 & BIT_15); + tps_p2 = !!(tps_p3_p0 & BIT_14); + tps_p1 = !!(tps_p3_p0 & BIT_13); + tps_p0 = !!(tps_p3_p0 & BIT_12); + mdi_reverse = rtl8127_mac_ocp_read(tp, 0xD442); + + if ((mdi_reverse & BIT_5) && tps_p3_p0 == 0xA000) + return; + + if (!(mdi_reverse & BIT_5)) + val = tps_p0 << 8 | + tps_p1 << 9 | + tps_p2 << 10 | + tps_p3 << 11; + else + val = tps_p3 << 8 | + tps_p2 << 9 | + tps_p1 << 10 | + tps_p0 << 11; + + for (i=8; i<12; i++) { + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, reg); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + BIT(i), + val & BIT(i)); + } +} + +static int rtl8127_vcd_test(struct rtl8127_private *tp) +{ + u16 val; + u32 wait_cnt; + int ret = -1; + + rtl8127_mdi_swap(tp); + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA422, BIT(0)); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA422, 0x00F0); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA422, BIT(0)); + + wait_cnt = 0; + do { + mdelay(1); + val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA422); + wait_cnt++; + } while (!(val & BIT_15) && (wait_cnt < 5000)); + + if (wait_cnt == 5000) + goto exit; + + ret = 0; + +exit: + return ret; +} + +static void rtl8127_get_cp_len(struct rtl8127_private *tp, + int cp_len[RTL8127_CP_NUM]) +{ + int i; + u16 status; + int tmp_cp_len; + + status = RTL_R16(tp, PHYstatus); + if (status & LinkStatus) { + if (status & _10bps) { + tmp_cp_len = -1; + } else if (status & (_100bps | _1000bpsF)) { + rtl8127_mdio_write(tp, 0x1f, 0x0a88); + tmp_cp_len = rtl8127_mdio_read(tp, 0x10); + } else if (status & _2500bpsF) { + rtl8127_mdio_write(tp, 0x1f, 0x0acb); + tmp_cp_len = rtl8127_mdio_read(tp, 0x15); + tmp_cp_len >>= 2; + } else + tmp_cp_len = 0; + } else + tmp_cp_len = 0; + + if (tmp_cp_len > 0) + tmp_cp_len &= 0xff; + for (i=0; i RTL8127_MAX_SUPPORT_CP_LEN) + cp_len[i] = RTL8127_MAX_SUPPORT_CP_LEN; + + return; +} + +static int __rtl8127_get_cp_status(u16 val) +{ + switch (val) { + case 0x0060: + return rtl8127_cp_normal; + case 0x0048: + return rtl8127_cp_open; + case 0x0050: + return rtl8127_cp_short; + case 0x0042: + case 0x0044: + return rtl8127_cp_mismatch; + default: + return rtl8127_cp_normal; + } +} + +static int _rtl8127_get_cp_status(struct rtl8127_private *tp, u8 pair_num) +{ + u16 val; + int cp_status = rtl8127_cp_unknown; + + if (pair_num > 3) + goto exit; + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8027 + 4 * pair_num); + val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA438); + + cp_status = __rtl8127_get_cp_status(val); + +exit: + return cp_status; +} + +static const char * rtl8127_get_cp_status_string(int cp_status) +{ + switch(cp_status) { + case rtl8127_cp_normal: + return "normal "; + case rtl8127_cp_short: + return "short "; + case rtl8127_cp_open: + return "open "; + case rtl8127_cp_mismatch: + return "mismatch"; + default: + return "unknown "; + } +} + +static u16 rtl8127_get_cp_pp(struct rtl8127_private *tp, u8 pair_num) +{ + u16 pp = 0; + + if (pair_num > 3) + goto exit; + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8029 + 4 * pair_num); + pp = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA438); + + pp &= 0x3fff; + pp /= 80; + +exit: + return pp; +} + +static void rtl8127_get_cp_status(struct rtl8127_private *tp, + int cp_status[RTL8127_CP_NUM], + bool poe_mode) +{ + u16 status; + int i; + + status = RTL_R16(tp, PHYstatus); + if (status & LinkStatus && !(status & (_10bps | _100bps))) { + for (i=0; i= KERNEL_VERSION(3,10,0) +static int proc_get_driver_variable(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + struct rtl8127_private *tp = netdev_priv(dev); + + seq_puts(m, "\nDump Driver Variable\n"); + + rtnl_lock(); + + seq_puts(m, "Variable\tValue\n----------\t-----\n"); + seq_printf(m, "MODULENAME\t%s\n", MODULENAME); + seq_printf(m, "driver version\t%s\n", RTL8127_VERSION); + seq_printf(m, "mcfg\t%d\n", tp->mcfg); + seq_printf(m, "chipset\t%d\n", tp->chipset); + seq_printf(m, "chipset_name\t%s\n", rtl_chip_info[tp->chipset].name); + seq_printf(m, "mtu\t%d\n", dev->mtu); + seq_printf(m, "NUM_RX_DESC\t0x%x\n", tp->rx_ring[0].num_rx_desc); + seq_printf(m, "cur_rx0\t0x%x\n", tp->rx_ring[0].cur_rx); + seq_printf(m, "dirty_rx0\t0x%x\n", tp->rx_ring[0].dirty_rx); + seq_printf(m, "cur_rx1\t0x%x\n", tp->rx_ring[1].cur_rx); + seq_printf(m, "dirty_rx1\t0x%x\n", tp->rx_ring[1].dirty_rx); + seq_printf(m, "cur_rx2\t0x%x\n", tp->rx_ring[2].cur_rx); + seq_printf(m, "dirty_rx2\t0x%x\n", tp->rx_ring[2].dirty_rx); + seq_printf(m, "cur_rx3\t0x%x\n", tp->rx_ring[3].cur_rx); + seq_printf(m, "dirty_rx3\t0x%x\n", tp->rx_ring[3].dirty_rx); + seq_printf(m, "NUM_TX_DESC\t0x%x\n", tp->tx_ring[0].num_tx_desc); + seq_printf(m, "cur_tx0\t0x%x\n", tp->tx_ring[0].cur_tx); + seq_printf(m, "dirty_tx0\t0x%x\n", tp->tx_ring[0].dirty_tx); + seq_printf(m, "cur_tx1\t0x%x\n", tp->tx_ring[1].cur_tx); + seq_printf(m, "dirty_tx1\t0x%x\n", tp->tx_ring[1].dirty_tx); + seq_printf(m, "rx_buf_sz\t0x%x\n", tp->rx_buf_sz); +#ifdef ENABLE_PAGE_REUSE + seq_printf(m, "rx_buf_page_order\t0x%x\n", tp->rx_buf_page_order); + seq_printf(m, "rx_buf_page_size\t0x%x\n", tp->rx_buf_page_size); + seq_printf(m, "page_reuse_fail_cnt\t0x%x\n", tp->page_reuse_fail_cnt); +#endif //ENABLE_PAGE_REUSE + seq_printf(m, "esd_flag\t0x%x\n", tp->esd_flag); + seq_printf(m, "pci_cfg_is_read\t0x%x\n", tp->pci_cfg_is_read); + seq_printf(m, "rtl8127_rx_config\t0x%x\n", tp->rtl8127_rx_config); + seq_printf(m, "cp_cmd\t0x%x\n", tp->cp_cmd); + seq_printf(m, "intr_mask\t0x%x\n", tp->intr_mask); + seq_printf(m, "timer_intr_mask\t0x%x\n", tp->timer_intr_mask); + seq_printf(m, "wol_enabled\t0x%x\n", tp->wol_enabled); + seq_printf(m, "wol_opts\t0x%x\n", tp->wol_opts); + seq_printf(m, "efuse_ver\t0x%x\n", tp->efuse_ver); + seq_printf(m, "eeprom_type\t0x%x\n", tp->eeprom_type); + seq_printf(m, "autoneg\t0x%x\n", tp->autoneg); + seq_printf(m, "duplex\t0x%x\n", tp->duplex); + seq_printf(m, "speed\t%d\n", tp->speed); + seq_printf(m, "advertising\t0x%llx\n", tp->advertising); + seq_printf(m, "eeprom_len\t0x%x\n", tp->eeprom_len); + seq_printf(m, "cur_page\t0x%x\n", tp->cur_page); + seq_printf(m, "features\t0x%x\n", tp->features); + seq_printf(m, "org_pci_offset_99\t0x%x\n", tp->org_pci_offset_99); + seq_printf(m, "org_pci_offset_180\t0x%x\n", tp->org_pci_offset_180); + seq_printf(m, "issue_offset_99_event\t0x%x\n", tp->issue_offset_99_event); + seq_printf(m, "org_pci_offset_80\t0x%x\n", tp->org_pci_offset_80); + seq_printf(m, "org_pci_offset_81\t0x%x\n", tp->org_pci_offset_81); + seq_printf(m, "use_timer_interrupt\t0x%x\n", tp->use_timer_interrupt); + seq_printf(m, "HwIcVerUnknown\t0x%x\n", tp->HwIcVerUnknown); + seq_printf(m, "NotWrRamCodeToMicroP\t0x%x\n", tp->NotWrRamCodeToMicroP); + seq_printf(m, "NotWrMcuPatchCode\t0x%x\n", tp->NotWrMcuPatchCode); + seq_printf(m, "HwHasWrRamCodeToMicroP\t0x%x\n", tp->HwHasWrRamCodeToMicroP); + seq_printf(m, "sw_ram_code_ver\t0x%x\n", tp->sw_ram_code_ver); + seq_printf(m, "hw_ram_code_ver\t0x%x\n", tp->hw_ram_code_ver); + seq_printf(m, "rtk_enable_diag\t0x%x\n", tp->rtk_enable_diag); + seq_printf(m, "ShortPacketSwChecksum\t0x%x\n", tp->ShortPacketSwChecksum); + seq_printf(m, "UseSwPaddingShortPkt\t0x%x\n", tp->UseSwPaddingShortPkt); + seq_printf(m, "RequireAdcBiasPatch\t0x%x\n", tp->RequireAdcBiasPatch); + seq_printf(m, "AdcBiasPatchIoffset\t0x%x\n", tp->AdcBiasPatchIoffset); + seq_printf(m, "RequireAdjustUpsTxLinkPulseTiming\t0x%x\n", tp->RequireAdjustUpsTxLinkPulseTiming); + seq_printf(m, "SwrCnt1msIni\t0x%x\n", tp->SwrCnt1msIni); + seq_printf(m, "HwSuppNowIsOobVer\t0x%x\n", tp->HwSuppNowIsOobVer); + seq_printf(m, "HwFiberModeVer\t0x%x\n", tp->HwFiberModeVer); + seq_printf(m, "HwFiberStat\t0x%x\n", tp->HwFiberStat); + seq_printf(m, "HwSwitchMdiToFiber\t0x%x\n", tp->HwSwitchMdiToFiber); + seq_printf(m, "NicCustLedValue\t0x%x\n", tp->NicCustLedValue); + seq_printf(m, "RequiredSecLanDonglePatch\t0x%x\n", tp->RequiredSecLanDonglePatch); + seq_printf(m, "HwSuppDashVer\t0x%x\n", tp->HwSuppDashVer); + seq_printf(m, "DASH\t0x%x\n", tp->DASH); + seq_printf(m, "dash_printer_enabled\t0x%x\n", tp->dash_printer_enabled); + seq_printf(m, "HwSuppKCPOffloadVer\t0x%x\n", tp->HwSuppKCPOffloadVer); + seq_printf(m, "speed_mode\t0x%x\n", speed_mode); + seq_printf(m, "duplex_mode\t0x%x\n", duplex_mode); + seq_printf(m, "autoneg_mode\t0x%x\n", autoneg_mode); + seq_printf(m, "aspm\t0x%x\n", aspm); + seq_printf(m, "s5wol\t0x%x\n", s5wol); + seq_printf(m, "s5_keep_curr_mac\t0x%x\n", s5_keep_curr_mac); + seq_printf(m, "eee_enable\t0x%x\n", tp->eee.eee_enabled); + seq_printf(m, "hwoptimize\t0x%lx\n", hwoptimize); + seq_printf(m, "proc_init_num\t0x%x\n", proc_init_num); + seq_printf(m, "s0_magic_packet\t0x%x\n", s0_magic_packet); + seq_printf(m, "disable_wol_support\t0x%x\n", disable_wol_support); + seq_printf(m, "enable_double_vlan\t0x%x\n", enable_double_vlan); + seq_printf(m, "eee_giga_lite\t0x%x\n", eee_giga_lite); + seq_printf(m, "HwSuppMagicPktVer\t0x%x\n", tp->HwSuppMagicPktVer); + seq_printf(m, "HwSuppLinkChgWakeUpVer\t0x%x\n", tp->HwSuppLinkChgWakeUpVer); + seq_printf(m, "HwSuppD0SpeedUpVer\t0x%x\n", tp->HwSuppD0SpeedUpVer); + seq_printf(m, "D0SpeedUpSpeed\t0x%x\n", tp->D0SpeedUpSpeed); + seq_printf(m, "HwSuppCheckPhyDisableModeVer\t0x%x\n", tp->HwSuppCheckPhyDisableModeVer); + seq_printf(m, "HwPkgDet\t0x%x\n", tp->HwPkgDet); + seq_printf(m, "HwSuppTxNoCloseVer\t0x%x\n", tp->HwSuppTxNoCloseVer); + seq_printf(m, "EnableTxNoClose\t0x%x\n", tp->EnableTxNoClose); + seq_printf(m, "NextHwDesCloPtr0\t0x%x\n", tp->tx_ring[0].NextHwDesCloPtr); + seq_printf(m, "BeginHwDesCloPtr0\t0x%x\n", tp->tx_ring[0].BeginHwDesCloPtr); + seq_printf(m, "hw_clo_ptr_reg0\t0x%x\n", rtl8127_get_hw_clo_ptr(&tp->tx_ring[0])); + seq_printf(m, "sw_tail_ptr_reg0\t0x%x\n", rtl8127_get_sw_tail_ptr(&tp->tx_ring[0])); + seq_printf(m, "NextHwDesCloPtr1\t0x%x\n", tp->tx_ring[1].NextHwDesCloPtr); + seq_printf(m, "BeginHwDesCloPtr1\t0x%x\n", tp->tx_ring[1].BeginHwDesCloPtr); + seq_printf(m, "hw_clo_ptr_reg1\t0x%x\n", rtl8127_get_hw_clo_ptr(&tp->tx_ring[1])); + seq_printf(m, "sw_tail_ptr_reg1\t0x%x\n", rtl8127_get_sw_tail_ptr(&tp->tx_ring[1])); + seq_printf(m, "InitRxDescType\t0x%x\n", tp->InitRxDescType); + seq_printf(m, "RxDescLength\t0x%x\n", tp->RxDescLength); + seq_printf(m, "num_rx_rings\t0x%x\n", tp->num_rx_rings); + seq_printf(m, "num_tx_rings\t0x%x\n", tp->num_tx_rings); + seq_printf(m, "tot_rx_rings\t0x%x\n", rtl8127_tot_rx_rings(tp)); + seq_printf(m, "tot_tx_rings\t0x%x\n", rtl8127_tot_tx_rings(tp)); + seq_printf(m, "HwSuppNumRxQueues\t0x%x\n", tp->HwSuppNumRxQueues); + seq_printf(m, "HwSuppNumTxQueues\t0x%x\n", tp->HwSuppNumTxQueues); + seq_printf(m, "EnableRss\t0x%x\n", tp->EnableRss); + seq_printf(m, "EnablePtp\t0x%x\n", tp->EnablePtp); + seq_printf(m, "min_irq_nvecs\t0x%x\n", tp->min_irq_nvecs); + seq_printf(m, "irq_nvecs\t0x%x\n", tp->irq_nvecs); + seq_printf(m, "hw_supp_irq_nvecs\t0x%x\n", tp->hw_supp_irq_nvecs); + seq_printf(m, "ring_lib_enabled\t0x%x\n", tp->ring_lib_enabled); + seq_printf(m, "HwSuppIsrVer\t0x%x\n", tp->HwSuppIsrVer); + seq_printf(m, "HwCurrIsrVer\t0x%x\n", tp->HwCurrIsrVer); + seq_printf(m, "HwSuppMacMcuVer\t0x%x\n", tp->HwSuppMacMcuVer); + seq_printf(m, "MacMcuPageSize\t0x%x\n", tp->MacMcuPageSize); + seq_printf(m, "hw_mcu_patch_code_ver\t0x%llx\n", tp->hw_mcu_patch_code_ver); + seq_printf(m, "bin_mcu_patch_code_ver\t0x%llx\n", tp->bin_mcu_patch_code_ver); +#ifdef ENABLE_PTP_SUPPORT + seq_printf(m, "tx_hwtstamp_timeouts\t0x%x\n", tp->tx_hwtstamp_timeouts); + seq_printf(m, "tx_hwtstamp_skipped\t0x%x\n", tp->tx_hwtstamp_skipped); +#endif + seq_printf(m, "random_mac\t0x%x\n", tp->random_mac); + seq_printf(m, "org_mac_addr\t%pM\n", tp->org_mac_addr); +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,13) + seq_printf(m, "perm_addr\t%pM\n", dev->perm_addr); +#endif + seq_printf(m, "dev_addr\t%pM\n", dev->dev_addr); + + rtnl_unlock(); + + seq_putc(m, '\n'); + return 0; +} + +static int proc_get_tally_counter(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + struct rtl8127_private *tp = netdev_priv(dev); + struct rtl8127_counters *counters; + dma_addr_t paddr; + + seq_puts(m, "\nDump Tally Counter\n"); + + rtnl_lock(); + + counters = tp->tally_vaddr; + paddr = tp->tally_paddr; + if (!counters) { + seq_puts(m, "\nDump Tally Counter Fail\n"); + goto out_unlock; + } + + rtl8127_dump_tally_counter(tp, paddr); + + seq_puts(m, "Statistics\tValue\n----------\t-----\n"); + seq_printf(m, "tx_packets\t%lld\n", le64_to_cpu(counters->tx_packets)); + seq_printf(m, "rx_packets\t%lld\n", le64_to_cpu(counters->rx_packets)); + seq_printf(m, "tx_errors\t%lld\n", le64_to_cpu(counters->tx_errors)); + seq_printf(m, "rx_errors\t%d\n", le32_to_cpu(counters->rx_errors)); + seq_printf(m, "rx_missed\t%d\n", le16_to_cpu(counters->rx_missed)); + seq_printf(m, "align_errors\t%d\n", le16_to_cpu(counters->align_errors)); + seq_printf(m, "tx_one_collision\t%d\n", le32_to_cpu(counters->tx_one_collision)); + seq_printf(m, "tx_multi_collision\t%d\n", le32_to_cpu(counters->tx_multi_collision)); + seq_printf(m, "rx_unicast\t%lld\n", le64_to_cpu(counters->rx_unicast)); + seq_printf(m, "rx_broadcast\t%lld\n", le64_to_cpu(counters->rx_broadcast)); + seq_printf(m, "rx_multicast\t%d\n", le32_to_cpu(counters->rx_multicast)); + seq_printf(m, "tx_aborted\t%d\n", le16_to_cpu(counters->tx_aborted)); + seq_printf(m, "tx_underrun\t%d\n", le16_to_cpu(counters->tx_underrun)); + + seq_printf(m, "tx_octets\t%lld\n", le64_to_cpu(counters->tx_octets)); + seq_printf(m, "rx_octets\t%lld\n", le64_to_cpu(counters->rx_octets)); + seq_printf(m, "rx_multicast64\t%lld\n", le64_to_cpu(counters->rx_multicast64)); + seq_printf(m, "tx_unicast64\t%lld\n", le64_to_cpu(counters->tx_unicast64)); + seq_printf(m, "tx_broadcast64\t%lld\n", le64_to_cpu(counters->tx_broadcast64)); + seq_printf(m, "tx_multicast64\t%lld\n", le64_to_cpu(counters->tx_multicast64)); + seq_printf(m, "tx_pause_on\t%d\n", le32_to_cpu(counters->tx_pause_on)); + seq_printf(m, "tx_pause_off\t%d\n", le32_to_cpu(counters->tx_pause_off)); + seq_printf(m, "tx_pause_all\t%d\n", le32_to_cpu(counters->tx_pause_all)); + seq_printf(m, "tx_deferred\t%d\n", le32_to_cpu(counters->tx_deferred)); + seq_printf(m, "tx_late_collision\t%d\n", le32_to_cpu(counters->tx_late_collision)); + seq_printf(m, "tx_all_collision\t%d\n", le32_to_cpu(counters->tx_all_collision)); + seq_printf(m, "tx_aborted32\t%d\n", le32_to_cpu(counters->tx_aborted32)); + seq_printf(m, "align_errors32\t%d\n", le32_to_cpu(counters->align_errors32)); + seq_printf(m, "rx_frame_too_long\t%d\n", le32_to_cpu(counters->rx_frame_too_long)); + seq_printf(m, "rx_runt\t%d\n", le32_to_cpu(counters->rx_runt)); + seq_printf(m, "rx_pause_on\t%d\n", le32_to_cpu(counters->rx_pause_on)); + seq_printf(m, "rx_pause_off\t%d\n", le32_to_cpu(counters->rx_pause_off)); + seq_printf(m, "rx_pause_all\t%d\n", le32_to_cpu(counters->rx_pause_all)); + seq_printf(m, "rx_unknown_opcode\t%d\n", le32_to_cpu(counters->rx_unknown_opcode)); + seq_printf(m, "rx_mac_error\t%d\n", le32_to_cpu(counters->rx_mac_error)); + seq_printf(m, "tx_underrun32\t%d\n", le32_to_cpu(counters->tx_underrun32)); + seq_printf(m, "rx_mac_missed\t%d\n", le32_to_cpu(counters->rx_mac_missed)); + seq_printf(m, "rx_tcam_dropped\t%d\n", le32_to_cpu(counters->rx_tcam_dropped)); + seq_printf(m, "tdu\t%d\n", le32_to_cpu(counters->tdu)); + seq_printf(m, "rdu\t%d\n", le32_to_cpu(counters->rdu)); + + seq_putc(m, '\n'); + +out_unlock: + rtnl_unlock(); + + return 0; +} + +static int proc_get_registers(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + int i, n, max = R8127_MAC_REGS_SIZE; + u8 byte_rd; + struct rtl8127_private *tp = netdev_priv(dev); + void __iomem *ioaddr = tp->mmio_addr; + + seq_puts(m, "\nDump MAC Registers\n"); + seq_puts(m, "Offset\tValue\n------\t-----\n"); + + rtnl_lock(); + + for (n = 0; n < max;) { + seq_printf(m, "\n0x%04x:\t", n); + + for (i = 0; i < 16 && n < max; i++, n++) { + byte_rd = readb(ioaddr + n); + seq_printf(m, "%02x ", byte_rd); + } + } + + max = 0xB00; + for (n = 0xA00; n < max;) { + seq_printf(m, "\n0x%04x:\t", n); + + for (i = 0; i < 16 && n < max; i++, n++) { + byte_rd = readb(ioaddr + n); + seq_printf(m, "%02x ", byte_rd); + } + } + + max = 0xD40; + for (n = 0xD00; n < max;) { + seq_printf(m, "\n0x%04x:\t", n); + + for (i = 0; i < 16 && n < max; i++, n++) { + byte_rd = readb(ioaddr + n); + seq_printf(m, "%02x ", byte_rd); + } + } + + max = 0x2840; + for (n = 0x2800; n < max;) { + seq_printf(m, "\n0x%04x:\t", n); + + for (i = 0; i < 16 && n < max; i++, n++) { + byte_rd = readb(ioaddr + n); + seq_printf(m, "%02x ", byte_rd); + } + } + + rtnl_unlock(); + + seq_putc(m, '\n'); + return 0; +} + +static int proc_get_all_registers(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + int i, n, max; + u8 byte_rd; + struct rtl8127_private *tp = netdev_priv(dev); + void __iomem *ioaddr = tp->mmio_addr; + struct pci_dev *pdev = tp->pci_dev; + + seq_puts(m, "\nDump All MAC Registers\n"); + seq_puts(m, "Offset\tValue\n------\t-----\n"); + + rtnl_lock(); + + max = pci_resource_len(pdev, 2); + + for (n = 0; n < max;) { + seq_printf(m, "\n0x%04x:\t", n); + + for (i = 0; i < 16 && n < max; i++, n++) { + byte_rd = readb(ioaddr + n); + seq_printf(m, "%02x ", byte_rd); + } + } + + rtnl_unlock(); + + seq_printf(m, "\nTotal length:0x%X", max); + + seq_putc(m, '\n'); + return 0; +} + +static int proc_get_pcie_phy(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + int i, n, max = R8127_EPHY_REGS_SIZE/2; + u16 word_rd; + struct rtl8127_private *tp = netdev_priv(dev); + + seq_puts(m, "\nDump PCIE PHY\n"); + seq_puts(m, "\nOffset\tValue\n------\t-----\n "); + + rtnl_lock(); + + for (n = 0; n < max;) { + seq_printf(m, "\n0x%02x:\t", n); + + for (i = 0; i < 8 && n < max; i++, n++) { + word_rd = rtl8127_ephy_read(tp, n); + seq_printf(m, "%04x ", word_rd); + } + } + + rtnl_unlock(); + + seq_putc(m, '\n'); + return 0; +} + +static int proc_get_eth_phy(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + int i, n, max = R8127_PHY_REGS_SIZE/2; + unsigned long flags; + u16 word_rd; + struct rtl8127_private *tp = netdev_priv(dev); + + seq_puts(m, "\nDump Ethernet PHY\n"); + seq_puts(m, "\nOffset\tValue\n------\t-----\n "); + + spin_lock_irqsave(&tp->phy_lock, flags); + + seq_puts(m, "\n####################page 0##################\n "); + rtl8127_mdio_write(tp, 0x1f, 0x0000); + for (n = 0; n < max;) { + seq_printf(m, "\n0x%02x:\t", n); + + for (i = 0; i < 8 && n < max; i++, n++) { + word_rd = rtl8127_mdio_read(tp, n); + seq_printf(m, "%04x ", word_rd); + } + } + + seq_puts(m, "\n####################extra reg##################\n "); + n = 0xA400; + seq_printf(m, "\n0x%02x:\t", n); + for (i = 0; i < 8; i++, n+=2) { + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + seq_printf(m, "%04x ", word_rd); + } + + n = 0xA410; + seq_printf(m, "\n0x%02x:\t", n); + for (i = 0; i < 3; i++, n+=2) { + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + seq_printf(m, "%04x ", word_rd); + } + + n = 0xA434; + seq_printf(m, "\n0x%02x:\t", n); + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + seq_printf(m, "%04x ", word_rd); + + n = 0xA5D0; + seq_printf(m, "\n0x%02x:\t", n); + for (i = 0; i < 4; i++, n+=2) { + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + seq_printf(m, "%04x ", word_rd); + } + + n = 0xA61A; + seq_printf(m, "\n0x%02x:\t", n); + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + seq_printf(m, "%04x ", word_rd); + + n = 0xA6D0; + seq_printf(m, "\n0x%02x:\t", n); + for (i = 0; i < 3; i++, n+=2) { + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + seq_printf(m, "%04x ", word_rd); + } + + spin_unlock_irqrestore(&tp->phy_lock, flags); + + seq_putc(m, '\n'); + return 0; +} + +static int proc_get_extended_registers(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + int i, n, max = R8127_ERI_REGS_SIZE; + u32 dword_rd; + struct rtl8127_private *tp = netdev_priv(dev); + + seq_puts(m, "\nDump Extended Registers\n"); + seq_puts(m, "\nOffset\tValue\n------\t-----\n "); + + rtnl_lock(); + + for (n = 0; n < max;) { + seq_printf(m, "\n0x%02x:\t", n); + + for (i = 0; i < 4 && n < max; i++, n+=4) { + dword_rd = rtl8127_eri_read(tp, n, 4, ERIAR_ExGMAC); + seq_printf(m, "%08x ", dword_rd); + } + } + + rtnl_unlock(); + + seq_putc(m, '\n'); + return 0; +} + +static int proc_get_pci_registers(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + int i, n, max = R8127_PCI_REGS_SIZE; + u32 dword_rd; + struct rtl8127_private *tp = netdev_priv(dev); + + seq_puts(m, "\nDump PCI Registers\n"); + seq_puts(m, "\nOffset\tValue\n------\t-----\n "); + + rtnl_lock(); + + for (n = 0; n < max;) { + seq_printf(m, "\n0x%03x:\t", n); + + for (i = 0; i < 4 && n < max; i++, n+=4) { + pci_read_config_dword(tp->pci_dev, n, &dword_rd); + seq_printf(m, "%08x ", dword_rd); + } + } + + n = 0x110; + pci_read_config_dword(tp->pci_dev, n, &dword_rd); + seq_printf(m, "\n0x%03x:\t%08x ", n, dword_rd); + n = 0x70c; + pci_read_config_dword(tp->pci_dev, n, &dword_rd); + seq_printf(m, "\n0x%03x:\t%08x ", n, dword_rd); + + rtnl_unlock(); + + seq_putc(m, '\n'); + return 0; +} + +static int proc_get_temperature(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + struct rtl8127_private *tp = netdev_priv(dev); + u16 ts_digout, tj, fah; + + seq_puts(m, "\nChip Temperature\n"); + + rtnl_lock(); + + if (!rtl8127_sysfs_testmode_on(tp)) { + seq_puts(m, "\nPlease turn on ""/sys/class/net//rtk_adv/testmode"".\n\n"); + rtnl_unlock(); + return 0; + } + + netif_testing_on(dev); + ts_digout = rtl8127_read_thermal_sensor(tp); + netif_testing_off(dev); + + rtnl_unlock(); + + tj = ts_digout / 2; + if (ts_digout <= 512) { + tj = ts_digout / 2; + seq_printf(m, "Cel:%d\n", tj); + fah = tj * (9/5) + 32; + seq_printf(m, "Fah:%d\n", fah); + } else { + tj = (512 - ((ts_digout / 2) - 512)) / 2; + seq_printf(m, "Cel:-%d\n", tj); + fah = tj * (9/5) + 32; + seq_printf(m, "Fah:-%d\n", fah); + } + + seq_putc(m, '\n'); + return 0; +} + +static int _proc_get_cable_info(struct seq_file *m, void *v, bool poe_mode) +{ + int i; + u16 status; + int cp_status[RTL8127_CP_NUM]; + int cp_len[RTL8127_CP_NUM] = {0}; + struct net_device *dev = m->private; + struct rtl8127_private *tp = netdev_priv(dev); + const char *pair_str[RTL8127_CP_NUM] = {"1-2", "3-6", "4-5", "7-8"}; + int ret; + + switch (tp->mcfg) { + default: + ret = -EOPNOTSUPP; + goto error_out; + } + + rtnl_lock(); + + if (!rtl8127_sysfs_testmode_on(tp)) { + seq_puts(m, "\nPlease turn on ""/sys/class/net//rtk_adv/testmode"".\n\n"); + ret = 0; + goto error_unlock; + } + + rtl8127_mdio_write(tp, 0x1F, 0x0000); + if (rtl8127_mdio_read(tp, MII_BMCR) & BMCR_PDOWN) { + ret = -EIO; + goto error_unlock; + } + + netif_testing_on(dev); + + status = RTL_R16(tp, PHYstatus); + if (status & LinkStatus) + seq_printf(m, "\nlink speed:%d", + rtl8127_convert_link_speed(status)); + else + seq_puts(m, "\nlink status:off"); + + rtl8127_get_cp_len(tp, cp_len); + + rtl8127_get_cp_status(tp, cp_status, poe_mode); + + seq_puts(m, "\npair\tlength\tstatus \tpp\n"); + + for (i=0; iprivate; + struct rtl8127_private *tp = netdev_priv(dev); + int i; + + rtnl_lock(); + + for (i = 0; i < tp->num_rx_rings; i++) { + struct rtl8127_rx_ring *ring = &tp->rx_ring[i]; + + if (!ring) + continue; + + seq_printf(m, "\ndump rx %d desc:%d\n", i, ring->num_rx_desc); + + _proc_dump_desc(m, (void*)ring->RxDescArray, ring->RxDescAllocSize); + } + +#ifdef ENABLE_LIB_SUPPORT + if (rtl8127_num_lib_rx_rings(tp) > 0) { + for (i = 0; i < tp->HwSuppNumRxQueues; i++) { + struct rtl8127_ring *lib_ring = &tp->lib_rx_ring[i]; + if (lib_ring->enabled) { + seq_printf(m, "\ndump lib rx %d desc:%d\n", i, + lib_ring->ring_size); + _proc_dump_desc(m, (void*)lib_ring->desc_addr, + lib_ring->desc_size); + } + } + } +#endif //ENABLE_LIB_SUPPORT + + rtnl_unlock(); + + seq_putc(m, '\n'); + return 0; +} + +static int proc_dump_tx_desc(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + struct rtl8127_private *tp = netdev_priv(dev); + int i; + + rtnl_lock(); + + for (i = 0; i < tp->num_tx_rings; i++) { + struct rtl8127_tx_ring *ring = &tp->tx_ring[i]; + + if (!ring) + continue; + + seq_printf(m, "\ndump tx %d desc:%d\n", i, ring->num_tx_desc); + + _proc_dump_desc(m, (void*)ring->TxDescArray, ring->TxDescAllocSize); + } + +#ifdef ENABLE_LIB_SUPPORT + if (rtl8127_num_lib_tx_rings(tp) > 0) { + for (i = 0; i < tp->HwSuppNumTxQueues; i++) { + struct rtl8127_ring *lib_ring = &tp->lib_tx_ring[i]; + if (lib_ring->enabled) { + seq_printf(m, "\ndump lib tx %d desc:%d\n", i, + lib_ring->ring_size); + _proc_dump_desc(m, (void*)lib_ring->desc_addr, + lib_ring->desc_size); + } + } + } +#endif //ENABLE_LIB_SUPPORT + + rtnl_unlock(); + + seq_putc(m, '\n'); + return 0; +} + +static int proc_dump_msix_tbl(struct seq_file *m, void *v) +{ + int i, j; + void __iomem *ioaddr; + struct net_device *dev = m->private; + struct rtl8127_private *tp = netdev_priv(dev); + + /* ioremap MMIO region */ + ioaddr = ioremap(pci_resource_start(tp->pci_dev, 4), pci_resource_len(tp->pci_dev, 4)); + if (!ioaddr) + return -EFAULT; + + rtnl_lock(); + + seq_printf(m, "\ndump MSI-X Table. Total Entry %d. \n", tp->hw_supp_irq_nvecs); + + for (i=0; ihw_supp_irq_nvecs; i++) { + seq_printf(m, "\n%04x ", i); + for (j=0; j<4; j++) + seq_printf(m, "%08x ", + readl(ioaddr + i*0x10 + 4*j)); + } + + rtnl_unlock(); + + iounmap(ioaddr); + + seq_putc(m, '\n'); + return 0; +} + +#else //LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) + +static int proc_get_driver_variable(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + struct net_device *dev = data; + struct rtl8127_private *tp = netdev_priv(dev); + int len = 0; + + len += snprintf(page + len, count - len, + "\nDump Driver Driver\n"); + + rtnl_lock(); + + len += snprintf(page + len, count - len, + "Variable\tValue\n----------\t-----\n"); + + len += snprintf(page + len, count - len, + "MODULENAME\t%s\n" + "driver version\t%s\n" + "mcfg\t%d\n" + "chipset\t%d\n" + "chipset_name\t%s\n" + "mtu\t%d\n" + "NUM_RX_DESC\t0x%x\n" + "cur_rx0\t0x%x\n" + "dirty_rx0\t0x%x\n" + "cur_rx1\t0x%x\n" + "dirty_rx1\t0x%x\n" + "cur_rx2\t0x%x\n" + "dirty_rx2\t0x%x\n" + "cur_rx3\t0x%x\n" + "dirty_rx3\t0x%x\n" + "NUM_TX_DESC\t0x%x\n" + "cur_tx0\t0x%x\n" + "dirty_tx0\t0x%x\n" + "cur_tx1\t0x%x\n" + "dirty_tx1\t0x%x\n" + "rx_buf_sz\t0x%x\n" +#ifdef ENABLE_PAGE_REUSE + "rx_buf_page_order\t0x%x\n" + "rx_buf_page_size\t0x%x\n" + "page_reuse_fail_cnt\t0x%x\n" +#endif //ENABLE_PAGE_REUSE + "esd_flag\t0x%x\n" + "pci_cfg_is_read\t0x%x\n" + "rtl8127_rx_config\t0x%x\n" + "cp_cmd\t0x%x\n" + "intr_mask\t0x%x\n" + "timer_intr_mask\t0x%x\n" + "wol_enabled\t0x%x\n" + "wol_opts\t0x%x\n" + "efuse_ver\t0x%x\n" + "eeprom_type\t0x%x\n" + "autoneg\t0x%x\n" + "duplex\t0x%x\n" + "speed\t%d\n" + "advertising\t0x%llx\n" + "eeprom_len\t0x%x\n" + "cur_page\t0x%x\n" + "features\t0x%x\n" + "org_pci_offset_99\t0x%x\n" + "org_pci_offset_180\t0x%x\n" + "issue_offset_99_event\t0x%x\n" + "org_pci_offset_80\t0x%x\n" + "org_pci_offset_81\t0x%x\n" + "use_timer_interrupt\t0x%x\n" + "HwIcVerUnknown\t0x%x\n" + "NotWrRamCodeToMicroP\t0x%x\n" + "NotWrMcuPatchCode\t0x%x\n" + "HwHasWrRamCodeToMicroP\t0x%x\n" + "sw_ram_code_ver\t0x%x\n" + "hw_ram_code_ver\t0x%x\n" + "rtk_enable_diag\t0x%x\n" + "ShortPacketSwChecksum\t0x%x\n" + "UseSwPaddingShortPkt\t0x%x\n" + "RequireAdcBiasPatch\t0x%x\n" + "AdcBiasPatchIoffset\t0x%x\n" + "RequireAdjustUpsTxLinkPulseTiming\t0x%x\n" + "SwrCnt1msIni\t0x%x\n" + "HwSuppNowIsOobVer\t0x%x\n" + "HwFiberModeVer\t0x%x\n" + "HwFiberStat\t0x%x\n" + "HwSwitchMdiToFiber\t0x%x\n" + "NicCustLedValue\t0x%x\n" + "RequiredSecLanDonglePatch\t0x%x\n" + "HwSuppDashVer\t0x%x\n" + "DASH\t0x%x\n" + "dash_printer_enabled\t0x%x\n" + "HwSuppKCPOffloadVer\t0x%x\n" + "speed_mode\t0x%x\n" + "duplex_mode\t0x%x\n" + "autoneg_mode\t0x%x\n" + "aspm\t0x%x\n" + "s5wol\t0x%x\n" + "s5_keep_curr_mac\t0x%x\n" + "eee_enable\t0x%x\n" + "hwoptimize\t0x%lx\n" + "proc_init_num\t0x%x\n" + "s0_magic_packet\t0x%x\n" + "disable_wol_support\t0x%x\n" + "enable_double_vlan\t0x%x\n" + "eee_giga_lite\t0x%x\n" + "HwSuppMagicPktVer\t0x%x\n" + "HwSuppLinkChgWakeUpVer\t0x%x\n" + "HwSuppD0SpeedUpVer\t0x%x\n" + "D0SpeedUpSpeed\t0x%x\n" + "HwSuppCheckPhyDisableModeVer\t0x%x\n" + "HwPkgDet\t0x%x\n" + "HwSuppTxNoCloseVer\t0x%x\n" + "EnableTxNoClose\t0x%x\n" + "NextHwDesCloPtr0\t0x%x\n" + "BeginHwDesCloPtr0\t0x%x\n" + "hw_clo_ptr_reg0\t0x%x\n" + "sw_tail_ptr_reg0\t0x%x\n" + "NextHwDesCloPtr1\t0x%x\n" + "BeginHwDesCloPtr1\t0x%x\n" + "hw_clo_ptr_reg1\t0x%x\n" + "sw_tail_ptr_reg1\t0x%x\n" + "InitRxDescType\t0x%x\n" + "RxDescLength\t0x%x\n" + "num_rx_rings\t0x%x\n" + "num_tx_rings\t0x%x\n" + "tot_rx_rings\t0x%x\n" + "tot_tx_rings\t0x%x\n" + "HwSuppNumRxQueues\t0x%x\n" + "HwSuppNumTxQueues\t0x%x\n" + "EnableRss\t0x%x\n" + "EnablePtp\t0x%x\n" + "min_irq_nvecs\t0x%x\n" + "irq_nvecs\t0x%x\n" + "hw_supp_irq_nvecs\t0x%x\n" + "ring_lib_enabled\t0x%x\n" + "HwSuppIsrVer\t0x%x\n" + "HwCurrIsrVer\t0x%x\n" + "HwSuppMacMcuVer\t0x%x\n" + "MacMcuPageSize\t0x%x\n" + "hw_mcu_patch_code_ver\t0x%llx\n" + "bin_mcu_patch_code_ver\t0x%llx\n" +#ifdef ENABLE_PTP_SUPPORT + "tx_hwtstamp_timeouts\t0x%x\n" + "tx_hwtstamp_skipped\t0x%x\n" +#endif + "random_mac\t0x%x\n" + "org_mac_addr\t%pM\n" +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,13) + "perm_addr\t%pM\n" +#endif + "dev_addr\t%pM\n", + MODULENAME, + RTL8127_VERSION, + tp->mcfg, + tp->chipset, + rtl_chip_info[tp->chipset].name, + dev->mtu, + tp->rx_ring[0].num_rx_desc, + tp->rx_ring[0].cur_rx, + tp->rx_ring[0].dirty_rx, + tp->rx_ring[1].cur_rx, + tp->rx_ring[1].dirty_rx, + tp->rx_ring[2].cur_rx, + tp->rx_ring[2].dirty_rx, + tp->rx_ring[3].cur_rx, + tp->rx_ring[3].dirty_rx, + tp->tx_ring[0].num_tx_desc, + tp->tx_ring[0].cur_tx, + tp->tx_ring[0].dirty_tx, + tp->tx_ring[1].cur_tx, + tp->tx_ring[1].dirty_tx, + tp->rx_buf_sz, +#ifdef ENABLE_PAGE_REUSE + tp->rx_buf_page_order, + tp->rx_buf_page_size, + tp->page_reuse_fail_cnt, +#endif //ENABLE_PAGE_REUSE + tp->esd_flag, + tp->pci_cfg_is_read, + tp->rtl8127_rx_config, + tp->cp_cmd, + tp->intr_mask, + tp->timer_intr_mask, + tp->wol_enabled, + tp->wol_opts, + tp->efuse_ver, + tp->eeprom_type, + tp->autoneg, + tp->duplex, + tp->speed, + tp->advertising, + tp->eeprom_len, + tp->cur_page, + tp->features, + tp->org_pci_offset_99, + tp->org_pci_offset_180, + tp->issue_offset_99_event, + tp->org_pci_offset_80, + tp->org_pci_offset_81, + tp->use_timer_interrupt, + tp->HwIcVerUnknown, + tp->NotWrRamCodeToMicroP, + tp->NotWrMcuPatchCode, + tp->HwHasWrRamCodeToMicroP, + tp->sw_ram_code_ver, + tp->hw_ram_code_ver, + tp->rtk_enable_diag, + tp->ShortPacketSwChecksum, + tp->UseSwPaddingShortPkt, + tp->RequireAdcBiasPatch, + tp->AdcBiasPatchIoffset, + tp->RequireAdjustUpsTxLinkPulseTiming, + tp->SwrCnt1msIni, + tp->HwSuppNowIsOobVer, + tp->HwFiberModeVer, + tp->HwFiberStat, + tp->HwSwitchMdiToFiber, + tp->NicCustLedValue, + tp->RequiredSecLanDonglePatch, + tp->HwSuppDashVer, + tp->DASH, + tp->dash_printer_enabled, + tp->HwSuppKCPOffloadVer, + speed_mode, + duplex_mode, + autoneg_mode, + aspm, + s5wol, + s5_keep_curr_mac, + tp->eee.eee_enabled, + hwoptimize, + proc_init_num, + s0_magic_packet, + disable_wol_support, + enable_double_vlan, + eee_giga_lite, + tp->HwSuppMagicPktVer, + tp->HwSuppLinkChgWakeUpVer, + tp->HwSuppD0SpeedUpVer, + tp->D0SpeedUpSpeed, + tp->HwSuppCheckPhyDisableModeVer, + tp->HwPkgDet, + tp->HwSuppTxNoCloseVer, + tp->EnableTxNoClose, + tp->tx_ring[0].NextHwDesCloPtr, + tp->tx_ring[0].BeginHwDesCloPtr, + rtl8127_get_hw_clo_ptr(&tp->tx_ring[0]), + rtl8127_get_sw_tail_ptr(&tp->tx_ring[0]), + tp->tx_ring[1].NextHwDesCloPtr, + tp->tx_ring[1].BeginHwDesCloPtr, + rtl8127_get_hw_clo_ptr(&tp->tx_ring[1]), + rtl8127_get_sw_tail_ptr(&tp->tx_ring[1]), + tp->InitRxDescType, + tp->RxDescLength, + tp->num_rx_rings, + tp->num_tx_rings, + rtl8127_tot_rx_rings(tp), + rtl8127_tot_tx_rings(tp), + tp->HwSuppNumRxQueues, + tp->HwSuppNumTxQueues, + tp->EnableRss, + tp->EnablePtp, + tp->min_irq_nvecs, + tp->irq_nvecs, + tp->hw_supp_irq_nvecs, + tp->ring_lib_enabled, + tp->HwSuppIsrVer, + tp->HwCurrIsrVer, + tp->HwSuppMacMcuVer, + tp->MacMcuPageSize, + tp->hw_mcu_patch_code_ver, + tp->bin_mcu_patch_code_ver, +#ifdef ENABLE_PTP_SUPPORT + tp->tx_hwtstamp_timeouts, + tp->tx_hwtstamp_skipped, +#endif + tp->random_mac, + tp->org_mac_addr, +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,13) + dev->perm_addr, +#endif + dev->dev_addr); + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + return len; +} + +static int proc_get_tally_counter(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + struct net_device *dev = data; + struct rtl8127_private *tp = netdev_priv(dev); + struct rtl8127_counters *counters; + dma_addr_t paddr; + int len = 0; + + len += snprintf(page + len, count - len, + "\nDump Tally Counter\n"); + + rtnl_lock(); + + counters = tp->tally_vaddr; + paddr = tp->tally_paddr; + if (!counters) { + len += snprintf(page + len, count - len, + "\nDump Tally Counter Fail\n"); + goto out_unlock; + } + + rtl8127_dump_tally_counter(tp, paddr); + + len += snprintf(page + len, count - len, + "Statistics\tValue\n----------\t-----\n"); + + len += snprintf(page + len, count - len, + "tx_packets\t%lld\n" + "rx_packets\t%lld\n" + "tx_errors\t%lld\n" + "rx_errors\t%d\n" + "rx_missed\t%d\n" + "align_errors\t%d\n" + "tx_one_collision\t%d\n" + "tx_multi_collision\t%d\n" + "rx_unicast\t%lld\n" + "rx_broadcast\t%lld\n" + "rx_multicast\t%d\n" + "tx_aborted\t%d\n" + "tx_underrun\t%d\n" + + "tx_octets\t%lld\n" + "rx_octets\t%lld\n" + "rx_multicast64\t%lld\n" + "tx_unicast64\t%lld\n" + "tx_broadcast64\t%lld\n" + "tx_multicast64\t%lld\n" + "tx_pause_on\t%d\n" + "tx_pause_off\t%d\n" + "tx_pause_all\t%d\n" + "tx_deferred\t%d\n" + "tx_late_collision\t%d\n" + "tx_all_collision\t%d\n" + "tx_aborted32\t%d\n" + "align_errors32\t%d\n" + "rx_frame_too_long\t%d\n" + "rx_runt\t%d\n" + "rx_pause_on\t%d\n" + "rx_pause_off\t%d\n" + "rx_pause_all\t%d\n" + "rx_unknown_opcode\t%d\n" + "rx_mac_error\t%d\n" + "tx_underrun32\t%d\n" + "rx_mac_missed\t%d\n" + "rx_tcam_dropped\t%d\n" + "tdu\t%d\n" + "rdu\t%d\n", + le64_to_cpu(counters->tx_packets), + le64_to_cpu(counters->rx_packets), + le64_to_cpu(counters->tx_errors), + le32_to_cpu(counters->rx_errors), + le16_to_cpu(counters->rx_missed), + le16_to_cpu(counters->align_errors), + le32_to_cpu(counters->tx_one_collision), + le32_to_cpu(counters->tx_multi_collision), + le64_to_cpu(counters->rx_unicast), + le64_to_cpu(counters->rx_broadcast), + le32_to_cpu(counters->rx_multicast), + le16_to_cpu(counters->tx_aborted), + le16_to_cpu(counters->tx_underrun), + + le64_to_cpu(counters->tx_octets), + le64_to_cpu(counters->rx_octets), + le64_to_cpu(counters->rx_multicast64), + le64_to_cpu(counters->tx_unicast64), + le64_to_cpu(counters->tx_broadcast64), + le64_to_cpu(counters->tx_multicast64), + le32_to_cpu(counters->tx_pause_on), + le32_to_cpu(counters->tx_pause_off), + le32_to_cpu(counters->tx_pause_all), + le32_to_cpu(counters->tx_deferred), + le32_to_cpu(counters->tx_late_collision), + le32_to_cpu(counters->tx_all_collision), + le32_to_cpu(counters->tx_aborted32), + le32_to_cpu(counters->align_errors32), + le32_to_cpu(counters->rx_frame_too_long), + le32_to_cpu(counters->rx_runt), + le32_to_cpu(counters->rx_pause_on), + le32_to_cpu(counters->rx_pause_off), + le32_to_cpu(counters->rx_pause_all), + le32_to_cpu(counters->rx_unknown_opcode), + le32_to_cpu(counters->rx_mac_error), + le32_to_cpu(counters->tx_underrun32), + le32_to_cpu(counters->rx_mac_missed), + le32_to_cpu(counters->rx_tcam_dropped), + le32_to_cpu(counters->tdu), + le32_to_cpu(counters->rdu)); + + len += snprintf(page + len, count - len, "\n"); +out_unlock: + rtnl_unlock(); + + *eof = 1; + return len; +} + +static int proc_get_registers(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + struct net_device *dev = data; + int i, n, max = R8127_MAC_REGS_SIZE; + u8 byte_rd; + struct rtl8127_private *tp = netdev_priv(dev); + void __iomem *ioaddr = tp->mmio_addr; + int len = 0; + + len += snprintf(page + len, count - len, + "\nDump MAC Registers\n" + "Offset\tValue\n------\t-----\n"); + + rtnl_lock(); + + for (n = 0; n < max;) { + len += snprintf(page + len, count - len, + "\n0x%04x:\t", + n); + + for (i = 0; i < 16 && n < max; i++, n++) { + byte_rd = readb(ioaddr + n); + len += snprintf(page + len, count - len, + "%02x ", + byte_rd); + } + } + + max = 0xB00; + for (n = 0xA00; n < max;) { + len += snprintf(page + len, count - len, + "\n0x%04x:\t", + n); + + for (i = 0; i < 16 && n < max; i++, n++) { + byte_rd = readb(ioaddr + n); + len += snprintf(page + len, count - len, + "%02x ", + byte_rd); + } + } + + max = 0xD40; + for (n = 0xD00; n < max;) { + len += snprintf(page + len, count - len, + "\n0x%04x:\t", + n); + + for (i = 0; i < 16 && n < max; i++, n++) { + byte_rd = readb(ioaddr + n); + len += snprintf(page + len, count - len, + "%02x ", + byte_rd); + } + } + + max = 0x2840; + for (n = 0x2800; n < max;) { + len += snprintf(page + len, count - len, + "\n0x%04x:\t", + n); + + for (i = 0; i < 16 && n < max; i++, n++) { + byte_rd = readb(ioaddr + n); + len += snprintf(page + len, count - len, + "%02x ", + byte_rd); + } + } + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + return len; +} + +static int proc_get_all_registers(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + struct net_device *dev = data; + int i, n, max; + u8 byte_rd; + struct rtl8127_private *tp = netdev_priv(dev); + void __iomem *ioaddr = tp->mmio_addr; + struct pci_dev *pdev = tp->pci_dev; + int len = 0; + + len += snprintf(page + len, count - len, + "\nDump All MAC Registers\n" + "Offset\tValue\n------\t-----\n"); + + rtnl_lock(); + + max = pci_resource_len(pdev, 2); + + for (n = 0; n < max;) { + len += snprintf(page + len, count - len, + "\n0x%04x:\t", + n); + + for (i = 0; i < 16 && n < max; i++, n++) { + byte_rd = readb(ioaddr + n); + len += snprintf(page + len, count - len, + "%02x ", + byte_rd); + } + } + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\nTotal length:0x%X", max); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + return len; +} + +static int proc_get_pcie_phy(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + struct net_device *dev = data; + int i, n, max = R8127_EPHY_REGS_SIZE/2; + u16 word_rd; + struct rtl8127_private *tp = netdev_priv(dev); + int len = 0; + + len += snprintf(page + len, count - len, + "\nDump PCIE PHY\n" + "Offset\tValue\n------\t-----\n"); + + rtnl_lock(); + + for (n = 0; n < max;) { + len += snprintf(page + len, count - len, + "\n0x%02x:\t", + n); + + for (i = 0; i < 8 && n < max; i++, n++) { + word_rd = rtl8127_ephy_read(tp, n); + len += snprintf(page + len, count - len, + "%04x ", + word_rd); + } + } + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + return len; +} + +static int proc_get_eth_phy(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + struct net_device *dev = data; + int i, n, max = R8127_PHY_REGS_SIZE/2; + u16 word_rd; + struct rtl8127_private *tp = netdev_priv(dev); + int len = 0; + + len += snprintf(page + len, count - len, + "\nDump Ethernet PHY\n" + "Offset\tValue\n------\t-----\n"); + + rtnl_lock(); + + len += snprintf(page + len, count - len, + "\n####################page 0##################\n"); + rtl8127_mdio_write(tp, 0x1f, 0x0000); + for (n = 0; n < max;) { + len += snprintf(page + len, count - len, + "\n0x%02x:\t", + n); + + for (i = 0; i < 8 && n < max; i++, n++) { + word_rd = rtl8127_mdio_read(tp, n); + len += snprintf(page + len, count - len, + "%04x ", + word_rd); + } + } + + len += snprintf(page + len, count - len, + "\n####################extra reg##################\n"); + n = 0xA400; + len += snprintf(page + len, count - len, + "\n0x%02x:\t", + n); + for (i = 0; i < 8; i++, n+=2) { + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + len += snprintf(page + len, count - len, + "%04x ", + word_rd); + } + + n = 0xA410; + len += snprintf(page + len, count - len, + "\n0x%02x:\t", + n); + for (i = 0; i < 3; i++, n+=2) { + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + len += snprintf(page + len, count - len, + "%04x ", + word_rd); + } + + n = 0xA434; + len += snprintf(page + len, count - len, + "\n0x%02x:\t", + n); + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + len += snprintf(page + len, count - len, + "%04x ", + word_rd); + + n = 0xA5D0; + len += snprintf(page + len, count - len, + "\n0x%02x:\t", + n); + for (i = 0; i < 4; i++, n+=2) { + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + len += snprintf(page + len, count - len, + "%04x ", + word_rd); + } + + n = 0xA61A; + len += snprintf(page + len, count - len, + "\n0x%02x:\t", + n); + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + len += snprintf(page + len, count - len, + "%04x ", + word_rd); + + n = 0xA6D0; + len += snprintf(page + len, count - len, + "\n0x%02x:\t", + n); + for (i = 0; i < 3; i++, n+=2) { + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + len += snprintf(page + len, count - len, + "%04x ", + word_rd); + } + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + return len; +} + +static int proc_get_extended_registers(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + struct net_device *dev = data; + int i, n, max = R8127_ERI_REGS_SIZE; + u32 dword_rd; + struct rtl8127_private *tp = netdev_priv(dev); + int len = 0; + + len += snprintf(page + len, count - len, + "\nDump Extended Registers\n" + "Offset\tValue\n------\t-----\n"); + + rtnl_lock(); + + for (n = 0; n < max;) { + len += snprintf(page + len, count - len, + "\n0x%02x:\t", + n); + + for (i = 0; i < 4 && n < max; i++, n+=4) { + dword_rd = rtl8127_eri_read(tp, n, 4, ERIAR_ExGMAC); + len += snprintf(page + len, count - len, + "%08x ", + dword_rd); + } + } + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + return len; +} + +static int proc_get_pci_registers(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + struct net_device *dev = data; + int i, n, max = R8127_PCI_REGS_SIZE; + u32 dword_rd; + struct rtl8127_private *tp = netdev_priv(dev); + int len = 0; + + len += snprintf(page + len, count - len, + "\nDump PCI Registers\n" + "Offset\tValue\n------\t-----\n"); + + rtnl_lock(); + + for (n = 0; n < max;) { + len += snprintf(page + len, count - len, + "\n0x%03x:\t", + n); + + for (i = 0; i < 4 && n < max; i++, n+=4) { + pci_read_config_dword(tp->pci_dev, n, &dword_rd); + len += snprintf(page + len, count - len, + "%08x ", + dword_rd); + } + } + + n = 0x110; + pci_read_config_dword(tp->pci_dev, n, &dword_rd); + len += snprintf(page + len, count - len, + "\n0x%03x:\t%08x ", + n, + dword_rd); + n = 0x70c; + pci_read_config_dword(tp->pci_dev, n, &dword_rd); + len += snprintf(page + len, count - len, + "\n0x%03x:\t%08x ", + n, + dword_rd); + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + return len; +} + +static int proc_get_temperature(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + struct net_device *dev = data; + struct rtl8127_private *tp = netdev_priv(dev); + u16 ts_digout, tj, fah; + int len = 0; + + len += snprintf(page + len, count - len, + "\nChip Temperature\n"); + + rtnl_lock(); + + if (!rtl8127_sysfs_testmode_on(tp)) { + len += snprintf(page + len, count - len, + "\nPlease turn on ""/sys/class/net//rtk_adv/testmode"".\n\n"); + goto out_unlock; + } + + ts_digout = rtl8127_read_thermal_sensor(tp); + + tj = ts_digout / 2; + if (ts_digout <= 512) { + tj = ts_digout / 2; + len += snprintf(page + len, count - len, + "Cel:%d\n", + tj); + fah = tj * (9/5) + 32; + len += snprintf(page + len, count - len, + "Fah:%d\n", + fah); + + } else { + tj = (512 - ((ts_digout / 2) - 512)) / 2; + len += snprintf(page + len, count - len, + "Cel:-%d\n", + tj); + fah = tj * (9/5) + 32; + len += snprintf(page + len, count - len, + "Fah:-%d\n", + fah); + } + + len += snprintf(page + len, count - len, "\n"); + +out_unlock: + rtnl_unlock(); + + *eof = 1; + return len; +} + +static int _proc_get_cable_info(char *page, char **start, + off_t offset, int count, + int *eof, void *data, + bool poe_mode) +{ + int i; + u16 status; + int len = 0; + struct net_device *dev = data; + int cp_status[RTL8127_CP_NUM] = {0}; + int cp_len[RTL8127_CP_NUM] = {0}; + struct rtl8127_private *tp = netdev_priv(dev); + const char *pair_str[RTL8127_CP_NUM] = {"1-2", "3-6", "4-5", "7-8"}; + + switch (tp->mcfg) { + default: + return -EOPNOTSUPP; + } + + spin_lock_irqsave(&tp->phy_lock, flags); + + if (!rtl8127_sysfs_testmode_on(tp)) { + len += snprintf(page + len, count - len, + "\nPlease turn on ""/sys/class/net//rtk_adv/testmode"".\n\n"); + goto out_unlock; + } + + status = RTL_R16(tp, PHYstatus); + if (status & LinkStatus) + len += snprintf(page + len, count - len, + "\nlink speed:%d", + rtl8127_convert_link_speed(status)); + else + len += snprintf(page + len, count - len, + "\nlink status:off"); + + rtl8127_get_cp_len(tp, cp_len); + + rtl8127_get_cp_status(tp, cp_status, poe_mode); + + len += snprintf(page + len, count - len, + "\npair\tlength\tstatus \tpp\n"); + + for (i=0; iphy_lock, flags); + + *eof = 1; + return len; +} + +static int proc_get_cable_info(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + return _proc_get_cable_info(page, start, offset, count, eof, data, 0); +} + +static int proc_get_poe_cable_info(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + return _proc_get_cable_info(page, start, offset, count, eof, data, 1); +} + +static void _proc_dump_desc(char *page, int *page_len, int *count, void *desc_base, + u32 alloc_size) +{ + u32 *pdword; + int i, len; + + if (desc_base == NULL || + alloc_size == 0) + return; + + len = *page_len; + pdword = (u32*)desc_base; + for (i=0; i<(alloc_size/4); i++) { + if (!(i % 4)) + len += snprintf(page + len, *count - len, + "\n%04x ", + i); + len += snprintf(page + len, *count - len, + "%08x ", + pdword[i]); + } + + len += snprintf(page + len, *count - len, "\n"); + + *page_len = len; + return; +} + +static int proc_dump_rx_desc(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + int i; + int len = 0; + struct net_device *dev = data; + struct rtl8127_private *tp = netdev_priv(dev); + + rtnl_lock(); + + for (i = 0; i < tp->num_rx_rings; i++) { + struct rtl8127_rx_ring *ring = &tp->rx_ring[i]; + + if (!ring) + continue; + + len += snprintf(page + len, count - len, + "\ndump rx %d desc:%d", + i, ring->num_rx_desc); + + _proc_dump_desc(page, &len, &count, + ring->RxDescArray, + ring->RxDescAllocSize); + } + +#ifdef ENABLE_LIB_SUPPORT + if (rtl8127_num_lib_rx_rings(tp) > 0) { + for (i = 0; i < tp->HwSuppNumRxQueues; i++) { + struct rtl8127_ring *lib_ring = &tp->lib_rx_ring[i]; + if (lib_ring->enabled) { + len += snprintf(page + len, count - len, + "\ndump lib rx %d desc:%d", + i, + ring->ring_size); + _proc_dump_desc(page, &len, &count, + (void*)lib_ring->desc_addr, + lib_ring->desc_size); + } + } + } +#endif //ENABLE_LIB_SUPPORT + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + + return len; +} + +static int proc_dump_tx_desc(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + int len = 0; + struct net_device *dev = data; + struct rtl8127_private *tp = netdev_priv(dev); + int i; + + rtnl_lock(); + + for (i = 0; i < tp->num_tx_rings; i++) { + struct rtl8127_tx_ring *ring = &tp->tx_ring[i]; + + if (!ring) + continue; + + len += snprintf(page + len, count - len, + "\ndump tx desc:%d", + ring->num_tx_desc); + + _proc_dump_desc(page, &len, &count, + ring->TxDescArray, + ring->TxDescAllocSize); + } + +#ifdef ENABLE_LIB_SUPPORT + if (rtl8127_num_lib_tx_rings(tp) > 0) { + for (i = 0; i < tp->HwSuppNumTxQueues; i++) { + struct rtl8127_ring *lib_ring = &tp->lib_tx_ring[i]; + if (lib_ring->enabled) { + len += snprintf(page + len, count - len, + "\ndump lib tx %d desc:%d", + i, + ring->ring_size); + _proc_dump_desc(page, &len, &count, + (void*)lib_ring->desc_addr, + lib_ring->desc_size); + } + } + } +#endif //ENABLE_LIB_SUPPORT + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + + return len; +} + +static int proc_dump_msix_tbl(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + int i, j; + int len = 0; + void __iomem *ioaddr; + struct net_device *dev = data; + struct rtl8127_private *tp = netdev_priv(dev); + + /* ioremap MMIO region */ + ioaddr = ioremap(pci_resource_start(tp->pci_dev, 4), pci_resource_len(tp->pci_dev, 4)); + if (!ioaddr) + return -EFAULT; + + rtnl_lock(); + + len += snprintf(page + len, count - len, + "\ndump MSI-X Table. Total Entry %d. \n", + tp->hw_supp_irq_nvecs); + + for (i=0; ihw_supp_irq_nvecs; i++) { + len += snprintf(page + len, count - len, + "\n%04x ", i); + for (j=0; j<4; j++) + len += snprintf(page + len, count - len, "%08x ", + readl(ioaddr + i*0x10 + 4*j)); + } + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + return 0; +} + +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) + +static void rtl8127_proc_module_init(void) +{ + //create /proc/net/r8127 +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,32) + rtl8127_proc = proc_mkdir(MODULENAME, init_net.proc_net); +#else + rtl8127_proc = proc_mkdir(MODULENAME, proc_net); +#endif + if (!rtl8127_proc) + dprintk("cannot create %s proc entry \n", MODULENAME); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) +/* + * seq_file wrappers for procfile show routines. + */ +static int rtl8127_proc_open(struct inode *inode, struct file *file) +{ + struct net_device *dev = proc_get_parent_data(inode); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0) + int (*show)(struct seq_file *, void *) = pde_data(inode); +#else + int (*show)(struct seq_file *, void *) = PDE_DATA(inode); +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0) + + return single_open(file, show, dev); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0) +static const struct proc_ops rtl8127_proc_fops = { + .proc_open = rtl8127_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, +}; +#else +static const struct file_operations rtl8127_proc_fops = { + .open = rtl8127_proc_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; +#endif + +#endif + +/* + * Table of proc files we need to create. + */ +struct rtl8127_proc_file { + char name[16]; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) + int (*show)(struct seq_file *, void *); +#else + int (*show)(char *, char **, off_t, int, int *, void *); +#endif +}; + +static const struct rtl8127_proc_file rtl8127_debug_proc_files[] = { + { "driver_var", &proc_get_driver_variable }, + { "tally", &proc_get_tally_counter }, + { "registers", &proc_get_registers }, + { "registers2", &proc_get_all_registers }, + { "pcie_phy", &proc_get_pcie_phy }, + { "eth_phy", &proc_get_eth_phy }, + { "ext_regs", &proc_get_extended_registers }, + { "pci_regs", &proc_get_pci_registers }, + { "tx_desc", &proc_dump_tx_desc }, + { "rx_desc", &proc_dump_rx_desc }, + { "msix_tbl", &proc_dump_msix_tbl }, + { "", NULL } +}; + +static const struct rtl8127_proc_file rtl8127_test_proc_files[] = { + { "temp", &proc_get_temperature }, + { "cdt", &proc_get_cable_info }, + { "cdt_poe", &proc_get_poe_cable_info }, + { "", NULL } +}; + +#define R8127_PROC_DEBUG_DIR "debug" +#define R8127_PROC_TEST_DIR "test" + +static void rtl8127_proc_init(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + const struct rtl8127_proc_file *f; + struct proc_dir_entry *dir; + + if (!rtl8127_proc) + return; + + if (tp->proc_dir_debug || tp->proc_dir_test) + return; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) + dir = proc_mkdir_data(dev->name, 0, rtl8127_proc, dev); + if (!dir) { + printk("Unable to initialize /proc/net/%s/%s\n", + MODULENAME, dev->name); + return; + } + tp->proc_dir = dir; + proc_init_num++; + + /* create debug entry */ + dir = proc_mkdir_data(R8127_PROC_DEBUG_DIR, 0, tp->proc_dir, dev); + if (!dir) { + printk("Unable to initialize /proc/net/%s/%s/%s\n", + MODULENAME, dev->name, R8127_PROC_DEBUG_DIR); + return; + } + + tp->proc_dir_debug = dir; + for (f = rtl8127_debug_proc_files; f->name[0]; f++) { + if (!proc_create_data(f->name, S_IFREG | S_IRUGO, dir, + &rtl8127_proc_fops, f->show)) { + printk("Unable to initialize " + "/proc/net/%s/%s/%s/%s\n", + MODULENAME, dev->name, R8127_PROC_DEBUG_DIR, + f->name); + return; + } + } + + /* create test entry */ + dir = proc_mkdir_data(R8127_PROC_TEST_DIR, 0, tp->proc_dir, dev); + if (!dir) { + printk("Unable to initialize /proc/net/%s/%s/%s\n", + MODULENAME, dev->name, R8127_PROC_TEST_DIR); + return; + } + + tp->proc_dir_test = dir; + for (f = rtl8127_test_proc_files; f->name[0]; f++) { + if (!proc_create_data(f->name, S_IFREG | S_IRUGO, dir, + &rtl8127_proc_fops, f->show)) { + printk("Unable to initialize " + "/proc/net/%s/%s/%s/%s\n", + MODULENAME, dev->name, R8127_PROC_TEST_DIR, + f->name); + return; + } + } +#else + dir = proc_mkdir(dev->name, rtl8127_proc); + if (!dir) { + printk("Unable to initialize /proc/net/%s/%s\n", + MODULENAME, dev->name); + return; + } + + tp->proc_dir = dir; + proc_init_num++; + + /* create debug entry */ + dir = proc_mkdir(R8127_PROC_DEBUG_DIR, tp->proc_dir); + if (!dir) { + printk("Unable to initialize /proc/net/%s/%s/%s\n", + MODULENAME, dev->name, R8127_PROC_DEBUG_DIR); + return; + } + + tp->proc_dir_debug = dir; + for (f = rtl8127_debug_proc_files; f->name[0]; f++) { + if (!create_proc_read_entry(f->name, S_IFREG | S_IRUGO, + dir, f->show, dev)) { + printk("Unable to initialize " + "/proc/net/%s/%s/%s/%s\n", + MODULENAME, dev->name, R8127_PROC_DEBUG_DIR, + f->name); + return; + } + } + + /* create test entry */ + dir = proc_mkdir(R8127_PROC_TEST_DIR, tp->proc_dir); + if (!dir) { + printk("Unable to initialize /proc/net/%s/%s/%s\n", + MODULENAME, dev->name, R8127_PROC_TEST_DIR); + return; + } + + tp->proc_dir_test = dir; + for (f = rtl8127_test_proc_files; f->name[0]; f++) { + if (!create_proc_read_entry(f->name, S_IFREG | S_IRUGO, + dir, f->show, dev)) { + printk("Unable to initialize " + "/proc/net/%s/%s/%s/%s\n", + MODULENAME, dev->name, R8127_PROC_TEST_DIR, + f->name); + return; + } + } +#endif +} + +static void rtl8127_proc_remove(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (tp->proc_dir) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) + remove_proc_subtree(dev->name, rtl8127_proc); +#else + const struct rtl8127_proc_file *f; + struct rtl8127_private *tp = netdev_priv(dev); + + if (tp->proc_dir_debug) { + for (f = rtl8127_debug_proc_files; f->name[0]; f++) + remove_proc_entry(f->name, tp->proc_dir_debug); + remove_proc_entry(R8127_PROC_DEBUG_DIR, tp->proc_dir); + } + + if (tp->proc_dir_test) { + for (f = rtl8127_test_proc_files; f->name[0]; f++) + remove_proc_entry(f->name, tp->proc_dir_test); + remove_proc_entry(R8127_PROC_TEST_DIR, tp->proc_dir); + } + + remove_proc_entry(dev->name, rtl8127_proc); +#endif + proc_init_num--; + + tp->proc_dir_debug = NULL; + tp->proc_dir_test = NULL; + tp->proc_dir = NULL; + } +} + +#endif //ENABLE_R8127_PROCFS + +#ifdef ENABLE_R8127_SYSFS +/**************************************************************************** +* -----------------------------SYSFS STUFF------------------------- +***************************************************************************** +*/ +static ssize_t testmode_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct net_device *netdev = to_net_dev(dev); + struct rtl8127_private *tp = netdev_priv(netdev); + + sprintf(buf, "%u\n", tp->testmode); + + return strlen(buf); +} + +static ssize_t testmode_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct net_device *netdev = to_net_dev(dev); + struct rtl8127_private *tp = netdev_priv(netdev); + u32 testmode; + + if (sscanf(buf, "%u\n", &testmode) != 1) + return -EINVAL; + + if (tp->testmode != testmode) { + rtnl_lock(); + tp->testmode = testmode; + rtnl_unlock(); + } + + return count; +} + +static DEVICE_ATTR_RW(testmode); + +static struct attribute *rtk_adv_attrs[] = { + &dev_attr_testmode.attr, + NULL +}; + +static struct attribute_group rtk_adv_grp = { + .name = "rtl_adv", + .attrs = rtk_adv_attrs, +}; + +static void rtl8127_sysfs_init(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int ret; + + /* init rtl_adv */ +#ifdef ENABLE_LIB_SUPPORT + tp->testmode = 0; +#else + tp->testmode = 1; +#endif //ENABLE_LIB_SUPPORT + + ret = sysfs_create_group(&dev->dev.kobj, &rtk_adv_grp); + if (ret < 0) + netif_warn(tp, probe, dev, "create rtk_adv_grp fail\n"); + else + set_bit(R8127_SYSFS_RTL_ADV, tp->sysfs_flag); +} + +static void rtl8127_sysfs_remove(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (test_and_clear_bit(R8127_SYSFS_RTL_ADV, tp->sysfs_flag)) + sysfs_remove_group(&dev->dev.kobj, &rtk_adv_grp); +} +#endif //ENABLE_R8127_SYSFS + +static inline u16 map_phy_ocp_addr(u16 PageNum, u8 RegNum) +{ + u16 OcpPageNum = 0; + u8 OcpRegNum = 0; + u16 OcpPhyAddress = 0; + + if (PageNum == 0) { + OcpPageNum = OCP_STD_PHY_BASE_PAGE + (RegNum / 8); + OcpRegNum = 0x10 + (RegNum % 8); + } else { + OcpPageNum = PageNum; + OcpRegNum = RegNum; + } + + OcpPageNum <<= 4; + + if (OcpRegNum < 16) { + OcpPhyAddress = 0; + } else { + OcpRegNum -= 16; + OcpRegNum <<= 1; + + OcpPhyAddress = OcpPageNum + OcpRegNum; + } + + + return OcpPhyAddress; +} + +static void mdio_real_direct_write_phy_ocp(struct rtl8127_private *tp, + u16 RegAddr, + u16 value) +{ + u32 data32; + int i; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18) + WARN_ON_ONCE(RegAddr % 2); +#endif + data32 = RegAddr/2; + data32 <<= OCPR_Addr_Reg_shift; + data32 |= OCPR_Write | value; + + RTL_W32(tp, PHYOCP, data32); + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { + udelay(R8127_CHANNEL_WAIT_TIME); + + if (!(RTL_R32(tp, PHYOCP) & OCPR_Flag)) + break; + } +} + +void rtl8127_mdio_direct_write_phy_ocp(struct rtl8127_private *tp, + u16 RegAddr, + u16 value) +{ + if (tp->rtk_enable_diag) + return; + + mdio_real_direct_write_phy_ocp(tp, RegAddr, value); +} + +/* +static void rtl8127_mdio_write_phy_ocp(struct rtl8127_private *tp, + u16 PageNum, + u32 RegAddr, + u32 value) +{ + u16 ocp_addr; + + ocp_addr = map_phy_ocp_addr(PageNum, RegAddr); + + rtl8127_mdio_direct_write_phy_ocp(tp, ocp_addr, value); +} +*/ + +static void rtl8127_mdio_real_write_phy_ocp(struct rtl8127_private *tp, + u16 PageNum, + u32 RegAddr, + u32 value) +{ + u16 ocp_addr; + + ocp_addr = map_phy_ocp_addr(PageNum, RegAddr); + + mdio_real_direct_write_phy_ocp(tp, ocp_addr, value); +} + +static void mdio_real_write(struct rtl8127_private *tp, + u16 RegAddr, + u16 value) +{ + if (RegAddr == 0x1F) { + tp->cur_page = value; + return; + } + rtl8127_mdio_real_write_phy_ocp(tp, tp->cur_page, RegAddr, value); +} + +void rtl8127_mdio_write(struct rtl8127_private *tp, + u16 RegAddr, + u16 value) +{ + if (tp->rtk_enable_diag) + return; + + mdio_real_write(tp, RegAddr, value); +} + +void rtl8127_mdio_prot_write(struct rtl8127_private *tp, + u32 RegAddr, + u32 value) +{ + mdio_real_write(tp, RegAddr, value); +} + +void rtl8127_mdio_prot_direct_write_phy_ocp(struct rtl8127_private *tp, + u32 RegAddr, + u32 value) +{ + mdio_real_direct_write_phy_ocp(tp, RegAddr, value); +} + +static u32 mdio_real_direct_read_phy_ocp(struct rtl8127_private *tp, + u16 RegAddr) +{ + u32 data32; + int i, value = 0; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18) + WARN_ON_ONCE(RegAddr % 2); +#endif + data32 = RegAddr/2; + data32 <<= OCPR_Addr_Reg_shift; + + RTL_W32(tp, PHYOCP, data32); + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { + udelay(R8127_CHANNEL_WAIT_TIME); + + if (RTL_R32(tp, PHYOCP) & OCPR_Flag) + break; + } + value = RTL_R32(tp, PHYOCP) & OCPDR_Data_Mask; + + return value; +} + +u32 rtl8127_mdio_direct_read_phy_ocp(struct rtl8127_private *tp, + u16 RegAddr) +{ + if (tp->rtk_enable_diag) + return 0xffffffff; + + return mdio_real_direct_read_phy_ocp(tp, RegAddr); +} + +/* +static u32 rtl8127_mdio_read_phy_ocp(struct rtl8127_private *tp, + u16 PageNum, + u32 RegAddr) +{ + u16 ocp_addr; + + ocp_addr = map_phy_ocp_addr(PageNum, RegAddr); + + return rtl8127_mdio_direct_read_phy_ocp(tp, ocp_addr); +} +*/ + +static u32 rtl8127_mdio_real_read_phy_ocp(struct rtl8127_private *tp, + u16 PageNum, + u32 RegAddr) +{ + u16 ocp_addr; + + ocp_addr = map_phy_ocp_addr(PageNum, RegAddr); + + return mdio_real_direct_read_phy_ocp(tp, ocp_addr); +} + +static u32 mdio_real_read(struct rtl8127_private *tp, + u16 RegAddr) +{ + return rtl8127_mdio_real_read_phy_ocp(tp, tp->cur_page, RegAddr); +} + +u32 rtl8127_mdio_read(struct rtl8127_private *tp, + u16 RegAddr) +{ + if (tp->rtk_enable_diag) + return 0xffffffff; + + return mdio_real_read(tp, RegAddr); +} + +u32 rtl8127_mdio_prot_read(struct rtl8127_private *tp, + u32 RegAddr) +{ + return mdio_real_read(tp, RegAddr); +} + +u32 rtl8127_mdio_prot_direct_read_phy_ocp(struct rtl8127_private *tp, + u32 RegAddr) +{ + return mdio_real_direct_read_phy_ocp(tp, RegAddr); +} + +static void rtl8127_clear_and_set_eth_phy_bit(struct rtl8127_private *tp, u8 addr, u16 clearmask, u16 setmask) +{ + u16 PhyRegValue; + + PhyRegValue = rtl8127_mdio_read(tp, addr); + PhyRegValue &= ~clearmask; + PhyRegValue |= setmask; + rtl8127_mdio_write(tp, addr, PhyRegValue); +} + +void rtl8127_clear_eth_phy_bit(struct rtl8127_private *tp, u8 addr, u16 mask) +{ + rtl8127_clear_and_set_eth_phy_bit(tp, + addr, + mask, + 0); +} + +void rtl8127_set_eth_phy_bit(struct rtl8127_private *tp, u8 addr, u16 mask) +{ + rtl8127_clear_and_set_eth_phy_bit(tp, + addr, + 0, + mask); +} + +void rtl8127_clear_and_set_eth_phy_ocp_bit(struct rtl8127_private *tp, u16 addr, u16 clearmask, u16 setmask) +{ + u16 PhyRegValue; + + PhyRegValue = rtl8127_mdio_direct_read_phy_ocp(tp, addr); + PhyRegValue &= ~clearmask; + PhyRegValue |= setmask; + rtl8127_mdio_direct_write_phy_ocp(tp, addr, PhyRegValue); +} + +void rtl8127_clear_eth_phy_ocp_bit(struct rtl8127_private *tp, u16 addr, u16 mask) +{ + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + addr, + mask, + 0); +} + +void rtl8127_set_eth_phy_ocp_bit(struct rtl8127_private *tp, u16 addr, u16 mask) +{ + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + addr, + 0, + mask); +} + +void rtl8127_mac_ocp_write(struct rtl8127_private *tp, u16 reg_addr, u16 value) +{ + u32 data32; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18) + WARN_ON_ONCE(reg_addr % 2); +#endif + + data32 = reg_addr/2; + data32 <<= OCPR_Addr_Reg_shift; + data32 += value; + data32 |= OCPR_Write; + + RTL_W32(tp, MACOCP, data32); +} + +u16 rtl8127_mac_ocp_read(struct rtl8127_private *tp, u16 reg_addr) +{ + u32 data32; + u16 data16 = 0; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18) + WARN_ON_ONCE(reg_addr % 2); +#endif + + data32 = reg_addr/2; + data32 <<= OCPR_Addr_Reg_shift; + + RTL_W32(tp, MACOCP, data32); + data16 = (u16)RTL_R32(tp, MACOCP); + + return data16; +} + +#ifdef ENABLE_USE_FIRMWARE_FILE +static void mac_mcu_write(struct rtl8127_private *tp, u16 reg, u16 value) +{ + if (reg == 0x1f) { + tp->ocp_base = value << 4; + return; + } + + rtl8127_mac_ocp_write(tp, tp->ocp_base + reg, value); +} + +static u32 mac_mcu_read(struct rtl8127_private *tp, u16 reg) +{ + return rtl8127_mac_ocp_read(tp, tp->ocp_base + reg); +} +#endif + +static void +rtl8127_clear_set_mac_ocp_bit( + struct rtl8127_private *tp, + u16 addr, + u16 clearmask, + u16 setmask +) +{ + u16 PhyRegValue; + + PhyRegValue = rtl8127_mac_ocp_read(tp, addr); + PhyRegValue &= ~clearmask; + PhyRegValue |= setmask; + rtl8127_mac_ocp_write(tp, addr, PhyRegValue); +} + +void +rtl8127_clear_mac_ocp_bit( + struct rtl8127_private *tp, + u16 addr, + u16 mask +) +{ + rtl8127_clear_set_mac_ocp_bit(tp, + addr, + mask, + 0); +} + +static void +rtl8127_set_mac_ocp_bit( + struct rtl8127_private *tp, + u16 addr, + u16 mask +) +{ + rtl8127_clear_set_mac_ocp_bit(tp, + addr, + 0, + mask); +} + +u32 rtl8127_ocp_read_with_oob_base_address(struct rtl8127_private *tp, u16 addr, u8 len, const u32 base_address) +{ + return rtl8127_eri_read_with_oob_base_address(tp, addr, len, ERIAR_OOB, base_address); +} + +u32 rtl8127_ocp_read(struct rtl8127_private *tp, u16 addr, u8 len) +{ + u32 value = 0; + + if (!tp->AllowAccessDashOcp) + return 0xffffffff; + + if (HW_DASH_SUPPORT_TYPE_2(tp)) + value = rtl8127_ocp_read_with_oob_base_address(tp, addr, len, NO_BASE_ADDRESS); + else if (HW_DASH_SUPPORT_TYPE_3(tp)) + value = rtl8127_ocp_read_with_oob_base_address(tp, addr, len, RTL8168FP_OOBMAC_BASE); + + return value; +} + +u32 rtl8127_ocp_write_with_oob_base_address(struct rtl8127_private *tp, u16 addr, u8 len, u32 value, const u32 base_address) +{ + return rtl8127_eri_write_with_oob_base_address(tp, addr, len, value, ERIAR_OOB, base_address); +} + +void rtl8127_ocp_write(struct rtl8127_private *tp, u16 addr, u8 len, u32 value) +{ + if (!tp->AllowAccessDashOcp) + return; + + if (HW_DASH_SUPPORT_TYPE_2(tp)) + rtl8127_ocp_write_with_oob_base_address(tp, addr, len, value, NO_BASE_ADDRESS); + else if (HW_DASH_SUPPORT_TYPE_3(tp)) + rtl8127_ocp_write_with_oob_base_address(tp, addr, len, value, RTL8168FP_OOBMAC_BASE); +} + +void rtl8127_oob_mutex_lock(struct rtl8127_private *tp) +{ + u8 reg_16, reg_a0; + u32 wait_cnt_0, wait_Cnt_1; + u16 ocp_reg_mutex_ib; + u16 ocp_reg_mutex_oob; + u16 ocp_reg_mutex_prio; + + if (!tp->DASH) + return; + + switch (tp->mcfg) { + default: + return; + } + + rtl8127_ocp_write(tp, ocp_reg_mutex_ib, 1, BIT_0); + reg_16 = rtl8127_ocp_read(tp, ocp_reg_mutex_oob, 1); + wait_cnt_0 = 0; + while(reg_16) { + reg_a0 = rtl8127_ocp_read(tp, ocp_reg_mutex_prio, 1); + if (reg_a0) { + rtl8127_ocp_write(tp, ocp_reg_mutex_ib, 1, 0x00); + reg_a0 = rtl8127_ocp_read(tp, ocp_reg_mutex_prio, 1); + wait_Cnt_1 = 0; + while(reg_a0) { + reg_a0 = rtl8127_ocp_read(tp, ocp_reg_mutex_prio, 1); + + wait_Cnt_1++; + + if (wait_Cnt_1 > 2000) + break; + }; + rtl8127_ocp_write(tp, ocp_reg_mutex_ib, 1, BIT_0); + + } + reg_16 = rtl8127_ocp_read(tp, ocp_reg_mutex_oob, 1); + + wait_cnt_0++; + + if (wait_cnt_0 > 2000) + break; + }; +} + +void rtl8127_oob_mutex_unlock(struct rtl8127_private *tp) +{ + //u16 ocp_reg_mutex_ib; + //u16 ocp_reg_mutex_oob; + //u16 ocp_reg_mutex_prio; + + if (!tp->DASH) + return; + + switch (tp->mcfg) { + default: + return; + } + + //rtl8127_ocp_write(tp, ocp_reg_mutex_prio, 1, BIT_0); + //rtl8127_ocp_write(tp, ocp_reg_mutex_ib, 1, 0x00); +} + +static bool +rtl8127_is_allow_access_dash_ocp(struct rtl8127_private *tp) +{ + bool allow_access = false; + + if (!HW_DASH_SUPPORT_DASH(tp)) + goto exit; + + allow_access = true; + switch (tp->mcfg) { + default: + goto exit; + } +exit: + return allow_access; +} + +static int rtl8127_check_dash(struct rtl8127_private *tp) +{ + if (!tp->AllowAccessDashOcp) + return 0; + + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) { + if (rtl8127_ocp_read(tp, 0x128, 1) & BIT_0) + return 1; + } + + return 0; +} + +void rtl8127_dash2_disable_tx(struct rtl8127_private *tp) +{ + if (!tp->DASH) + return; + + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) { + u16 WaitCnt; + u8 TmpUchar; + + //Disable oob Tx + RTL_CMAC_W8(tp, CMAC_IBCR2, RTL_CMAC_R8(tp, CMAC_IBCR2) & ~(BIT_0)); + WaitCnt = 0; + + //wait oob tx disable + do { + TmpUchar = RTL_CMAC_R8(tp, CMAC_IBISR0); + + if (TmpUchar & ISRIMR_DASH_TYPE2_TX_DISABLE_IDLE) { + break; + } + + fsleep(50); + WaitCnt++; + } while(WaitCnt < 2000); + + //Clear ISRIMR_DASH_TYPE2_TX_DISABLE_IDLE + RTL_CMAC_W8(tp, CMAC_IBISR0, RTL_CMAC_R8(tp, CMAC_IBISR0) | ISRIMR_DASH_TYPE2_TX_DISABLE_IDLE); + } +} + +void rtl8127_dash2_enable_tx(struct rtl8127_private *tp) +{ + if (!tp->DASH) + return; + + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) + RTL_CMAC_W8(tp, CMAC_IBCR2, RTL_CMAC_R8(tp, CMAC_IBCR2) | BIT_0); +} + +void rtl8127_dash2_disable_rx(struct rtl8127_private *tp) +{ + if (!tp->DASH) + return; + + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) + RTL_CMAC_W8(tp, CMAC_IBCR0, RTL_CMAC_R8(tp, CMAC_IBCR0) & ~(BIT_0)); +} + +void rtl8127_dash2_enable_rx(struct rtl8127_private *tp) +{ + if (!tp->DASH) + return; + + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) + RTL_CMAC_W8(tp, CMAC_IBCR0, RTL_CMAC_R8(tp, CMAC_IBCR0) | BIT_0); +} + +static void rtl8127_dash2_disable_txrx(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) { + rtl8127_dash2_disable_tx(tp); + rtl8127_dash2_disable_rx(tp); + } +} + +static int rtl8127_wait_dash_fw_ready(struct rtl8127_private *tp) +{ + int rc = -1; + int timeout; + + if (HW_DASH_SUPPORT_TYPE_2(tp) == FALSE && + HW_DASH_SUPPORT_TYPE_3(tp) == FALSE) + goto out; + + if (!tp->DASH) + goto out; + + for (timeout = 0; timeout < 10; timeout++) { + fsleep(10000); + if (rtl8127_ocp_read(tp, 0x124, 1) & BIT_0) { + rc = 1; + goto out; + } + } + + rc = 0; + +out: + return rc; +} + +static void rtl8127_driver_start(struct rtl8127_private *tp) +{ + u32 tmp_value; + + if (HW_DASH_SUPPORT_TYPE_2(tp) == FALSE && + HW_DASH_SUPPORT_TYPE_3(tp) == FALSE) + return; + + if (!tp->AllowAccessDashOcp) + return; + + rtl8127_ocp_write(tp, 0x180, 1, OOB_CMD_DRIVER_START); + tmp_value = rtl8127_ocp_read(tp, 0x30, 1); + tmp_value |= BIT_0; + rtl8127_ocp_write(tp, 0x30, 1, tmp_value); + + rtl8127_wait_dash_fw_ready(tp); +} + +static void rtl8127_driver_stop(struct rtl8127_private *tp) +{ + u32 tmp_value; + struct net_device *dev = tp->dev; + + if (HW_DASH_SUPPORT_TYPE_2(tp) == FALSE && + HW_DASH_SUPPORT_TYPE_3(tp) == FALSE) + return; + + if (!tp->AllowAccessDashOcp) + return; + + rtl8127_dash2_disable_txrx(dev); + + rtl8127_ocp_write(tp, 0x180, 1, OOB_CMD_DRIVER_STOP); + tmp_value = rtl8127_ocp_read(tp, 0x30, 1); + tmp_value |= BIT_0; + rtl8127_ocp_write(tp, 0x30, 1, tmp_value); + + rtl8127_wait_dash_fw_ready(tp); +} + +static void _rtl8127_ephy_write(struct rtl8127_private *tp, int addr, int data) +{ + int i; + + RTL_W32(tp, EPHYAR, + EPHYAR_Write | + (addr & EPHYAR_Reg_Mask_v2) << EPHYAR_Reg_shift | + (data & EPHYAR_Data_Mask)); + + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { + fsleep(R8127_CHANNEL_WAIT_TIME); + + /* Check if the RTL8125 has completed EPHY write */ + if (!(RTL_R32(tp, EPHYAR) & EPHYAR_Flag)) + break; + } + + fsleep(R8127_CHANNEL_EXIT_DELAY_TIME); +} + +static void rtl8127_set_ephy_ext_addr(struct rtl8127_private *tp, int addr) +{ + _rtl8127_ephy_write(tp, EPHYAR_EXT_ADDR, addr); +} + +static int rtl8127_check_ephy_ext_addr(struct rtl8127_private *tp, int addr) +{ + int data; + + data = ((u16)addr >> 12); + + rtl8127_set_ephy_ext_addr(tp, data); + + return (addr & 0xfff); +} + +void rtl8127_ephy_write(struct rtl8127_private *tp, int addr, int data) +{ + _rtl8127_ephy_write(tp, rtl8127_check_ephy_ext_addr(tp, addr), data); +} + +static u16 _rtl8127_ephy_read(struct rtl8127_private *tp, int addr) +{ + int i; + u16 data = 0xffff; + + RTL_W32(tp, EPHYAR, + EPHYAR_Read | (addr & EPHYAR_Reg_Mask_v2) << EPHYAR_Reg_shift); + + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { + fsleep(R8127_CHANNEL_WAIT_TIME); + + /* Check if the RTL8125 has completed EPHY read */ + if (RTL_R32(tp, EPHYAR) & EPHYAR_Flag) { + data = (u16) (RTL_R32(tp, EPHYAR) & EPHYAR_Data_Mask); + break; + } + } + + fsleep(R8127_CHANNEL_EXIT_DELAY_TIME); + + return data; +} + +u16 rtl8127_ephy_read(struct rtl8127_private *tp, int addr) +{ + return _rtl8127_ephy_read(tp, rtl8127_check_ephy_ext_addr(tp, addr)); +} + +/* +static void ClearAndSetPCIePhyBit(struct rtl8127_private *tp, u8 addr, u16 clearmask, u16 setmask) +{ + u16 EphyValue; + + EphyValue = rtl8127_ephy_read(tp, addr); + EphyValue &= ~clearmask; + EphyValue |= setmask; + rtl8127_ephy_write(tp, addr, EphyValue); +} + +static void ClearPCIePhyBit(struct rtl8127_private *tp, u8 addr, u16 mask) +{ + ClearAndSetPCIePhyBit(tp, + addr, + mask, + 0); +} + +static void SetPCIePhyBit(struct rtl8127_private *tp, u8 addr, u16 mask) +{ + ClearAndSetPCIePhyBit(tp, + addr, + 0, + mask); +} +*/ + +static u32 +rtl8127_csi_other_fun_read(struct rtl8127_private *tp, + u8 multi_fun_sel_bit, + u32 addr) +{ + u32 cmd; + int i; + u32 value = 0xffffffff; + + cmd = CSIAR_Read | CSIAR_ByteEn << CSIAR_ByteEn_shift | (addr & CSIAR_Addr_Mask); + + if (tp->mcfg == CFG_METHOD_DEFAULT) + multi_fun_sel_bit = 0; + + if (multi_fun_sel_bit > 7) + goto exit; + + cmd |= multi_fun_sel_bit << 16; + + RTL_W32(tp, CSIAR, cmd); + + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { + fsleep(R8127_CHANNEL_WAIT_TIME); + + /* Check if the RTL8125 has completed CSI read */ + if (RTL_R32(tp, CSIAR) & CSIAR_Flag) { + value = (u32)RTL_R32(tp, CSIDR); + break; + } + } + + fsleep(R8127_CHANNEL_EXIT_DELAY_TIME); + +exit: + return value; +} + +static void +rtl8127_csi_other_fun_write(struct rtl8127_private *tp, + u8 multi_fun_sel_bit, + u32 addr, + u32 value) +{ + u32 cmd; + int i; + + RTL_W32(tp, CSIDR, value); + cmd = CSIAR_Write | CSIAR_ByteEn << CSIAR_ByteEn_shift | (addr & CSIAR_Addr_Mask); + if (tp->mcfg == CFG_METHOD_DEFAULT) + multi_fun_sel_bit = 0; + + if (multi_fun_sel_bit > 7) + return; + + cmd |= multi_fun_sel_bit << 16; + + RTL_W32(tp, CSIAR, cmd); + + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { + fsleep(R8127_CHANNEL_WAIT_TIME); + + /* Check if the RTL8125 has completed CSI write */ + if (!(RTL_R32(tp, CSIAR) & CSIAR_Flag)) + break; + } + + fsleep(R8127_CHANNEL_EXIT_DELAY_TIME); +} + +static u32 +rtl8127_csi_read(struct rtl8127_private *tp, + u32 addr) +{ + u8 multi_fun_sel_bit; + + multi_fun_sel_bit = 0; + + return rtl8127_csi_other_fun_read(tp, multi_fun_sel_bit, addr); +} + +static void +rtl8127_csi_write(struct rtl8127_private *tp, + u32 addr, + u32 value) +{ + u8 multi_fun_sel_bit; + + multi_fun_sel_bit = 0; + + rtl8127_csi_other_fun_write(tp, multi_fun_sel_bit, addr, value); +} + +static u8 +rtl8127_csi_fun0_read_byte(struct rtl8127_private *tp, + u32 addr) +{ + u8 RetVal = 0; + + if (tp->mcfg == CFG_METHOD_DEFAULT) { + struct pci_dev *pdev = tp->pci_dev; + + pci_read_config_byte(pdev, addr, &RetVal); + } else { + u32 TmpUlong; + u16 RegAlignAddr; + u8 ShiftByte; + + RegAlignAddr = addr & ~(0x3); + ShiftByte = addr & (0x3); + TmpUlong = rtl8127_csi_other_fun_read(tp, 0, RegAlignAddr); + TmpUlong >>= (8*ShiftByte); + RetVal = (u8)TmpUlong; + } + + fsleep(R8127_CHANNEL_EXIT_DELAY_TIME); + + return RetVal; +} + +static void +rtl8127_csi_fun0_write_byte(struct rtl8127_private *tp, + u32 addr, + u8 value) +{ + if (tp->mcfg == CFG_METHOD_DEFAULT) { + struct pci_dev *pdev = tp->pci_dev; + + pci_write_config_byte(pdev, addr, value); + } else { + u32 TmpUlong; + u16 RegAlignAddr; + u8 ShiftByte; + + RegAlignAddr = addr & ~(0x3); + ShiftByte = addr & (0x3); + TmpUlong = rtl8127_csi_other_fun_read(tp, 0, RegAlignAddr); + TmpUlong &= ~(0xFF << (8*ShiftByte)); + TmpUlong |= (value << (8*ShiftByte)); + rtl8127_csi_other_fun_write(tp, 0, RegAlignAddr, TmpUlong); + } + + fsleep(R8127_CHANNEL_EXIT_DELAY_TIME); +} + +u32 rtl8127_eri_read_with_oob_base_address(struct rtl8127_private *tp, int addr, int len, int type, const u32 base_address) +{ + int i, val_shift, shift = 0; + u32 value1 = 0, value2 = 0, mask; + u32 eri_cmd; + const u32 transformed_base_address = ((base_address & 0x00FFF000) << 6) | (base_address & 0x000FFF); + + if (len > 4 || len <= 0) + return -1; + + while (len > 0) { + val_shift = addr % ERIAR_Addr_Align; + addr = addr & ~0x3; + + eri_cmd = ERIAR_Read | + transformed_base_address | + type << ERIAR_Type_shift | + ERIAR_ByteEn << ERIAR_ByteEn_shift | + (addr & 0x0FFF); + if (addr & 0xF000) { + u32 tmp; + + tmp = addr & 0xF000; + tmp >>= 12; + eri_cmd |= (tmp << 20) & 0x00F00000; + } + + RTL_W32(tp, ERIAR, eri_cmd); + + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { + fsleep(R8127_CHANNEL_WAIT_TIME); + + /* Check if the RTL8125 has completed ERI read */ + if (RTL_R32(tp, ERIAR) & ERIAR_Flag) + break; + } + + if (len == 1) mask = (0xFF << (val_shift * 8)) & 0xFFFFFFFF; + else if (len == 2) mask = (0xFFFF << (val_shift * 8)) & 0xFFFFFFFF; + else if (len == 3) mask = (0xFFFFFF << (val_shift * 8)) & 0xFFFFFFFF; + else mask = (0xFFFFFFFF << (val_shift * 8)) & 0xFFFFFFFF; + + value1 = RTL_R32(tp, ERIDR) & mask; + value2 |= (value1 >> val_shift * 8) << shift * 8; + + if (len <= 4 - val_shift) { + len = 0; + } else { + len -= (4 - val_shift); + shift = 4 - val_shift; + addr += 4; + } + } + + fsleep(R8127_CHANNEL_EXIT_DELAY_TIME); + + return value2; +} + +u32 rtl8127_eri_read(struct rtl8127_private *tp, int addr, int len, int type) +{ + return rtl8127_eri_read_with_oob_base_address(tp, addr, len, type, 0); +} + +int rtl8127_eri_write_with_oob_base_address(struct rtl8127_private *tp, int addr, int len, u32 value, int type, const u32 base_address) +{ + int i, val_shift, shift = 0; + u32 value1 = 0, mask; + u32 eri_cmd; + const u32 transformed_base_address = ((base_address & 0x00FFF000) << 6) | (base_address & 0x000FFF); + + if (len > 4 || len <= 0) + return -1; + + while (len > 0) { + val_shift = addr % ERIAR_Addr_Align; + addr = addr & ~0x3; + + if (len == 1) mask = (0xFF << (val_shift * 8)) & 0xFFFFFFFF; + else if (len == 2) mask = (0xFFFF << (val_shift * 8)) & 0xFFFFFFFF; + else if (len == 3) mask = (0xFFFFFF << (val_shift * 8)) & 0xFFFFFFFF; + else mask = (0xFFFFFFFF << (val_shift * 8)) & 0xFFFFFFFF; + + value1 = rtl8127_eri_read_with_oob_base_address(tp, addr, 4, type, base_address) & ~mask; + value1 |= ((value << val_shift * 8) >> shift * 8); + + RTL_W32(tp, ERIDR, value1); + + eri_cmd = ERIAR_Write | + transformed_base_address | + type << ERIAR_Type_shift | + ERIAR_ByteEn << ERIAR_ByteEn_shift | + (addr & 0x0FFF); + if (addr & 0xF000) { + u32 tmp; + + tmp = addr & 0xF000; + tmp >>= 12; + eri_cmd |= (tmp << 20) & 0x00F00000; + } + + RTL_W32(tp, ERIAR, eri_cmd); + + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { + fsleep(R8127_CHANNEL_WAIT_TIME); + + /* Check if the RTL8125 has completed ERI write */ + if (!(RTL_R32(tp, ERIAR) & ERIAR_Flag)) + break; + } + + if (len <= 4 - val_shift) { + len = 0; + } else { + len -= (4 - val_shift); + shift = 4 - val_shift; + addr += 4; + } + } + + fsleep(R8127_CHANNEL_EXIT_DELAY_TIME); + + return 0; +} + +int rtl8127_eri_write(struct rtl8127_private *tp, int addr, int len, u32 value, int type) +{ + return rtl8127_eri_write_with_oob_base_address(tp, addr, len, value, type, NO_BASE_ADDRESS); +} + +static void +rtl8127_enable_rxdvgate(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + RTL_W8(tp, 0xF2, RTL_R8(tp, 0xF2) | BIT_3); +} + +static void +rtl8127_disable_rxdvgate(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + RTL_W8(tp, 0xF2, RTL_R8(tp, 0xF2) & ~BIT_3); +} + +static u8 +rtl8127_is_gpio_low(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u8 gpio_low = FALSE; + + switch (tp->HwSuppCheckPhyDisableModeVer) { + case 3: + if (!(rtl8127_mac_ocp_read(tp, 0xDC04) & BIT_13)) + gpio_low = TRUE; + break; + } + + if (gpio_low) + dprintk("gpio is low.\n"); + + return gpio_low; +} + +static u8 +rtl8127_is_phy_disable_mode_enabled(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u8 phy_disable_mode_enabled = FALSE; + + switch (tp->HwSuppCheckPhyDisableModeVer) { + case 3: + if (RTL_R8(tp, 0xF2) & BIT_5) + phy_disable_mode_enabled = TRUE; + break; + } + + if (phy_disable_mode_enabled) + dprintk("phy disable mode enabled.\n"); + + return phy_disable_mode_enabled; +} + +static u8 +rtl8127_is_in_phy_disable_mode(struct net_device *dev) +{ + u8 in_phy_disable_mode = FALSE; + + if (rtl8127_is_phy_disable_mode_enabled(dev) && rtl8127_is_gpio_low(dev)) + in_phy_disable_mode = TRUE; + + if (in_phy_disable_mode) + dprintk("Hardware is in phy disable mode.\n"); + + return in_phy_disable_mode; +} + +static void +rtl8127_stop_all_request(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) | StopReq); + fsleep(200); +} + +static void +rtl8127_clear_stop_all_request(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) & (CmdTxEnb | CmdRxEnb)); +} + +void +rtl8127_wait_txrx_fifo_empty(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i; + + /* Txfifo_empty require StopReq been set */ + for (i = 0; i < 3000; i++) { + fsleep(50); + if ((RTL_R8(tp, MCUCmd_reg) & (Txfifo_empty | Rxfifo_empty)) == (Txfifo_empty | Rxfifo_empty)) + break; + } + + for (i = 0; i < 3000; i++) { + fsleep(50); + if ((RTL_R16(tp, IntrMitigate) & (BIT_0 | BIT_1 | BIT_8)) == (BIT_0 | BIT_1 | BIT_8)) + break; + } +} + +#ifdef ENABLE_DASH_SUPPORT + +static inline void +rtl8127_enable_dash2_interrupt(struct rtl8127_private *tp) +{ + if (!tp->DASH) + return; + + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) + RTL_CMAC_W8(tp, CMAC_IBIMR0, (ISRIMR_DASH_TYPE2_ROK | ISRIMR_DASH_TYPE2_TOK | ISRIMR_DASH_TYPE2_TDU | ISRIMR_DASH_TYPE2_RDU | ISRIMR_DASH_TYPE2_RX_DISABLE_IDLE)); +} + +static inline void +rtl8127_disable_dash2_interrupt(struct rtl8127_private *tp) +{ + if (!tp->DASH) + return; + + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) + RTL_CMAC_W8(tp, CMAC_IBIMR0, 0); +} +#endif + +void +rtl8127_enable_hw_linkchg_interrupt(struct rtl8127_private *tp) +{ + switch (tp->HwCurrIsrVer) { + case 6: + RTL_W32(tp, IMR_V2_SET_REG_8125, ISRIMR_V6_LINKCHG); + break; + case 5: + RTL_W32(tp, IMR_V2_SET_REG_8125, ISRIMR_V5_LINKCHG); + break; + case 4: + RTL_W32(tp, IMR_V2_SET_REG_8125, ISRIMR_V4_LINKCHG); + break; + case 2: + case 3: + RTL_W32(tp, IMR_V2_SET_REG_8125, ISRIMR_V2_LINKCHG); + break; + case 1: + RTL_W32(tp, tp->imr_reg[0], LinkChg | RTL_R32(tp, tp->imr_reg[0])); + break; + } + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) + rtl8127_enable_dash2_interrupt(tp); +#endif +} + +static inline void +rtl8127_enable_hw_interrupt(struct rtl8127_private *tp) +{ + switch (tp->HwCurrIsrVer) { + case 2: + case 3: + case 4: + case 5: + case 6: + RTL_W32(tp, IMR_V2_SET_REG_8125, tp->intr_mask); + break; + case 1: + RTL_W32(tp, tp->imr_reg[0], tp->intr_mask); + + if (R8127_MULTI_RX_Q(tp)) { + int i; + for (i=1; inum_rx_rings; i++) + RTL_W16(tp, tp->imr_reg[i], other_q_intr_mask); + } + break; + } + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) + rtl8127_enable_dash2_interrupt(tp); +#endif +} + +static inline void rtl8127_clear_hw_isr_v2(struct rtl8127_private *tp, + u32 message_id) +{ + RTL_W32(tp, ISR_V2_8125, BIT(message_id)); +} + +static inline void +rtl8127_disable_hw_interrupt(struct rtl8127_private *tp) +{ + if (tp->HwCurrIsrVer > 1) { + RTL_W32(tp, IMR_V2_CLEAR_REG_8125, 0xFFFFFFFF); + if (tp->HwCurrIsrVer > 3) + RTL_W32(tp, IMR_V4_L2_CLEAR_REG_8125, 0xFFFFFFFF); + } else { + RTL_W32(tp, tp->imr_reg[0], 0x0000); + + if (R8127_MULTI_RX_Q(tp)) { + int i; + for (i=1; inum_rx_rings; i++) + RTL_W16(tp, tp->imr_reg[i], 0); + } + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) + rtl8127_disable_dash2_interrupt(tp); +#endif + } +} + +static inline void +rtl8127_switch_to_hw_interrupt(struct rtl8127_private *tp) +{ + RTL_W32(tp, TIMER_INT0_8125, 0x0000); + + rtl8127_enable_hw_interrupt(tp); +} + +static inline void +rtl8127_switch_to_timer_interrupt(struct rtl8127_private *tp) +{ + if (tp->use_timer_interrupt) { + RTL_W32(tp, TIMER_INT0_8125, timer_count); + RTL_W32(tp, TCTR0_8125, timer_count); + RTL_W32(tp, tp->imr_reg[0], tp->timer_intr_mask); + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) + rtl8127_enable_dash2_interrupt(tp); +#endif + } else { + rtl8127_switch_to_hw_interrupt(tp); + } +} + +static void +rtl8127_irq_mask_and_ack(struct rtl8127_private *tp) +{ + rtl8127_disable_hw_interrupt(tp); + + if (tp->HwCurrIsrVer > 1) { + RTL_W32(tp, ISR_V2_8125, 0xFFFFFFFF); + if (tp->HwCurrIsrVer > 3) + RTL_W32(tp, ISR_V4_L2_8125, 0xFFFFFFFF); + } else { +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) { + if (tp->dash_printer_enabled) { + RTL_W32(tp, tp->isr_reg[0], RTL_R32(tp, tp->isr_reg[0]) & + ~(ISRIMR_DASH_INTR_EN | ISRIMR_DASH_INTR_CMAC_RESET)); + } else { + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) { + RTL_CMAC_W8(tp, CMAC_IBISR0, RTL_CMAC_R8(tp, CMAC_IBISR0)); + } + } + } else { + RTL_W32(tp, tp->isr_reg[0], RTL_R32(tp, tp->isr_reg[0])); + } +#else + RTL_W32(tp, tp->isr_reg[0], RTL_R32(tp, tp->isr_reg[0])); +#endif + if (R8127_MULTI_RX_Q(tp)) { + int i; + for (i=1; inum_rx_rings; i++) + RTL_W16(tp, tp->isr_reg[i], RTL_R16(tp, tp->isr_reg[i])); + } + } +} + +static void +rtl8127_disable_rx_packet_filter(struct rtl8127_private *tp) +{ + + RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) & + ~(AcceptErr | AcceptRunt |AcceptBroadcast | AcceptMulticast | + AcceptMyPhys | AcceptAllPhys)); +} + +static void +rtl8127_nic_reset(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i; + + rtl8127_disable_rx_packet_filter(tp); + + rtl8127_enable_rxdvgate(dev); + + rtl8127_stop_all_request(dev); + + rtl8127_wait_txrx_fifo_empty(dev); + + rtl8127_clear_stop_all_request(dev); + + /* Soft reset the chip. */ + RTL_W8(tp, ChipCmd, CmdReset); + + /* Check that the chip has finished the reset. */ + for (i = 100; i > 0; i--) { + fsleep(100); + if ((RTL_R8(tp, ChipCmd) & CmdReset) == 0) + break; + } + + /* reset rcr */ + RTL_W32(tp, RxConfig, (RX_DMA_BURST_512 << RxCfgDMAShift)); +} + +static void +rtl8127_hw_set_interrupt_type(struct rtl8127_private *tp, u8 isr_ver) +{ + u8 tmp; + + if (tp->HwSuppIsrVer < 2) + return; + + tmp = RTL_R8(tp, INT_CFG0_8125); + + switch (tp->HwSuppIsrVer) { + case 6: + tmp &= ~INT_CFG0_AVOID_MISS_INTR; + fallthrough; + case 4: + case 5: + if (tp->HwSuppIsrVer == 6) + tmp &= ~INT_CFG0_AUTO_CLEAR_IMR; + else + tmp &= ~INT_CFG0_MSIX_ENTRY_NUM_MODE; + fallthrough; + case 2: + case 3: + tmp &= ~(INT_CFG0_ENABLE_8125); + if (isr_ver > 1) + tmp |= INT_CFG0_ENABLE_8125; + break; + default: + return; + } + + RTL_W8(tp, INT_CFG0_8125, tmp); +} + +static void +rtl8127_hw_clear_timer_int(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + RTL_W32(tp, TIMER_INT0_8125, 0x0000); + RTL_W32(tp, TIMER_INT1_8125, 0x0000); + RTL_W32(tp, TIMER_INT2_8125, 0x0000); + RTL_W32(tp, TIMER_INT3_8125, 0x0000); +} + +static void +rtl8127_hw_clear_int_miti(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i; + + switch (tp->HwSuppIntMitiVer) { + case 3: + case 6: + //IntMITI_0-IntMITI_31 + for (i=0xA00; i<0xB00; i+=4) + RTL_W32(tp, i, 0x0000); + break; + case 4: + case 5: + //IntMITI_0-IntMITI_15 + for (i = 0xA00; i < 0xA80; i += 4) + RTL_W32(tp, i, 0x0000); + + if (tp->HwSuppIntMitiVer == 5) + RTL_W8(tp, INT_CFG0_8125, RTL_R8(tp, INT_CFG0_8125) & + ~(INT_CFG0_TIMEOUT0_BYPASS_8125 | + INT_CFG0_MITIGATION_BYPASS_8125 | + INT_CFG0_RDU_BYPASS_8126)); + else + RTL_W8(tp, INT_CFG0_8125, RTL_R8(tp, INT_CFG0_8125) & + ~(INT_CFG0_TIMEOUT0_BYPASS_8125 | INT_CFG0_MITIGATION_BYPASS_8125)); + + RTL_W16(tp, INT_CFG1_8125, 0x0000); + break; + } +} + +static bool +rtl8127_vec_2_tx_q_num( + struct rtl8127_private *tp, + u32 messageId, + u32 *qnum +) +{ + u32 whichQ = 0xffffffff; + bool rc = false; + + switch (tp->HwSuppIsrVer) { + case 2: + if (messageId == 0x10) + whichQ = 0; + else if (messageId == 0x12 && tp->num_tx_rings > 1) + whichQ = 1; + break; + case 3: + case 4: + if (messageId == 0x00) + whichQ = 0; + else if (messageId == 0x01 && tp->num_tx_rings > 1) + whichQ = 1; + break; + case 5: + if (messageId == 0x10) + whichQ = 0; + else if (messageId == 0x11 && tp->num_tx_rings > 1) + whichQ = 1; + break; + case 6: + if (messageId == 0x08) + whichQ = 0; + else if (messageId == 0x09 && tp->num_tx_rings > 1) + whichQ = 1; + break; + case 7: + if (messageId == 0x1B) + whichQ = 0; + else if (messageId == 0x1C && tp->num_tx_rings > 1) + whichQ = 1; + break; + } + + if (whichQ != 0xffffffff) { + *qnum = whichQ; + rc = true; + } + + return rc; +} + +static bool +rtl8127_vec_2_rx_q_num( + struct rtl8127_private *tp, + u32 messageId, + u32 *qnum +) +{ + u32 whichQ = 0xffffffff; + bool rc = false; + + switch (tp->HwSuppIsrVer) { + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + if (messageId < tp->HwSuppNumRxQueues) + whichQ = messageId; + break; + } + + if (whichQ != 0xffffffff) { + *qnum = whichQ; + rc = true; + } + + return rc; +} + +void +rtl8127_hw_set_timer_int(struct rtl8127_private *tp, + u32 message_id, + u8 timer_intmiti_val) +{ + u32 qnum; + + switch (tp->HwSuppIntMitiVer) { + case 4: + case 5: + case 6: + //ROK + if (rtl8127_vec_2_rx_q_num(tp, message_id, &qnum)) + RTL_W8(tp,INT_MITI_V2_0_RX + 8 * qnum, timer_intmiti_val); + //TOK + if (rtl8127_vec_2_tx_q_num(tp, message_id, &qnum)) + RTL_W8(tp,INT_MITI_V2_0_TX + 8 * qnum, timer_intmiti_val); + break; + } +} + +void +rtl8127_hw_reset(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + rtl8127_lib_reset_prepare(tp); + + /* Disable interrupts */ + rtl8127_irq_mask_and_ack(tp); + + rtl8127_hw_clear_timer_int(dev); + + rtl8127_nic_reset(dev); +} + +static unsigned int +rtl8127_xmii_reset_pending(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned int retval; + unsigned long flags; + + spin_lock_irqsave(&tp->phy_lock, flags); + rtl8127_mdio_write(tp, 0x1f, 0x0000); + retval = rtl8127_mdio_read(tp, MII_BMCR) & BMCR_RESET; + spin_unlock_irqrestore(&tp->phy_lock, flags); + + return retval; +} + +static unsigned int +rtl8127_xmii_link_ok(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u16 status; + + status = RTL_R16(tp, PHYstatus); + if (status == 0xffff) + return 0; + + return (status & LinkStatus) ? 1 : 0; +} + +static int +rtl8127_wait_phy_reset_complete(struct rtl8127_private *tp) +{ + int i, val; + + for (i = 0; i < 2500; i++) { + val = rtl8127_mdio_read(tp, MII_BMCR) & BMCR_RESET; + if (!val) + return 0; + + mdelay(1); + } + + return -1; +} + +static void +rtl8127_xmii_reset_enable(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned long flags; + int ret; + + if (rtl8127_is_in_phy_disable_mode(dev)) + return; + + spin_lock_irqsave(&tp->phy_lock, flags); + + rtl8127_mdio_write(tp, 0x1f, 0x0000); + rtl8127_mdio_write(tp, MII_ADVERTISE, rtl8127_mdio_read(tp, MII_ADVERTISE) & + ~(ADVERTISE_10HALF | ADVERTISE_10FULL | + ADVERTISE_100HALF | ADVERTISE_100FULL)); + rtl8127_mdio_write(tp, MII_CTRL1000, rtl8127_mdio_read(tp, MII_CTRL1000) & + ~(ADVERTISE_1000HALF | ADVERTISE_1000FULL)); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA5D4, rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D4) & + ~(RTK_ADVERTISE_2500FULL | RTK_ADVERTISE_5000FULL | + RTK_ADVERTISE_10000FULL)); + rtl8127_mdio_write(tp, MII_BMCR, BMCR_RESET | BMCR_ANENABLE); + + ret = rtl8127_wait_phy_reset_complete(tp); + + spin_unlock_irqrestore(&tp->phy_lock, flags); + + if (ret != 0 && netif_msg_link(tp)) + printk(KERN_ERR "%s: PHY reset failed.\n", dev->name); +} + +void +rtl8127_init_ring_indexes(struct rtl8127_private *tp) +{ + int i; + + for (i = 0; i < tp->HwSuppNumTxQueues; i++) { + struct rtl8127_tx_ring *ring = &tp->tx_ring[i]; + ring->dirty_tx = ring->cur_tx = 0; + ring->NextHwDesCloPtr = 0; + ring->BeginHwDesCloPtr = 0; + ring->index = i; + ring->priv = tp; + ring->netdev = tp->dev; + + /* reset BQL for queue */ + netdev_tx_reset_queue(txring_txq(ring)); + } + + for (i = 0; i < tp->HwSuppNumRxQueues; i++) { + struct rtl8127_rx_ring *ring = &tp->rx_ring[i]; + ring->dirty_rx = ring->cur_rx = 0; + ring->index = i; + ring->priv = tp; + ring->netdev = tp->dev; + } + +#ifdef ENABLE_LIB_SUPPORT + for (i = 0; i < tp->HwSuppNumTxQueues; i++) { + struct rtl8127_ring *ring = &tp->lib_tx_ring[i]; + ring->direction = RTL8127_CH_DIR_TX; + ring->queue_num = i; + ring->private = tp; + } + + for (i = 0; i < tp->HwSuppNumRxQueues; i++) { + struct rtl8127_ring *ring = &tp->lib_rx_ring[i]; + ring->direction = RTL8127_CH_DIR_RX; + ring->queue_num = i; + ring->private = tp; + } +#endif +} + +static void +rtl8127_issue_offset_99_event(struct rtl8127_private *tp) +{ + rtl8127_mac_ocp_write(tp, 0xE09A, rtl8127_mac_ocp_read(tp, 0xE09A) | BIT_0); +} + +#ifdef ENABLE_DASH_SUPPORT +static void +NICChkTypeEnableDashInterrupt(struct rtl8127_private *tp) +{ + if (tp->DASH) { + // + // even disconnected, enable 3 dash interrupt mask bits for in-band/out-band communication + // + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) { + rtl8127_enable_dash2_interrupt(tp); + RTL_W16(tp, IntrMask, (ISRIMR_DASH_INTR_EN | ISRIMR_DASH_INTR_CMAC_RESET)); + } + } +} +#endif + +static int rtl8127_enable_eee_plus(struct rtl8127_private *tp) +{ + rtl8127_mac_ocp_write(tp, 0xE080, rtl8127_mac_ocp_read(tp, 0xE080)|BIT_1); + + return 0; +} + +static int rtl8127_disable_eee_plus(struct rtl8127_private *tp) +{ + rtl8127_mac_ocp_write(tp, 0xE080, rtl8127_mac_ocp_read(tp, 0xE080)&~BIT_1); + + return 0; +} + +static void rtl8127_enable_double_vlan(struct rtl8127_private *tp) +{ + RTL_W16(tp, DOUBLE_VLAN_CONFIG, 0xf002); +} + +static void rtl8127_disable_double_vlan(struct rtl8127_private *tp) +{ + RTL_W16(tp, DOUBLE_VLAN_CONFIG, 0); +} + +static void +rtl8127_link_on_patch(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned long flags; + + rtl8127_hw_config(dev); + + if (RTL_R8(tp, PHYstatus) & _10bps) + rtl8127_enable_eee_plus(tp); + + rtl8127_hw_start(dev); + + netif_carrier_on(dev); + + netif_tx_wake_all_queues(dev); + + spin_lock_irqsave(&tp->phy_lock, flags); + tp->phy_reg_aner = rtl8127_mdio_read(tp, MII_EXPANSION); + tp->phy_reg_anlpar = rtl8127_mdio_read(tp, MII_LPA); + tp->phy_reg_gbsr = rtl8127_mdio_read(tp, MII_STAT1000); + tp->phy_reg_status_2500 = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D6); + spin_unlock_irqrestore(&tp->phy_lock, flags); + +#ifdef ENABLE_PTP_SUPPORT + if (tp->EnablePtp) + rtl8127_set_local_time(tp); +#endif // ENABLE_PTP_SUPPORT +} + +static void +rtl8127_link_down_patch(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + tp->phy_reg_aner = 0; + tp->phy_reg_anlpar = 0; + tp->phy_reg_gbsr = 0; + tp->phy_reg_status_2500 = 0; + + rtl8127_disable_eee_plus(tp); + + netif_carrier_off(dev); + + netif_tx_disable(dev); + + rtl8127_hw_reset(dev); + + rtl8127_tx_clear(tp); + + rtl8127_rx_clear(tp); + + rtl8127_init_ring(dev); + + rtl8127_enable_hw_linkchg_interrupt(tp); + + //rtl8127_set_speed(dev, tp->autoneg, tp->speed, tp->duplex, tp->advertising); + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) + NICChkTypeEnableDashInterrupt(tp); +#endif +} + +static void +_rtl8127_check_link_status(struct net_device *dev, unsigned int link_state) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (link_state != R8127_LINK_STATE_OFF && + link_state != R8127_LINK_STATE_ON) + link_state = tp->link_ok(dev); + + if (link_state == R8127_LINK_STATE_ON) { + rtl8127_link_on_patch(dev); + + if (netif_msg_ifup(tp)) + printk(KERN_INFO PFX "%s: link up\n", dev->name); + } else { + if (netif_msg_ifdown(tp)) + printk(KERN_INFO PFX "%s: link down\n", dev->name); + + rtl8127_link_down_patch(dev); + } +} + +static void +rtl8127_check_link_status(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned int link_status_on; + + tp->resume_not_chg_speed = 0; + + link_status_on = tp->link_ok(dev); + if (netif_carrier_ok(dev) == link_status_on) + return; + + _rtl8127_check_link_status(dev, link_status_on); +} + +static bool +rtl8127_is_autoneg_mode_valid(u32 autoneg) +{ + switch(autoneg) { + case AUTONEG_ENABLE: + case AUTONEG_DISABLE: + return true; + default: + return false; + } +} + +static bool +rtl8127_is_speed_mode_valid(u32 speed) +{ + switch(speed) { + case SPEED_10000: + case SPEED_5000: + case SPEED_2500: + case SPEED_1000: + case SPEED_100: + case SPEED_10: + return true; + default: + return false; + } +} + +static bool +rtl8127_is_duplex_mode_valid(u8 duplex) +{ + switch(duplex) { + case DUPLEX_FULL: + case DUPLEX_HALF: + return true; + default: + return false; + } +} + +static void +rtl8127_set_link_option(struct rtl8127_private *tp, + u8 autoneg, + u32 speed, + u8 duplex, + enum rtl8127_fc_mode fc) +{ + u64 adv; + + if (!rtl8127_is_speed_mode_valid(speed)) + speed = SPEED_10000; + + if (!rtl8127_is_duplex_mode_valid(duplex)) + duplex = DUPLEX_FULL; + + if (!rtl8127_is_autoneg_mode_valid(autoneg)) + autoneg = AUTONEG_ENABLE; + + speed = min(speed, tp->HwSuppMaxPhyLinkSpeed); + + adv = 0; + switch(speed) { + case SPEED_10000: + adv |= ADVERTISED_10000baseT_Full; + fallthrough; + case SPEED_5000: + adv |= RTK_ADVERTISED_5000baseX_Full; + fallthrough; + case SPEED_2500: + adv |= ADVERTISED_2500baseX_Full; + fallthrough; + default: + adv |= (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | + ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | + ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full); + break; + } + + tp->autoneg = autoneg; + tp->speed = speed; + tp->duplex = duplex; + tp->advertising = adv; + tp->fcpause = fc; +} + +static void +rtl8127_wait_ll_share_fifo_ready(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i; + + for (i = 0; i < 10; i++) { + fsleep(100); + if (RTL_R16(tp, 0xD2) & BIT_9) + break; + } +} + +static void +rtl8127_disable_pci_offset_99(struct rtl8127_private *tp) +{ + rtl8127_mac_ocp_write(tp, 0xE032, rtl8127_mac_ocp_read(tp, 0xE032) & ~(BIT_0 | BIT_1)); + + rtl8127_csi_fun0_write_byte(tp, 0x99, 0x00); +} + +static void +rtl8127_enable_pci_offset_99(struct rtl8127_private *tp) +{ + u32 csi_tmp; + + rtl8127_csi_fun0_write_byte(tp, 0x99, tp->org_pci_offset_99); + + csi_tmp = rtl8127_mac_ocp_read(tp, 0xE032); + csi_tmp &= ~(BIT_0 | BIT_1); + if (tp->org_pci_offset_99 & (BIT_5 | BIT_6)) + csi_tmp |= BIT_1; + if (tp->org_pci_offset_99 & BIT_2) + csi_tmp |= BIT_0; + rtl8127_mac_ocp_write(tp, 0xE032, csi_tmp); +} + +static void +rtl8127_init_pci_offset_99(struct rtl8127_private *tp) +{ + rtl8127_mac_ocp_write(tp, 0xCDD0, 0x9003); + rtl8127_set_mac_ocp_bit(tp, 0xE034, (BIT_15 | BIT_14)); + rtl8127_mac_ocp_write(tp, 0xCDD2, 0x8C17); + rtl8127_mac_ocp_write(tp, 0xCDD8, 0x9003); + rtl8127_mac_ocp_write(tp, 0xCDD4, 0x9003); + rtl8127_mac_ocp_write(tp, 0xCDDA, 0x9003); + rtl8127_mac_ocp_write(tp, 0xCDD6, 0x9003); + rtl8127_mac_ocp_write(tp, 0xCDDC, 0x9003); + rtl8127_mac_ocp_write(tp, 0xCDE8, 0x8C08); + rtl8127_mac_ocp_write(tp, 0xCDEA, 0x9003); + rtl8127_mac_ocp_write(tp, 0xCDEC, 0x8C12); + rtl8127_mac_ocp_write(tp, 0xCDEE, 0x9003); + rtl8127_mac_ocp_write(tp, 0xCDF0, 0x8C2E); + rtl8127_mac_ocp_write(tp, 0xCDF2, 0x9003); + rtl8127_mac_ocp_write(tp, 0xCDF4, 0x8892); + rtl8127_mac_ocp_write(tp, 0xCDF6, 0x9003); + rtl8127_mac_ocp_write(tp, 0xCDF4, 0x8849); + rtl8127_mac_ocp_write(tp, 0xCDF6, 0x9003); + rtl8127_set_mac_ocp_bit(tp, 0xE032, BIT_14); + rtl8127_set_mac_ocp_bit(tp, 0xE0A2, BIT_0); + + rtl8127_enable_pci_offset_99(tp); +} + +static void +rtl8127_disable_pci_offset_180(struct rtl8127_private *tp) +{ + rtl8127_clear_mac_ocp_bit(tp, 0xE092, 0x00FF); +} + +static void +rtl8127_enable_pci_offset_180(struct rtl8127_private *tp) +{ + rtl8127_clear_mac_ocp_bit(tp, 0xE094, 0xFF00); + + rtl8127_clear_set_mac_ocp_bit(tp, 0xE092, 0x00FF, BIT_2); +} + +static void +rtl8127_init_pci_offset_180(struct rtl8127_private *tp) +{ + rtl8127_enable_pci_offset_180(tp); +} + +static void +rtl8127_set_pci_99_exit_driver_para(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (tp->org_pci_offset_99 & BIT_2) + rtl8127_issue_offset_99_event(tp); + rtl8127_disable_pci_offset_99(tp); +} + +static void +rtl8127_enable_cfg9346_write(struct rtl8127_private *tp) +{ + RTL_W8(tp, Cfg9346, RTL_R8(tp, Cfg9346) | Cfg9346_Unlock); +} + +static void +rtl8127_disable_cfg9346_write(struct rtl8127_private *tp) +{ + RTL_W8(tp, Cfg9346, RTL_R8(tp, Cfg9346) & ~Cfg9346_Unlock); +} + +static void +rtl8127_enable_exit_l1_mask(struct rtl8127_private *tp) +{ + //(1)ERI(0xD4)(OCP 0xC0AC).bit[7:12]=6'b111111, L1 Mask + rtl8127_set_mac_ocp_bit(tp, 0xC0AC, (BIT_7 | BIT_8 | BIT_9 | BIT_10 | BIT_11 | BIT_12)); +} + +static void +rtl8127_disable_exit_l1_mask(struct rtl8127_private *tp) +{ + //(1)ERI(0xD4)(OCP 0xC0AC).bit[7:12]=6'b000000, L1 Mask + rtl8127_clear_mac_ocp_bit(tp, 0xC0AC, (BIT_7 | BIT_8 | BIT_9 | BIT_10 | BIT_11 | BIT_12)); +} + +static void +rtl8127_enable_extend_tally_couter(struct rtl8127_private *tp) +{ + switch (tp->HwSuppExtendTallyCounterVer) { + case 1: + rtl8127_set_mac_ocp_bit(tp, 0xEA84, (BIT_1 | BIT_0)); + break; + } +} + +static void +rtl8127_disable_extend_tally_couter(struct rtl8127_private *tp) +{ + switch (tp->HwSuppExtendTallyCounterVer) { + case 1: + rtl8127_clear_mac_ocp_bit(tp, 0xEA84, (BIT_1 | BIT_0)); + break; + } +} + +static void +rtl8127_enable_force_clkreq(struct rtl8127_private *tp, bool enable) +{ + if (enable) + RTL_W8(tp, 0xF1, RTL_R8(tp, 0xF1) | BIT_7); + else + RTL_W8(tp, 0xF1, RTL_R8(tp, 0xF1) & ~BIT_7); +} + +static void +rtl8127_enable_aspm_clkreq_lock(struct rtl8127_private *tp, bool enable) +{ + bool unlock_cfg_wr; + + if ((RTL_R8(tp, Cfg9346) & Cfg9346_EEM_MASK) == Cfg9346_Unlock) + unlock_cfg_wr = false; + else + unlock_cfg_wr = true; + + if (unlock_cfg_wr) + rtl8127_enable_cfg9346_write(tp); + + if (enable) { + RTL_W8(tp, INT_CFG0_8125, RTL_R8(tp, INT_CFG0_8125) | BIT_3); + RTL_W8(tp, Config5, RTL_R8(tp, Config5) | BIT_0); + } else { + RTL_W8(tp, INT_CFG0_8125, RTL_R8(tp, INT_CFG0_8125) & ~BIT_3); + RTL_W8(tp, Config5, RTL_R8(tp, Config5) & ~BIT_0); + } + + if (unlock_cfg_wr) + rtl8127_disable_cfg9346_write(tp); +} + +static void +rtl8127_hw_d3_para(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + RTL_W16(tp, RxMaxSize, RX_BUF_SIZE); + + rtl8127_enable_force_clkreq(tp, 0); + rtl8127_enable_aspm_clkreq_lock(tp, 0); + + rtl8127_disable_exit_l1_mask(tp); + +#ifdef ENABLE_REALWOW_SUPPORT + rtl8127_set_realwow_d3_para(dev); +#endif + + rtl8127_set_pci_99_exit_driver_para(dev); + + rtl8127_disable_rxdvgate(dev); + + rtl8127_disable_extend_tally_couter(tp); +} + +static void +rtl8127_enable_magic_packet(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + switch (tp->HwSuppMagicPktVer) { + case WAKEUP_MAGIC_PACKET_V3: + rtl8127_mac_ocp_write(tp, 0xC0B6, rtl8127_mac_ocp_read(tp, 0xC0B6) | BIT_0); + break; + } +} +static void +rtl8127_disable_magic_packet(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + switch (tp->HwSuppMagicPktVer) { + case WAKEUP_MAGIC_PACKET_V3: + rtl8127_mac_ocp_write(tp, 0xC0B6, rtl8127_mac_ocp_read(tp, 0xC0B6) & ~BIT_0); + break; + } +} + +static void +rtl8127_enable_linkchg_wakeup(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + switch (tp->HwSuppLinkChgWakeUpVer) { + case 3: + RTL_W8(tp, Config3, RTL_R8(tp, Config3) | LinkUp); + rtl8127_clear_set_mac_ocp_bit(tp, 0xE0C6, (BIT_5 | BIT_3 | BIT_2), (BIT_4 | BIT_1 | BIT_0)); + break; + } +} + +static void +rtl8127_disable_linkchg_wakeup(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + switch (tp->HwSuppLinkChgWakeUpVer) { + case 3: + RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~LinkUp); + if (!(rtl8127_mac_ocp_read(tp, 0xE0C6) & BIT_0)) + rtl8127_clear_set_mac_ocp_bit(tp, 0xE0C6, (BIT_5 | BIT_3 | BIT_2 | BIT_1), BIT_4); + break; + } +} + +#define WAKE_ANY (WAKE_PHY | WAKE_MAGIC | WAKE_UCAST | WAKE_BCAST | WAKE_MCAST) + +static u32 +rtl8127_get_hw_wol(struct rtl8127_private *tp) +{ + u8 options; + u32 csi_tmp; + u32 wol_opts = 0; + + if (disable_wol_support) + goto out; + + options = RTL_R8(tp, Config1); + if (!(options & PMEnable)) + goto out; + + options = RTL_R8(tp, Config3); + if (options & LinkUp) + wol_opts |= WAKE_PHY; + + switch (tp->HwSuppMagicPktVer) { + case WAKEUP_MAGIC_PACKET_V3: + csi_tmp = rtl8127_mac_ocp_read(tp, 0xC0B6); + if (csi_tmp & BIT_0) + wol_opts |= WAKE_MAGIC; + break; + } + + options = RTL_R8(tp, Config5); + if (options & UWF) + wol_opts |= WAKE_UCAST; + if (options & BWF) + wol_opts |= WAKE_BCAST; + if (options & MWF) + wol_opts |= WAKE_MCAST; + +out: + return wol_opts; +} + +static void +rtl8127_enable_d0_speedup(struct rtl8127_private *tp) +{ + u16 clearmask; + u16 setmask; + + if (FALSE == HW_SUPPORT_D0_SPEED_UP(tp)) + return; + + if (tp->D0SpeedUpSpeed == D0_SPEED_UP_SPEED_DISABLE) + return; + + if (tp->HwSuppD0SpeedUpVer == 1 || tp->HwSuppD0SpeedUpVer == 2) { + //speed up speed + clearmask = (BIT_10 | BIT_9 | BIT_8 | BIT_7); + if (tp->D0SpeedUpSpeed == D0_SPEED_UP_SPEED_2500) + setmask = BIT_7; + else if (tp->D0SpeedUpSpeed == D0_SPEED_UP_SPEED_5000) + setmask = BIT_8; + else if (tp->D0SpeedUpSpeed == D0_SPEED_UP_SPEED_10000) + setmask = BIT_7 | BIT_8; + else + setmask = 0; + rtl8127_clear_set_mac_ocp_bit(tp, 0xE10A, clearmask, setmask); + + //speed up flowcontrol + clearmask = (BIT_15 | BIT_14); + if (tp->HwSuppD0SpeedUpVer == 2) + clearmask |= BIT_13; + + if (tp->fcpause == rtl8127_fc_full) { + setmask = (BIT_15 | BIT_14); + if (tp->HwSuppD0SpeedUpVer == 2) + setmask |= BIT_13; + } else + setmask = 0; + rtl8127_clear_set_mac_ocp_bit(tp, 0xE860, clearmask, setmask); + } + + RTL_W8(tp, 0xD0, RTL_R8(tp, 0xD0) | BIT_3); +} + +static void +rtl8127_disable_d0_speedup(struct rtl8127_private *tp) +{ + if (FALSE == HW_SUPPORT_D0_SPEED_UP(tp)) + return; + + RTL_W8(tp, 0xD0, RTL_R8(tp, 0xD0) & ~BIT_3); +} + +static void +rtl8127_set_hw_wol(struct net_device *dev, u32 wolopts) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i,tmp = 0; + static struct { + u32 opt; + u16 reg; + u8 mask; + } cfg[] = { + { WAKE_PHY, Config3, LinkUp }, + { WAKE_UCAST, Config5, UWF }, + { WAKE_BCAST, Config5, BWF }, + { WAKE_MCAST, Config5, MWF }, + { WAKE_ANY, Config5, LanWake }, + { WAKE_MAGIC, Config3, MagicPacket }, + }; + + switch (tp->HwSuppMagicPktVer) { + case WAKEUP_MAGIC_PACKET_V3: + tmp = ARRAY_SIZE(cfg) - 1; + + if (wolopts & WAKE_MAGIC) + rtl8127_enable_magic_packet(dev); + else + rtl8127_disable_magic_packet(dev); + break; + default: + break; + } + + rtl8127_enable_cfg9346_write(tp); + + for (i = 0; i < tmp; i++) { + u8 options = RTL_R8(tp, cfg[i].reg) & ~cfg[i].mask; + if (wolopts & cfg[i].opt) + options |= cfg[i].mask; + RTL_W8(tp, cfg[i].reg, options); + } + + switch (tp->HwSuppLinkChgWakeUpVer) { + case 3: + if (wolopts & WAKE_PHY) + rtl8127_enable_linkchg_wakeup(dev); + else + rtl8127_disable_linkchg_wakeup(dev); + break; + } + + rtl8127_disable_cfg9346_write(tp); +} + +static void +rtl8127_phy_restart_nway(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (rtl8127_is_in_phy_disable_mode(dev)) + return; + + rtl8127_mdio_write(tp, 0x1F, 0x0000); + rtl8127_mdio_write(tp, MII_BMCR, BMCR_ANENABLE | BMCR_ANRESTART); +} + +static void +rtl8127_phy_setup_force_mode(struct net_device *dev, u32 speed, u8 duplex) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u16 bmcr_true_force = 0; + + if (rtl8127_is_in_phy_disable_mode(dev)) + return; + + if ((speed == SPEED_10) && (duplex == DUPLEX_HALF)) { + bmcr_true_force = BMCR_SPEED10; + } else if ((speed == SPEED_10) && (duplex == DUPLEX_FULL)) { + bmcr_true_force = BMCR_SPEED10 | BMCR_FULLDPLX; + } else if ((speed == SPEED_100) && (duplex == DUPLEX_HALF)) { + bmcr_true_force = BMCR_SPEED100; + } else if ((speed == SPEED_100) && (duplex == DUPLEX_FULL)) { + bmcr_true_force = BMCR_SPEED100 | BMCR_FULLDPLX; + } else { + netif_err(tp, drv, dev, "Failed to set phy force mode!\n"); + return; + } + + rtl8127_mdio_write(tp, 0x1F, 0x0000); + rtl8127_mdio_write(tp, MII_BMCR, bmcr_true_force); +} + +static void +rtl8127_set_pci_pme(struct rtl8127_private *tp, int set) +{ + struct pci_dev *pdev = tp->pci_dev; + u16 pmc; + + if (!pdev->pm_cap) + return; + + pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &pmc); + pmc |= PCI_PM_CTRL_PME_STATUS; + if (set) + pmc |= PCI_PM_CTRL_PME_ENABLE; + else + pmc &= ~PCI_PM_CTRL_PME_ENABLE; + pci_write_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, pmc); +} + +static void +rtl8127_enable_giga_lite(struct rtl8127_private *tp, u64 adv) +{ + if (adv & ADVERTISED_1000baseT_Full) + rtl8127_set_eth_phy_ocp_bit(tp, 0xA428, BIT_9); + else + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA428, BIT_9); + + if (adv & ADVERTISED_2500baseX_Full) + rtl8127_set_eth_phy_ocp_bit(tp, 0xA5EA, BIT_0); + else + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA5EA, BIT_0); + + if (adv & RTK_ADVERTISED_5000baseX_Full) + rtl8127_set_eth_phy_ocp_bit(tp, 0xA5EA, BIT_1); + else + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA5EA, BIT_1); + + if (adv & ADVERTISED_10000baseT_Full) + rtl8127_set_eth_phy_ocp_bit(tp, 0xA5EA, BIT_2); + else + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA5EA, BIT_2); +} + +static void +rtl8127_disable_giga_lite(struct rtl8127_private *tp) +{ + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA428, BIT_9); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA5EA, BIT_0 | BIT_1 | BIT_2); +} + +static void +rtl8127_set_wol_link_speed(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned long flags; + int auto_nego; + int giga_ctrl; + int ctrl_2500; + u64 adv; + u16 anlpar; + u16 gbsr; + u16 status_2500; + u16 aner; + + spin_lock_irqsave(&tp->phy_lock, flags); + + if (tp->autoneg != AUTONEG_ENABLE) + goto exit; + + rtl8127_mdio_write(tp, 0x1F, 0x0000); + + auto_nego = rtl8127_mdio_read(tp, MII_ADVERTISE); + auto_nego &= ~(ADVERTISE_10HALF | ADVERTISE_10FULL + | ADVERTISE_100HALF | ADVERTISE_100FULL); + + giga_ctrl = rtl8127_mdio_read(tp, MII_CTRL1000); + giga_ctrl &= ~(ADVERTISE_1000HALF | ADVERTISE_1000FULL); + + ctrl_2500 = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D4); + ctrl_2500 &= ~(RTK_ADVERTISE_2500FULL | RTK_ADVERTISE_5000FULL | + RTK_ADVERTISE_10000FULL); + + aner = tp->phy_reg_aner; + anlpar = tp->phy_reg_anlpar; + gbsr = tp->phy_reg_gbsr; + status_2500 = tp->phy_reg_status_2500; + if (tp->link_ok(dev)) { + aner = rtl8127_mdio_read(tp, MII_EXPANSION); + anlpar = rtl8127_mdio_read(tp, MII_LPA); + gbsr = rtl8127_mdio_read(tp, MII_STAT1000); + status_2500 = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D6); + } + + adv = tp->advertising; + if ((aner | anlpar | gbsr | status_2500) == 0) { + int auto_nego_tmp = 0; + if (adv & ADVERTISED_10baseT_Half) + auto_nego_tmp |= ADVERTISE_10HALF; + if (adv & ADVERTISED_10baseT_Full) + auto_nego_tmp |= ADVERTISE_10FULL; + if (adv & ADVERTISED_100baseT_Half) + auto_nego_tmp |= ADVERTISE_100HALF; + if (adv & ADVERTISED_100baseT_Full) + auto_nego_tmp |= ADVERTISE_100FULL; + + if (auto_nego_tmp == 0) + goto exit; + + auto_nego |= auto_nego_tmp; + goto skip_check_lpa; + } + if (!(aner & EXPANSION_NWAY)) + goto exit; + + if ((adv & ADVERTISED_10baseT_Half) && (anlpar & LPA_10HALF)) + auto_nego |= ADVERTISE_10HALF; + else if ((adv & ADVERTISED_10baseT_Full) && (anlpar & LPA_10FULL)) + auto_nego |= ADVERTISE_10FULL; + else if ((adv & ADVERTISED_100baseT_Half) && (anlpar & LPA_100HALF)) + auto_nego |= ADVERTISE_100HALF; + else if ((adv & ADVERTISED_100baseT_Full) && (anlpar & LPA_100FULL)) + auto_nego |= ADVERTISE_100FULL; + else if (adv & ADVERTISED_1000baseT_Half && (gbsr & LPA_1000HALF)) + giga_ctrl |= ADVERTISE_1000HALF; + else if (adv & ADVERTISED_1000baseT_Full && (gbsr & LPA_1000FULL)) + giga_ctrl |= ADVERTISE_1000FULL; + else if (adv & ADVERTISED_2500baseX_Full && (status_2500 & RTK_LPA_ADVERTISE_2500FULL)) + ctrl_2500 |= RTK_ADVERTISE_2500FULL; + else if (adv & RTK_ADVERTISED_5000baseX_Full && (status_2500 & RTK_LPA_ADVERTISE_5000FULL)) + ctrl_2500 |= RTK_ADVERTISE_5000FULL; + else if (adv & ADVERTISED_10000baseT_Full && (status_2500 & RTK_LPA_ADVERTISE_10000FULL)) + ctrl_2500 |= RTK_ADVERTISE_10000FULL; + else + goto exit; + +skip_check_lpa: + if (tp->DASH) + auto_nego |= (ADVERTISE_100FULL | ADVERTISE_100HALF | ADVERTISE_10HALF | ADVERTISE_10FULL); + +#ifdef CONFIG_DOWN_SPEED_100 + auto_nego |= (ADVERTISE_100FULL | ADVERTISE_100HALF | ADVERTISE_10HALF | ADVERTISE_10FULL); +#endif + + rtl8127_mdio_write(tp, MII_ADVERTISE, auto_nego); + rtl8127_mdio_write(tp, MII_CTRL1000, giga_ctrl); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA5D4, ctrl_2500); + + rtl8127_disable_giga_lite(tp); + + rtl8127_phy_restart_nway(dev); + +exit: + spin_unlock_irqrestore(&tp->phy_lock, flags); + + return; +} + +static bool +rtl8127_keep_wol_link_speed(struct net_device *dev, u8 from_suspend) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (from_suspend && tp->link_ok(dev) && (tp->wol_opts & WAKE_PHY)) + return 1; + + if (!from_suspend && tp->resume_not_chg_speed) + return 1; + + return 0; +} +static void +rtl8127_powerdown_pll(struct net_device *dev, u8 from_suspend) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + /* Reboot not set wol link speed */ + if (system_state == SYSTEM_RESTART) + return; + + tp->check_keep_link_speed = 0; + if (tp->wol_enabled == WOL_ENABLED || tp->DASH || tp->EnableKCPOffload) { + rtl8127_set_hw_wol(dev, tp->wol_opts); + + rtl8127_enable_cfg9346_write(tp); + RTL_W8(tp, Config2, RTL_R8(tp, Config2) | PMSTS_En); + rtl8127_disable_cfg9346_write(tp); + + /* Enable the PME and clear the status */ + rtl8127_set_pci_pme(tp, 1); + + if (rtl8127_keep_wol_link_speed(dev, from_suspend)) { + tp->check_keep_link_speed = 1; + } else { + if (tp->D0SpeedUpSpeed != D0_SPEED_UP_SPEED_DISABLE) { + rtl8127_enable_d0_speedup(tp); + tp->check_keep_link_speed = 1; + } + + rtl8127_set_wol_link_speed(dev); + } + + RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) | AcceptBroadcast | AcceptMulticast | AcceptMyPhys); + + return; + } + + if (tp->DASH) + return; + + rtl8127_phy_power_down(dev); + + RTL_W8(tp, 0xF2, RTL_R8(tp, 0xF2) & ~BIT_6); +} + +static void rtl8127_powerup_pll(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) | BIT_7 | BIT_6); + + if (tp->resume_not_chg_speed) + return; + + rtl8127_phy_power_up(dev); +} + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) +static void +rtl8127_get_wol(struct net_device *dev, + struct ethtool_wolinfo *wol) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u8 options; + + wol->wolopts = 0; + + if (tp->mcfg == CFG_METHOD_DEFAULT || disable_wol_support) { + wol->supported = 0; + return; + } else { + wol->supported = WAKE_ANY; + } + + options = RTL_R8(tp, Config1); + if (!(options & PMEnable)) + return; + + wol->wolopts = tp->wol_opts; +} + +static int +rtl8127_set_wol(struct net_device *dev, + struct ethtool_wolinfo *wol) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (tp->mcfg == CFG_METHOD_DEFAULT || disable_wol_support) + return -EOPNOTSUPP; + + tp->wol_opts = wol->wolopts; + + tp->wol_enabled = (tp->wol_opts) ? WOL_ENABLED : WOL_DISABLED; + + device_set_wakeup_enable(tp_to_dev(tp), wol->wolopts); + + return 0; +} + +static void +rtl8127_get_drvinfo(struct net_device *dev, + struct ethtool_drvinfo *info) +{ + struct rtl8127_private *tp = netdev_priv(dev); + struct rtl8127_fw *rtl_fw = tp->rtl_fw; + + strscpy(info->driver, MODULENAME, sizeof(info->driver)); + strscpy(info->version, RTL8127_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(tp->pci_dev), sizeof(info->bus_info)); + info->regdump_len = R8127_REGS_DUMP_SIZE; + info->eedump_len = tp->eeprom_len; + BUILD_BUG_ON(sizeof(info->fw_version) < sizeof(rtl_fw->version)); + if (rtl_fw) + strscpy(info->fw_version, rtl_fw->version, + sizeof(info->fw_version)); +} + +static int +rtl8127_get_regs_len(struct net_device *dev) +{ + return R8127_REGS_DUMP_SIZE; +} +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) + +static void +rtl8127_set_d0_speedup_speed(struct rtl8127_private *tp) +{ + if (FALSE == HW_SUPPORT_D0_SPEED_UP(tp)) + return; + + tp->D0SpeedUpSpeed = D0_SPEED_UP_SPEED_DISABLE; + if (tp->autoneg == AUTONEG_ENABLE) { + if (tp->speed == SPEED_10000) + tp->D0SpeedUpSpeed = D0_SPEED_UP_SPEED_10000; + else if (tp->speed == SPEED_5000) + tp->D0SpeedUpSpeed = D0_SPEED_UP_SPEED_5000; + else if (tp->speed == SPEED_2500) + tp->D0SpeedUpSpeed = D0_SPEED_UP_SPEED_2500; + else if (tp->speed == SPEED_1000) + tp->D0SpeedUpSpeed = D0_SPEED_UP_SPEED_1000; + } +} + +static int +rtl8127_set_speed_xmii(struct net_device *dev, + u8 autoneg, + u32 speed, + u8 duplex, + u64 adv) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned long flags; + int auto_nego = 0; + int giga_ctrl = 0; + int ctrl_2500 = 0; + int rc = -EINVAL; + + spin_lock_irqsave(&tp->phy_lock, flags); + + if (!rtl8127_is_speed_mode_valid(speed)) { + speed = SPEED_10000; + duplex = DUPLEX_FULL; + adv |= tp->advertising; + } + + if (eee_giga_lite && (autoneg == AUTONEG_ENABLE)) + rtl8127_enable_giga_lite(tp, adv); + else + rtl8127_disable_giga_lite(tp); + + giga_ctrl = rtl8127_mdio_read(tp, MII_CTRL1000); + giga_ctrl &= ~(ADVERTISE_1000HALF | ADVERTISE_1000FULL); + ctrl_2500 = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D4); + ctrl_2500 &= ~(RTK_ADVERTISE_2500FULL | RTK_ADVERTISE_5000FULL | + RTK_ADVERTISE_10000FULL); + + if (autoneg == AUTONEG_ENABLE) { + /*n-way force*/ + auto_nego = rtl8127_mdio_read(tp, MII_ADVERTISE); + auto_nego &= ~(ADVERTISE_10HALF | ADVERTISE_10FULL | + ADVERTISE_100HALF | ADVERTISE_100FULL | + ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM); + + if (adv & ADVERTISED_10baseT_Half) + auto_nego |= ADVERTISE_10HALF; + if (adv & ADVERTISED_10baseT_Full) + auto_nego |= ADVERTISE_10FULL; + if (adv & ADVERTISED_100baseT_Half) + auto_nego |= ADVERTISE_100HALF; + if (adv & ADVERTISED_100baseT_Full) + auto_nego |= ADVERTISE_100FULL; + if (adv & ADVERTISED_1000baseT_Half) + giga_ctrl |= ADVERTISE_1000HALF; + if (adv & ADVERTISED_1000baseT_Full) + giga_ctrl |= ADVERTISE_1000FULL; + if (adv & ADVERTISED_2500baseX_Full) + ctrl_2500 |= RTK_ADVERTISE_2500FULL; + if (HW_SUPP_PHY_LINK_SPEED_5000M(tp)) { + if (adv & RTK_ADVERTISED_5000baseX_Full) + ctrl_2500 |= RTK_ADVERTISE_5000FULL; + } + if (HW_SUPP_PHY_LINK_SPEED_10000M(tp)) { + if (adv & ADVERTISED_10000baseT_Full) + ctrl_2500 |= RTK_ADVERTISE_10000FULL; + } + + //flow control + if (tp->fcpause == rtl8127_fc_full) + auto_nego |= ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM; + + tp->phy_auto_nego_reg = auto_nego; + tp->phy_1000_ctrl_reg = giga_ctrl; + + tp->phy_2500_ctrl_reg = ctrl_2500; + + rtl8127_mdio_write(tp, 0x1f, 0x0000); + rtl8127_mdio_write(tp, MII_ADVERTISE, auto_nego); + rtl8127_mdio_write(tp, MII_CTRL1000, giga_ctrl); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA5D4, ctrl_2500); + rtl8127_phy_restart_nway(dev); + } else { + /*true force*/ + if (speed == SPEED_10 || speed == SPEED_100) + rtl8127_phy_setup_force_mode(dev, speed, duplex); + else + goto out; + } + + tp->autoneg = autoneg; + tp->speed = speed; + tp->duplex = duplex; + tp->advertising = adv; + + rtl8127_set_d0_speedup_speed(tp); + + rc = 0; +out: + spin_unlock_irqrestore(&tp->phy_lock, flags); + + return rc; +} + +static int +rtl8127_set_speed(struct net_device *dev, + u8 autoneg, + u32 speed, + u8 duplex, + u64 adv) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int ret; + + if (tp->resume_not_chg_speed) + return 0; + + ret = tp->set_speed(dev, autoneg, speed, duplex, adv); + + return ret; +} + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) +static int +rtl8127_set_settings(struct net_device *dev, +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) + struct ethtool_cmd *cmd +#else + const struct ethtool_link_ksettings *cmd +#endif + ) +{ + int ret; + u8 autoneg; + u32 speed; + u8 duplex; + u64 supported = 0, advertising = 0; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) + autoneg = cmd->autoneg; + speed = cmd->speed; + duplex = cmd->duplex; + supported = cmd->supported; + advertising = cmd->advertising; +#else + struct rtl8127_private *tp = netdev_priv(dev); + const struct ethtool_link_settings *base = &cmd->base; + autoneg = base->autoneg; + speed = base->speed; + duplex = base->duplex; + ethtool_convert_link_mode_to_legacy_u32((u32*)&supported, + cmd->link_modes.supported); + ethtool_convert_link_mode_to_legacy_u32((u32*)&advertising, + cmd->link_modes.advertising); + if (test_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, + cmd->link_modes.supported)) + supported |= ADVERTISED_2500baseX_Full; + if (test_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, + cmd->link_modes.advertising)) + advertising |= ADVERTISED_2500baseX_Full; + if (HW_SUPP_PHY_LINK_SPEED_5000M(tp)) { + if (test_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, + cmd->link_modes.supported)) + supported |= RTK_ADVERTISED_5000baseX_Full; + if (test_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, + cmd->link_modes.advertising)) + advertising |= RTK_ADVERTISED_5000baseX_Full; + } + if (HW_SUPP_PHY_LINK_SPEED_10000M(tp)) { + if (test_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, + cmd->link_modes.supported)) + supported |= ADVERTISED_10000baseT_Full; + if (test_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, + cmd->link_modes.advertising)) + advertising |= ADVERTISED_10000baseT_Full; + } +#endif + if (advertising & ~supported) + return -EINVAL; + + ret = rtl8127_set_speed(dev, autoneg, speed, duplex, advertising); + + return ret; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) +static u32 +rtl8127_get_tx_csum(struct net_device *dev) +{ + u32 ret; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + ret = ((dev->features & NETIF_F_IP_CSUM) != 0); +#else + ret = ((dev->features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) != 0); +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + + return ret; +} + +static u32 +rtl8127_get_rx_csum(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u32 ret; + + ret = tp->cp_cmd & RxChkSum; + + return ret; +} + +static int +rtl8127_set_tx_csum(struct net_device *dev, + u32 data) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (tp->mcfg == CFG_METHOD_DEFAULT) + return -EOPNOTSUPP; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + if (data) + dev->features |= NETIF_F_IP_CSUM; + else + dev->features &= ~NETIF_F_IP_CSUM; +#else + if (data) + dev->features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM); + else + dev->features &= ~(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM); +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + + return 0; +} + +static int +rtl8127_set_rx_csum(struct net_device *dev, + u32 data) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (tp->mcfg == CFG_METHOD_DEFAULT) + return -EOPNOTSUPP; + + if (data) + tp->cp_cmd |= RxChkSum; + else + tp->cp_cmd &= ~RxChkSum; + + RTL_W16(tp, CPlusCmd, tp->cp_cmd); + + return 0; +} +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) + +static u32 +rtl8127_rx_desc_opts1(struct rtl8127_private *tp, + struct RxDesc *desc) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + return READ_ONCE(((struct RxDescV3 *)desc)->RxDescNormalDDWord4.opts1); + case RX_DESC_RING_TYPE_4: + return READ_ONCE(((struct RxDescV4 *)desc)->RxDescNormalDDWord2.opts1); + default: + return READ_ONCE(desc->opts1); + } +} + +static u32 +rtl8127_rx_desc_opts2(struct rtl8127_private *tp, + struct RxDesc *desc) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + return ((struct RxDescV3 *)desc)->RxDescNormalDDWord4.opts2; + case RX_DESC_RING_TYPE_4: + return ((struct RxDescV4 *)desc)->RxDescNormalDDWord2.opts2; + default: + return desc->opts2; + } +} + +#ifdef CONFIG_R8127_VLAN + +static void +rtl8127_clear_rx_desc_opts2(struct rtl8127_private *tp, + struct RxDesc *desc) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + ((struct RxDescV3 *)desc)->RxDescNormalDDWord4.opts2 = 0; + break; + case RX_DESC_RING_TYPE_4: + ((struct RxDescV4 *)desc)->RxDescNormalDDWord2.opts2 = 0; + break; + default: + desc->opts2 = 0; + break; + } +} + + +static inline u32 +rtl8127_tx_vlan_tag(struct rtl8127_private *tp, + struct sk_buff *skb) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + return (tp->vlgrp && vlan_tx_tag_present(skb)) ? + TxVlanTag | swab16(vlan_tx_tag_get(skb)) : 0x00; +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,0,0) + return (vlan_tx_tag_present(skb)) ? + TxVlanTag | swab16(vlan_tx_tag_get(skb)) : 0x00; +#else + return (skb_vlan_tag_present(skb)) ? + TxVlanTag | swab16(skb_vlan_tag_get(skb)) : 0x00; +#endif + + return 0; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + +static void +rtl8127_vlan_rx_register(struct net_device *dev, + struct vlan_group *grp) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + tp->vlgrp = grp; + + if (tp->vlgrp) { + tp->rtl8127_rx_config |= (EnableInnerVlan | EnableOuterVlan); + RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) | (EnableInnerVlan | EnableOuterVlan)) + } else { + tp->rtl8127_rx_config &= ~(EnableInnerVlan | EnableOuterVlan); + RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) & ~(EnableInnerVlan | EnableOuterVlan)) + } +} + +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) +static void +rtl8127_vlan_rx_kill_vid(struct net_device *dev, + unsigned short vid) +{ + struct rtl8127_private *tp = netdev_priv(dev); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,21) + if (tp->vlgrp) + tp->vlgrp->vlan_devices[vid] = NULL; +#else + vlan_group_set_device(tp->vlgrp, vid, NULL); +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,21) +} +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) + +static int +rtl8127_rx_vlan_skb(struct rtl8127_private *tp, + struct RxDesc *desc, + struct sk_buff *skb) +{ + u32 opts2 = le32_to_cpu(rtl8127_rx_desc_opts2(tp, desc)); + int ret = -1; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + if (tp->vlgrp && (opts2 & RxVlanTag)) { + rtl8127_rx_hwaccel_skb(skb, tp->vlgrp, + swab16(opts2 & 0xffff)); + ret = 0; + } +#elif LINUX_VERSION_CODE < KERNEL_VERSION(3,10,0) + if (opts2 & RxVlanTag) + __vlan_hwaccel_put_tag(skb, swab16(opts2 & 0xffff)); +#else + if (opts2 & RxVlanTag) + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), swab16(opts2 & 0xffff)); +#endif + + rtl8127_clear_rx_desc_opts2(tp, desc); + return ret; +} + +#else /* !CONFIG_R8127_VLAN */ + +static inline u32 +rtl8127_tx_vlan_tag(struct rtl8127_private *tp, + struct sk_buff *skb) +{ + return 0; +} + +static int +rtl8127_rx_vlan_skb(struct rtl8127_private *tp, + struct RxDesc *desc, + struct sk_buff *skb) +{ + return -1; +} + +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0) + +static netdev_features_t rtl8127_fix_features(struct net_device *dev, + netdev_features_t features) +{ + if (dev->mtu > MSS_MAX) + features &= ~NETIF_F_ALL_TSO; + if (dev->mtu > ETH_DATA_LEN) { + features &= ~NETIF_F_ALL_TSO; + features &= ~NETIF_F_ALL_CSUM; + } +#ifndef CONFIG_R8127_VLAN + features &= ~NETIF_F_ALL_CSUM; +#endif + + return features; +} + +static int rtl8127_hw_set_features(struct net_device *dev, + netdev_features_t features) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u32 rx_config; + + rx_config = RTL_R32(tp, RxConfig); + if (features & NETIF_F_RXALL) { + tp->rtl8127_rx_config |= (AcceptErr | AcceptRunt); + rx_config |= (AcceptErr | AcceptRunt); + } else { + tp->rtl8127_rx_config &= ~(AcceptErr | AcceptRunt); + rx_config &= ~(AcceptErr | AcceptRunt); + } + + if (features & NETIF_F_HW_VLAN_RX) { + tp->rtl8127_rx_config |= (EnableInnerVlan | EnableOuterVlan); + rx_config |= (EnableInnerVlan | EnableOuterVlan); + } else { + tp->rtl8127_rx_config &= ~(EnableInnerVlan | EnableOuterVlan); + rx_config &= ~(EnableInnerVlan | EnableOuterVlan); + } + + RTL_W32(tp, RxConfig, rx_config); + + if (features & NETIF_F_RXCSUM) + tp->cp_cmd |= RxChkSum; + else + tp->cp_cmd &= ~RxChkSum; + + RTL_W16(tp, CPlusCmd, tp->cp_cmd); + RTL_R16(tp, CPlusCmd); + + return 0; +} + +static int rtl8127_set_features(struct net_device *dev, + netdev_features_t features) +{ + features &= NETIF_F_RXALL | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_RX; + + rtl8127_hw_set_features(dev, features); + + return 0; +} + +#endif + +static u8 rtl8127_get_mdi_status(struct rtl8127_private *tp) +{ + if (!tp->link_ok(tp->dev)) + return ETH_TP_MDI_INVALID; + + if (rtl8127_mdio_direct_read_phy_ocp(tp, 0xA444) & BIT_1) + return ETH_TP_MDI; + else + return ETH_TP_MDI_X; +} + +static void rtl8127_gset_xmii(struct net_device *dev, +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) + struct ethtool_cmd *cmd +#else + struct ethtool_link_ksettings *cmd +#endif + ) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u16 aner = tp->phy_reg_aner; + u16 anlpar = tp->phy_reg_anlpar; + u16 gbsr = tp->phy_reg_gbsr; + u16 status_2500 = tp->phy_reg_status_2500; + unsigned long flags; + u64 lpa_adv = 0; + u32 status; + u8 autoneg, duplex; + u32 speed = 0; + u16 bmcr; + u64 supported, advertising; + u8 report_lpa = 0; + + supported = SUPPORTED_10baseT_Half | + SUPPORTED_10baseT_Full | + SUPPORTED_100baseT_Half | + SUPPORTED_100baseT_Full | + SUPPORTED_1000baseT_Full | + SUPPORTED_2500baseX_Full | + RTK_SUPPORTED_5000baseX_Full | + SUPPORTED_10000baseT_Full | + SUPPORTED_Autoneg | + SUPPORTED_TP | + SUPPORTED_Pause | + SUPPORTED_Asym_Pause; + + if (!HW_SUPP_PHY_LINK_SPEED_2500M(tp)) + supported &= ~SUPPORTED_2500baseX_Full; + + if (!HW_SUPP_PHY_LINK_SPEED_5000M(tp)) + supported &= ~RTK_SUPPORTED_5000baseX_Full; + + if (!HW_SUPP_PHY_LINK_SPEED_10000M(tp)) + supported &= ~SUPPORTED_10000baseT_Full; + + advertising = tp->advertising; + if (tp->phy_auto_nego_reg || tp->phy_1000_ctrl_reg || + tp->phy_2500_ctrl_reg) { + advertising = 0; + if (tp->phy_auto_nego_reg & ADVERTISE_10HALF) + advertising |= ADVERTISED_10baseT_Half; + if (tp->phy_auto_nego_reg & ADVERTISE_10FULL) + advertising |= ADVERTISED_10baseT_Full; + if (tp->phy_auto_nego_reg & ADVERTISE_100HALF) + advertising |= ADVERTISED_100baseT_Half; + if (tp->phy_auto_nego_reg & ADVERTISE_100FULL) + advertising |= ADVERTISED_100baseT_Full; + if (tp->phy_1000_ctrl_reg & ADVERTISE_1000FULL) + advertising |= ADVERTISED_1000baseT_Full; + if (tp->phy_2500_ctrl_reg & RTK_ADVERTISE_2500FULL) + advertising |= ADVERTISED_2500baseX_Full; + if (tp->phy_2500_ctrl_reg & RTK_ADVERTISE_5000FULL) + advertising |= RTK_ADVERTISED_5000baseX_Full; + if (tp->phy_2500_ctrl_reg & RTK_ADVERTISE_10000FULL) + advertising |= ADVERTISED_10000baseT_Full; + } + + spin_lock_irqsave(&tp->phy_lock, flags); + rtl8127_mdio_write(tp, 0x1F, 0x0000); + bmcr = rtl8127_mdio_read(tp, MII_BMCR); + spin_unlock_irqrestore(&tp->phy_lock, flags); + if (bmcr & BMCR_ANENABLE) { + autoneg = AUTONEG_ENABLE; + advertising |= ADVERTISED_Autoneg; + } else { + autoneg = AUTONEG_DISABLE; + } + + advertising |= ADVERTISED_TP; + + status = RTL_R32(tp, PHYstatus); + if (netif_running(dev) && (status & LinkStatus)) + report_lpa = 1; + + if (report_lpa) { + /*link on*/ + speed = rtl8127_convert_link_speed(status); + + if (status & TxFlowCtrl) + advertising |= ADVERTISED_Asym_Pause; + + if (status & RxFlowCtrl) + advertising |= ADVERTISED_Pause; + + duplex = ((status & (_1000bpsF | _2500bpsF | _5000bpsF | _10000bpsF)) || + (status & FullDup)) ? + DUPLEX_FULL : DUPLEX_HALF; + + /*link partner*/ + if (aner & EXPANSION_NWAY) + lpa_adv |= ADVERTISED_Autoneg; + if (anlpar & LPA_10HALF) + lpa_adv |= ADVERTISED_10baseT_Half; + if (anlpar & LPA_10FULL) + lpa_adv |= ADVERTISED_10baseT_Full; + if (anlpar & LPA_100HALF) + lpa_adv |= ADVERTISED_100baseT_Half; + if (anlpar & LPA_100FULL) + lpa_adv |= ADVERTISED_100baseT_Full; + if (anlpar & LPA_PAUSE_CAP) + lpa_adv |= ADVERTISED_Pause; + if (anlpar & LPA_PAUSE_ASYM) + lpa_adv |= ADVERTISED_Asym_Pause; + if (gbsr & LPA_1000HALF) + lpa_adv |= ADVERTISED_1000baseT_Half; + if (gbsr & LPA_1000FULL) + lpa_adv |= ADVERTISED_1000baseT_Full; + if (status_2500 & RTK_LPA_ADVERTISE_2500FULL) + lpa_adv |= ADVERTISED_2500baseX_Full; + if (status_2500 & RTK_LPA_ADVERTISE_5000FULL) + lpa_adv |= RTK_ADVERTISED_5000baseX_Full; + if (status_2500 & RTK_LPA_ADVERTISE_10000FULL) + lpa_adv |= ADVERTISED_10000baseT_Full; + } else { + /*link down*/ + speed = SPEED_UNKNOWN; + duplex = DUPLEX_UNKNOWN; + lpa_adv = 0; + } + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) + cmd->supported = (u32)supported; + cmd->advertising = (u32)advertising; + cmd->autoneg = autoneg; + cmd->speed = speed; + cmd->duplex = duplex; + cmd->port = PORT_TP; + cmd->lp_advertising = (u32)lpa_adv; + cmd->eth_tp_mdix = rtl8127_get_mdi_status(tp); +#else + ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, + supported); + ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising, + advertising); + ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.lp_advertising, + lpa_adv); + + if (supported & SUPPORTED_2500baseX_Full) { + linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, + cmd->link_modes.supported, 1); + } + if (advertising & ADVERTISED_2500baseX_Full) { + linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, + cmd->link_modes.advertising, 1); + } + if (supported & RTK_SUPPORTED_5000baseX_Full) { + linkmode_mod_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, + cmd->link_modes.supported, 1); + } + if (advertising & RTK_ADVERTISED_5000baseX_Full) { + linkmode_mod_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, + cmd->link_modes.advertising, 1); + } + if (supported & SUPPORTED_10000baseT_Full) { + linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, + cmd->link_modes.supported, 1); + } + if (advertising & ADVERTISED_10000baseT_Full) { + linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, + cmd->link_modes.advertising, 1); + } + if (report_lpa) { + if (lpa_adv & ADVERTISED_2500baseX_Full) { + linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, + cmd->link_modes.lp_advertising, 1); + } + if (lpa_adv & RTK_ADVERTISED_5000baseX_Full) + linkmode_mod_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, + cmd->link_modes.lp_advertising, 1); + if (lpa_adv & ADVERTISED_10000baseT_Full) + linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, + cmd->link_modes.lp_advertising, 1); + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0) + /* Use ETHTOOL_LINK_MODE_2500baseT_Full_BIT instead of + ETHTOOL_LINK_MODE_2500baseX_Full_BIT. */ + linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseX_Full_BIT, + cmd->link_modes.supported, 0); + + linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseX_Full_BIT, + cmd->link_modes.advertising, 0); + + linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseX_Full_BIT, + cmd->link_modes.lp_advertising, 0); +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0) */ + + cmd->base.autoneg = autoneg; + cmd->base.speed = speed; + cmd->base.duplex = duplex; + cmd->base.port = PORT_TP; + cmd->base.eth_tp_mdix = rtl8127_get_mdi_status(tp); +#endif +} + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) +static int +rtl8127_get_settings(struct net_device *dev, +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) + struct ethtool_cmd *cmd +#else + struct ethtool_link_ksettings *cmd +#endif + ) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + tp->get_settings(dev, cmd); + + return 0; +} + +static void rtl8127_get_regs(struct net_device *dev, struct ethtool_regs *regs, + void *p) +{ + struct rtl8127_private *tp = netdev_priv(dev); + void __iomem *ioaddr = tp->mmio_addr; + unsigned int i; + u8 *data = p; + + if (regs->len < R8127_REGS_DUMP_SIZE) + return /* -EINVAL */; + + memset(p, 0, regs->len); + + for (i = 0; i < R8127_MAC_REGS_SIZE; i++) + *data++ = readb(ioaddr + i); + data = (u8*)p + 256; + + rtl8127_mdio_write(tp, 0x1F, 0x0000); + for (i = 0; i < R8127_PHY_REGS_SIZE/2; i++) { + *(u16*)data = rtl8127_mdio_read(tp, i); + data += 2; + } + data = (u8*)p + 256 * 2; + + for (i = 0; i < R8127_EPHY_REGS_SIZE/2; i++) { + *(u16*)data = rtl8127_ephy_read(tp, i); + data += 2; + } + data = (u8*)p + 256 * 3; + + for (i = 0; i < R8127_ERI_REGS_SIZE; i+=4) { + *(u32*)data = rtl8127_eri_read(tp, i , 4, ERIAR_ExGMAC); + data += 4; + } +} + +static void rtl8127_get_pauseparam(struct net_device *dev, + struct ethtool_pauseparam *pause) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + pause->autoneg = (tp->autoneg ? AUTONEG_ENABLE : AUTONEG_DISABLE); + if (tp->fcpause == rtl8127_fc_rx_pause) + pause->rx_pause = 1; + else if (tp->fcpause == rtl8127_fc_tx_pause) + pause->tx_pause = 1; + else if (tp->fcpause == rtl8127_fc_full) { + pause->rx_pause = 1; + pause->tx_pause = 1; + } +} + +static int rtl8127_set_pauseparam(struct net_device *dev, + struct ethtool_pauseparam *pause) +{ + struct rtl8127_private *tp = netdev_priv(dev); + enum rtl8127_fc_mode newfc; + + if (pause->tx_pause || pause->rx_pause) + newfc = rtl8127_fc_full; + else + newfc = rtl8127_fc_none; + + if (tp->fcpause != newfc) { + tp->fcpause = newfc; + + rtl8127_set_speed(dev, tp->autoneg, tp->speed, tp->duplex, tp->advertising); + } + + return 0; + +} + +static u32 +rtl8127_get_msglevel(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + return tp->msg_enable; +} + +static void +rtl8127_set_msglevel(struct net_device *dev, + u32 value) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + tp->msg_enable = value; +} + +static const char rtl8127_gstrings[][ETH_GSTRING_LEN] = { + /* legacy */ + "tx_packets", + "rx_packets", + "tx_errors", + "rx_errors", + "rx_missed", + "align_errors", + "tx_single_collisions", + "tx_multi_collisions", + "unicast", + "broadcast", + "multicast", + "tx_aborted", + "tx_underrun", + + /* extended */ + "tx_octets", + "rx_octets", + "rx_multicast64", + "tx_unicast64", + "tx_broadcast64", + "tx_multicast64", + "tx_pause_on", + "tx_pause_off", + "tx_pause_all", + "tx_deferred", + "tx_late_collision", + "tx_all_collision", + "tx_aborted32", + "align_errors32", + "rx_frame_too_long", + "rx_runt", + "rx_pause_on", + "rx_pause_off", + "rx_pause_all", + "rx_unknown_opcode", + "rx_mac_error", + "tx_underrun32", + "rx_mac_missed", + "rx_tcam_dropped", + "tdu", + "rdu", +}; +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,33) +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) +static int rtl8127_get_stats_count(struct net_device *dev) +{ + return ARRAY_SIZE(rtl8127_gstrings); +} +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) +#else +static int rtl8127_get_sset_count(struct net_device *dev, int sset) +{ + switch (sset) { + case ETH_SS_STATS: + return ARRAY_SIZE(rtl8127_gstrings); + default: + return -EOPNOTSUPP; + } +} +#endif + +static void +rtl8127_set_ring_size(struct rtl8127_private *tp, u32 rx, u32 tx) +{ + int i; + + for (i = 0; i < R8127_MAX_RX_QUEUES; i++) + tp->rx_ring[i].num_rx_desc = rx; + + for (i = 0; i < R8127_MAX_TX_QUEUES; i++) + tp->tx_ring[i].num_tx_desc = tx; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0) +static void rtl8127_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) +#else +static void rtl8127_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring) +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + ring->rx_max_pending = MAX_NUM_TX_DESC; + ring->tx_max_pending = MAX_NUM_RX_DESC; + ring->rx_pending = tp->rx_ring[0].num_rx_desc; + ring->tx_pending = tp->tx_ring[0].num_tx_desc; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0) +static int rtl8127_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) +#else +static int rtl8127_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring) +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u32 new_rx_count, new_tx_count; + int rc = 0; + + if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) + return -EINVAL; + + new_tx_count = clamp_t(u32, ring->tx_pending, + MIN_NUM_TX_DESC, MAX_NUM_TX_DESC); + + new_rx_count = clamp_t(u32, ring->rx_pending, + MIN_NUM_RX_DESC, MAX_NUM_RX_DESC); + + if ((new_rx_count == tp->rx_ring[0].num_rx_desc) && + (new_tx_count == tp->tx_ring[0].num_tx_desc)) { + /* nothing to do */ + return 0; + } + + if (netif_running(dev)) { + rtl8127_wait_for_quiescence(dev); + rtl8127_close(dev); + } + + rtl8127_set_ring_size(tp, new_rx_count, new_tx_count); + + if (netif_running(dev)) + rc = rtl8127_open(dev); + + return rc; +} +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) +static void +rtl8127_get_ethtool_stats(struct net_device *dev, + struct ethtool_stats *stats, + u64 *data) +{ + struct rtl8127_private *tp = netdev_priv(dev); + struct rtl8127_counters *counters; + dma_addr_t paddr; + + ASSERT_RTNL(); + + counters = tp->tally_vaddr; + paddr = tp->tally_paddr; + if (!counters) + return; + + rtl8127_dump_tally_counter(tp, paddr); + + data[0] = le64_to_cpu(counters->tx_packets); + data[1] = le64_to_cpu(counters->rx_packets); + data[2] = le64_to_cpu(counters->tx_errors); + data[3] = le32_to_cpu(counters->rx_errors); + data[4] = le16_to_cpu(counters->rx_missed); + data[5] = le16_to_cpu(counters->align_errors); + data[6] = le32_to_cpu(counters->tx_one_collision); + data[7] = le32_to_cpu(counters->tx_multi_collision); + data[8] = le64_to_cpu(counters->rx_unicast); + data[9] = le64_to_cpu(counters->rx_broadcast); + data[10] = le32_to_cpu(counters->rx_multicast); + data[11] = le16_to_cpu(counters->tx_aborted); + data[12] = le16_to_cpu(counters->tx_underrun); + + data[13] = le64_to_cpu(counters->tx_octets); + data[14] = le64_to_cpu(counters->rx_octets); + data[15] = le64_to_cpu(counters->rx_multicast64); + data[16] = le64_to_cpu(counters->tx_unicast64); + data[17] = le64_to_cpu(counters->tx_broadcast64); + data[18] = le64_to_cpu(counters->tx_multicast64); + data[19] = le32_to_cpu(counters->tx_pause_on); + data[20] = le32_to_cpu(counters->tx_pause_off); + data[21] = le32_to_cpu(counters->tx_pause_all); + data[22] = le32_to_cpu(counters->tx_deferred); + data[23] = le32_to_cpu(counters->tx_late_collision); + data[24] = le32_to_cpu(counters->tx_all_collision); + data[25] = le32_to_cpu(counters->tx_aborted32); + data[26] = le32_to_cpu(counters->align_errors32); + data[27] = le32_to_cpu(counters->rx_frame_too_long); + data[28] = le32_to_cpu(counters->rx_runt); + data[29] = le32_to_cpu(counters->rx_pause_on); + data[30] = le32_to_cpu(counters->rx_pause_off); + data[31] = le32_to_cpu(counters->rx_pause_all); + data[32] = le32_to_cpu(counters->rx_unknown_opcode); + data[33] = le32_to_cpu(counters->rx_mac_error); + data[34] = le32_to_cpu(counters->tx_underrun32); + data[35] = le32_to_cpu(counters->rx_mac_missed); + data[36] = le32_to_cpu(counters->rx_tcam_dropped); + data[37] = le32_to_cpu(counters->tdu); + data[38] = le32_to_cpu(counters->rdu); +} + +static void +rtl8127_get_strings(struct net_device *dev, + u32 stringset, + u8 *data) +{ + switch (stringset) { + case ETH_SS_STATS: + memcpy(data, rtl8127_gstrings, sizeof(rtl8127_gstrings)); + break; + } +} +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) + +static int rtl_get_eeprom_len(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + return tp->eeprom_len; +} + +static int rtl_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *buf) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i,j,ret; + int start_w, end_w; + int VPD_addr, VPD_data; + u32 *eeprom_buff; + u16 tmp; + + if (tp->eeprom_type == EEPROM_TYPE_NONE) { + dev_printk(KERN_DEBUG, tp_to_dev(tp), "Detect none EEPROM\n"); + return -EOPNOTSUPP; + } else if (eeprom->len == 0 || (eeprom->offset+eeprom->len) > tp->eeprom_len) { + dev_printk(KERN_DEBUG, tp_to_dev(tp), "Invalid parameter\n"); + return -EINVAL; + } + + VPD_addr = 0xD2; + VPD_data = 0xD4; + + start_w = eeprom->offset >> 2; + end_w = (eeprom->offset + eeprom->len - 1) >> 2; + + eeprom_buff = kmalloc(sizeof(u32)*(end_w - start_w + 1), GFP_KERNEL); + if (!eeprom_buff) + return -ENOMEM; + + rtl8127_enable_cfg9346_write(tp); + ret = -EFAULT; + for (i=start_w; i<=end_w; i++) { + pci_write_config_word(tp->pci_dev, VPD_addr, (u16)i*4); + ret = -EFAULT; + for (j = 0; j < 10; j++) { + fsleep(400); + pci_read_config_word(tp->pci_dev, VPD_addr, &tmp); + if (tmp&0x8000) { + ret = 0; + break; + } + } + + if (ret) + break; + + pci_read_config_dword(tp->pci_dev, VPD_data, &eeprom_buff[i-start_w]); + } + rtl8127_disable_cfg9346_write(tp); + + if (!ret) + memcpy(buf, (u8 *)eeprom_buff + (eeprom->offset & 3), eeprom->len); + + kfree(eeprom_buff); + + return ret; +} + +#undef ethtool_op_get_link +#define ethtool_op_get_link _kc_ethtool_op_get_link +static u32 _kc_ethtool_op_get_link(struct net_device *dev) +{ + return netif_carrier_ok(dev) ? 1 : 0; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) +#undef ethtool_op_get_sg +#define ethtool_op_get_sg _kc_ethtool_op_get_sg +static u32 _kc_ethtool_op_get_sg(struct net_device *dev) +{ +#ifdef NETIF_F_SG + return (dev->features & NETIF_F_SG) != 0; +#else + return 0; +#endif +} + +#undef ethtool_op_set_sg +#define ethtool_op_set_sg _kc_ethtool_op_set_sg +static int _kc_ethtool_op_set_sg(struct net_device *dev, u32 data) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (tp->mcfg == CFG_METHOD_DEFAULT) + return -EOPNOTSUPP; + +#ifdef NETIF_F_SG + if (data) + dev->features |= NETIF_F_SG; + else + dev->features &= ~NETIF_F_SG; +#endif + + return 0; +} +#endif + +static void +rtl8127_set_eee_lpi_timer(struct rtl8127_private *tp) +{ + u16 dev_lpi_timer; + + dev_lpi_timer = tp->eee.tx_lpi_timer; + + RTL_W16(tp, EEE_TXIDLE_TIMER_8125, dev_lpi_timer); +} + +static bool rtl8127_is_adv_eee_enabled(struct rtl8127_private *tp) +{ + if (rtl8127_mdio_direct_read_phy_ocp(tp, 0xA430) & BIT_15) + return true; + else + return false; +} + +static void rtl8127_disable_adv_eee(struct rtl8127_private *tp) +{ + bool lock; + + if (rtl8127_is_adv_eee_enabled(tp)) + lock = true; + else + lock = false; + + if (lock) + rtl8127_set_phy_mcu_patch_request(tp); + + rtl8127_clear_mac_ocp_bit(tp, 0xE052, BIT_0); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA442, BIT_12 | BIT_13); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA430, BIT_15); + + if (lock) + rtl8127_clear_phy_mcu_patch_request(tp); +} + +static int rtl8127_enable_eee(struct rtl8127_private *tp) +{ + struct ethtool_keee *eee = &tp->eee; + u16 eee_adv_cap1_t = rtl8127_ethtool_adv_to_mmd_eee_adv_cap1_t(eee->advertised); + u16 eee_adv_cap2_t = rtl8127_ethtool_adv_to_mmd_eee_adv_cap2_t(eee->advertised); + + rtl8127_set_mac_ocp_bit(tp, 0xE040, (BIT_1|BIT_0)); + + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA5D0, + MDIO_EEE_100TX | MDIO_EEE_1000T | MDIO_EEE_10GT, + eee_adv_cap1_t); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA6D4, + MDIO_EEE_2_5GT | MDIO_EEE_5GT, + eee_adv_cap2_t); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA6D8, BIT_4); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA428, BIT_7); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA4A2, BIT_9); + + /*Advanced EEE*/ + rtl8127_disable_adv_eee(tp); + + return 0; +} + +static int rtl8127_disable_eee(struct rtl8127_private *tp) +{ + rtl8127_clear_mac_ocp_bit(tp, 0xE040, (BIT_1|BIT_0)); + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA5D0, + (MDIO_EEE_100TX | MDIO_EEE_1000T | MDIO_EEE_10GT)); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA6D4, + (MDIO_EEE_2_5GT | MDIO_EEE_5GT)); + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA6D8, BIT_4); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA428, BIT_7); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA4A2, BIT_9); + + /*Advanced EEE*/ + rtl8127_disable_adv_eee(tp); + + return 0; +} + +static int rtl_nway_reset(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int ret, bmcr; + + if (unlikely(tp->rtk_enable_diag)) + return -EBUSY; + + /* if autoneg is off, it's an error */ + rtl8127_mdio_write(tp, 0x1F, 0x0000); + bmcr = rtl8127_mdio_read(tp, MII_BMCR); + + if (bmcr & BMCR_ANENABLE) { + bmcr |= BMCR_ANRESTART; + rtl8127_mdio_write(tp, MII_BMCR, bmcr); + ret = 0; + } else { + ret = -EINVAL; + } + + return ret; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0) +static u32 +rtl8127_device_lpi_t_to_ethtool_lpi_t(struct rtl8127_private *tp , u32 lpi_timer) +{ + u32 to_us; + u16 status; + + to_us = lpi_timer * 80; + status = RTL_R16(tp, PHYstatus); + if (status & LinkStatus) { + /*link on*/ + if (HW_SUPP_PHY_LINK_SPEED_10000M(tp)) { + //5G : lpi_timer * 12.8ns + //2.5G : lpi_timer * 25.6ns + //Giga: lpi_timer * 8ns + //100M : lpi_timer * 80ns + if (status & (_10000bpsF)) + to_us = (lpi_timer * 128) / 10; + else if (status & (_5000bpsF)) + to_us = (lpi_timer * 128) / 10; + else if (status & _2500bpsF) + to_us = (lpi_timer * 256) / 10; + else if (status & _1000bpsF) + to_us = lpi_timer * 8; + } else if (HW_SUPP_PHY_LINK_SPEED_5000M(tp)) { + //5G : lpi_timer * 12.8ns + //2.5G : lpi_timer * 25.6ns + //Giga: lpi_timer * 8ns + //100M : lpi_timer * 80ns + if (status & (_5000bpsF)) + to_us = (lpi_timer * 128) / 10; + else if (status & _2500bpsF) + to_us = (lpi_timer * 256) / 10; + else if (status & _1000bpsF) + to_us = lpi_timer * 8; + } else { + //2.5G : lpi_timer * 3.2ns + //Giga: lpi_timer * 8ns + //100M : lpi_timer * 80ns + if (status & _2500bpsF) + to_us = (lpi_timer * 32) / 10; + else if (status & _1000bpsF) + to_us = lpi_timer * 8; + } + } + + //ns to us + to_us /= 1000; + + return to_us; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,9,0) +static void +rtl8127_adv_to_linkmode(unsigned long *mode, u64 adv) +{ + linkmode_zero(mode); + + if (adv & ADVERTISED_10baseT_Half) + linkmode_set_bit(ETHTOOL_LINK_MODE_10baseT_Half_BIT, mode); + if (adv & ADVERTISED_10baseT_Full) + linkmode_set_bit(ETHTOOL_LINK_MODE_10baseT_Full_BIT, mode); + if (adv & ADVERTISED_100baseT_Half) + linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Half_BIT, mode); + if (adv & ADVERTISED_100baseT_Full) + linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, mode); + if (adv & ADVERTISED_1000baseT_Half) + linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseT_Half_BIT, mode); + if (adv & ADVERTISED_1000baseT_Full) + linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT, mode); + if (adv & ADVERTISED_2500baseX_Full) + linkmode_set_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, mode); + if (adv & RTK_ADVERTISED_5000baseX_Full) + linkmode_set_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, mode); + if (adv & ADVERTISED_10000baseT_Full) + linkmode_set_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, mode); +} + +static int +rtl_ethtool_get_eee(struct net_device *net, struct ethtool_keee *edata) +{ + __ETHTOOL_DECLARE_LINK_MODE_MASK(common); + struct rtl8127_private *tp = netdev_priv(net); + struct ethtool_keee *eee = &tp->eee; + u32 tx_lpi_timer; + u16 val; + + if (unlikely(tp->rtk_enable_diag)) + return -EBUSY; + + /* Get LP advertisement EEE */ + val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D2); + mii_eee_cap1_mod_linkmode_t(edata->lp_advertised, val); + val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA6D0); + mii_eee_cap2_mod_linkmode_sup_t(edata->lp_advertised, val); + + /* Get EEE Tx LPI timer*/ + tx_lpi_timer = rtl8127_device_lpi_t_to_ethtool_lpi_t(tp, eee->tx_lpi_timer); + + val = rtl8127_mac_ocp_read(tp, 0xE040); + val &= BIT_1 | BIT_0; + + edata->eee_enabled = !!val; + linkmode_copy(edata->supported, eee->supported); + linkmode_copy(edata->advertised, eee->advertised); + edata->tx_lpi_enabled = edata->eee_enabled; + edata->tx_lpi_timer = tx_lpi_timer; + linkmode_and(common, edata->advertised, edata->lp_advertised); + edata->eee_active = !linkmode_empty(common); + + return 0; +} + +static int +rtl_ethtool_set_eee(struct net_device *net, struct ethtool_keee *edata) +{ + __ETHTOOL_DECLARE_LINK_MODE_MASK(advertising); + __ETHTOOL_DECLARE_LINK_MODE_MASK(tmp); + struct rtl8127_private *tp = netdev_priv(net); + struct ethtool_keee *eee = &tp->eee; + int rc = 0; + + if (!HW_HAS_WRITE_PHY_MCU_RAM_CODE(tp) || + tp->DASH) + return -EOPNOTSUPP; + + if (unlikely(tp->rtk_enable_diag)) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "Diag Enabled\n"); + rc = -EBUSY; + goto out; + } + + if (tp->autoneg != AUTONEG_ENABLE) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "EEE requires autoneg\n"); + rc = -EINVAL; + goto out; + } + + /* + if (edata->tx_lpi_enabled) { + if (edata->tx_lpi_timer > tp->max_jumbo_frame_size || + edata->tx_lpi_timer < ETH_MIN_MTU) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "Valid LPI timer range is %d to %d. \n", + ETH_MIN_MTU, tp->max_jumbo_frame_size); + rc = -EINVAL; + goto out; + } + } + */ + + rtl8127_adv_to_linkmode(advertising, tp->advertising); + if (linkmode_empty(edata->advertised)) { + linkmode_and(edata->advertised, advertising, eee->supported); + } else if (linkmode_andnot(tmp, edata->advertised, advertising)) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "EEE advertised must be a subset of autoneg advertised speeds\n"); + rc = -EINVAL; + goto out; + } + + if (linkmode_andnot(tmp, edata->advertised, eee->supported)) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "EEE advertised must be a subset of support \n"); + rc = -EINVAL; + goto out; + } + + //tp->eee.eee_enabled = edata->eee_enabled; + //tp->eee_adv_t = rtl8127_ethtool_adv_to_mmd_eee_adv_cap1_t(edata->advertised); + + linkmode_copy(eee->advertised, edata->advertised); + //eee->tx_lpi_enabled = edata->tx_lpi_enabled; + //eee->tx_lpi_timer = edata->tx_lpi_timer; + eee->eee_enabled = edata->eee_enabled; + + if (eee->eee_enabled) + rtl8127_enable_eee(tp); + else + rtl8127_disable_eee(tp); + + rtl_nway_reset(net); + +out: + return rc; +} +#else +static int +rtl_ethtool_get_eee(struct net_device *net, struct ethtool_eee *edata) +{ + struct rtl8127_private *tp = netdev_priv(net); + struct ethtool_eee *eee = &tp->eee; + u32 lp, adv, tx_lpi_timer, supported = 0; + u16 val; + + if (unlikely(tp->rtk_enable_diag)) + return -EBUSY; + + /* Get Supported EEE */ + //val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5C4); + //supported = mmd_eee_cap_to_ethtool_sup_t(val); + supported = eee->supported; + + /* Get advertisement EEE */ + adv = eee->advertised; + + /* Get LP advertisement EEE */ + val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D2); + lp = mmd_eee_adv_to_ethtool_adv_t(val); + val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA6D0); + if (val & RTK_LPA_EEE_ADVERTISE_2500FULL) + lp |= ADVERTISED_2500baseX_Full; + + /* Get EEE Tx LPI timer*/ + tx_lpi_timer = rtl8127_device_lpi_t_to_ethtool_lpi_t(tp, eee->tx_lpi_timer); + + val = rtl8127_mac_ocp_read(tp, 0xE040); + val &= BIT_1 | BIT_0; + + edata->eee_enabled = !!val; + edata->eee_active = !!(supported & adv & lp); + edata->supported = supported; + edata->advertised = adv; + edata->lp_advertised = lp; + edata->tx_lpi_enabled = edata->eee_enabled; + edata->tx_lpi_timer = tx_lpi_timer; + + return 0; +} + +static int +rtl_ethtool_set_eee(struct net_device *net, struct ethtool_eee *edata) +{ + struct rtl8127_private *tp = netdev_priv(net); + struct ethtool_eee *eee = &tp->eee; + u32 advertising; + int rc = 0; + + if (!HW_HAS_WRITE_PHY_MCU_RAM_CODE(tp) || + tp->DASH) + return -EOPNOTSUPP; + + if (unlikely(tp->rtk_enable_diag)) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "Diag Enabled\n"); + rc = -EBUSY; + goto out; + } + + if (tp->autoneg != AUTONEG_ENABLE) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "EEE requires autoneg\n"); + rc = -EINVAL; + goto out; + } + + /* + if (edata->tx_lpi_enabled) { + if (edata->tx_lpi_timer > tp->max_jumbo_frame_size || + edata->tx_lpi_timer < ETH_MIN_MTU) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "Valid LPI timer range is %d to %d. \n", + ETH_MIN_MTU, tp->max_jumbo_frame_size); + rc = -EINVAL; + goto out; + } + } + */ + + advertising = tp->advertising; + if (!edata->advertised) { + edata->advertised = advertising & eee->supported; + } else if (edata->advertised & ~advertising) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "EEE advertised %x must be a subset of autoneg advertised speeds %x\n", + edata->advertised, advertising); + rc = -EINVAL; + goto out; + } + + if (edata->advertised & ~eee->supported) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "EEE advertised %x must be a subset of support %x\n", + edata->advertised, eee->supported); + rc = -EINVAL; + goto out; + } + + //tp->eee.eee_enabled = edata->eee_enabled; + //tp->eee_adv_t = ethtool_adv_to_mmd_eee_adv_t(edata->advertised); + + eee->advertised = edata->advertised; + //eee->tx_lpi_enabled = edata->tx_lpi_enabled; + //eee->tx_lpi_timer = edata->tx_lpi_timer; + eee->eee_enabled = edata->eee_enabled; + + if (eee->eee_enabled) + rtl8127_enable_eee(tp); + else + rtl8127_disable_eee(tp); + + rtl_nway_reset(net); + +out: + return rc; +} +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6,9,0) */ +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0) */ + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0) +static void rtl8127_get_channels(struct net_device *dev, + struct ethtool_channels *channel) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + channel->max_rx = tp->HwSuppNumRxQueues; + channel->max_tx = tp->HwSuppNumTxQueues; + channel->rx_count = tp->num_rx_rings; + channel->tx_count = tp->num_tx_rings; +} +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0) */ + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) +static const struct ethtool_ops rtl8127_ethtool_ops = { + .get_drvinfo = rtl8127_get_drvinfo, + .get_regs_len = rtl8127_get_regs_len, + .get_link = ethtool_op_get_link, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) + .get_ringparam = rtl8127_get_ringparam, + .set_ringparam = rtl8127_set_ringparam, +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) + .get_settings = rtl8127_get_settings, + .set_settings = rtl8127_set_settings, +#else + .get_link_ksettings = rtl8127_get_settings, + .set_link_ksettings = rtl8127_set_settings, +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) + .get_pauseparam = rtl8127_get_pauseparam, + .set_pauseparam = rtl8127_set_pauseparam, +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) + .get_msglevel = rtl8127_get_msglevel, + .set_msglevel = rtl8127_set_msglevel, +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) + .get_rx_csum = rtl8127_get_rx_csum, + .set_rx_csum = rtl8127_set_rx_csum, + .get_tx_csum = rtl8127_get_tx_csum, + .set_tx_csum = rtl8127_set_tx_csum, + .get_sg = ethtool_op_get_sg, + .set_sg = ethtool_op_set_sg, +#ifdef NETIF_F_TSO + .get_tso = ethtool_op_get_tso, + .set_tso = ethtool_op_set_tso, +#endif //NETIF_F_TSO +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) + .get_regs = rtl8127_get_regs, + .get_wol = rtl8127_get_wol, + .set_wol = rtl8127_set_wol, + .get_strings = rtl8127_get_strings, +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,33) + .get_stats_count = rtl8127_get_stats_count, +#else + .get_sset_count = rtl8127_get_sset_count, +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,33) + .get_ethtool_stats = rtl8127_get_ethtool_stats, +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,23) +#ifdef ETHTOOL_GPERMADDR + .get_perm_addr = ethtool_op_get_perm_addr, +#endif //ETHTOOL_GPERMADDR +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,23) + .get_eeprom = rtl_get_eeprom, + .get_eeprom_len = rtl_get_eeprom_len, +#ifdef ENABLE_RSS_SUPPORT + .get_rxnfc = rtl8127_get_rxnfc, + .set_rxnfc = rtl8127_set_rxnfc, + .get_rxfh_indir_size = rtl8127_rss_indir_size, + .get_rxfh_key_size = rtl8127_get_rxfh_key_size, + .get_rxfh = rtl8127_get_rxfh, + .set_rxfh = rtl8127_set_rxfh, +#endif //ENABLE_RSS_SUPPORT +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0) +#ifdef ENABLE_PTP_SUPPORT + .get_ts_info = rtl8127_get_ts_info, +#else + .get_ts_info = ethtool_op_get_ts_info, +#endif //ENABLE_PTP_SUPPORT +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0) + .get_eee = rtl_ethtool_get_eee, + .set_eee = rtl_ethtool_set_eee, +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0) */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0) + .get_channels = rtl8127_get_channels, +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0) */ + .nway_reset = rtl_nway_reset, + +}; +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) + +static void rtl8127_get_mac_version(struct rtl8127_private *tp) +{ + u32 reg,val32; + u32 ICVerID; + + val32 = RTL_R32(tp, TxConfig); + reg = val32 & 0x7c800000; + ICVerID = val32 & 0x00700000; + + switch (reg) { + case 0x6C800000: + if (ICVerID == 0x00000000) { + tp->mcfg = CFG_METHOD_1; + } else if (ICVerID == 0x100000) { + tp->mcfg = CFG_METHOD_2; + } else { + tp->mcfg = CFG_METHOD_2; + tp->HwIcVerUnknown = TRUE; + } + + tp->efuse_ver = EFUSE_SUPPORT_V4; + break; + default: + printk("unknown chip version (%x)\n",reg); + tp->mcfg = CFG_METHOD_DEFAULT; + tp->HwIcVerUnknown = TRUE; + tp->efuse_ver = EFUSE_NOT_SUPPORT; + break; + } +} + +static void +rtl8127_print_mac_version(struct rtl8127_private *tp) +{ + int i; + for (i = ARRAY_SIZE(rtl_chip_info) - 1; i >= 0; i--) { + if (tp->mcfg == rtl_chip_info[i].mcfg) { + dprintk("Realtek %s Ethernet controller mcfg = %04d\n", + MODULENAME, rtl_chip_info[i].mcfg); + return; + } + } + + dprintk("mac_version == Unknown\n"); +} + +static void +rtl8127_tally_counter_addr_fill(struct rtl8127_private *tp) +{ + if (!tp->tally_paddr) + return; + + RTL_W32(tp, CounterAddrHigh, (u64)tp->tally_paddr >> 32); + RTL_W32(tp, CounterAddrLow, (u64)tp->tally_paddr & (DMA_BIT_MASK(32))); +} + +static void +rtl8127_tally_counter_clear(struct rtl8127_private *tp) +{ + if (!tp->tally_paddr) + return; + + RTL_W32(tp, CounterAddrHigh, (u64)tp->tally_paddr >> 32); + RTL_W32(tp, CounterAddrLow, ((u64)tp->tally_paddr & (DMA_BIT_MASK(32))) | CounterReset); +} + +static void +rtl8127_clear_phy_ups_reg(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA466, BIT_0); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA468, BIT_3 | BIT_1); +} + +static int +rtl8127_is_ups_resume(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + return (rtl8127_mac_ocp_read(tp, 0xD42C) & BIT_8); +} + +static void +rtl8127_clear_ups_resume_bit(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + rtl8127_clear_mac_ocp_bit(tp, 0xD42C, BIT_8); +} + +static u8 +rtl8127_get_phy_state(struct rtl8127_private *tp) +{ + return (rtl8127_mdio_direct_read_phy_ocp(tp, 0xA420) & 0x7); +} + +static void +rtl8127_wait_phy_ups_resume(struct net_device *dev, u16 PhyState) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i; + + for (i=0; i< 100; i++) { + if (rtl8127_get_phy_state(tp) == PhyState) + break; + else + mdelay(1); + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18) + WARN_ON_ONCE(i == 100); +#endif +} + +void +rtl8127_enable_now_is_oob(struct rtl8127_private *tp) +{ + if (tp->HwSuppNowIsOobVer == 1) + RTL_W8(tp, MCUCmd_reg, RTL_R8(tp, MCUCmd_reg) | Now_is_oob); +} + +void +rtl8127_disable_now_is_oob(struct rtl8127_private *tp) +{ + if (tp->HwSuppNowIsOobVer == 1) + RTL_W8(tp, MCUCmd_reg, RTL_R8(tp, MCUCmd_reg) & ~Now_is_oob); +} + +static void +rtl8127_exit_oob(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u16 data16; + + rtl8127_disable_rx_packet_filter(tp); + + if (HW_DASH_SUPPORT_DASH(tp)) { + rtl8127_driver_start(tp); + rtl8127_dash2_disable_txrx(dev); +#ifdef ENABLE_DASH_SUPPORT + DashHwInit(dev); +#endif + } + +#ifdef ENABLE_REALWOW_SUPPORT + rtl8127_realwow_hw_init(dev); +#else + rtl8127_mac_ocp_write(tp, 0xC0BC, 0x00FF); +#endif //ENABLE_REALWOW_SUPPORT + + rtl8127_nic_reset(dev); + + rtl8127_disable_now_is_oob(tp); + + data16 = rtl8127_mac_ocp_read(tp, 0xE8DE) & ~BIT_14; + rtl8127_mac_ocp_write(tp, 0xE8DE, data16); + rtl8127_wait_ll_share_fifo_ready(dev); + + rtl8127_mac_ocp_write(tp, 0xC0AA, 0x07D0); +#ifdef ENABLE_LIB_SUPPORT + rtl8127_mac_ocp_write(tp, 0xC0A6, 0x04E2); +#else + rtl8127_mac_ocp_write(tp, 0xC0A6, 0x01B5); +#endif + rtl8127_mac_ocp_write(tp, 0xC01E, 0x5555); + + rtl8127_wait_ll_share_fifo_ready(dev); + + //wait ups resume (phy state 2) + if (rtl8127_is_ups_resume(dev)) { + rtl8127_wait_phy_ups_resume(dev, 2); + rtl8127_clear_ups_resume_bit(dev); + rtl8127_clear_phy_ups_reg(dev); + } +} + +void +rtl8127_hw_disable_mac_mcu_bps(struct net_device *dev) +{ + u16 regAddr; + + struct rtl8127_private *tp = netdev_priv(dev); + + rtl8127_enable_aspm_clkreq_lock(tp, 0); + + rtl8127_mac_ocp_write(tp, 0xFC48, 0x0000); + + for (regAddr = 0xFC28; regAddr < 0xFC48; regAddr += 2) { + rtl8127_mac_ocp_write(tp, regAddr, 0x0000); + } + + fsleep(3000); + + rtl8127_mac_ocp_write(tp, 0xFC26, 0x0000); +} + +#ifndef ENABLE_USE_FIRMWARE_FILE +static void +rtl8127_switch_mac_mcu_ram_code_page(struct rtl8127_private *tp, u16 page) +{ + u16 tmpUshort; + + page &= (BIT_1 | BIT_0); + tmpUshort = rtl8127_mac_ocp_read(tp, 0xE446); + tmpUshort &= ~(BIT_1 | BIT_0); + tmpUshort |= page; + rtl8127_mac_ocp_write(tp, 0xE446, tmpUshort); +} + +static void +_rtl8127_set_hw_mcu_patch_code_ver(struct rtl8127_private *tp, u64 ver) +{ + int i; + + /* Switch to page 2 */ + rtl8127_switch_mac_mcu_ram_code_page(tp, 2); + + for (i = 0; i < 8; i += 2) { + rtl8127_mac_ocp_write(tp, 0xF9F8 + 6 - i, (u16)ver); + ver >>= 16; + } + + /* Switch back to page 0 */ + rtl8127_switch_mac_mcu_ram_code_page(tp, 0); +} + +static void +rtl8127_set_hw_mcu_patch_code_ver(struct rtl8127_private *tp, u64 ver) +{ + _rtl8127_set_hw_mcu_patch_code_ver(tp, ver); + + tp->hw_mcu_patch_code_ver = ver; +} + +static u64 +rtl8127_get_hw_mcu_patch_code_ver(struct rtl8127_private *tp) +{ + u64 ver; + int i; + + /* Switch to page 2 */ + rtl8127_switch_mac_mcu_ram_code_page(tp, 2); + + ver = 0; + for (i = 0; i < 8; i += 2) { + ver <<= 16; + ver |= rtl8127_mac_ocp_read(tp, 0xF9F8 + i); + } + + /* Switch back to page 0 */ + rtl8127_switch_mac_mcu_ram_code_page(tp, 0); + + return ver; +} + +static u64 +rtl8127_get_bin_mcu_patch_code_ver(const u16 *entry, u16 entry_cnt) +{ + u64 ver; + int i; + + if (entry == NULL || entry_cnt == 0 || entry_cnt < 4) + return 0; + + ver = 0; + for (i = 0; i < 4; i++) { + ver <<= 16; + ver |= entry[entry_cnt - 4 + i]; + } + + return ver; +} + +static void +_rtl8127_write_mac_mcu_ram_code(struct rtl8127_private *tp, const u16 *entry, u16 entry_cnt) +{ + u16 i; + + for (i = 0; i < entry_cnt; i++) + rtl8127_mac_ocp_write(tp, 0xF800 + i * 2, entry[i]); +} + +static void +_rtl8127_write_mac_mcu_ram_code_with_page(struct rtl8127_private *tp, const u16 *entry, u16 entry_cnt, u16 page_size) +{ + u16 i; + u16 offset; + + if (page_size == 0) + return; + + for (i = 0; i < entry_cnt; i++) { + offset = i % page_size; + if (offset == 0) { + u16 page = (i / page_size); + rtl8127_switch_mac_mcu_ram_code_page(tp, page); + } + rtl8127_mac_ocp_write(tp, 0xF800 + offset * 2, entry[i]); + } +} + +static void +rtl8127_write_mac_mcu_ram_code(struct rtl8127_private *tp, const u16 *entry, u16 entry_cnt) +{ + if (FALSE == HW_SUPPORT_MAC_MCU(tp)) + return; + + if (entry == NULL || entry_cnt == 0) + return; + + if (tp->MacMcuPageSize > 0) + _rtl8127_write_mac_mcu_ram_code_with_page(tp, entry, entry_cnt, tp->MacMcuPageSize); + else + _rtl8127_write_mac_mcu_ram_code(tp, entry, entry_cnt); + + if (tp->bin_mcu_patch_code_ver > 0) + rtl8127_set_hw_mcu_patch_code_ver(tp, tp->bin_mcu_patch_code_ver); +} + +static void +rtl8127_set_mac_mcu_8127a_tc(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + static const u16 mcu_patch_code[] = { + 0xE010, 0xE019, 0xE01B, 0xE01D, 0xE029, 0xE02C, 0xE0E1, 0xE192, 0xE194, + 0xE196, 0xE198, 0xE19A, 0xE19C, 0xE19E, 0xE1A0, 0xE1A2, 0xC008, 0x7100, + 0x4897, 0x9900, 0xC005, 0xC602, 0xBE00, 0x3D8E, 0xD428, 0xD400, 0xC602, + 0xBE00, 0x0000, 0xC502, 0xBD00, 0x0000, 0x48C3, 0x4847, 0x48C1, 0x8CF8, + 0x74F8, 0x74F8, 0x74F8, 0x4842, 0x8CF8, 0x1E10, 0xC502, 0xBD00, 0x14BA, + 0x1E10, 0xC502, 0xBD00, 0x14EE, 0xC643, 0x76C0, 0x49E1, 0xF13F, 0xC140, + 0x7720, 0x49E0, 0xF003, 0x1B00, 0xE00A, 0x49E2, 0xF003, 0x1B04, 0xE006, + 0x49E4, 0xF003, 0x1B08, 0xE002, 0x1B0C, 0x21B8, 0x1A0E, 0x44DA, 0xE893, + 0x481C, 0xE884, 0xE001, 0x49E0, 0xF003, 0x1B00, 0xE00A, 0x49E2, 0xF003, + 0x1B04, 0xE006, 0x49E4, 0xF003, 0x1B08, 0xE002, 0x1B0C, 0x21B8, 0x1A12, + 0x44DA, 0xE87F, 0x481F, 0xE870, 0xE001, 0x49E0, 0xF003, 0x1B00, 0xE00A, + 0x49E2, 0xF003, 0x1B04, 0xE006, 0x49E4, 0xF003, 0x1B08, 0xE002, 0x1B0C, + 0x21B8, 0x1A1C, 0x44DA, 0xE86B, 0x481F, 0xE85C, 0xE004, 0xE04F, 0xDD98, + 0xD450, 0x49E0, 0xF003, 0x1B00, 0xE00A, 0x49E2, 0xF003, 0x1B04, 0xE006, + 0x49E4, 0xF003, 0x1B08, 0xE002, 0x1B0C, 0x21B8, 0x1A0E, 0x44DA, 0xE854, + 0x489E, 0x481F, 0xE844, 0xE001, 0x1908, 0xE83E, 0x49E0, 0xF003, 0x1B00, + 0xE00A, 0x49E2, 0xF003, 0x1B04, 0xE006, 0x49E4, 0xF003, 0x1B08, 0xE002, + 0x1B0C, 0x21B8, 0x1A8A, 0x44DA, 0xE83D, 0x4813, 0xE82E, 0x49F9, 0xF106, + 0x4838, 0xE837, 0x4813, 0xE828, 0xE001, 0x49E0, 0xF003, 0x1B00, 0xE00A, + 0x49E2, 0xF003, 0x1B04, 0xE006, 0x49E4, 0xF003, 0x1B08, 0xE002, 0x1B0C, + 0x21B8, 0x1A84, 0x44DA, 0xE823, 0x4890, 0x4811, 0xE813, 0x49F9, 0xF106, + 0x4838, 0xE81C, 0x4890, 0x4811, 0xE80C, 0xC207, 0x7440, 0xC602, 0xBE00, + 0x14CC, 0x0FFE, 0xDE20, 0xE092, 0xC3FD, 0xE802, 0xFF80, 0xC0FB, 0x7202, + 0x49AE, 0xF1FE, 0x9900, 0x44D3, 0x4413, 0x482F, 0x9A02, 0x7202, 0x49AE, + 0xF1FE, 0xFF80, 0xC0EE, 0x7202, 0x49AE, 0xF1FE, 0x44D3, 0x4413, 0x48AF, + 0x9A02, 0x7202, 0x49AE, 0xF1FE, 0x7100, 0xFF80, 0xB401, 0xB402, 0xB404, + 0xB407, 0xC61F, 0x76C0, 0x49E1, 0xF164, 0xC11C, 0x7720, 0x1906, 0xE88A, + 0x1B0C, 0x21B8, 0x1A40, 0x44DA, 0xE895, 0x4810, 0xE886, 0x190C, 0xE881, + 0x1B08, 0x21B8, 0x1A26, 0x44DA, 0xE88C, 0x4890, 0x4891, 0xE87C, 0x49F9, + 0xF107, 0x4898, 0x4899, 0xE877, 0xE003, 0xDD98, 0xD450, 0x1908, 0xE86F, + 0x49E0, 0xF003, 0x1B00, 0xE00A, 0x49E2, 0xF003, 0x1B04, 0xE006, 0x49E2, + 0xF003, 0x1B08, 0xE002, 0x1B0C, 0x21B8, 0x1A5C, 0x44DA, 0xE86E, 0x4897, + 0x4898, 0x4819, 0x481A, 0xE85C, 0x49F9, 0xF109, 0x4838, 0xE865, 0x4897, + 0x4898, 0x4819, 0x481A, 0xE853, 0xE001, 0x190A, 0xE84D, 0x1B00, 0xE85B, + 0x44E1, 0x4838, 0xE858, 0x44E9, 0x1908, 0xE845, 0x49E0, 0xF003, 0x1B00, + 0xE00A, 0x49E2, 0xF003, 0x1B04, 0xE006, 0x49E4, 0xF003, 0x1B08, 0xE002, + 0x1B0C, 0x21B8, 0x1A86, 0x44DA, 0xE844, 0x44CC, 0xE835, 0x49F9, 0xF108, + 0x4838, 0xE83E, 0x44CD, 0xE82F, 0xE003, 0xE021, 0xFFC0, 0x190A, 0xE827, + 0x1B00, 0x4839, 0xE834, 0x249A, 0x1C00, 0x44E1, 0x1909, 0xE81F, 0x49E0, + 0xF003, 0x1B00, 0xE00A, 0x49E2, 0xF003, 0x1B04, 0xE006, 0x49E4, 0xF003, + 0x1B08, 0xE002, 0x1B0C, 0x21B8, 0x1A1A, 0x44DA, 0xE81E, 0xC5E4, 0x414D, + 0x418C, 0xE80D, 0xB007, 0xB004, 0xB002, 0xB001, 0xC602, 0xBE00, 0x14B2, + 0x0FFE, 0xDE20, 0xC3FE, 0xE802, 0xFF80, 0xC0FC, 0x7202, 0x49AE, 0xF1FE, + 0x9900, 0x44D3, 0x4413, 0x482F, 0x9A02, 0x7202, 0x49AE, 0xF1FE, 0xFF80, + 0xC0EF, 0x7202, 0x49AE, 0xF1FE, 0x44D3, 0x4413, 0x48AF, 0x9A02, 0x7202, + 0x49AE, 0xF1FE, 0x7100, 0xFF80, 0xC502, 0xBD00, 0x0000, 0xC502, 0xBD00, + 0x0000, 0xC502, 0xBD00, 0x0000, 0xC302, 0xBB00, 0x0000, 0xC602, 0xBE00, + 0x0000, 0xC102, 0xB900, 0x0000, 0xC102, 0xB900, 0x0000, 0xC602, 0xBE00, + 0x0000, 0xC602, 0xBE00, 0x0000, 0x1332, 0x0018, 0x0C05, 0x140D + }; + + /* Get BIN mac mcu patch code version */ + tp->bin_mcu_patch_code_ver = rtl8127_get_bin_mcu_patch_code_ver(mcu_patch_code, ARRAY_SIZE(mcu_patch_code)); + + if (tp->hw_mcu_patch_code_ver != tp->bin_mcu_patch_code_ver) + rtl8127_write_mac_mcu_ram_code(tp, mcu_patch_code, ARRAY_SIZE(mcu_patch_code)); + + rtl8127_mac_ocp_write(tp, 0xFC26, 0x8000); + + rtl8127_mac_ocp_write(tp, 0xFC2E, 0x14B8); + rtl8127_mac_ocp_write(tp, 0xFC30, 0x14EC); + + rtl8127_mac_ocp_write(tp, 0xFC48, 0x0018); +} + +static void +_rtl8127_set_mac_mcu_8127a_1(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + static const u16 mcu_patch_code[] = { + 0xE010, 0xE014, 0xE018, 0xE01C, 0xE020, 0xE033, 0xE035, 0xE037, 0xE039, + 0xE03B, 0xE03D, 0xE03F, 0xE041, 0xE043, 0xE045, 0xE047, 0x7020, 0x4809, + 0xC502, 0xBD00, 0x1522, 0x7760, 0x4879, 0xC002, 0xB800, 0x41E2, 0x7160, + 0x4819, 0xC302, 0xBB00, 0x508E, 0x7720, 0x4879, 0xC102, 0xB900, 0x50F8, + 0x9F86, 0xB400, 0xB401, 0xB402, 0xB403, 0xC00D, 0x7100, 0xC20C, 0x7340, + 0x418B, 0x9900, 0xB003, 0xB002, 0xB001, 0xB000, 0xC702, 0xBF00, 0x3550, + 0xFC48, 0xD482, 0xC602, 0xBE00, 0x0000, 0xC602, 0xBE00, 0x0000, 0xC102, + 0xB900, 0x0000, 0xC302, 0xBB00, 0x0000, 0xC002, 0xB800, 0x0000, 0xC002, + 0xB800, 0x0000, 0xC502, 0xBD00, 0x0000, 0xC102, 0xB900, 0x0000, 0xC102, + 0xB900, 0x0000, 0xC602, 0xBE00, 0x0000, 0xC602, 0xBE00, 0x0000, 0x6961, + 0x0019, 0x0311, 0x1431 + }; + + /* Get BIN mac mcu patch code version */ + tp->bin_mcu_patch_code_ver = rtl8127_get_bin_mcu_patch_code_ver(mcu_patch_code, ARRAY_SIZE(mcu_patch_code)); + + if (tp->hw_mcu_patch_code_ver != tp->bin_mcu_patch_code_ver) + rtl8127_write_mac_mcu_ram_code(tp, mcu_patch_code, ARRAY_SIZE(mcu_patch_code)); + + rtl8127_mac_ocp_write(tp, 0xFC26, 0x8000); + + rtl8127_mac_ocp_write(tp, 0xFC28, 0x1520); + rtl8127_mac_ocp_write(tp, 0xFC2A, 0x41E0); + rtl8127_mac_ocp_write(tp, 0xFC2C, 0x508C); + rtl8127_mac_ocp_write(tp, 0xFC2E, 0x50F6); + rtl8127_mac_ocp_write(tp, 0xFC30, 0x354E); + + rtl8127_mac_ocp_write(tp, 0xFC48, 0x001F); +} + +static void +rtl8127_set_mac_mcu_8127a_1(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u8 tmp = (u8)rtl8127_mac_ocp_read(tp, 0xD006); + + if (tmp != 0x04) + return; + + _rtl8127_set_mac_mcu_8127a_1(dev); +} + +static void +rtl8127_hw_mac_mcu_config(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (tp->NotWrMcuPatchCode == TRUE) + return; + + rtl8127_hw_disable_mac_mcu_bps(dev); + + /* Get H/W mac mcu patch code version */ + tp->hw_mcu_patch_code_ver = rtl8127_get_hw_mcu_patch_code_ver(tp); + + switch (tp->mcfg) { + case CFG_METHOD_1: + rtl8127_set_mac_mcu_8127a_tc(dev); + break; + case CFG_METHOD_2: + rtl8127_set_mac_mcu_8127a_1(dev); + break; + default: + break; + } +} +#endif + +#ifdef ENABLE_USE_FIRMWARE_FILE +static void rtl8127_release_firmware(struct rtl8127_private *tp) +{ + if (tp->rtl_fw) { + rtl8127_fw_release_firmware(tp->rtl_fw); + kfree(tp->rtl_fw); + tp->rtl_fw = NULL; + } +} + +static void rtl8127_apply_firmware(struct rtl8127_private *tp) +{ + /* TODO: release firmware if rtl_fw_write_firmware signals failure. */ + if (tp->rtl_fw) { + rtl8127_fw_write_firmware(tp, tp->rtl_fw); + /* At least one firmware doesn't reset tp->ocp_base. */ + tp->ocp_base = OCP_STD_PHY_BASE; + + /* PHY soft reset may still be in progress */ + //phy_read_poll_timeout(tp->phydev, MII_BMCR, val, + // !(val & BMCR_RESET), + // 50000, 600000, true); + rtl8127_wait_phy_reset_complete(tp); + + tp->hw_ram_code_ver = rtl8127_get_hw_phy_mcu_code_ver(tp); + tp->sw_ram_code_ver = tp->hw_ram_code_ver; + tp->HwHasWrRamCodeToMicroP = TRUE; + } +} +#endif + +static void +rtl8127_hw_init(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u32 csi_tmp; + + rtl8127_enable_aspm_clkreq_lock(tp, 0); + rtl8127_enable_force_clkreq(tp, 0); + + //Disable UPS + rtl8127_mac_ocp_write(tp, 0xD40A, rtl8127_mac_ocp_read(tp, 0xD40A) & ~(BIT_4)); + +#ifndef ENABLE_USE_FIRMWARE_FILE + if (!tp->rtl_fw) + rtl8127_hw_mac_mcu_config(dev); +#endif + + //Set PCIE uncorrectable error status mask pcie 0x108 + csi_tmp = rtl8127_csi_read(tp, 0x108); + csi_tmp |= BIT_20; + rtl8127_csi_write(tp, 0x108, csi_tmp); + + rtl8127_enable_cfg9346_write(tp); + rtl8127_disable_linkchg_wakeup(dev); + rtl8127_disable_cfg9346_write(tp); + rtl8127_disable_magic_packet(dev); + rtl8127_disable_d0_speedup(tp); + rtl8127_set_pci_pme(tp, 0); + if (s0_magic_packet == 1) + rtl8127_enable_magic_packet(dev); + +#ifdef ENABLE_USE_FIRMWARE_FILE + if (tp->rtl_fw && + !tp->resume_not_chg_speed && + !(HW_DASH_SUPPORT_TYPE_3(tp) && + tp->HwPkgDet == 0x06)) + rtl8127_apply_firmware(tp); +#endif +} + +static void +rtl8127_clear_ephy_ext_addr(struct rtl8127_private *tp) +{ + rtl8127_set_ephy_ext_addr(tp, 0x0000); +} + +static void +rtl8127_hw_ephy_config_8127_1(struct rtl8127_private *tp) +{ + rtl8127_ephy_write(tp, 0x8088, 0x0064); + rtl8127_ephy_write(tp, 0x8488, 0x0064); + rtl8127_ephy_write(tp, 0x8888, 0x0064); + rtl8127_ephy_write(tp, 0x8C88, 0x0064); + rtl8127_ephy_write(tp, 0x8188, 0x0064); + rtl8127_ephy_write(tp, 0x8588, 0x0064); + rtl8127_ephy_write(tp, 0x8988, 0x0064); + rtl8127_ephy_write(tp, 0x8D88, 0x0064); + rtl8127_ephy_write(tp, 0x808C, 0x09B0); + rtl8127_ephy_write(tp, 0x848C, 0x09B0); + rtl8127_ephy_write(tp, 0x888C, 0x0F90); + rtl8127_ephy_write(tp, 0x8C8C, 0x0F90); + rtl8127_ephy_write(tp, 0x818C, 0x09B0); + rtl8127_ephy_write(tp, 0x858C, 0x09B0); + rtl8127_ephy_write(tp, 0x898C, 0x0F90); + rtl8127_ephy_write(tp, 0x8D8C, 0x0F90); + rtl8127_ephy_write(tp, 0x808A, 0x09B8); + rtl8127_ephy_write(tp, 0x848A, 0x09B8); + rtl8127_ephy_write(tp, 0x888A, 0x0F98); + rtl8127_ephy_write(tp, 0x8C8A, 0x0F98); + rtl8127_ephy_write(tp, 0x818A, 0x09B8); + rtl8127_ephy_write(tp, 0x858A, 0x09B8); + rtl8127_ephy_write(tp, 0x898A, 0x0F98); + rtl8127_ephy_write(tp, 0x8D8A, 0x0F98); + rtl8127_ephy_write(tp, 0x9020, 0x0080); + rtl8127_ephy_write(tp, 0x9420, 0x0080); + rtl8127_ephy_write(tp, 0x9820, 0x0080); + rtl8127_ephy_write(tp, 0x9C20, 0x0080); + rtl8127_ephy_write(tp, 0x901E, 0x0190); + rtl8127_ephy_write(tp, 0x941E, 0x0190); + rtl8127_ephy_write(tp, 0x981E, 0x0140); + rtl8127_ephy_write(tp, 0x9C1E, 0x0140); + rtl8127_ephy_write(tp, 0x901C, 0x0190); + rtl8127_ephy_write(tp, 0x941C, 0x0190); + rtl8127_ephy_write(tp, 0x981C, 0x0140); + rtl8127_ephy_write(tp, 0x9C1C, 0x0140); + + /* Clear extended address */ + rtl8127_clear_ephy_ext_addr(tp); +} + +static void +rtl8127_hw_ephy_config(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + switch (tp->mcfg) { + case CFG_METHOD_2: + rtl8127_hw_ephy_config_8127_1(tp); + break; + default: + /* nothing to do */ + break; + } +} + +static u16 +rtl8127_get_hw_phy_mcu_code_ver(struct rtl8127_private *tp) +{ + u16 hw_ram_code_ver; + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x801E); + hw_ram_code_ver = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA438); + + return hw_ram_code_ver; +} + +static int +rtl8127_check_hw_phy_mcu_code_ver(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int ram_code_ver_match = 0; + + tp->hw_ram_code_ver = rtl8127_get_hw_phy_mcu_code_ver(tp); + + if (tp->hw_ram_code_ver == tp->sw_ram_code_ver) { + ram_code_ver_match = 1; + tp->HwHasWrRamCodeToMicroP = TRUE; + } + + return ram_code_ver_match; +} + +bool +rtl8127_set_phy_mcu_patch_request(struct rtl8127_private *tp) +{ + u16 gphy_val; + u16 WaitCount; + bool bSuccess = TRUE; + + rtl8127_set_eth_phy_ocp_bit(tp, 0xB820, BIT_4); + + WaitCount = 0; + do { + gphy_val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xB800); + udelay(100); + WaitCount++; + } while (!(gphy_val & BIT_6) && (WaitCount < 1000)); + + if (!(gphy_val & BIT_6) && (WaitCount == 1000)) + bSuccess = FALSE; + + if (!bSuccess) + dprintk("rtl8127_set_phy_mcu_patch_request fail.\n"); + + return bSuccess; +} + +bool +rtl8127_clear_phy_mcu_patch_request(struct rtl8127_private *tp) +{ + u16 gphy_val; + u16 WaitCount; + bool bSuccess = TRUE; + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xB820, BIT_4); + + WaitCount = 0; + do { + gphy_val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xB800); + udelay(100); + WaitCount++; + } while ((gphy_val & BIT_6) && (WaitCount < 1000)); + + if ((gphy_val & BIT_6) && (WaitCount == 1000)) + bSuccess = FALSE; + + if (!bSuccess) + dprintk("rtl8127_clear_phy_mcu_patch_request fail.\n"); + + return bSuccess; +} + +#ifndef ENABLE_USE_FIRMWARE_FILE +static void +rtl8127_write_hw_phy_mcu_code_ver(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x801E); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, tp->sw_ram_code_ver); + tp->hw_ram_code_ver = tp->sw_ram_code_ver; +} + +static void +rtl8127_set_phy_mcu_ram_code(struct net_device *dev, const u16 *ramcode, u16 codesize) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u16 i; + u16 addr; + u16 val; + + if (ramcode == NULL || codesize % 2) + goto out; + + for (i = 0; i < codesize; i += 2) { + addr = ramcode[i]; + val = ramcode[i + 1]; + if (addr == 0xFFFF && val == 0xFFFF) + break; + rtl8127_mdio_direct_write_phy_ocp(tp, addr, val); + } + +out: + return; +} + +static void +rtl8127_enable_phy_disable_mode(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + switch (tp->HwSuppCheckPhyDisableModeVer) { + case 3: + RTL_W8(tp, 0xF2, RTL_R8(tp, 0xF2) | BIT_5); + break; + } + + dprintk("enable phy disable mode.\n"); +} + +static void +rtl8127_disable_phy_disable_mode(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + switch (tp->HwSuppCheckPhyDisableModeVer) { + case 3: + RTL_W8(tp, 0xF2, RTL_R8(tp, 0xF2) & ~BIT_5); + break; + } + + mdelay(1); + + dprintk("disable phy disable mode.\n"); +} + +static const u16 phy_mcu_ram_code_8127a_tc_1[] = { + 0xa436, 0x8023, 0xa438, 0x3200, 0xa436, 0xB82E, 0xa438, 0x0001, + 0xBF8E, 0x1410, 0xBF8E, 0x1410, 0xBF90, 0xC20E, 0xBF9E, 0xBFFC, + 0xBFAA, 0x1924, 0xBFB4, 0x0B1E, 0xBFB6, 0x3740, 0xBFB8, 0x460E, + 0xBFBE, 0x000D, 0xBF8A, 0x3FF7, 0xBF9A, 0x0007, 0xBF1E, 0x01FF, + 0xBF1E, 0x01FF, 0xBF1E, 0x01FF, 0xBF2E, 0x454D, 0xbc10, 0xD50C, + 0xbc10, 0x950C, 0xbc10, 0xD50C, 0xbc10, 0x550C, 0xbddE, 0xEF00, + 0xbd32, 0xF000, 0xbd2C, 0x0800, 0xbdc8, 0x04B0, 0xbdc8, 0x0CB0, + 0xBD92, 0x0003, 0xBD94, 0x8000, 0xBD96, 0x000f, 0xBD96, 0x0000, + 0xBD92, 0x0016, 0xBD94, 0x1000, 0xBD96, 0x000f, 0xBD96, 0x0000, + 0xBD92, 0x0016, 0xBD94, 0x1200, 0xBD96, 0x000f, 0xBD96, 0x0000, + 0xBD92, 0x0000, 0xBD94, 0x1F1F, 0xBD96, 0x000f, 0xBD96, 0x0000, + 0xBD92, 0x0001, 0xBD94, 0x1F00, 0xBD96, 0x000f, 0xBD96, 0x0000, + 0xBD92, 0x0003, 0xBD94, 0x0000, 0xBD96, 0x000f, 0xBD96, 0x0000, + 0xBD92, 0x0016, 0xBD94, 0x0000, 0xBD96, 0x000f, 0xBD96, 0x0000, + 0xBD92, 0x0000, 0xBD94, 0x0000, 0xBD96, 0x000f, 0xBD96, 0x0000, + 0xbdc8, 0x08B0, 0xbdc8, 0x00B0, 0xbd32, 0x0000, 0xbd2C, 0x0000, + 0xbc10, 0x750C, 0xbc10, 0x650C, 0xbc10, 0x750C, 0xbc10, 0x550C, + 0xb820, 0x0090, 0xa436, 0xA016, 0xa438, 0x0000, 0xa436, 0xA012, + 0xa438, 0x0000, 0xa436, 0xA014, 0xa438, 0x1800, 0xa438, 0x8010, + 0xa438, 0x1800, 0xa438, 0x809b, 0xa438, 0x1800, 0xa438, 0x8145, + 0xa438, 0x1800, 0xa438, 0x8197, 0xa438, 0x1800, 0xa438, 0x81d4, + 0xa438, 0x1800, 0xa438, 0x8214, 0xa438, 0x1800, 0xa438, 0x8226, + 0xa438, 0x1800, 0xa438, 0x8232, 0xa438, 0xd707, 0xa438, 0x4141, + 0xa438, 0xd70a, 0xa438, 0x4115, 0xa438, 0xd705, 0xa438, 0x40da, + 0xa438, 0xb808, 0xa438, 0xd028, 0xa438, 0xd1c1, 0xa438, 0x1800, + 0xa438, 0x801e, 0xa438, 0x9808, 0xa438, 0xd07b, 0xa438, 0xd1c5, + 0xa438, 0xbe10, 0xa438, 0xd503, 0xa438, 0xa108, 0xa438, 0xd505, + 0xa438, 0x8103, 0xa438, 0xd504, 0xa438, 0xa002, 0xa438, 0xa302, + 0xa438, 0xd707, 0xa438, 0x4061, 0xa438, 0xd503, 0xa438, 0x8b01, + 0xa438, 0xd500, 0xa438, 0xc48a, 0xa438, 0xd503, 0xa438, 0xcc09, + 0xa438, 0xcd58, 0xa438, 0xaf01, 0xa438, 0xd500, 0xa438, 0x1000, + 0xa438, 0x1764, 0xa438, 0xd719, 0xa438, 0x606c, 0xa438, 0xd704, + 0xa438, 0x645c, 0xa438, 0xd75e, 0xa438, 0x604d, 0xa438, 0xfff8, + 0xa438, 0x9e10, 0xa438, 0x1000, 0xa438, 0x1764, 0xa438, 0xd719, + 0xa438, 0x606c, 0xa438, 0xd704, 0xa438, 0x631c, 0xa438, 0xd75e, + 0xa438, 0x404d, 0xa438, 0xfff8, 0xa438, 0xd504, 0xa438, 0xaa18, + 0xa438, 0xa001, 0xa438, 0xa1e0, 0xa438, 0xd500, 0xa438, 0x1000, + 0xa438, 0x1764, 0xa438, 0xd719, 0xa438, 0x7fac, 0xa438, 0xd504, + 0xa438, 0xa001, 0xa438, 0xd500, 0xa438, 0x1000, 0xa438, 0x1764, + 0xa438, 0xd704, 0xa438, 0x5f5c, 0xa438, 0xd719, 0xa438, 0x3aaf, + 0xa438, 0x8058, 0xa438, 0xf016, 0xa438, 0xd707, 0xa438, 0x6121, + 0xa438, 0x1000, 0xa438, 0x16f8, 0xa438, 0xd503, 0xa438, 0xcd59, + 0xa438, 0xaf01, 0xa438, 0xd500, 0xa438, 0x1800, 0xa438, 0x0e49, + 0xa438, 0xd503, 0xa438, 0x8040, 0xa438, 0xd500, 0xa438, 0x1000, + 0xa438, 0x16f8, 0xa438, 0xd503, 0xa438, 0xcd5a, 0xa438, 0xaf01, + 0xa438, 0xd500, 0xa438, 0x1800, 0xa438, 0x0e2f, 0xa438, 0xd504, + 0xa438, 0xa008, 0xa438, 0xa204, 0xa438, 0xd500, 0xa438, 0x1000, + 0xa438, 0x1764, 0xa438, 0xd701, 0xa438, 0x5fa0, 0xa438, 0xd503, + 0xa438, 0xa082, 0xa438, 0xd500, 0xa438, 0xd71e, 0xa438, 0x4097, + 0xa438, 0xd078, 0xa438, 0xd1aa, 0xa438, 0xf003, 0xa438, 0xd078, + 0xa438, 0xd1aa, 0xa438, 0xd707, 0xa438, 0x4081, 0xa438, 0xd70a, + 0xa438, 0x4055, 0xa438, 0xf014, 0xa438, 0xd706, 0xa438, 0x6065, + 0xa438, 0xcc89, 0xa438, 0xf002, 0xa438, 0xcc8b, 0xa438, 0x1000, + 0xa438, 0x0bb2, 0xa438, 0xd705, 0xa438, 0x2ad0, 0xa438, 0x808f, + 0xa438, 0xf003, 0xa438, 0x1000, 0xa438, 0x0bb8, 0xa438, 0x1000, + 0xa438, 0x0bbe, 0xa438, 0x607a, 0xa438, 0x9c01, 0xa438, 0xf002, + 0xa438, 0xbc01, 0xa438, 0x1000, 0xa438, 0x0cc9, 0xa438, 0x1800, + 0xa438, 0x132f, 0xa438, 0x9a10, 0xa438, 0x9d02, 0xa438, 0xd706, + 0xa438, 0x629a, 0xa438, 0x61bb, 0xa438, 0xd707, 0xa438, 0x60d7, + 0xa438, 0xd70a, 0xa438, 0x6196, 0xa438, 0xd0e5, 0xa438, 0xd1e7, + 0xa438, 0xf012, 0xa438, 0xd70a, 0xa438, 0x5f55, 0xa438, 0xd060, + 0xa438, 0xd1e8, 0xa438, 0xf00d, 0xa438, 0xd056, 0xa438, 0xd1e8, + 0xa438, 0xf00a, 0xa438, 0xd043, 0xa438, 0xd1e8, 0xa438, 0xf007, + 0xa438, 0x609b, 0xa438, 0xd078, 0xa438, 0xd1e9, 0xa438, 0xf003, + 0xa438, 0xd07f, 0xa438, 0xd1e9, 0xa438, 0xd503, 0xa438, 0xab01, + 0xa438, 0xd500, 0xa438, 0xd706, 0xa438, 0x6139, 0xa438, 0xd503, + 0xa438, 0x6065, 0xa438, 0xa7f0, 0xa438, 0xf003, 0xa438, 0x0cf0, + 0xa438, 0x0750, 0xa438, 0x8908, 0xa438, 0xf004, 0xa438, 0xd503, + 0xa438, 0xa908, 0xa438, 0x87f0, 0xa438, 0xd503, 0xa438, 0xa040, + 0xa438, 0xd500, 0xa438, 0xd705, 0xa438, 0x407b, 0xa438, 0x1000, + 0xa438, 0x1ac8, 0xa438, 0xd503, 0xa438, 0x0c07, 0xa438, 0x0902, + 0xa438, 0xa008, 0xa438, 0xd500, 0xa438, 0x9d80, 0xa438, 0xc48c, + 0xa438, 0xd73e, 0xa438, 0x6000, 0xa438, 0xd706, 0xa438, 0x419b, + 0xa438, 0xd503, 0xa438, 0x0c87, 0xa438, 0x0981, 0xa438, 0xd500, + 0xa438, 0xd073, 0xa438, 0xd1b7, 0xa438, 0xc490, 0xa438, 0x1000, + 0xa438, 0x1764, 0xa438, 0xd704, 0xa438, 0x5fbb, 0xa438, 0xd503, + 0xa438, 0xd706, 0xa438, 0x607a, 0xa438, 0x617c, 0xa438, 0xf013, + 0xa438, 0xd70b, 0xa438, 0x40b4, 0xa438, 0xd706, 0xa438, 0x6309, + 0xa438, 0x6348, 0xa438, 0xf025, 0xa438, 0xd706, 0xa438, 0x6408, + 0xa438, 0xf022, 0xa438, 0xd70b, 0xa438, 0x40b4, 0xa438, 0xd706, + 0xa438, 0x6429, 0xa438, 0x6468, 0xa438, 0xf028, 0xa438, 0xd706, + 0xa438, 0x6468, 0xa438, 0xf025, 0xa438, 0xd70b, 0xa438, 0x67d4, + 0xa438, 0xd706, 0xa438, 0x6488, 0xa438, 0x6589, 0xa438, 0x2c69, + 0xa438, 0x8135, 0xa438, 0x66ab, 0xa438, 0xf037, 0xa438, 0xc320, + 0xa438, 0xc420, 0xa438, 0xf03c, 0xa438, 0xd707, 0xa438, 0x6077, + 0xa438, 0xc310, 0xa438, 0xf004, 0xa438, 0xd70a, 0xa438, 0x5fb5, + 0xa438, 0xc340, 0xa438, 0xc410, 0xa438, 0xf033, 0xa438, 0xc340, + 0xa438, 0xc420, 0xa438, 0xf030, 0xa438, 0xc340, 0xa438, 0xc440, + 0xa438, 0xf02d, 0xa438, 0xc304, 0xa438, 0xc408, 0xa438, 0xf02a, + 0xa438, 0xc302, 0xa438, 0xc404, 0xa438, 0xf027, 0xa438, 0xc308, + 0xa438, 0xc408, 0xa438, 0xf024, 0xa438, 0xc308, 0xa438, 0xc410, + 0xa438, 0xf021, 0xa438, 0xd707, 0xa438, 0x6077, 0xa438, 0xc304, + 0xa438, 0xf004, 0xa438, 0xd70a, 0xa438, 0x5fb5, 0xa438, 0xc308, + 0xa438, 0xc408, 0xa438, 0xf018, 0xa438, 0xd707, 0xa438, 0x6077, + 0xa438, 0xc308, 0xa438, 0xf004, 0xa438, 0xd70a, 0xa438, 0x5fb5, + 0xa438, 0xc310, 0xa438, 0xc410, 0xa438, 0xf00f, 0xa438, 0xc304, + 0xa438, 0xc408, 0xa438, 0xf00c, 0xa438, 0xc302, 0xa438, 0xc404, + 0xa438, 0xf009, 0xa438, 0xd707, 0xa438, 0x6077, 0xa438, 0xc308, + 0xa438, 0xf004, 0xa438, 0xd70a, 0xa438, 0x5fb5, 0xa438, 0xc310, + 0xa438, 0xc410, 0xa438, 0x1800, 0xa438, 0x14e7, 0xa438, 0xd07b, + 0xa438, 0xd1c5, 0xa438, 0xd503, 0xa438, 0xa028, 0xa438, 0x8970, + 0xa438, 0x880f, 0xa438, 0x0c0f, 0xa438, 0x0909, 0xa438, 0x87f0, + 0xa438, 0xc600, 0xa438, 0xa521, 0xa438, 0xd501, 0xa438, 0xce01, + 0xa438, 0xa202, 0xa438, 0xa201, 0xa438, 0x8201, 0xa438, 0xce00, + 0xa438, 0xd500, 0xa438, 0xd706, 0xa438, 0x4425, 0xa438, 0xd503, + 0xa438, 0xab80, 0xa438, 0xd500, 0xa438, 0x1000, 0xa438, 0x0c1d, + 0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0xaac0, 0xa438, 0xd00d, + 0xa438, 0xd1a2, 0xa438, 0xd700, 0xa438, 0x401a, 0xa438, 0xa60c, + 0xa438, 0xd010, 0xa438, 0xd1a2, 0xa438, 0xd700, 0xa438, 0x401a, + 0xa438, 0xa70c, 0xa438, 0xd09e, 0xa438, 0xd1a2, 0xa438, 0xd700, + 0xa438, 0x401a, 0xa438, 0xce00, 0xa438, 0xd500, 0xa438, 0xd505, + 0xa438, 0xab01, 0xa438, 0xd500, 0xa438, 0x1000, 0xa438, 0x0c30, + 0xa438, 0xd505, 0xa438, 0x8b01, 0xa438, 0xd500, 0xa438, 0xbc02, + 0xa438, 0xc482, 0xa438, 0xd503, 0xa438, 0xcc01, 0xa438, 0xcd0c, + 0xa438, 0xaf01, 0xa438, 0xd500, 0xa438, 0xd75e, 0xa438, 0x4000, + 0xa438, 0xd706, 0xa438, 0x4245, 0xa438, 0x1000, 0xa438, 0x0c1d, + 0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0x8ac0, 0xa438, 0x860c, + 0xa438, 0x870c, 0xa438, 0xce00, 0xa438, 0xd500, 0xa438, 0xd505, + 0xa438, 0xab01, 0xa438, 0xd500, 0xa438, 0x1000, 0xa438, 0x0c30, + 0xa438, 0xd505, 0xa438, 0x8b01, 0xa438, 0xd500, 0xa438, 0x1800, + 0xa438, 0x0e74, 0xa438, 0xd707, 0xa438, 0x43cf, 0xa438, 0x1000, + 0xa438, 0x0c1d, 0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0xaac0, + 0xa438, 0xd00d, 0xa438, 0xd1a2, 0xa438, 0xd700, 0xa438, 0x401a, + 0xa438, 0xa60c, 0xa438, 0xd010, 0xa438, 0xd1a2, 0xa438, 0xd700, + 0xa438, 0x401a, 0xa438, 0xa70c, 0xa438, 0xd09e, 0xa438, 0xd1a2, + 0xa438, 0xd700, 0xa438, 0x401a, 0xa438, 0xce00, 0xa438, 0xd500, + 0xa438, 0xd505, 0xa438, 0xab01, 0xa438, 0xd500, 0xa438, 0x1000, + 0xa438, 0x0c30, 0xa438, 0xd505, 0xa438, 0x8b01, 0xa438, 0xd500, + 0xa438, 0xd501, 0xa438, 0xce00, 0xa438, 0xab10, 0xa438, 0xbb10, + 0xa438, 0x9c02, 0xa438, 0xd75e, 0xa438, 0x6000, 0xa438, 0xd707, + 0xa438, 0x424f, 0xa438, 0x1000, 0xa438, 0x0c1d, 0xa438, 0xd501, + 0xa438, 0xce01, 0xa438, 0x8ac0, 0xa438, 0x860c, 0xa438, 0x870c, + 0xa438, 0xce00, 0xa438, 0xd500, 0xa438, 0xd505, 0xa438, 0xab01, + 0xa438, 0xd500, 0xa438, 0x1000, 0xa438, 0x0c30, 0xa438, 0xd505, + 0xa438, 0x8b01, 0xa438, 0xd500, 0xa438, 0x9808, 0xa438, 0x9c01, + 0xa438, 0x1800, 0xa438, 0x172a, 0xa438, 0xd707, 0xa438, 0x43cf, + 0xa438, 0x1000, 0xa438, 0x0c1d, 0xa438, 0xd501, 0xa438, 0xce01, + 0xa438, 0xaac0, 0xa438, 0xd00d, 0xa438, 0xd1a2, 0xa438, 0xd700, + 0xa438, 0x401a, 0xa438, 0xa60c, 0xa438, 0xd010, 0xa438, 0xd1a2, + 0xa438, 0xd700, 0xa438, 0x401a, 0xa438, 0xa70c, 0xa438, 0xd09e, + 0xa438, 0xd1a2, 0xa438, 0xd700, 0xa438, 0x401a, 0xa438, 0xce00, + 0xa438, 0xd500, 0xa438, 0xd505, 0xa438, 0xab01, 0xa438, 0xd500, + 0xa438, 0x1000, 0xa438, 0x0c30, 0xa438, 0xd505, 0xa438, 0x8b01, + 0xa438, 0xd500, 0xa438, 0x1000, 0xa438, 0x1a6b, 0xa438, 0x9808, + 0xa438, 0x9c01, 0xa438, 0xd707, 0xa438, 0x424f, 0xa438, 0x1000, + 0xa438, 0x0c1d, 0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0x8ac0, + 0xa438, 0x860c, 0xa438, 0x870c, 0xa438, 0xce00, 0xa438, 0xd500, + 0xa438, 0xd505, 0xa438, 0xab01, 0xa438, 0xd500, 0xa438, 0x1000, + 0xa438, 0x0c30, 0xa438, 0xd505, 0xa438, 0x8b01, 0xa438, 0xd500, + 0xa438, 0xd503, 0xa438, 0x8b80, 0xa438, 0xd500, 0xa438, 0x1000, + 0xa438, 0x1aa5, 0xa438, 0xd503, 0xa438, 0xcda3, 0xa438, 0xaf01, + 0xa438, 0x1800, 0xa438, 0x181c, 0xa438, 0xd700, 0xa438, 0x41f6, + 0xa438, 0xd703, 0xa438, 0x41a3, 0xa438, 0x1000, 0xa438, 0x0c1d, + 0xa438, 0xd501, 0xa438, 0xa580, 0xa438, 0xa701, 0xa438, 0xd500, + 0xa438, 0xd014, 0xa438, 0xd1c3, 0xa438, 0xd703, 0xa438, 0x401c, + 0xa438, 0x1000, 0xa438, 0x0c30, 0xa438, 0x1800, 0xa438, 0x069b, + 0xa438, 0xba10, 0xa438, 0xd70c, 0xa438, 0x4107, 0xa438, 0xd702, + 0xa438, 0x40d0, 0xa438, 0xd504, 0xa438, 0xa110, 0xa438, 0xd500, + 0xa438, 0x1800, 0xa438, 0x1447, 0xa438, 0x1800, 0xa438, 0x1420, + 0xa438, 0xd70c, 0xa438, 0x60a6, 0xa438, 0xd501, 0xa438, 0xce01, + 0xa438, 0x840f, 0xa438, 0xce00, 0xa438, 0xd503, 0xa438, 0x8008, + 0xa438, 0xd500, 0xa438, 0x1800, 0xa438, 0x1360, 0xa436, 0xA026, + 0xa438, 0x1359, 0xa436, 0xA024, 0xa438, 0x141f, 0xa436, 0xA022, + 0xa438, 0x068f, 0xa436, 0xA020, 0xa438, 0x1815, 0xa436, 0xA006, + 0xa438, 0x1723, 0xa436, 0xA004, 0xa438, 0x0e59, 0xa436, 0xA002, + 0xa438, 0x1452, 0xa436, 0xA000, 0xa438, 0x12bb, 0xa436, 0xA008, + 0xa438, 0xff00, 0xa436, 0xA016, 0xa438, 0x0010, 0xa436, 0xA012, + 0xa438, 0x0000, 0xa436, 0xA014, 0xa438, 0x1800, 0xa438, 0x8010, + 0xa438, 0x1800, 0xa438, 0x8010, 0xa438, 0x1800, 0xa438, 0x8019, + 0xa438, 0x1800, 0xa438, 0x8019, 0xa438, 0x1800, 0xa438, 0x8019, + 0xa438, 0x1800, 0xa438, 0x8019, 0xa438, 0x1800, 0xa438, 0x8019, + 0xa438, 0x1800, 0xa438, 0x8019, 0xa438, 0xd700, 0xa438, 0x2a59, + 0xa438, 0x0101, 0xa438, 0x2841, 0xa438, 0x0122, 0xa438, 0x2d69, + 0xa438, 0x00e6, 0xa438, 0x1800, 0xa438, 0x00e0, 0xa436, 0xA08E, + 0xa438, 0x0000, 0xa436, 0xA08C, 0xa438, 0x0000, 0xa436, 0xA08A, + 0xa438, 0x0000, 0xa436, 0xA088, 0xa438, 0x0000, 0xa436, 0xA086, + 0xa438, 0x0000, 0xa436, 0xA084, 0xa438, 0x0000, 0xa436, 0xA082, + 0xa438, 0x00e2, 0xa436, 0xA080, 0xa438, 0x0000, 0xa436, 0xA090, + 0xa438, 0x0002, 0xa436, 0xA016, 0xa438, 0x0020, 0xa436, 0xA012, + 0xa438, 0x0000, 0xa436, 0xA014, 0xa438, 0x1800, 0xa438, 0x8010, + 0xa438, 0x1800, 0xa438, 0x9dfb, 0xa438, 0x1800, 0xa438, 0x9e02, + 0xa438, 0x1800, 0xa438, 0x9e02, 0xa438, 0x1800, 0xa438, 0x9e02, + 0xa438, 0x1800, 0xa438, 0x9e02, 0xa438, 0x1800, 0xa438, 0x9e02, + 0xa438, 0x1800, 0xa438, 0x9e02, 0xa438, 0xd71f, 0xa438, 0x626d, + 0xa438, 0xd71e, 0xa438, 0x4103, 0xa438, 0xa70c, 0xa438, 0xb801, + 0xa438, 0xba04, 0xa438, 0xd71f, 0xa438, 0x6001, 0xa438, 0x1800, + 0xa438, 0x001c, 0xa438, 0xb801, 0xa438, 0xd704, 0xa438, 0x2211, + 0xa438, 0x9d54, 0xa438, 0xd700, 0xa438, 0x2739, 0xa438, 0x837a, + 0xa438, 0x1800, 0xa438, 0x814b, 0xa438, 0xcd70, 0xa438, 0xb801, + 0xa438, 0xa708, 0xa438, 0xd700, 0xa438, 0x60d7, 0xa438, 0x6073, + 0xa438, 0xce02, 0xa438, 0xf004, 0xa438, 0xce01, 0xa438, 0xf002, + 0xa438, 0xce01, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0ccf, 0xa438, 0x0b04, 0xa438, 0xcc22, + 0xa438, 0xcd01, 0xa438, 0xa702, 0xa438, 0x9503, 0xa438, 0xa501, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fbd, + 0xa438, 0x8501, 0xa438, 0x1000, 0xa438, 0x9c02, 0xa438, 0xa610, + 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xcd71, 0xa438, 0xa501, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fbd, + 0xa438, 0x8501, 0xa438, 0xd707, 0xa438, 0x61cf, 0xa438, 0xa502, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd702, 0xa438, 0x5fbe, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, + 0xa438, 0xd403, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x8502, + 0xa438, 0xa340, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108, + 0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000, + 0xa438, 0x9c5b, 0xa438, 0xa304, 0xa438, 0xa440, 0xa438, 0xa8c0, + 0xa438, 0xa2fc, 0xa438, 0xa120, 0xa438, 0x0ca0, 0xa438, 0x0480, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa370, 0xa438, 0x9503, + 0xa438, 0xcd72, 0xa438, 0xd1f5, 0xa438, 0xd057, 0xa438, 0xd1c4, + 0xa438, 0xd066, 0xa438, 0xd1c4, 0xa438, 0xd077, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x607c, 0xa438, 0x613d, + 0xa438, 0xfffb, 0xa438, 0xa310, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fbd, 0xa438, 0xa607, 0xa438, 0xf007, + 0xa438, 0xa607, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fbc, 0xa438, 0xa310, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fbb, 0xa438, 0x8840, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0cf8, 0xa438, 0x0d48, 0xa438, 0x8320, + 0xa438, 0xa180, 0xa438, 0x9503, 0xa438, 0xd1c4, 0xa438, 0xd055, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fbb, + 0xa438, 0xd706, 0xa438, 0x6227, 0xa438, 0xd700, 0xa438, 0x5f3a, + 0xa438, 0x88c0, 0xa438, 0x82fc, 0xa438, 0x8120, 0xa438, 0x8350, + 0xa438, 0x84a0, 0xa438, 0x8607, 0xa438, 0xa510, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8df8, 0xa438, 0x8370, 0xa438, 0x8180, + 0xa438, 0x9503, 0xa438, 0xff96, 0xa438, 0x8510, 0xa438, 0xa508, + 0xa438, 0x8508, 0xa438, 0xcd73, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x9920, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x6065, + 0xa438, 0x5f94, 0xa438, 0xffdf, 0xa438, 0xb820, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, + 0xa438, 0x8120, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108, + 0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000, + 0xa438, 0x9c5b, 0xa438, 0xa304, 0xa438, 0x8880, 0xa438, 0x8480, + 0xa438, 0x8606, 0xa438, 0xcd74, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8b0f, 0xa438, 0x8c3f, 0xa438, 0x9503, 0xa438, 0xa810, + 0xa438, 0xa120, 0xa438, 0xa310, 0xa438, 0xa4a0, 0xa438, 0xa606, + 0xa438, 0xd700, 0xa438, 0x37cd, 0xa438, 0x80e5, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8310, 0xa438, 0x9503, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x80f0, 0xa438, 0x33a9, 0xa438, 0x80ed, + 0xa438, 0xd17a, 0xa438, 0xd04c, 0xa438, 0xf006, 0xa438, 0xd199, + 0xa438, 0xd04c, 0xa438, 0xf003, 0xa438, 0xd1d6, 0xa438, 0xd04c, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, + 0xa438, 0xd706, 0xa438, 0x5f69, 0xa438, 0xd700, 0xa438, 0x60d7, + 0xa438, 0x6073, 0xa438, 0xce05, 0xa438, 0xf004, 0xa438, 0xce04, + 0xa438, 0xf002, 0xa438, 0xce04, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, 0xa438, 0x0b02, + 0xa438, 0x0c38, 0xa438, 0x0c10, 0xa438, 0x9503, 0xa438, 0xa180, + 0xa438, 0xa680, 0xa438, 0xcd75, 0xa438, 0xd199, 0xa438, 0xd04b, + 0xa438, 0xd13b, 0xa438, 0xd055, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fbb, 0xa438, 0xa302, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd706, + 0xa438, 0x5f6a, 0xa438, 0xbb50, 0xa438, 0xcd76, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb5, 0xa438, 0x9b10, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb5, + 0xa438, 0xa120, 0xa438, 0x0c12, 0xa438, 0x0310, 0xa438, 0x8480, + 0xa438, 0x0c84, 0xa438, 0x0604, 0xa438, 0xcd77, 0xa438, 0xd1a0, + 0xa438, 0xd04b, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x422a, 0xa438, 0xa1a0, + 0xa438, 0xa312, 0xa438, 0xa480, 0xa438, 0xa684, 0xa438, 0xcd78, + 0xa438, 0xd148, 0xa438, 0xd048, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xcd79, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x7f8a, 0xa438, 0x9b40, + 0xa438, 0xcd7f, 0xa438, 0xd71f, 0xa438, 0x7fe1, 0xa438, 0x1800, + 0xa438, 0x001c, 0xa438, 0xa708, 0xa438, 0xd700, 0xa438, 0x60cf, + 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, + 0xa438, 0xf00b, 0xa438, 0xce02, 0xa438, 0xf00a, 0xa438, 0xce02, + 0xa438, 0xf008, 0xa438, 0xce01, 0xa438, 0xf006, 0xa438, 0xce01, + 0xa438, 0xf004, 0xa438, 0xce01, 0xa438, 0xf002, 0xa438, 0xce01, + 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0ccf, 0xa438, 0x0b04, 0xa438, 0xcc22, 0xa438, 0xcd01, + 0xa438, 0xa702, 0xa438, 0x9503, 0xa438, 0xcd15, 0xa438, 0xa501, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fbd, + 0xa438, 0x8501, 0xa438, 0xba20, 0xa438, 0xd701, 0xa438, 0x4154, + 0xa438, 0xd115, 0xa438, 0xd04f, 0xa438, 0x1000, 0xa438, 0x9c02, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, + 0xa438, 0xf003, 0xa438, 0x1000, 0xa438, 0x9c02, 0xa438, 0xa610, + 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xcd16, 0xa438, 0xa501, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fbd, + 0xa438, 0x8501, 0xa438, 0xd707, 0xa438, 0x61cf, 0xa438, 0xa502, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd702, 0xa438, 0x5fbe, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, + 0xa438, 0xd403, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x8502, + 0xa438, 0xa340, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108, + 0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000, + 0xa438, 0x9c5b, 0xa438, 0xa304, 0xa438, 0xa440, 0xa438, 0xa8c0, + 0xa438, 0xd705, 0xa438, 0x40d3, 0xa438, 0xd707, 0xa438, 0x4082, + 0xa438, 0x0cfc, 0xa438, 0x02bc, 0xa438, 0xf002, 0xa438, 0xa2fc, + 0xa438, 0xa120, 0xa438, 0x0ca0, 0xa438, 0x0480, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa370, 0xa438, 0x9503, 0xa438, 0xcd17, + 0xa438, 0xd1f5, 0xa438, 0xd057, 0xa438, 0xd1c4, 0xa438, 0xd066, + 0xa438, 0xd1c4, 0xa438, 0xd077, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x607c, 0xa438, 0x613d, 0xa438, 0xfffb, + 0xa438, 0xa310, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fbd, 0xa438, 0xa607, 0xa438, 0xf007, 0xa438, 0xa607, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fbc, + 0xa438, 0xa310, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fbb, 0xa438, 0x8840, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0cf8, 0xa438, 0x0d48, 0xa438, 0x8320, 0xa438, 0xa180, + 0xa438, 0x9503, 0xa438, 0xd1c4, 0xa438, 0xd055, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fbb, 0xa438, 0xd706, + 0xa438, 0x6227, 0xa438, 0xd700, 0xa438, 0x5f3a, 0xa438, 0x88c0, + 0xa438, 0x82fc, 0xa438, 0x8120, 0xa438, 0x8350, 0xa438, 0x84a0, + 0xa438, 0x8607, 0xa438, 0xa510, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8df8, 0xa438, 0x8370, 0xa438, 0x8180, 0xa438, 0x9503, + 0xa438, 0xff8f, 0xa438, 0x8510, 0xa438, 0xa508, 0xa438, 0x8508, + 0xa438, 0xcd18, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x5fb4, 0xa438, 0xb920, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x9920, 0xa438, 0x9a20, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x6065, + 0xa438, 0x5f94, 0xa438, 0xffde, 0xa438, 0xb820, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, + 0xa438, 0x8120, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108, + 0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000, + 0xa438, 0x9c5b, 0xa438, 0xa304, 0xa438, 0x8880, 0xa438, 0x8480, + 0xa438, 0x8606, 0xa438, 0xcd19, 0xa438, 0xd705, 0xa438, 0x40d3, + 0xa438, 0xd707, 0xa438, 0x4082, 0xa438, 0x8310, 0xa438, 0x1000, + 0xa438, 0x9c0c, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8b0f, + 0xa438, 0x8c3f, 0xa438, 0x9503, 0xa438, 0xa810, 0xa438, 0xa120, + 0xa438, 0xd705, 0xa438, 0x40b3, 0xa438, 0xd707, 0xa438, 0x4062, + 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa4a0, + 0xa438, 0xa606, 0xa438, 0xd700, 0xa438, 0x37cd, 0xa438, 0x8236, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8310, 0xa438, 0x9503, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8241, 0xa438, 0x33a9, + 0xa438, 0x823e, 0xa438, 0xd17a, 0xa438, 0xd04c, 0xa438, 0xf006, + 0xa438, 0xd199, 0xa438, 0xd04c, 0xa438, 0xf003, 0xa438, 0xd1d6, + 0xa438, 0xd04c, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706, + 0xa438, 0x5f29, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce05, 0xa438, 0xf00a, 0xa438, 0xce05, 0xa438, 0xf008, + 0xa438, 0xce04, 0xa438, 0xf006, 0xa438, 0xce04, 0xa438, 0xf004, + 0xa438, 0xce04, 0xa438, 0xf002, 0xa438, 0xce04, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, + 0xa438, 0x0b02, 0xa438, 0x0c38, 0xa438, 0x0c10, 0xa438, 0x9503, + 0xa438, 0xa180, 0xa438, 0xa680, 0xa438, 0xcd1a, 0xa438, 0xd199, + 0xa438, 0xd04b, 0xa438, 0xd13b, 0xa438, 0xd055, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7b, 0xa438, 0xa302, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, + 0xa438, 0xd706, 0xa438, 0x5f2a, 0xa438, 0xbb10, 0xa438, 0xcd1b, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd71f, 0xa438, 0x5f75, 0xa438, 0xa704, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce03, 0xa438, 0xf00a, + 0xa438, 0xce03, 0xa438, 0xf008, 0xa438, 0xce03, 0xa438, 0xf006, + 0xa438, 0xce03, 0xa438, 0xf004, 0xa438, 0xce03, 0xa438, 0xf002, + 0xa438, 0xce03, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xa00a, + 0xa438, 0x81a0, 0xa438, 0x8312, 0xa438, 0x8480, 0xa438, 0xa686, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503, + 0xa438, 0x9b10, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x7fb5, 0xa438, 0xcd2a, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x5fa4, 0xa438, 0xd17b, 0xa438, 0xd04a, + 0xa438, 0xa980, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f68, 0xa438, 0x800a, + 0xa438, 0x8604, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8801, + 0xa438, 0x9503, 0xa438, 0xd40c, 0xa438, 0x1000, 0xa438, 0x9bbf, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, + 0xa438, 0xd417, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xa00a, + 0xa438, 0xa604, 0xa438, 0xd700, 0xa438, 0x39b9, 0xa438, 0x82e4, + 0xa438, 0xd707, 0xa438, 0x432f, 0xa438, 0xd700, 0xa438, 0x608f, + 0xa438, 0x60b1, 0xa438, 0x60d3, 0xa438, 0x60f5, 0xa438, 0xce08, + 0xa438, 0xf007, 0xa438, 0xce08, 0xa438, 0xf005, 0xa438, 0xce08, + 0xa438, 0xf003, 0xa438, 0xce08, 0xa438, 0xf001, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xa1a0, 0xa438, 0xa302, 0xa438, 0xa480, + 0xa438, 0xd707, 0xa438, 0x409f, 0xa438, 0x4062, 0xa438, 0x8310, + 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xd17b, 0xa438, 0xd049, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, + 0xa438, 0xd706, 0xa438, 0x5f68, 0xa438, 0xd700, 0xa438, 0x37cd, + 0xa438, 0x8302, 0xa438, 0x800a, 0xa438, 0x8604, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd417, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xa00a, 0xa438, 0xa604, + 0xa438, 0xd17b, 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f68, + 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, + 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, + 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, 0xa438, 0xce08, + 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, 0xa438, 0xce08, + 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0xa1a0, 0xa438, 0xd707, 0xa438, 0x40df, 0xa438, 0x40a2, + 0xa438, 0x0cfc, 0xa438, 0x02bc, 0xa438, 0x8310, 0xa438, 0xf004, + 0xa438, 0x0cfc, 0xa438, 0x02fc, 0xa438, 0xa310, 0xa438, 0xa302, + 0xa438, 0xa480, 0xa438, 0xa686, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa340, 0xa438, 0x9503, 0xa438, 0xcd2b, 0xa438, 0xd199, + 0xa438, 0xd04a, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f68, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d02, 0xa438, 0x9503, + 0xa438, 0xcd2c, 0xa438, 0xd199, 0xa438, 0xd04b, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd706, + 0xa438, 0x5f6a, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8312, + 0xa438, 0x8604, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, + 0xa438, 0x9503, 0xa438, 0xd417, 0xa438, 0x1000, 0xa438, 0x9bbf, + 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xd707, 0xa438, 0x409f, + 0xa438, 0x4062, 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, + 0xa438, 0xa302, 0xa438, 0xa604, 0xa438, 0xd409, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x5fb4, 0xa438, 0xb920, 0xa438, 0xcd2d, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x9920, + 0xa438, 0xbb10, 0xa438, 0xcd2e, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x5fb5, 0xa438, 0x800a, 0xa438, 0x81a0, + 0xa438, 0x8312, 0xa438, 0x8480, 0xa438, 0x0c86, 0xa438, 0x0680, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8340, 0xa438, 0xa140, + 0xa438, 0x9503, 0xa438, 0x9b10, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fb5, 0xa438, 0x1800, 0xa438, 0x8740, + 0xa438, 0xa70c, 0xa438, 0x8510, 0xa438, 0xd700, 0xa438, 0x60cf, + 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, + 0xa438, 0xf00b, 0xa438, 0xce03, 0xa438, 0xf00a, 0xa438, 0xce03, + 0xa438, 0xf008, 0xa438, 0xce03, 0xa438, 0xf006, 0xa438, 0xce03, + 0xa438, 0xf004, 0xa438, 0xce03, 0xa438, 0xf002, 0xa438, 0xce03, + 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0ccf, 0xa438, 0x0b04, 0xa438, 0xcc21, 0xa438, 0xcd10, + 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x406d, 0xa438, 0x0cc0, + 0xa438, 0x0080, 0xa438, 0xd198, 0xa438, 0xd07f, 0xa438, 0xcd11, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fa4, + 0xa438, 0xd102, 0xa438, 0xd040, 0xa438, 0xcd12, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0x1000, + 0xa438, 0x9c02, 0xa438, 0xa340, 0xa438, 0x1000, 0xa438, 0x9c52, + 0xa438, 0xa110, 0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8110, + 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xa304, 0xa438, 0xa224, + 0xa438, 0xa00a, 0xa438, 0xa802, 0xa438, 0xa980, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa370, 0xa438, 0x9503, 0xa438, 0xcd13, + 0xa438, 0xd700, 0xa438, 0x2469, 0xa438, 0x83db, 0xa438, 0x634b, + 0xa438, 0x39b9, 0xa438, 0x83c5, 0xa438, 0xf017, 0xa438, 0xd17a, + 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f64, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d02, 0xa438, 0x9503, + 0xa438, 0xcd14, 0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd706, + 0xa438, 0x5f65, 0xa438, 0x800a, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8340, 0xa438, 0xd700, 0xa438, 0x37cd, 0xa438, 0x83e3, + 0xa438, 0xa180, 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x2469, + 0xa438, 0x8409, 0xa438, 0xd700, 0xa438, 0x65eb, 0xa438, 0xd700, + 0xa438, 0x6609, 0xa438, 0xd705, 0xa438, 0x6051, 0xa438, 0xf009, + 0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xa00a, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd705, 0xa438, 0x7fb1, + 0xa438, 0xd700, 0xa438, 0x39b9, 0xa438, 0x83fa, 0xa438, 0xf01e, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa801, 0xa438, 0x9503, + 0xa438, 0xd40c, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd417, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xf00f, 0xa438, 0xd700, + 0xa438, 0x39b9, 0xa438, 0x840d, 0xa438, 0xf00e, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa801, 0xa438, 0x9503, 0xa438, 0xd40c, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd700, 0xa438, 0x6044, + 0xa438, 0xf004, 0xa438, 0xd407, 0xa438, 0x1000, 0xa438, 0x9bbf, + 0xa438, 0xa508, 0xa438, 0x8508, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8d07, 0xa438, 0x9503, 0xa438, 0xa00a, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503, 0xa438, 0xd704, + 0xa438, 0x6091, 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf003, + 0xa438, 0xd1c4, 0xa438, 0xd045, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x6a8b, 0xa438, 0xd700, 0xa438, 0x5f7a, + 0xa438, 0xd706, 0xa438, 0x5f24, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c07, 0xa438, 0x0d02, 0xa438, 0x9503, 0xa438, 0xd704, + 0xa438, 0x6091, 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf003, + 0xa438, 0xd1c4, 0xa438, 0xd045, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f65, + 0xa438, 0x800a, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8340, + 0xa438, 0xa180, 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x2d49, + 0xa438, 0x848d, 0xa438, 0x37c9, 0xa438, 0x8459, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd417, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xf023, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd417, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8d07, 0xa438, 0x9503, 0xa438, 0xa00a, 0xa438, 0xd17a, + 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f64, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d02, 0xa438, 0x9503, + 0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f65, + 0xa438, 0x800a, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, + 0xa438, 0x9503, 0xa438, 0xd417, 0xa438, 0x1000, 0xa438, 0x9bbf, + 0xa438, 0xf00b, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x5fab, 0xa438, 0xba08, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7f8b, 0xa438, 0x9a08, 0xa438, 0xa00a, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb4, + 0xa438, 0x800a, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8340, + 0xa438, 0xa180, 0xa438, 0x9503, 0xa438, 0xb920, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x9920, + 0xa438, 0xd700, 0xa438, 0x296d, 0xa438, 0x84bc, 0xa438, 0x1000, + 0xa438, 0x9c52, 0xa438, 0xa004, 0xa438, 0x1000, 0xa438, 0x9be6, + 0xa438, 0x8004, 0xa438, 0xa001, 0xa438, 0x1000, 0xa438, 0x9be6, + 0xa438, 0x8001, 0xa438, 0xa020, 0xa438, 0x1000, 0xa438, 0x9be6, + 0xa438, 0x8020, 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xd120, + 0xa438, 0xd040, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0x8704, 0xa438, 0xcd21, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fad, 0xa438, 0xa501, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fbd, + 0xa438, 0x8501, 0xa438, 0xba20, 0xa438, 0xd700, 0xa438, 0x2969, + 0xa438, 0x84d5, 0xa438, 0xd700, 0xa438, 0x612b, 0xa438, 0xd701, + 0xa438, 0x40f4, 0xa438, 0xd196, 0xa438, 0xd04d, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0x8802, + 0xa438, 0xcd22, 0xa438, 0xd17a, 0xa438, 0xd05a, 0xa438, 0xa501, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fbd, + 0xa438, 0x8501, 0xa438, 0xa502, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd702, 0xa438, 0x5fbe, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd403, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0x8502, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8801, 0xa438, 0x9503, 0xa438, 0xd40c, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0xd707, 0xa438, 0x428f, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce06, 0xa438, 0xf01d, + 0xa438, 0xce06, 0xa438, 0xf01b, 0xa438, 0xce06, 0xa438, 0xf019, + 0xa438, 0xce06, 0xa438, 0xf017, 0xa438, 0xce06, 0xa438, 0xf015, + 0xa438, 0xce06, 0xa438, 0xf013, 0xa438, 0xd700, 0xa438, 0x60cf, + 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, + 0xa438, 0xf00b, 0xa438, 0xce02, 0xa438, 0xf00a, 0xa438, 0xce02, + 0xa438, 0xf008, 0xa438, 0xce01, 0xa438, 0xf006, 0xa438, 0xce01, + 0xa438, 0xf004, 0xa438, 0xce01, 0xa438, 0xf002, 0xa438, 0xce01, + 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xa340, 0xa438, 0x1000, + 0xa438, 0x9c52, 0xa438, 0xa110, 0xa438, 0x1000, 0xa438, 0x9be6, + 0xa438, 0x8110, 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xa304, + 0xa438, 0xa440, 0xa438, 0xa8c0, 0xa438, 0xd707, 0xa438, 0x40cf, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8d07, 0xa438, 0x9503, + 0xa438, 0xa00a, 0xa438, 0xa120, 0xa438, 0xa310, 0xa438, 0x0ca0, + 0xa438, 0x0480, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x853b, + 0xa438, 0x60a4, 0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0xa308, + 0xa438, 0xf002, 0xa438, 0x8308, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c60, 0xa438, 0x0340, 0xa438, 0x9503, 0xa438, 0xcd23, + 0xa438, 0xd162, 0xa438, 0xd048, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0x8840, 0xa438, 0xd1c4, + 0xa438, 0xd045, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x6327, 0xa438, 0xd700, + 0xa438, 0x5f3b, 0xa438, 0x88c0, 0xa438, 0x800a, 0xa438, 0x8120, + 0xa438, 0x8358, 0xa438, 0x8308, 0xa438, 0x84a0, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8360, 0xa438, 0x9503, 0xa438, 0xd707, + 0xa438, 0x2f7d, 0xa438, 0x84d7, 0xa438, 0xd17a, 0xa438, 0xd05a, + 0xa438, 0xa501, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, + 0xa438, 0x5fbd, 0xa438, 0x8501, 0xa438, 0xff8b, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920, + 0xa438, 0xcd24, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x7fb4, 0xa438, 0x9920, 0xa438, 0x9a20, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x6065, 0xa438, 0x5f94, + 0xa438, 0xffd9, 0xa438, 0xb820, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0x800a, + 0xa438, 0x8120, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108, + 0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000, + 0xa438, 0x9c5b, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8598, + 0xa438, 0x6144, 0xa438, 0xd704, 0xa438, 0x60bd, 0xa438, 0xd707, + 0xa438, 0x417f, 0xa438, 0x4140, 0xa438, 0xf006, 0xa438, 0x82fc, + 0xa438, 0xa201, 0xa438, 0xf007, 0xa438, 0xa2fc, 0xa438, 0xf005, + 0xa438, 0x0cfc, 0xa438, 0x02bc, 0xa438, 0xf002, 0xa438, 0xa2fc, + 0xa438, 0xa304, 0xa438, 0x8880, 0xa438, 0x0cc0, 0xa438, 0x0440, + 0xa438, 0xcd25, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x85b7, + 0xa438, 0x6224, 0xa438, 0xd704, 0xa438, 0x605d, 0xa438, 0xf004, + 0xa438, 0xd704, 0xa438, 0x613b, 0xa438, 0xf00b, 0xa438, 0xd700, + 0xa438, 0x2969, 0xa438, 0x85b7, 0xa438, 0x60eb, 0xa438, 0xd707, + 0xa438, 0x40bf, 0xa438, 0x4080, 0xa438, 0x8310, 0xa438, 0x1000, + 0xa438, 0x9c0c, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, 0xa438, 0xf008, + 0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, 0xa438, 0xf004, + 0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8c07, + 0xa438, 0x0c07, 0xa438, 0x0d03, 0xa438, 0xd700, 0xa438, 0x2969, + 0xa438, 0x85d9, 0xa438, 0x60c4, 0xa438, 0xd704, 0xa438, 0x409d, + 0xa438, 0x8b0f, 0xa438, 0x8c38, 0xa438, 0xf003, 0xa438, 0xab07, + 0xa438, 0xac38, 0xa438, 0x0c07, 0xa438, 0x0d03, 0xa438, 0x9503, + 0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0xa120, 0xa438, 0xd700, + 0xa438, 0x2969, 0xa438, 0x85e7, 0xa438, 0x6064, 0xa438, 0xd704, + 0xa438, 0x607d, 0xa438, 0xa4a0, 0xa438, 0xa605, 0xa438, 0xd17a, + 0xa438, 0xd049, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x85f9, + 0xa438, 0x6164, 0xa438, 0xd704, 0xa438, 0x413d, 0xa438, 0xa00a, + 0xa438, 0xa120, 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, + 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa403, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x409a, 0xa438, 0xd706, 0xa438, 0x4046, 0xa438, 0xf014, + 0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff5, 0xa438, 0x1000, + 0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x8120, 0xa438, 0x8310, + 0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77, 0xa438, 0x8380, + 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdd, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8b0f, 0xa438, 0x8c38, 0xa438, 0x8d38, + 0xa438, 0x9503, 0xa438, 0xcd26, 0xa438, 0xd704, 0xa438, 0x6291, + 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x862e, 0xa438, 0x37c9, + 0xa438, 0x862b, 0xa438, 0x33a9, 0xa438, 0x8628, 0xa438, 0xd18a, + 0xa438, 0xd04b, 0xa438, 0xf00c, 0xa438, 0xd17a, 0xa438, 0xd04b, + 0xa438, 0xf009, 0xa438, 0xd1c6, 0xa438, 0xd04b, 0xa438, 0xf006, + 0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd1c4, + 0xa438, 0xd046, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8641, + 0xa438, 0x6164, 0xa438, 0xd704, 0xa438, 0x413d, 0xa438, 0xa00a, + 0xa438, 0xa120, 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, + 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa403, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x409a, 0xa438, 0xd706, 0xa438, 0x4048, 0xa438, 0xf014, + 0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff5, 0xa438, 0x1000, + 0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x8120, 0xa438, 0x8310, + 0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77, 0xa438, 0x8380, + 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdd, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d01, 0xa438, 0x9503, + 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, + 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, + 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, 0xa438, 0xce08, + 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, 0xa438, 0xce08, + 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0xa180, 0xa438, 0xcd27, 0xa438, 0xd704, 0xa438, 0x61f1, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8685, 0xa438, 0x33a9, + 0xa438, 0x8682, 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf009, + 0xa438, 0xd1b7, 0xa438, 0xd049, 0xa438, 0xf006, 0xa438, 0xd17a, + 0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd128, 0xa438, 0xd044, + 0xa438, 0xd13b, 0xa438, 0xd055, 0xa438, 0xd700, 0xa438, 0x2969, + 0xa438, 0x869a, 0xa438, 0x6164, 0xa438, 0xd704, 0xa438, 0x413d, + 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xd704, 0xa438, 0x407b, + 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa403, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd700, 0xa438, 0x405b, 0xa438, 0xf014, 0xa438, 0xd704, + 0xa438, 0x6055, 0xa438, 0xfff7, 0xa438, 0x1000, 0xa438, 0x9c52, + 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8310, 0xa438, 0xa380, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd705, 0xa438, 0x5f77, 0xa438, 0x8380, 0xa438, 0x1000, + 0xa438, 0x9c5b, 0xa438, 0xffdf, 0xa438, 0xa302, 0xa438, 0xd700, + 0xa438, 0x2969, 0xa438, 0x86c4, 0xa438, 0x6184, 0xa438, 0xd704, + 0xa438, 0x415d, 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xd704, + 0xa438, 0x407b, 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, + 0xa438, 0xa302, 0xa438, 0xa403, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x409a, + 0xa438, 0xd706, 0xa438, 0x4048, 0xa438, 0xf014, 0xa438, 0xd704, + 0xa438, 0x6055, 0xa438, 0xfff5, 0xa438, 0x1000, 0xa438, 0x9c52, + 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8312, 0xa438, 0xa380, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd705, 0xa438, 0x5f77, 0xa438, 0x8380, 0xa438, 0x1000, + 0xa438, 0x9c5b, 0xa438, 0xffdc, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c0f, 0xa438, 0x0b02, 0xa438, 0x0c38, 0xa438, 0x0c10, + 0xa438, 0x9503, 0xa438, 0xcd28, 0xa438, 0xd704, 0xa438, 0x61f1, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x86f5, 0xa438, 0x33a9, + 0xa438, 0x86f2, 0xa438, 0xd199, 0xa438, 0xd04a, 0xa438, 0xf009, + 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf006, 0xa438, 0xd199, + 0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd128, 0xa438, 0xd044, + 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8709, 0xa438, 0x6184, + 0xa438, 0xd704, 0xa438, 0x415d, 0xa438, 0xa00a, 0xa438, 0xa1a0, + 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, 0xa438, 0xf002, + 0xa438, 0xa310, 0xa438, 0xa302, 0xa438, 0xa403, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x409a, 0xa438, 0xd706, 0xa438, 0x404a, 0xa438, 0xf014, + 0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff5, 0xa438, 0x1000, + 0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8312, + 0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77, 0xa438, 0x8380, + 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdc, 0xa438, 0x8403, + 0xa438, 0xd409, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xbb10, + 0xa438, 0xcd29, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd71f, 0xa438, 0x5f75, 0xa438, 0x800a, + 0xa438, 0x81a0, 0xa438, 0x8312, 0xa438, 0x8480, 0xa438, 0x8604, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8340, 0xa438, 0xa140, + 0xa438, 0x9503, 0xa438, 0x9b10, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fb5, 0xa438, 0xcd30, 0xa438, 0xaa80, + 0xa438, 0xd704, 0xa438, 0x2319, 0xa438, 0x9d9a, 0xa438, 0xd700, + 0xa438, 0x40bd, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xaf01, + 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x5fb4, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8798, + 0xa438, 0xd700, 0xa438, 0x273d, 0xa438, 0x8794, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x876a, 0xa438, 0x33a9, 0xa438, 0x8762, + 0xa438, 0xd702, 0xa438, 0x6099, 0xa438, 0xd1b7, 0xa438, 0xd05c, + 0xa438, 0xf013, 0xa438, 0xd1b7, 0xa438, 0xd05c, 0xa438, 0xf010, + 0xa438, 0xd702, 0xa438, 0x6099, 0xa438, 0xd1b7, 0xa438, 0xd05c, + 0xa438, 0xf00b, 0xa438, 0xd1b7, 0xa438, 0xd05c, 0xa438, 0xf008, + 0xa438, 0xd702, 0xa438, 0x6099, 0xa438, 0xd199, 0xa438, 0xd05d, + 0xa438, 0xf003, 0xa438, 0xd199, 0xa438, 0xd05d, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x8786, 0xa438, 0x33a9, 0xa438, 0x877e, + 0xa438, 0xd702, 0xa438, 0x6099, 0xa438, 0xd1bf, 0xa438, 0xd06d, + 0xa438, 0xf013, 0xa438, 0xd1de, 0xa438, 0xd06d, 0xa438, 0xf010, + 0xa438, 0xd702, 0xa438, 0x6099, 0xa438, 0xd1bf, 0xa438, 0xd06d, + 0xa438, 0xf00b, 0xa438, 0xd1bf, 0xa438, 0xd06d, 0xa438, 0xf008, + 0xa438, 0xd702, 0xa438, 0x6099, 0xa438, 0xd199, 0xa438, 0xd06e, + 0xa438, 0xf003, 0xa438, 0xd199, 0xa438, 0xd06e, 0xa438, 0xd703, + 0xa438, 0x60d0, 0xa438, 0x1000, 0xa438, 0x9c20, 0xa438, 0xd41a, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd408, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0xcd31, 0xa438, 0xd700, 0xa438, 0x2fa9, + 0xa438, 0x879f, 0xa438, 0x33c9, 0xa438, 0x87a2, 0xa438, 0x6117, + 0xa438, 0xf00a, 0xa438, 0xd141, 0xa438, 0xd043, 0xa438, 0xf009, + 0xa438, 0xd121, 0xa438, 0xd043, 0xa438, 0xf006, 0xa438, 0xd122, + 0xa438, 0xd042, 0xa438, 0xf003, 0xa438, 0xd181, 0xa438, 0xd043, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, + 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8817, 0xa438, 0xd700, + 0xa438, 0x6cab, 0xa438, 0xd700, 0xa438, 0x6d47, 0xa438, 0x800a, + 0xa438, 0x81a0, 0xa438, 0x8312, 0xa438, 0x8480, 0xa438, 0x0c86, + 0xa438, 0x0680, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8702, + 0xa438, 0x9503, 0xa438, 0xd407, 0xa438, 0x1000, 0xa438, 0x9bbf, + 0xa438, 0xd703, 0xa438, 0x60d0, 0xa438, 0x1000, 0xa438, 0x9c20, + 0xa438, 0xd41a, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd406, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xcd39, 0xa438, 0xd404, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd700, 0xa438, 0x60cf, + 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, + 0xa438, 0xf00b, 0xa438, 0xce03, 0xa438, 0xf00a, 0xa438, 0xce03, + 0xa438, 0xf008, 0xa438, 0xce03, 0xa438, 0xf006, 0xa438, 0xce03, + 0xa438, 0xf004, 0xa438, 0xce03, 0xa438, 0xf002, 0xa438, 0xce03, + 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c07, 0xa438, 0x0c01, 0xa438, 0x8d07, 0xa438, 0x9503, + 0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0xa302, 0xa438, 0x0ca0, + 0xa438, 0x0480, 0xa438, 0xa684, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xb920, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x8810, + 0xa438, 0x9920, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x60e5, 0xa438, 0x5f94, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xfff0, 0xa438, 0xb820, + 0xa438, 0xa810, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd706, 0xa438, 0x5f69, + 0xa438, 0xf010, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xae80, + 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x2b6d, 0xa438, 0x88c0, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, + 0xa438, 0xd406, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xcd32, + 0xa438, 0xd701, 0xa438, 0x6191, 0xa438, 0xa504, 0xa438, 0xcd3a, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x6067, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd701, 0xa438, 0x5f3a, + 0xa438, 0x8504, 0xa438, 0xd700, 0xa438, 0x2739, 0xa438, 0x88b6, + 0xa438, 0xd707, 0xa438, 0x6061, 0xa438, 0x1800, 0xa438, 0x88a8, + 0xa438, 0xd193, 0xa438, 0xd047, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, + 0xa438, 0xd706, 0xa438, 0x5f29, 0xa438, 0xd700, 0xa438, 0x60cf, + 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, + 0xa438, 0xf00b, 0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, + 0xa438, 0xf008, 0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, + 0xa438, 0xf004, 0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, + 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c0f, 0xa438, 0x0b01, 0xa438, 0x0c3f, 0xa438, 0x0c08, + 0xa438, 0x9503, 0xa438, 0xd707, 0xa438, 0x409f, 0xa438, 0x4062, + 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa120, + 0xa438, 0xa420, 0xa438, 0xd193, 0xa438, 0xd048, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f29, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce03, 0xa438, 0xf00a, + 0xa438, 0xce03, 0xa438, 0xf008, 0xa438, 0xce03, 0xa438, 0xf006, + 0xa438, 0xce03, 0xa438, 0xf004, 0xa438, 0xce03, 0xa438, 0xf002, + 0xa438, 0xce03, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0c01, 0xa438, 0x9503, + 0xa438, 0x8420, 0xa438, 0x800a, 0xa438, 0x8120, 0xa438, 0x8312, + 0xa438, 0x8480, 0xa438, 0x8604, 0xa438, 0xd419, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0xd702, 0xa438, 0x4080, 0xa438, 0xbb20, + 0xa438, 0x1800, 0xa438, 0x8c32, 0xa438, 0xa00a, 0xa438, 0xa302, + 0xa438, 0xa480, 0xa438, 0xa604, 0xa438, 0xd193, 0xa438, 0xd047, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f29, + 0xa438, 0x1800, 0xa438, 0x888b, 0xa438, 0x800a, 0xa438, 0x8302, + 0xa438, 0x8480, 0xa438, 0x8604, 0xa438, 0xd405, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0xbb20, 0xa438, 0xa00a, 0xa438, 0xa302, + 0xa438, 0xa480, 0xa438, 0xa604, 0xa438, 0x1800, 0xa438, 0x8c32, + 0xa438, 0xd405, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd404, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd700, 0xa438, 0x2b69, + 0xa438, 0x88c0, 0xa438, 0xf06d, 0xa438, 0xd700, 0xa438, 0x60cf, + 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, + 0xa438, 0xf00b, 0xa438, 0xce03, 0xa438, 0xf00a, 0xa438, 0xce03, + 0xa438, 0xf008, 0xa438, 0xce03, 0xa438, 0xf006, 0xa438, 0xce03, + 0xa438, 0xf004, 0xa438, 0xce03, 0xa438, 0xf002, 0xa438, 0xce03, + 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c07, 0xa438, 0x0c01, 0xa438, 0x0c3f, 0xa438, 0x0d08, + 0xa438, 0x9503, 0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0xa302, + 0xa438, 0x0ca0, 0xa438, 0x0480, 0xa438, 0xd700, 0xa438, 0x2969, + 0xa438, 0x88e8, 0xa438, 0x60a4, 0xa438, 0xd704, 0xa438, 0x407d, + 0xa438, 0x8604, 0xa438, 0xf002, 0xa438, 0xa604, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0xd700, 0xa438, 0x37cd, + 0xa438, 0x88f0, 0xa438, 0x8310, 0xa438, 0x9503, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xb920, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, + 0xa438, 0x8810, 0xa438, 0x9920, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x60e5, 0xa438, 0x5f94, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xfff0, + 0xa438, 0xb820, 0xa438, 0xa810, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0xd700, + 0xa438, 0x2d59, 0xa438, 0x8c32, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8c07, 0xa438, 0x9503, 0xa438, 0xa120, 0xa438, 0xd700, + 0xa438, 0x2969, 0xa438, 0x891f, 0xa438, 0x60e4, 0xa438, 0xd704, + 0xa438, 0x40bd, 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, + 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xd700, 0xa438, 0x2969, + 0xa438, 0x8928, 0xa438, 0x60a4, 0xa438, 0xd704, 0xa438, 0x407d, + 0xa438, 0xa202, 0xa438, 0xf002, 0xa438, 0x8202, 0xa438, 0xa420, + 0xa438, 0x1800, 0xa438, 0x8d48, 0xa438, 0x8810, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fbb, 0xa438, 0xd17a, + 0xa438, 0xd05a, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, 0xa438, 0xf008, + 0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, 0xa438, 0xf004, + 0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, + 0xa438, 0x0b04, 0xa438, 0x0c3f, 0xa438, 0x0c21, 0xa438, 0x8d07, + 0xa438, 0x9503, 0xa438, 0xa340, 0xa438, 0x1000, 0xa438, 0x9c52, + 0xa438, 0xa110, 0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8110, + 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xa304, 0xa438, 0xa440, + 0xa438, 0xa8c0, 0xa438, 0x8810, 0xa438, 0xa00a, 0xa438, 0xa120, + 0xa438, 0xa310, 0xa438, 0xd704, 0xa438, 0x405d, 0xa438, 0xa308, + 0xa438, 0x0cfc, 0xa438, 0x0224, 0xa438, 0x0ca0, 0xa438, 0x0480, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0xd700, + 0xa438, 0x37cd, 0xa438, 0x896d, 0xa438, 0x8310, 0xa438, 0x9503, + 0xa438, 0xd162, 0xa438, 0xd048, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0x8840, 0xa438, 0xd1c4, + 0xa438, 0xd045, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x6127, 0xa438, 0xd700, + 0xa438, 0x5f3b, 0xa438, 0x88c0, 0xa438, 0x800a, 0xa438, 0x8120, + 0xa438, 0x8350, 0xa438, 0x84a0, 0xa438, 0xffad, 0xa438, 0xb920, + 0xa438, 0xcd33, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x7fb4, 0xa438, 0x9920, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x6065, 0xa438, 0x5f94, 0xa438, 0xffee, + 0xa438, 0xb820, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0x800a, 0xa438, 0x8120, + 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108, 0xa438, 0x1000, + 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000, 0xa438, 0x9c5b, + 0xa438, 0xd704, 0xa438, 0x60bd, 0xa438, 0xd707, 0xa438, 0x413f, + 0xa438, 0x4100, 0xa438, 0xf004, 0xa438, 0x0cfd, 0xa438, 0x0201, + 0xa438, 0xf005, 0xa438, 0x0cfc, 0xa438, 0x02bc, 0xa438, 0xf002, + 0xa438, 0xa2fc, 0xa438, 0xa304, 0xa438, 0x8880, 0xa438, 0x0cc0, + 0xa438, 0x0440, 0xa438, 0xcd34, 0xa438, 0xd704, 0xa438, 0x407d, + 0xa438, 0x405b, 0xa438, 0xf006, 0xa438, 0xd704, 0xa438, 0x60fd, + 0xa438, 0xd707, 0xa438, 0x40bf, 0xa438, 0x4080, 0xa438, 0x8310, + 0xa438, 0x1000, 0xa438, 0x9c0c, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8c07, 0xa438, 0xd704, 0xa438, 0x409d, 0xa438, 0x8b0f, + 0xa438, 0x8c38, 0xa438, 0xf004, 0xa438, 0x0c0f, 0xa438, 0x0b07, + 0xa438, 0xac38, 0xa438, 0x0c38, 0xa438, 0x0d10, 0xa438, 0x9503, + 0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0xa120, 0xa438, 0xd704, + 0xa438, 0x607d, 0xa438, 0xa4a0, 0xa438, 0xa604, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x89db, 0xa438, 0xd17a, 0xa438, 0xd049, + 0xa438, 0xf003, 0xa438, 0xd19f, 0xa438, 0xd049, 0xa438, 0xd704, + 0xa438, 0x413d, 0xa438, 0xa00a, 0xa438, 0xa120, 0xa438, 0xd704, + 0xa438, 0x407b, 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, + 0xa438, 0xa403, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x6a14, 0xa438, 0xd700, 0xa438, 0x409a, 0xa438, 0xd706, + 0xa438, 0x4988, 0xa438, 0xf014, 0xa438, 0xd704, 0xa438, 0x6055, + 0xa438, 0xfff5, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0x800a, + 0xa438, 0x8120, 0xa438, 0x8310, 0xa438, 0xa380, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd705, + 0xa438, 0x5f77, 0xa438, 0x8380, 0xa438, 0x1000, 0xa438, 0x9c5b, + 0xa438, 0xffdd, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8b0f, + 0xa438, 0x8c38, 0xa438, 0x8d38, 0xa438, 0x9503, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x8a14, 0xa438, 0x33a9, 0xa438, 0x8a11, + 0xa438, 0xd1f4, 0xa438, 0xd04b, 0xa438, 0xf006, 0xa438, 0xd1b7, + 0xa438, 0xd04b, 0xa438, 0xf003, 0xa438, 0xd1c6, 0xa438, 0xd04c, + 0xa438, 0xd704, 0xa438, 0x413d, 0xa438, 0xa00a, 0xa438, 0xa120, + 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, 0xa438, 0xf002, + 0xa438, 0xa310, 0xa438, 0xa403, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x62f4, 0xa438, 0xd700, 0xa438, 0x405a, + 0xa438, 0xf01a, 0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff7, + 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x8120, + 0xa438, 0x8310, 0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77, + 0xa438, 0x8380, 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdf, + 0xa438, 0x8403, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x629c, 0xa438, 0xfffb, 0xa438, 0x8403, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x6096, 0xa438, 0xd700, + 0xa438, 0x619c, 0xa438, 0xfffa, 0xa438, 0xd706, 0xa438, 0x4128, + 0xa438, 0xd702, 0xa438, 0x60b0, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xae80, 0xa438, 0x9503, 0xa438, 0x1800, 0xa438, 0x8b62, + 0xa438, 0xd17a, 0xa438, 0xd05a, 0xa438, 0xd700, 0xa438, 0x60cf, + 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, + 0xa438, 0xf00b, 0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, + 0xa438, 0xf008, 0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, + 0xa438, 0xf004, 0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, + 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c0f, 0xa438, 0x0b04, 0xa438, 0x0c3f, 0xa438, 0x0c21, + 0xa438, 0x8d07, 0xa438, 0x9503, 0xa438, 0xa340, 0xa438, 0x1000, + 0xa438, 0x9c52, 0xa438, 0xa110, 0xa438, 0x1000, 0xa438, 0x9be6, + 0xa438, 0x8110, 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xa304, + 0xa438, 0xa440, 0xa438, 0xa8c0, 0xa438, 0x8810, 0xa438, 0xa00a, + 0xa438, 0xa120, 0xa438, 0xa310, 0xa438, 0xd704, 0xa438, 0x405d, + 0xa438, 0xa308, 0xa438, 0x0cfc, 0xa438, 0x0224, 0xa438, 0x0ca0, + 0xa438, 0x0480, 0xa438, 0x8604, 0xa438, 0xcd35, 0xa438, 0xd162, + 0xa438, 0xd048, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0x8840, 0xa438, 0xd1c4, 0xa438, 0xd045, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, + 0xa438, 0xd706, 0xa438, 0x6127, 0xa438, 0xd700, 0xa438, 0x5f3b, + 0xa438, 0x88c0, 0xa438, 0x800a, 0xa438, 0x8120, 0xa438, 0x8350, + 0xa438, 0x84a0, 0xa438, 0xffb3, 0xa438, 0xbb80, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920, + 0xa438, 0xcd36, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x7fb4, 0xa438, 0x9920, 0xa438, 0x9b80, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x6065, 0xa438, 0x5f94, + 0xa438, 0xffe8, 0xa438, 0xb820, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0x800a, + 0xa438, 0x8120, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108, + 0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000, + 0xa438, 0x9c5b, 0xa438, 0xd704, 0xa438, 0x60bd, 0xa438, 0xd707, + 0xa438, 0x413f, 0xa438, 0x4100, 0xa438, 0xf004, 0xa438, 0x0cfd, + 0xa438, 0x0201, 0xa438, 0xf005, 0xa438, 0x0cfc, 0xa438, 0x02bc, + 0xa438, 0xf002, 0xa438, 0xa2fc, 0xa438, 0xa304, 0xa438, 0x8880, + 0xa438, 0x0cc0, 0xa438, 0x0440, 0xa438, 0xcd37, 0xa438, 0xd704, + 0xa438, 0x407d, 0xa438, 0x405b, 0xa438, 0xf006, 0xa438, 0xd704, + 0xa438, 0x60fd, 0xa438, 0xd707, 0xa438, 0x40bf, 0xa438, 0x4080, + 0xa438, 0x8310, 0xa438, 0x1000, 0xa438, 0x9c0c, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8c07, 0xa438, 0xd704, 0xa438, 0x409d, + 0xa438, 0x8b0f, 0xa438, 0x8c38, 0xa438, 0xf004, 0xa438, 0x0c0f, + 0xa438, 0x0b07, 0xa438, 0xac38, 0xa438, 0x0c38, 0xa438, 0x0d10, + 0xa438, 0x9503, 0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0xa120, + 0xa438, 0xd704, 0xa438, 0x607d, 0xa438, 0xa4a0, 0xa438, 0xa604, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8b01, 0xa438, 0x33a9, + 0xa438, 0x8afe, 0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0xf006, + 0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0xf003, 0xa438, 0xd17a, + 0xa438, 0xd048, 0xa438, 0xd704, 0xa438, 0x413d, 0xa438, 0xa00a, + 0xa438, 0xa120, 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, + 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa403, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x409a, 0xa438, 0xd706, 0xa438, 0x4048, 0xa438, 0xf014, + 0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff5, 0xa438, 0x1000, + 0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x8120, 0xa438, 0x8310, + 0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77, 0xa438, 0x8380, + 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdd, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8b0f, 0xa438, 0x8c38, 0xa438, 0x8d38, + 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8b3a, + 0xa438, 0x33a9, 0xa438, 0x8b37, 0xa438, 0xd199, 0xa438, 0xd04b, + 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf003, + 0xa438, 0xd189, 0xa438, 0xd04c, 0xa438, 0xd704, 0xa438, 0x413d, + 0xa438, 0xa00a, 0xa438, 0xa120, 0xa438, 0xd704, 0xa438, 0x407b, + 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa403, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd700, 0xa438, 0x409a, 0xa438, 0xd706, 0xa438, 0x4048, + 0xa438, 0xf014, 0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff5, + 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x8120, + 0xa438, 0x8310, 0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77, + 0xa438, 0x8380, 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdd, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d01, + 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce08, 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, + 0xa438, 0xce08, 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, + 0xa438, 0xce08, 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xa180, 0xa438, 0xcd38, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x8b92, 0xa438, 0x33a9, 0xa438, 0x8b8a, + 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd17a, 0xa438, 0xd04a, + 0xa438, 0xf013, 0xa438, 0xd100, 0xa438, 0xd049, 0xa438, 0xf010, + 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd1b7, 0xa438, 0xd049, + 0xa438, 0xf00b, 0xa438, 0xd100, 0xa438, 0xd048, 0xa438, 0xf008, + 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd17a, 0xa438, 0xd04b, + 0xa438, 0xf003, 0xa438, 0xd199, 0xa438, 0xd04a, 0xa438, 0xd13b, + 0xa438, 0xd055, 0xa438, 0xd704, 0xa438, 0x413d, 0xa438, 0xa00a, + 0xa438, 0xa1a0, 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, + 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa403, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x405b, 0xa438, 0xf014, 0xa438, 0xd704, 0xa438, 0x6055, + 0xa438, 0xfff7, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0x800a, + 0xa438, 0x81a0, 0xa438, 0x8310, 0xa438, 0xa380, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd705, + 0xa438, 0x5f77, 0xa438, 0x8380, 0xa438, 0x1000, 0xa438, 0x9c5b, + 0xa438, 0xffdf, 0xa438, 0xa302, 0xa438, 0xd704, 0xa438, 0x415d, + 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xd704, 0xa438, 0x407b, + 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa302, + 0xa438, 0xa403, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x409a, 0xa438, 0xd706, + 0xa438, 0x4048, 0xa438, 0xf014, 0xa438, 0xd704, 0xa438, 0x6055, + 0xa438, 0xfff5, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0x800a, + 0xa438, 0x81a0, 0xa438, 0x8312, 0xa438, 0xa380, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd705, + 0xa438, 0x5f77, 0xa438, 0x8380, 0xa438, 0x1000, 0xa438, 0x9c5b, + 0xa438, 0xffdc, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, + 0xa438, 0x0b02, 0xa438, 0x0c38, 0xa438, 0x0c10, 0xa438, 0x9503, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8c03, 0xa438, 0x33a9, + 0xa438, 0x8bfb, 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd199, + 0xa438, 0xd04a, 0xa438, 0xf013, 0xa438, 0xd100, 0xa438, 0xd04a, + 0xa438, 0xf010, 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd17a, + 0xa438, 0xd04a, 0xa438, 0xf00b, 0xa438, 0xd100, 0xa438, 0xd048, + 0xa438, 0xf008, 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd199, + 0xa438, 0xd04b, 0xa438, 0xf003, 0xa438, 0xd16b, 0xa438, 0xd04b, + 0xa438, 0xd704, 0xa438, 0x415d, 0xa438, 0xa00a, 0xa438, 0xa1a0, + 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, 0xa438, 0xf002, + 0xa438, 0xa310, 0xa438, 0xa302, 0xa438, 0xa403, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x40ba, 0xa438, 0xd706, 0xa438, 0x406a, 0xa438, 0x8403, + 0xa438, 0xf014, 0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff4, + 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x81a0, + 0xa438, 0x8312, 0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77, + 0xa438, 0x8380, 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdb, + 0xa438, 0x81a0, 0xa438, 0x8310, 0xa438, 0xa302, 0xa438, 0xa00a, + 0xa438, 0xa480, 0xa438, 0x8420, 0xa438, 0xd700, 0xa438, 0x2969, + 0xa438, 0x8c42, 0xa438, 0x60e4, 0xa438, 0xd700, 0xa438, 0x40a7, + 0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0x8604, 0xa438, 0xf002, + 0xa438, 0xa604, 0xa438, 0xd700, 0xa438, 0x60c7, 0xa438, 0xa602, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa702, 0xa438, 0x9503, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503, + 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, + 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce03, + 0xa438, 0xf00a, 0xa438, 0xce03, 0xa438, 0xf008, 0xa438, 0xce03, + 0xa438, 0xf006, 0xa438, 0xce03, 0xa438, 0xf004, 0xa438, 0xce03, + 0xa438, 0xf002, 0xa438, 0xce03, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0c01, + 0xa438, 0x0c3f, 0xa438, 0x0d08, 0xa438, 0x9503, 0xa438, 0xd700, + 0xa438, 0x646d, 0xa438, 0x37c9, 0xa438, 0x8c83, 0xa438, 0x33a9, + 0xa438, 0x8c79, 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, + 0xa438, 0x6098, 0xa438, 0xd100, 0xa438, 0xd048, 0xa438, 0xf01a, + 0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0xf017, 0xa438, 0xd700, + 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd100, + 0xa438, 0xd049, 0xa438, 0xf010, 0xa438, 0xd17a, 0xa438, 0xd04c, + 0xa438, 0xf00d, 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, + 0xa438, 0x6098, 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf006, + 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd17a, + 0xa438, 0xd048, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706, + 0xa438, 0x5f29, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, 0xa438, 0xf008, + 0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, 0xa438, 0xf004, + 0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, + 0xa438, 0x0b01, 0xa438, 0x0c3f, 0xa438, 0x0c08, 0xa438, 0x9503, + 0xa438, 0xa120, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8cc7, + 0xa438, 0x6224, 0xa438, 0xd700, 0xa438, 0x4147, 0xa438, 0xd704, + 0xa438, 0x409d, 0xa438, 0xd704, 0xa438, 0x613b, 0xa438, 0xf00a, + 0xa438, 0xd707, 0xa438, 0x411f, 0xa438, 0x40e0, 0xa438, 0xf004, + 0xa438, 0xd707, 0xa438, 0x409f, 0xa438, 0x4062, 0xa438, 0x8310, + 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xd700, 0xa438, 0x2969, + 0xa438, 0x8cd2, 0xa438, 0x60e4, 0xa438, 0xd700, 0xa438, 0x40a7, + 0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0xa202, 0xa438, 0xf002, + 0xa438, 0x8202, 0xa438, 0xa420, 0xa438, 0xcd3b, 0xa438, 0xd700, + 0xa438, 0x65ad, 0xa438, 0x43c7, 0xa438, 0xd700, 0xa438, 0x37c9, + 0xa438, 0x8ced, 0xa438, 0x33a9, 0xa438, 0x8ce5, 0xa438, 0xd702, + 0xa438, 0x4098, 0xa438, 0xd199, 0xa438, 0xd04c, 0xa438, 0xf024, + 0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0xf021, 0xa438, 0xd702, + 0xa438, 0x4098, 0xa438, 0xd1c6, 0xa438, 0xd04c, 0xa438, 0xf01c, + 0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0xf019, 0xa438, 0xd702, + 0xa438, 0x4098, 0xa438, 0xd199, 0xa438, 0xd04c, 0xa438, 0xf014, + 0xa438, 0xd17a, 0xa438, 0xd04c, 0xa438, 0xf011, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x8d00, 0xa438, 0x33a9, 0xa438, 0x8cfd, + 0xa438, 0xd1e5, 0xa438, 0xd04c, 0xa438, 0xf009, 0xa438, 0xd191, + 0xa438, 0xd04d, 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd04d, + 0xa438, 0xf003, 0xa438, 0xd16b, 0xa438, 0xd04c, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2c, 0xa438, 0xd700, + 0xa438, 0x40e7, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c07, + 0xa438, 0x0d01, 0xa438, 0x9503, 0xa438, 0xf006, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d02, 0xa438, 0x9503, + 0xa438, 0xcd3c, 0xa438, 0xd700, 0xa438, 0x644d, 0xa438, 0x43c7, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8d33, 0xa438, 0x33a9, + 0xa438, 0x8d2b, 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd17a, + 0xa438, 0xd04a, 0xa438, 0xf019, 0xa438, 0xd17a, 0xa438, 0xd048, + 0xa438, 0xf016, 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd17a, + 0xa438, 0xd04b, 0xa438, 0xf011, 0xa438, 0xd17a, 0xa438, 0xd048, + 0xa438, 0xf00e, 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd17a, + 0xa438, 0xd04a, 0xa438, 0xf009, 0xa438, 0xd17a, 0xa438, 0xd049, + 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd04b, 0xa438, 0xf003, + 0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, + 0xa438, 0xd706, 0xa438, 0x5f2d, 0xa438, 0xd700, 0xa438, 0x60cf, + 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, + 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf00a, 0xa438, 0xce08, + 0xa438, 0xf008, 0xa438, 0xce08, 0xa438, 0xf006, 0xa438, 0xce08, + 0xa438, 0xf004, 0xa438, 0xce08, 0xa438, 0xf002, 0xa438, 0xce08, + 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c0f, 0xa438, 0x0b02, 0xa438, 0x0c38, 0xa438, 0x0c10, + 0xa438, 0x9503, 0xa438, 0xa180, 0xa438, 0xcd3d, 0xa438, 0xd700, + 0xa438, 0x2969, 0xa438, 0x8d8a, 0xa438, 0x37c9, 0xa438, 0x8d80, + 0xa438, 0x33a9, 0xa438, 0x8d76, 0xa438, 0xd700, 0xa438, 0x40c7, + 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd15c, 0xa438, 0xd04a, + 0xa438, 0xf01a, 0xa438, 0xd199, 0xa438, 0xd04b, 0xa438, 0xf017, + 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, + 0xa438, 0xd13e, 0xa438, 0xd04a, 0xa438, 0xf010, 0xa438, 0xd199, + 0xa438, 0xd04b, 0xa438, 0xf00d, 0xa438, 0xd700, 0xa438, 0x40c7, + 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd17a, 0xa438, 0xd04c, + 0xa438, 0xf006, 0xa438, 0xd199, 0xa438, 0xd04c, 0xa438, 0xf003, + 0xa438, 0xd17a, 0xa438, 0xd04b, 0xa438, 0xd700, 0xa438, 0x37c9, + 0xa438, 0x8da5, 0xa438, 0x33a9, 0xa438, 0x8d9b, 0xa438, 0xd700, + 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd100, + 0xa438, 0xd050, 0xa438, 0xf017, 0xa438, 0xd17a, 0xa438, 0xd05a, + 0xa438, 0xf014, 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, + 0xa438, 0x6098, 0xa438, 0xd100, 0xa438, 0xd050, 0xa438, 0xf00d, + 0xa438, 0xd17a, 0xa438, 0xd05a, 0xa438, 0xf00a, 0xa438, 0xd700, + 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd17a, + 0xa438, 0xd05a, 0xa438, 0xf003, 0xa438, 0xd17a, 0xa438, 0xd05a, + 0xa438, 0xd707, 0xa438, 0x3ad1, 0xa438, 0x8e73, 0xa438, 0xd705, + 0xa438, 0x36b1, 0xa438, 0x8ddd, 0xa438, 0xd700, 0xa438, 0x2969, + 0xa438, 0x8dd5, 0xa438, 0x63c4, 0xa438, 0xd700, 0xa438, 0x40e7, + 0xa438, 0xd704, 0xa438, 0x40bd, 0xa438, 0xd704, 0xa438, 0x60dc, + 0xa438, 0x63fb, 0xa438, 0xf016, 0xa438, 0xd707, 0xa438, 0x429f, + 0xa438, 0xf001, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7b, 0xa438, 0xd700, + 0xa438, 0x40e7, 0xa438, 0xd704, 0xa438, 0x40bd, 0xa438, 0xd704, + 0xa438, 0x407c, 0xa438, 0x1800, 0xa438, 0x8e73, 0xa438, 0xd706, + 0xa438, 0x2b59, 0xa438, 0x8e73, 0xa438, 0xf009, 0xa438, 0xaa20, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd701, 0xa438, 0x5f72, 0xa438, 0x8a20, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd700, + 0xa438, 0x4287, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce08, 0xa438, 0xf01e, 0xa438, 0xce08, 0xa438, 0xf01c, + 0xa438, 0xce08, 0xa438, 0xf01a, 0xa438, 0xce08, 0xa438, 0xf018, + 0xa438, 0xce08, 0xa438, 0xf016, 0xa438, 0xce08, 0xa438, 0xf014, + 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, + 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, + 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf009, 0xa438, 0xce08, + 0xa438, 0xf007, 0xa438, 0xce08, 0xa438, 0xf005, 0xa438, 0xce08, + 0xa438, 0xf003, 0xa438, 0xce08, 0xa438, 0xf001, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8e23, + 0xa438, 0x6224, 0xa438, 0xd700, 0xa438, 0x4147, 0xa438, 0xd704, + 0xa438, 0x409d, 0xa438, 0xd704, 0xa438, 0x417b, 0xa438, 0xf008, + 0xa438, 0xd707, 0xa438, 0x411f, 0xa438, 0x40e0, 0xa438, 0xf004, + 0xa438, 0xd707, 0xa438, 0x409f, 0xa438, 0x4062, 0xa438, 0x8320, + 0xa438, 0xf002, 0xa438, 0xa320, 0xa438, 0x8310, 0xa438, 0xcd3e, + 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8e4b, 0xa438, 0x37c9, + 0xa438, 0x8e41, 0xa438, 0x33a9, 0xa438, 0x8e37, 0xa438, 0xd700, + 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd15d, + 0xa438, 0xd04b, 0xa438, 0xf01a, 0xa438, 0xd17a, 0xa438, 0xd04b, + 0xa438, 0xf017, 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, + 0xa438, 0x6098, 0xa438, 0xd16b, 0xa438, 0xd04b, 0xa438, 0xf010, + 0xa438, 0xd17a, 0xa438, 0xd04b, 0xa438, 0xf00d, 0xa438, 0xd700, + 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd17a, + 0xa438, 0xd04b, 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd04b, + 0xa438, 0xf003, 0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd700, + 0xa438, 0x2969, 0xa438, 0x8e6b, 0xa438, 0x6264, 0xa438, 0xd700, + 0xa438, 0x4187, 0xa438, 0xd704, 0xa438, 0x40bd, 0xa438, 0xd704, + 0xa438, 0x41bb, 0xa438, 0x1800, 0xa438, 0x8f25, 0xa438, 0xd707, + 0xa438, 0x413f, 0xa438, 0x4100, 0xa438, 0x1800, 0xa438, 0x8f25, + 0xa438, 0xd707, 0xa438, 0x409f, 0xa438, 0x4060, 0xa438, 0x1800, + 0xa438, 0x8f25, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd701, 0xa438, 0x5f77, 0xa438, 0x1800, + 0xa438, 0x8f25, 0xa438, 0xd700, 0xa438, 0x4287, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf01e, + 0xa438, 0xce08, 0xa438, 0xf01c, 0xa438, 0xce08, 0xa438, 0xf01a, + 0xa438, 0xce08, 0xa438, 0xf018, 0xa438, 0xce08, 0xa438, 0xf016, + 0xa438, 0xce08, 0xa438, 0xf014, 0xa438, 0xd700, 0xa438, 0x60cf, + 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, + 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf00b, 0xa438, 0xce08, + 0xa438, 0xf009, 0xa438, 0xce08, 0xa438, 0xf007, 0xa438, 0xce08, + 0xa438, 0xf005, 0xa438, 0xce08, 0xa438, 0xf003, 0xa438, 0xce08, + 0xa438, 0xf001, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa102, 0xa438, 0xa802, 0xa438, 0x9503, + 0xa438, 0xaa02, 0xa438, 0xd700, 0xa438, 0x40e7, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c0f, 0xa438, 0x0b05, 0xa438, 0xac38, + 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd704, + 0xa438, 0x60fd, 0xa438, 0xd707, 0xa438, 0x40a0, 0xa438, 0xf003, + 0xa438, 0xd707, 0xa438, 0x4042, 0xa438, 0xa308, 0xa438, 0xa310, + 0xa438, 0xcd3f, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8ed0, + 0xa438, 0x33a9, 0xa438, 0x8ec6, 0xa438, 0xd700, 0xa438, 0x40c7, + 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd1d6, 0xa438, 0xd04a, + 0xa438, 0xf017, 0xa438, 0xd1d6, 0xa438, 0xd04a, 0xa438, 0xf014, + 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, + 0xa438, 0xd1d6, 0xa438, 0xd04a, 0xa438, 0xf00d, 0xa438, 0xd1d6, + 0xa438, 0xd04a, 0xa438, 0xf00a, 0xa438, 0xd700, 0xa438, 0x40c7, + 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd199, 0xa438, 0xd04a, + 0xa438, 0xf003, 0xa438, 0xd199, 0xa438, 0xd04a, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0x0c30, 0xa438, 0x0320, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x8efa, 0xa438, 0x33a9, 0xa438, 0x8ef0, + 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, + 0xa438, 0xd199, 0xa438, 0xd04a, 0xa438, 0xf017, 0xa438, 0xd17a, + 0xa438, 0xd04b, 0xa438, 0xf014, 0xa438, 0xd700, 0xa438, 0x40c7, + 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd199, 0xa438, 0xd04a, + 0xa438, 0xf00d, 0xa438, 0xd17a, 0xa438, 0xd04b, 0xa438, 0xf00a, + 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, + 0xa438, 0xd17a, 0xa438, 0xd04b, 0xa438, 0xf003, 0xa438, 0xd17a, + 0xa438, 0xd04b, 0xa438, 0xd700, 0xa438, 0x4287, 0xa438, 0xd704, + 0xa438, 0x425d, 0xa438, 0xd704, 0xa438, 0x421c, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd701, + 0xa438, 0x5f77, 0xa438, 0x1000, 0xa438, 0x9c16, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xf009, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, + 0xa438, 0xd701, 0xa438, 0x5f37, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8102, 0xa438, 0x9503, 0xa438, 0x8a02, 0xa438, 0xcd40, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, + 0xa438, 0xd40a, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xbb10, + 0xa438, 0xcd41, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd71f, 0xa438, 0x5f75, 0xa438, 0x800a, + 0xa438, 0x81a0, 0xa438, 0x8302, 0xa438, 0x8480, 0xa438, 0xd700, + 0xa438, 0x6047, 0xa438, 0xa680, 0xa438, 0x8606, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8340, 0xa438, 0xd700, 0xa438, 0x6060, + 0xa438, 0xa120, 0xa438, 0xa302, 0xa438, 0x9503, 0xa438, 0x8c40, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8750, 0xa438, 0x8702, + 0xa438, 0x9503, 0xa438, 0x9b30, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8011, 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fb5, 0xa438, 0xcd42, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5f94, 0xa438, 0xd700, + 0xa438, 0x2969, 0xa438, 0x8f67, 0xa438, 0xd700, 0xa438, 0x6140, + 0xa438, 0xd705, 0xa438, 0x611e, 0xa438, 0xd703, 0xa438, 0x40d0, + 0xa438, 0x1000, 0xa438, 0x9c20, 0xa438, 0xd41a, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0x8340, 0xa438, 0xa801, 0xa438, 0xd700, + 0xa438, 0x2fa9, 0xa438, 0x8f70, 0xa438, 0x33c9, 0xa438, 0x8f73, + 0xa438, 0x6117, 0xa438, 0xf00a, 0xa438, 0xd141, 0xa438, 0xd043, + 0xa438, 0xf009, 0xa438, 0xd121, 0xa438, 0xd043, 0xa438, 0xf006, + 0xa438, 0xd122, 0xa438, 0xd042, 0xa438, 0xf003, 0xa438, 0xd181, + 0xa438, 0xd043, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce08, 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, + 0xa438, 0xce08, 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, + 0xa438, 0xce08, 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd705, 0xa438, 0x611e, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0ccf, 0xa438, 0x0b02, 0xa438, 0x8cc7, + 0xa438, 0x9503, 0xa438, 0xf008, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0ccf, 0xa438, 0x0b46, 0xa438, 0x0cc7, 0xa438, 0x0c03, + 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x40e7, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c3f, 0xa438, 0x0d09, 0xa438, 0x9503, + 0xa438, 0xf006, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c3f, + 0xa438, 0x0d0a, 0xa438, 0x9503, 0xa438, 0xd705, 0xa438, 0x607e, + 0xa438, 0xa302, 0xa438, 0xf00b, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa011, 0xa438, 0x9503, 0xa438, 0xd14f, 0xa438, 0xd043, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, + 0xa438, 0xa810, 0xa438, 0x8240, 0xa438, 0xa00a, 0xa438, 0xa1a0, + 0xa438, 0xa480, 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, + 0xa438, 0x407d, 0xa438, 0x8604, 0xa438, 0xf002, 0xa438, 0xa604, + 0xa438, 0xd700, 0xa438, 0x60c7, 0xa438, 0xa682, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa702, 0xa438, 0x9503, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xb920, + 0xa438, 0xcd43, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x7fb4, 0xa438, 0x8810, 0xa438, 0x9920, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x60e5, 0xa438, 0x5f94, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, + 0xa438, 0xffef, 0xa438, 0xb820, 0xa438, 0xa810, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, + 0xa438, 0xbb20, 0xa438, 0xcd44, 0xa438, 0xd700, 0xa438, 0x4060, + 0xa438, 0x1800, 0xa438, 0x9072, 0xa438, 0xd700, 0xa438, 0x37c9, + 0xa438, 0x9003, 0xa438, 0x33a9, 0xa438, 0x9000, 0xa438, 0xd17a, + 0xa438, 0xd047, 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd048, + 0xa438, 0xf003, 0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd700, + 0xa438, 0x2d49, 0xa438, 0x9014, 0xa438, 0xd701, 0xa438, 0x60b0, + 0xa438, 0x1800, 0xa438, 0x90c4, 0xa438, 0x1800, 0xa438, 0x911b, + 0xa438, 0x0c06, 0xa438, 0x0a06, 0xa438, 0x0cc0, 0xa438, 0x0cc0, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa7a0, 0xa438, 0x9503, + 0xa438, 0xcd45, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x902a, + 0xa438, 0x33a9, 0xa438, 0x9027, 0xa438, 0xd17a, 0xa438, 0xd047, + 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0xf003, + 0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, + 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0x8c40, 0xa438, 0xd418, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xac40, 0xa438, 0xaa01, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd706, 0xa438, 0x5f6f, 0xa438, 0xd700, 0xa438, 0x37c9, + 0xa438, 0x904b, 0xa438, 0x33a9, 0xa438, 0x9048, 0xa438, 0xd17a, + 0xa438, 0xd047, 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd048, + 0xa438, 0xf003, 0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xab20, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa750, 0xa438, 0x9503, + 0xa438, 0xcd46, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x9066, + 0xa438, 0x33a9, 0xa438, 0x9063, 0xa438, 0xd1b7, 0xa438, 0xd04d, + 0xa438, 0xf006, 0xa438, 0xd1b7, 0xa438, 0xd04d, 0xa438, 0xf003, + 0xa438, 0xd1b7, 0xa438, 0xd04d, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, + 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0x1800, 0xa438, 0x90c4, + 0xa438, 0xd701, 0xa438, 0x4830, 0xa438, 0xd700, 0xa438, 0x47e7, + 0xa438, 0xd704, 0xa438, 0x47bd, 0xa438, 0xd704, 0xa438, 0x477c, + 0xa438, 0x0c06, 0xa438, 0x0a06, 0xa438, 0x0cc0, 0xa438, 0x0cc0, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x9089, 0xa438, 0x33a9, + 0xa438, 0x9086, 0xa438, 0xd17a, 0xa438, 0xd047, 0xa438, 0xf006, + 0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0xf003, 0xa438, 0xd17a, + 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706, + 0xa438, 0x5f2f, 0xa438, 0x8c40, 0xa438, 0xd418, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0xac40, 0xa438, 0xaa01, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd706, + 0xa438, 0x5f6f, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x90aa, + 0xa438, 0x33a9, 0xa438, 0x90a7, 0xa438, 0xd17a, 0xa438, 0xd047, + 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0xf003, + 0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, + 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd701, 0xa438, 0x40f0, + 0xa438, 0xac40, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa750, + 0xa438, 0x9503, 0xa438, 0xab20, 0xa438, 0xd1c4, 0xa438, 0xd046, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x424f, + 0xa438, 0x8c40, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8750, + 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd417, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0xd701, 0xa438, 0x40d0, 0xa438, 0xac40, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa750, 0xa438, 0x9503, + 0xa438, 0xcd47, 0xa438, 0xd700, 0xa438, 0x686b, 0xa438, 0x6060, + 0xa438, 0x1800, 0xa438, 0x90f3, 0xa438, 0xd700, 0xa438, 0x37c9, + 0xa438, 0x90e7, 0xa438, 0x33a9, 0xa438, 0x90e4, 0xa438, 0xd15c, + 0xa438, 0xd04c, 0xa438, 0xf006, 0xa438, 0xd182, 0xa438, 0xd04c, + 0xa438, 0xf003, 0xa438, 0xd191, 0xa438, 0xd04c, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0x1800, + 0xa438, 0x911b, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0ccf, + 0xa438, 0x0b03, 0xa438, 0x8cc7, 0xa438, 0x9503, 0xa438, 0xd700, + 0xa438, 0x60e7, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c3f, + 0xa438, 0x0d1a, 0xa438, 0x9503, 0xa438, 0xf006, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c3f, 0xa438, 0x0d19, 0xa438, 0x9503, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x9111, 0xa438, 0x33a9, + 0xa438, 0x910e, 0xa438, 0xd19f, 0xa438, 0xd049, 0xa438, 0xf006, + 0xa438, 0xd199, 0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd199, + 0xa438, 0xd04b, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706, + 0xa438, 0x5f2f, 0xa438, 0xd416, 0xa438, 0x1000, 0xa438, 0x9bbf, + 0xa438, 0xbb10, 0xa438, 0xcd4f, 0xa438, 0xcd50, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd71f, + 0xa438, 0x5f75, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8302, + 0xa438, 0x8480, 0xa438, 0xd700, 0xa438, 0x6047, 0xa438, 0xa682, + 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, 0xa438, 0x407d, + 0xa438, 0x8604, 0xa438, 0xf002, 0xa438, 0xa604, 0xa438, 0x8818, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8340, 0xa438, 0xa120, + 0xa438, 0xa302, 0xa438, 0x9503, 0xa438, 0x8c40, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8750, 0xa438, 0x9503, 0xa438, 0x9b30, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8011, 0xa438, 0x9503, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb5, + 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, + 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, + 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, 0xa438, 0xce08, + 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, 0xa438, 0xce08, + 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0xcd51, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x5f94, 0xa438, 0xd71e, 0xa438, 0x6103, 0xa438, 0xd700, + 0xa438, 0x2fa9, 0xa438, 0x916f, 0xa438, 0x33c9, 0xa438, 0x9172, + 0xa438, 0x6177, 0xa438, 0x61b1, 0xa438, 0xd101, 0xa438, 0xd040, + 0xa438, 0xf00c, 0xa438, 0xd141, 0xa438, 0xd043, 0xa438, 0xf009, + 0xa438, 0xd121, 0xa438, 0xd043, 0xa438, 0xf006, 0xa438, 0xd122, + 0xa438, 0xd042, 0xa438, 0xf003, 0xa438, 0xd181, 0xa438, 0xd043, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, + 0xa438, 0xd705, 0xa438, 0x60fe, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0ccf, 0xa438, 0x0b03, 0xa438, 0x8c07, 0xa438, 0x9503, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8cc0, 0xa438, 0x9503, + 0xa438, 0x8106, 0xa438, 0xd700, 0xa438, 0x60e7, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c3f, 0xa438, 0x0d1a, 0xa438, 0x9503, + 0xa438, 0xf006, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c3f, + 0xa438, 0x0d19, 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x2f99, + 0xa438, 0x919c, 0xa438, 0xa804, 0xa438, 0xd705, 0xa438, 0x607e, + 0xa438, 0xa302, 0xa438, 0xf00b, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa017, 0xa438, 0x9503, 0xa438, 0xd14f, 0xa438, 0xd043, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, + 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xa480, 0xa438, 0xd700, + 0xa438, 0x40a7, 0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0x8644, + 0xa438, 0xf003, 0xa438, 0x0c44, 0xa438, 0x0604, 0xa438, 0xd701, + 0xa438, 0x40d0, 0xa438, 0xac40, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa750, 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x6047, + 0xa438, 0xa682, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, + 0xa438, 0xc5aa, 0xa438, 0x9503, 0xa438, 0xab80, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5faf, 0xa438, 0xb920, + 0xa438, 0xcd52, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x7fb4, 0xa438, 0x9920, 0xa438, 0xa00a, 0xa438, 0xa1a0, + 0xa438, 0xd705, 0xa438, 0x605e, 0xa438, 0xa302, 0xa438, 0xa480, + 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, 0xa438, 0x407d, + 0xa438, 0x8644, 0xa438, 0xf003, 0xa438, 0x0c44, 0xa438, 0x0604, + 0xa438, 0xa902, 0xa438, 0x8920, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa480, 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9c2a, + 0xa438, 0xd707, 0xa438, 0x40ae, 0xa438, 0x0c18, 0xa438, 0x0a08, + 0xa438, 0x1000, 0xa438, 0x9c30, 0xa438, 0xcd60, 0xa438, 0xd101, + 0xa438, 0xd040, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa230, + 0xa438, 0x9503, 0xa438, 0xd703, 0xa438, 0x68d1, 0xa438, 0xcd62, + 0xa438, 0x1000, 0xa438, 0x9a89, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd707, 0xa438, 0x4058, 0xa438, 0xa901, 0xa438, 0xd700, + 0xa438, 0x62dc, 0xa438, 0xd71f, 0xa438, 0x628e, 0xa438, 0xd704, + 0xa438, 0x4067, 0xa438, 0x1800, 0xa438, 0x96ac, 0xa438, 0xd704, + 0xa438, 0x40ab, 0xa438, 0xd705, 0xa438, 0x607f, 0xa438, 0x1800, + 0xa438, 0x974b, 0xa438, 0xd704, 0xa438, 0x609f, 0xa438, 0xd705, + 0xa438, 0x405d, 0xa438, 0xf005, 0xa438, 0xd704, 0xa438, 0x5c96, + 0xa438, 0xd75f, 0xa438, 0x5c40, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xc5aa, 0xa438, 0x9503, 0xa438, 0xd705, 0xa438, 0x60de, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c38, 0xa438, 0x0d18, + 0xa438, 0x9503, 0xa438, 0xd702, 0xa438, 0x4357, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8f80, 0xa438, 0x9503, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf00b, + 0xa438, 0xce08, 0xa438, 0xf009, 0xa438, 0xce08, 0xa438, 0xf007, + 0xa438, 0xce08, 0xa438, 0xf005, 0xa438, 0xce08, 0xa438, 0xf003, + 0xa438, 0xce08, 0xa438, 0xf001, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0xcd61, 0xa438, 0x1000, 0xa438, 0x9a89, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd707, 0xa438, 0x4058, 0xa438, 0xa901, + 0xa438, 0xd71f, 0xa438, 0x2e71, 0xa438, 0x925e, 0xa438, 0xd704, + 0xa438, 0x2739, 0xa438, 0x96ac, 0xa438, 0xd704, 0xa438, 0x40ab, + 0xa438, 0xd705, 0xa438, 0x607f, 0xa438, 0x1800, 0xa438, 0x974b, + 0xa438, 0xd704, 0xa438, 0x60bf, 0xa438, 0xd705, 0xa438, 0x407d, + 0xa438, 0x1800, 0xa438, 0x97da, 0xa438, 0xd704, 0xa438, 0x5cb6, + 0xa438, 0xd75f, 0xa438, 0x5c60, 0xa438, 0x1800, 0xa438, 0x997c, + 0xa438, 0xaa10, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8480, + 0xa438, 0x9503, 0xa438, 0x9904, 0xa438, 0xcd80, 0xa438, 0x1000, + 0xa438, 0x9c99, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8302, + 0xa438, 0x8480, 0xa438, 0x8646, 0xa438, 0x1000, 0xa438, 0x9d44, + 0xa438, 0xd707, 0xa438, 0x605d, 0xa438, 0x8320, 0xa438, 0x8c40, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8750, 0xa438, 0x8340, + 0xa438, 0x8120, 0xa438, 0x8302, 0xa438, 0xa61c, 0xa438, 0x9503, + 0xa438, 0xd701, 0xa438, 0x4050, 0xa438, 0xab20, 0xa438, 0x8b80, + 0xa438, 0x0c0c, 0xa438, 0x0808, 0xa438, 0xd702, 0xa438, 0x4191, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fa1, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8f02, 0xa438, 0x9503, + 0xa438, 0xd400, 0xa438, 0xb302, 0xa438, 0xd200, 0xa438, 0xb910, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fae, + 0xa438, 0x9910, 0xa438, 0xd71f, 0xa438, 0x409e, 0xa438, 0xd1b8, + 0xa438, 0xd049, 0xa438, 0xf012, 0xa438, 0xd700, 0xa438, 0x2fa9, + 0xa438, 0x92a1, 0xa438, 0x6131, 0xa438, 0x33c9, 0xa438, 0x92a7, + 0xa438, 0xd15d, 0xa438, 0xd040, 0xa438, 0xf009, 0xa438, 0xd193, + 0xa438, 0xd040, 0xa438, 0xf006, 0xa438, 0xd1db, 0xa438, 0xd040, + 0xa438, 0xf003, 0xa438, 0xd16f, 0xa438, 0xd040, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa380, 0xa438, 0x9503, 0xa438, 0xd417, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd700, 0xa438, 0x4127, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8bc0, 0xa438, 0x8cc7, + 0xa438, 0x0c3f, 0xa438, 0x0d08, 0xa438, 0x9503, 0xa438, 0xf007, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8bc0, 0xa438, 0x8cc7, + 0xa438, 0xcd48, 0xa438, 0x9503, 0xa438, 0x0c06, 0xa438, 0x0102, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xc555, 0xa438, 0x9503, + 0xa438, 0xcd81, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd707, 0xa438, 0x40fd, 0xa438, 0xd193, + 0xa438, 0xd047, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xa810, 0xa438, 0x8902, 0xa438, 0xd707, + 0xa438, 0x407b, 0xa438, 0x8801, 0xa438, 0xa340, 0xa438, 0xd71f, + 0xa438, 0x3ffd, 0xa438, 0x9521, 0xa438, 0xcd82, 0xa438, 0xd704, + 0xa438, 0x407f, 0xa438, 0x1800, 0xa438, 0x9390, 0xa438, 0x8810, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa204, 0xa438, 0xa108, + 0xa438, 0x9503, 0xa438, 0xa604, 0xa438, 0xd131, 0xa438, 0xd046, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, + 0xa438, 0xd707, 0xa438, 0x5f75, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8108, 0xa438, 0x9503, 0xa438, 0x0c07, 0xa438, 0x0b02, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c03, 0xa438, 0x0200, + 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce0b, 0xa438, 0xf00a, 0xa438, 0xce0b, 0xa438, 0xf008, + 0xa438, 0xce0a, 0xa438, 0xf006, 0xa438, 0xce0a, 0xa438, 0xf004, + 0xa438, 0xce09, 0xa438, 0xf002, 0xa438, 0xce09, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xab08, 0xa438, 0xcdc0, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd704, 0xa438, 0x5fb7, 0xa438, 0x8b08, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8204, 0xa438, 0xa380, + 0xa438, 0x9503, 0xa438, 0xd414, 0xa438, 0xb308, 0xa438, 0xd202, + 0xa438, 0xb302, 0xa438, 0xb301, 0xa438, 0xcdc1, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fa1, 0xa438, 0xd400, + 0xa438, 0xb302, 0xa438, 0xd200, 0xa438, 0xd71f, 0xa438, 0x6060, + 0xa438, 0xd704, 0xa438, 0x6074, 0xa438, 0x1800, 0xa438, 0x9390, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c91, 0xa438, 0x0080, + 0xa438, 0xa060, 0xa438, 0xa101, 0xa438, 0xa408, 0xa438, 0xa240, + 0xa438, 0x9503, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c26, + 0xa438, 0x0022, 0xa438, 0xa110, 0xa438, 0xa011, 0xa438, 0x9503, + 0xa438, 0xd17a, 0xa438, 0xd046, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c38, 0xa438, 0x0c18, 0xa438, 0x9503, 0xa438, 0xa318, + 0xa438, 0xd131, 0xa438, 0xd046, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c83, 0xa438, 0x0282, 0xa438, 0x9503, 0xa438, 0x0c07, + 0xa438, 0x0b02, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce11, 0xa438, 0xf00a, 0xa438, 0xce11, 0xa438, 0xf008, + 0xa438, 0xce10, 0xa438, 0xf006, 0xa438, 0xce10, 0xa438, 0xf004, + 0xa438, 0xce0f, 0xa438, 0xf002, 0xa438, 0xce0f, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa204, 0xa438, 0x9503, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xab08, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd704, 0xa438, 0x3ebc, 0xa438, 0x9370, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8284, 0xa438, 0x9503, 0xa438, 0x8b08, + 0xa438, 0xd704, 0xa438, 0x61b9, 0xa438, 0xd705, 0xa438, 0x407c, + 0xa438, 0x1800, 0xa438, 0x938e, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa402, 0xa438, 0x9503, 0xa438, 0x8310, 0xa438, 0xcdc3, + 0xa438, 0x1800, 0xa438, 0x9337, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8402, 0xa438, 0x9503, 0xa438, 0xcdc2, 0xa438, 0xf003, + 0xa438, 0x1000, 0xa438, 0x9c99, 0xa438, 0x8320, 0xa438, 0xd705, + 0xa438, 0x429e, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce02, 0xa438, 0xf01d, 0xa438, 0xce02, 0xa438, 0xf01b, + 0xa438, 0xce01, 0xa438, 0xf019, 0xa438, 0xce01, 0xa438, 0xf017, + 0xa438, 0xce01, 0xa438, 0xf015, 0xa438, 0xce01, 0xa438, 0xf013, + 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, + 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce06, + 0xa438, 0xf00a, 0xa438, 0xce06, 0xa438, 0xf008, 0xa438, 0xce06, + 0xa438, 0xf006, 0xa438, 0xce06, 0xa438, 0xf004, 0xa438, 0xce06, + 0xa438, 0xf002, 0xa438, 0xce06, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0xd705, 0xa438, 0x61be, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c0f, 0xa438, 0x0b05, 0xa438, 0x0c38, 0xa438, 0x0c28, + 0xa438, 0x9503, 0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0xa302, + 0xa438, 0xa4a0, 0xa438, 0xf012, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c0f, 0xa438, 0x0b06, 0xa438, 0x0c3f, 0xa438, 0x0c1b, + 0xa438, 0xd700, 0xa438, 0x6067, 0xa438, 0x0cc0, 0xa438, 0x0d80, + 0xa438, 0x9503, 0xa438, 0x8810, 0xa438, 0x800a, 0xa438, 0x8302, + 0xa438, 0xa8c0, 0xa438, 0x8420, 0xa438, 0xa480, 0xa438, 0xd700, + 0xa438, 0x40c7, 0xa438, 0xd704, 0xa438, 0x409d, 0xa438, 0x0ce3, + 0xa438, 0x0203, 0xa438, 0xf003, 0xa438, 0x0ce0, 0xa438, 0x02a0, + 0xa438, 0xa120, 0xa438, 0xd700, 0xa438, 0x4187, 0xa438, 0xd704, + 0xa438, 0x40dd, 0xa438, 0xd704, 0xa438, 0x617b, 0xa438, 0xd704, + 0xa438, 0x613c, 0xa438, 0xf00a, 0xa438, 0xd707, 0xa438, 0x411f, + 0xa438, 0x40e0, 0xa438, 0xf004, 0xa438, 0xd707, 0xa438, 0x409f, + 0xa438, 0x4062, 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa318, + 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, 0xa438, 0x407d, + 0xa438, 0x8604, 0xa438, 0xf002, 0xa438, 0xa604, 0xa438, 0xd700, + 0xa438, 0x6047, 0xa438, 0xa682, 0xa438, 0x1000, 0xa438, 0x9cc6, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb4, + 0xa438, 0xb920, 0xa438, 0xcd83, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x8810, 0xa438, 0x9920, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x60e5, + 0xa438, 0x5f94, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd706, + 0xa438, 0x5fa7, 0xa438, 0xffef, 0xa438, 0xb820, 0xa438, 0xa810, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fa5, + 0xa438, 0x9820, 0xa438, 0xbb20, 0xa438, 0xd705, 0xa438, 0x605e, + 0xa438, 0xf018, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, 0xa438, 0xf008, + 0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, 0xa438, 0xf004, + 0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xa00a, 0xa438, 0xa420, 0xa438, 0x88c0, + 0xa438, 0xd701, 0xa438, 0x40d0, 0xa438, 0xac40, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa750, 0xa438, 0x9503, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x9450, 0xa438, 0x33a9, 0xa438, 0x944d, + 0xa438, 0xd1b7, 0xa438, 0xd04b, 0xa438, 0xf006, 0xa438, 0xd1b7, + 0xa438, 0xd04b, 0xa438, 0xf003, 0xa438, 0xd1b7, 0xa438, 0xd04b, + 0xa438, 0xd707, 0xa438, 0x3ad1, 0xa438, 0x94d8, 0xa438, 0xd700, + 0xa438, 0x40c7, 0xa438, 0xd704, 0xa438, 0x409d, 0xa438, 0xd704, + 0xa438, 0x465c, 0xa438, 0xf004, 0xa438, 0xd707, 0xa438, 0x605f, + 0xa438, 0xf02e, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x946a, + 0xa438, 0x33a9, 0xa438, 0x9467, 0xa438, 0xd199, 0xa438, 0xd05a, + 0xa438, 0xf006, 0xa438, 0xd199, 0xa438, 0xd05a, 0xa438, 0xf003, + 0xa438, 0xd1d6, 0xa438, 0xd05a, 0xa438, 0xcd84, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7b, 0xa438, 0xd700, 0xa438, 0x42c7, 0xa438, 0xd704, + 0xa438, 0x429d, 0xa438, 0xd704, 0xa438, 0x425c, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd706, + 0xa438, 0x5f6f, 0xa438, 0x8a04, 0xa438, 0x8b20, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa380, 0xa438, 0x9503, 0xa438, 0xd401, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x1800, 0xa438, 0x94d8, + 0xa438, 0xd706, 0xa438, 0x69ab, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, + 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd700, 0xa438, 0x4167, + 0xa438, 0xd704, 0xa438, 0x40bd, 0xa438, 0xd704, 0xa438, 0x615b, + 0xa438, 0x1800, 0xa438, 0x94a5, 0xa438, 0xd707, 0xa438, 0x411f, + 0xa438, 0x40e0, 0xa438, 0xf004, 0xa438, 0xd707, 0xa438, 0x409f, + 0xa438, 0x4062, 0xa438, 0x8320, 0xa438, 0xf002, 0xa438, 0xa320, + 0xa438, 0x8310, 0xa438, 0xcd85, 0xa438, 0xd700, 0xa438, 0x37c9, + 0xa438, 0x94b3, 0xa438, 0x33a9, 0xa438, 0x94b0, 0xa438, 0xd1b7, + 0xa438, 0xd04a, 0xa438, 0xf006, 0xa438, 0xd1b7, 0xa438, 0xd04a, + 0xa438, 0xf003, 0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd700, + 0xa438, 0x41a7, 0xa438, 0xd704, 0xa438, 0x40dd, 0xa438, 0xd704, + 0xa438, 0x605b, 0xa438, 0xf00d, 0xa438, 0x1800, 0xa438, 0x8f25, + 0xa438, 0xd707, 0xa438, 0x413f, 0xa438, 0x4100, 0xa438, 0x1800, + 0xa438, 0x8f25, 0xa438, 0xd707, 0xa438, 0x409f, 0xa438, 0x4062, + 0xa438, 0x1800, 0xa438, 0x8f25, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd701, 0xa438, 0x5f77, + 0xa438, 0x1800, 0xa438, 0x8f25, 0xa438, 0xd705, 0xa438, 0x413e, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, 0xa438, 0x0b03, + 0xa438, 0x0c38, 0xa438, 0x0c00, 0xa438, 0xa104, 0xa438, 0x9503, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa102, 0xa438, 0x9503, + 0xa438, 0xaa02, 0xa438, 0xa310, 0xa438, 0xcd86, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x94f4, 0xa438, 0x33a9, 0xa438, 0x94f1, + 0xa438, 0xd1d6, 0xa438, 0xd04a, 0xa438, 0xf006, 0xa438, 0xd1d6, + 0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd199, 0xa438, 0xd04a, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0x0c30, 0xa438, 0x0320, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x9509, 0xa438, 0x33a9, + 0xa438, 0x9506, 0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0xf006, + 0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd1b7, + 0xa438, 0xd04a, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd701, + 0xa438, 0x5f37, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8102, + 0xa438, 0x8104, 0xa438, 0x9503, 0xa438, 0xd701, 0xa438, 0x40b0, + 0xa438, 0xaa02, 0xa438, 0xac80, 0xa438, 0x1800, 0xa438, 0x8f25, + 0xa438, 0x8a02, 0xa438, 0x1800, 0xa438, 0x8f25, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa708, 0xa438, 0x9503, 0xa438, 0xcd87, + 0xa438, 0xd704, 0xa438, 0x615f, 0xa438, 0x8810, 0xa438, 0xd702, + 0xa438, 0x40e6, 0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce06, 0xa438, 0xf00a, + 0xa438, 0xce06, 0xa438, 0xf008, 0xa438, 0xce06, 0xa438, 0xf006, + 0xa438, 0xce06, 0xa438, 0xf004, 0xa438, 0xce06, 0xa438, 0xf002, + 0xa438, 0xce06, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xd707, + 0xa438, 0x413b, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, + 0xa438, 0x0b03, 0xa438, 0x0c38, 0xa438, 0x0c18, 0xa438, 0x9503, + 0xa438, 0xf006, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, + 0xa438, 0x0b07, 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x40c7, + 0xa438, 0xd704, 0xa438, 0x409d, 0xa438, 0x0ce3, 0xa438, 0x0203, + 0xa438, 0xf003, 0xa438, 0x0ce0, 0xa438, 0x02a0, 0xa438, 0xa120, + 0xa438, 0xd707, 0xa438, 0x425b, 0xa438, 0xd700, 0xa438, 0x4167, + 0xa438, 0xd704, 0xa438, 0x40dd, 0xa438, 0xd704, 0xa438, 0x613b, + 0xa438, 0xd704, 0xa438, 0x60fc, 0xa438, 0xf008, 0xa438, 0xd707, + 0xa438, 0x40df, 0xa438, 0xf003, 0xa438, 0xd707, 0xa438, 0x407f, + 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa318, 0xa438, 0xa00a, + 0xa438, 0xa302, 0xa438, 0xa4a0, 0xa438, 0xd700, 0xa438, 0x40a7, + 0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0x8604, 0xa438, 0xf002, + 0xa438, 0xa604, 0xa438, 0xd700, 0xa438, 0x6047, 0xa438, 0xa682, + 0xa438, 0x1000, 0xa438, 0x9cc6, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920, 0xa438, 0xcd88, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, + 0xa438, 0x8810, 0xa438, 0x9920, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x60e5, 0xa438, 0x5f94, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xffef, + 0xa438, 0xb820, 0xa438, 0xa810, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0xbb20, + 0xa438, 0xd701, 0xa438, 0x40d0, 0xa438, 0xac40, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa750, 0xa438, 0x9503, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x95be, 0xa438, 0x33a9, 0xa438, 0x95b6, + 0xa438, 0xd702, 0xa438, 0x4086, 0xa438, 0xd161, 0xa438, 0xd049, + 0xa438, 0xf00e, 0xa438, 0xd19e, 0xa438, 0xd049, 0xa438, 0xf00b, + 0xa438, 0xd702, 0xa438, 0x4086, 0xa438, 0xd186, 0xa438, 0xd049, + 0xa438, 0xf006, 0xa438, 0xd1c3, 0xa438, 0xd049, 0xa438, 0xf003, + 0xa438, 0xd1cf, 0xa438, 0xd049, 0xa438, 0xcd89, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd707, + 0xa438, 0x40fb, 0xa438, 0x8310, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c0f, 0xa438, 0x0b05, 0xa438, 0x9503, 0xa438, 0xcd8a, + 0xa438, 0xd707, 0xa438, 0x419b, 0xa438, 0xd700, 0xa438, 0x33ad, + 0xa438, 0x95df, 0xa438, 0xd189, 0xa438, 0xd045, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xcd8b, 0xa438, 0xd40a, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0xbb10, 0xa438, 0xcd8c, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd71f, + 0xa438, 0x5f75, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8302, + 0xa438, 0x8480, 0xa438, 0xd700, 0xa438, 0x6047, 0xa438, 0xa680, + 0xa438, 0x8606, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8340, + 0xa438, 0x8750, 0xa438, 0x9503, 0xa438, 0x8c40, 0xa438, 0x9b30, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb5, + 0xa438, 0xcd8d, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x5f94, 0xa438, 0x8340, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8708, 0xa438, 0x9503, 0xa438, 0xa801, 0xa438, 0xd700, + 0xa438, 0x2fa9, 0xa438, 0x9610, 0xa438, 0x33c9, 0xa438, 0x9613, + 0xa438, 0x6117, 0xa438, 0xf00a, 0xa438, 0xd141, 0xa438, 0xd043, + 0xa438, 0xf009, 0xa438, 0xd121, 0xa438, 0xd043, 0xa438, 0xf006, + 0xa438, 0xd122, 0xa438, 0xd042, 0xa438, 0xf003, 0xa438, 0xd181, + 0xa438, 0xd043, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, 0xa438, 0xf008, + 0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, 0xa438, 0xf004, + 0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, + 0xa438, 0x0b02, 0xa438, 0x8cc7, 0xa438, 0x9503, 0xa438, 0xd700, + 0xa438, 0x40e7, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c3f, + 0xa438, 0x0d09, 0xa438, 0x9503, 0xa438, 0xf006, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c3f, 0xa438, 0x0d08, 0xa438, 0x9503, + 0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0x0ca0, 0xa438, 0x0120, + 0xa438, 0xa302, 0xa438, 0xa480, 0xa438, 0xd700, 0xa438, 0x40a7, + 0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0x8604, 0xa438, 0xf002, + 0xa438, 0xa604, 0xa438, 0xd700, 0xa438, 0x6047, 0xa438, 0xa682, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, + 0xa438, 0xb920, 0xa438, 0xcd8e, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x8810, 0xa438, 0x9920, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x60e5, + 0xa438, 0x5f94, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd706, + 0xa438, 0x5fa7, 0xa438, 0xffef, 0xa438, 0xb820, 0xa438, 0xa810, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fa5, + 0xa438, 0x9820, 0xa438, 0xbb20, 0xa438, 0xd701, 0xa438, 0x40d0, + 0xa438, 0xac40, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa750, + 0xa438, 0x9503, 0xa438, 0xd1c4, 0xa438, 0xd046, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x422f, 0xa438, 0x8c40, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8750, 0xa438, 0x8380, + 0xa438, 0x9503, 0xa438, 0xd417, 0xa438, 0x1000, 0xa438, 0x9bbf, + 0xa438, 0xd701, 0xa438, 0x40d0, 0xa438, 0xac40, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa750, 0xa438, 0x9503, 0xa438, 0xd192, + 0xa438, 0xd047, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706, + 0xa438, 0x5f2f, 0xa438, 0xd707, 0xa438, 0x409b, 0xa438, 0xd416, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xbb10, 0xa438, 0xcd8f, + 0xa438, 0x1800, 0xa438, 0x9120, 0xa438, 0x1000, 0xa438, 0x9c99, + 0xa438, 0x8302, 0xa438, 0x8646, 0xa438, 0xd700, 0xa438, 0x60c7, + 0xa438, 0xa680, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8702, + 0xa438, 0x9503, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa680, + 0xa438, 0x850f, 0xa438, 0x9503, 0xa438, 0xd701, 0xa438, 0x4050, + 0xa438, 0x8b20, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0ccf, + 0xa438, 0x0bc8, 0xa438, 0x0cc7, 0xa438, 0x0c44, 0xa438, 0x9503, + 0xa438, 0x0c06, 0xa438, 0x0102, 0xa438, 0xd700, 0xa438, 0x40e7, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c3f, 0xa438, 0x0d24, + 0xa438, 0x9503, 0xa438, 0xf005, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xcde4, 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9cdd, + 0xa438, 0xcd90, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa601, + 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x2fa9, 0xa438, 0x96e1, + 0xa438, 0x60d1, 0xa438, 0x6113, 0xa438, 0x6157, 0xa438, 0xd150, + 0xa438, 0xd040, 0xa438, 0xf009, 0xa438, 0xd1a0, 0xa438, 0xd040, + 0xa438, 0xf006, 0xa438, 0xd128, 0xa438, 0xd040, 0xa438, 0xf003, + 0xa438, 0xd114, 0xa438, 0xd040, 0xa438, 0x1000, 0xa438, 0x99a8, + 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd704, 0xa438, 0x7f6b, + 0xa438, 0xa81a, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8603, + 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9d31, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8480, + 0xa438, 0x1000, 0xa438, 0x99a8, 0xa438, 0xd707, 0xa438, 0x4058, + 0xa438, 0xa901, 0xa438, 0xd704, 0xa438, 0x2b59, 0xa438, 0x974b, + 0xa438, 0xd704, 0xa438, 0x604c, 0xa438, 0xfff6, 0xa438, 0xd701, + 0xa438, 0x6056, 0xa438, 0xa704, 0xa438, 0xa302, 0xa438, 0xd700, + 0xa438, 0x40a7, 0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0x8604, + 0xa438, 0xf002, 0xa438, 0xa604, 0xa438, 0xd700, 0xa438, 0x60c7, + 0xa438, 0xa602, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa702, + 0xa438, 0x9503, 0xa438, 0xd701, 0xa438, 0x4055, 0xa438, 0xa940, + 0xa438, 0xd701, 0xa438, 0x4050, 0xa438, 0xab20, 0xa438, 0xcd91, + 0xa438, 0x1000, 0xa438, 0x9cf4, 0xa438, 0x1000, 0xa438, 0x99a8, + 0xa438, 0xd704, 0xa438, 0x7f6c, 0xa438, 0xa120, 0xa438, 0xd707, + 0xa438, 0x4051, 0xa438, 0xa180, 0xa438, 0xa480, 0xa438, 0x8302, + 0xa438, 0x8606, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8702, + 0xa438, 0x9503, 0xa438, 0xd701, 0xa438, 0x6056, 0xa438, 0x8704, + 0xa438, 0x1000, 0xa438, 0x9cf4, 0xa438, 0xce93, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xd701, 0xa438, 0x4050, 0xa438, 0x8b20, + 0xa438, 0xcd92, 0xa438, 0x1000, 0xa438, 0x99a8, 0xa438, 0xd707, + 0xa438, 0x7fb7, 0xa438, 0x81a0, 0xa438, 0x8480, 0xa438, 0x1000, + 0xa438, 0x9d31, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x1800, + 0xa438, 0x96fe, 0xa438, 0xd705, 0xa438, 0x40d9, 0xa438, 0xd701, + 0xa438, 0x6056, 0xa438, 0xa704, 0xa438, 0xa010, 0xa438, 0xf002, + 0xa438, 0x8010, 0xa438, 0xd701, 0xa438, 0x4095, 0xa438, 0xd705, + 0xa438, 0x4059, 0xa438, 0xa940, 0xa438, 0xd705, 0xa438, 0x6119, + 0xa438, 0x0c06, 0xa438, 0x0102, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c07, 0xa438, 0x0d04, 0xa438, 0x9503, 0xa438, 0xd701, + 0xa438, 0x4050, 0xa438, 0xab20, 0xa438, 0xcd93, 0xa438, 0x1000, + 0xa438, 0x99a8, 0xa438, 0xd704, 0xa438, 0x464b, 0xa438, 0xd707, + 0xa438, 0x5f78, 0xa438, 0x1000, 0xa438, 0x9d13, 0xa438, 0x800a, + 0xa438, 0x81a0, 0xa438, 0xd700, 0xa438, 0x6117, 0xa438, 0x6093, + 0xa438, 0xd15e, 0xa438, 0xd040, 0xa438, 0xf006, 0xa438, 0xd132, + 0xa438, 0xd040, 0xa438, 0xf003, 0xa438, 0xd119, 0xa438, 0xd040, + 0xa438, 0x1000, 0xa438, 0x9d08, 0xa438, 0x1000, 0xa438, 0x99a8, + 0xa438, 0xd73e, 0xa438, 0x6065, 0xa438, 0xd700, 0xa438, 0x5f3a, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x85f0, 0xa438, 0x9503, + 0xa438, 0xa008, 0xa438, 0xd707, 0xa438, 0x4052, 0xa438, 0xa002, + 0xa438, 0x8010, 0xa438, 0xd705, 0xa438, 0x4099, 0xa438, 0xd701, + 0xa438, 0x6056, 0xa438, 0x8704, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0xd701, 0xa438, 0x4050, 0xa438, 0x8b20, 0xa438, 0xcd94, + 0xa438, 0x1000, 0xa438, 0x99a8, 0xa438, 0xd707, 0xa438, 0x7fb8, + 0xa438, 0x8010, 0xa438, 0xd705, 0xa438, 0x4099, 0xa438, 0xd701, + 0xa438, 0x6056, 0xa438, 0x8704, 0xa438, 0xd705, 0xa438, 0x4099, + 0xa438, 0xd701, 0xa438, 0x4050, 0xa438, 0x8b20, 0xa438, 0xd705, + 0xa438, 0x61f9, 0xa438, 0x8106, 0xa438, 0xd700, 0xa438, 0x60e7, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d02, + 0xa438, 0x9503, 0xa438, 0xf006, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c07, 0xa438, 0x0d01, 0xa438, 0x9503, 0xa438, 0x800a, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0cf0, 0xa438, 0x05a0, + 0xa438, 0x9503, 0xa438, 0xd705, 0xa438, 0x4099, 0xa438, 0x1000, + 0xa438, 0x9d31, 0xa438, 0xf014, 0xa438, 0xa1a0, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf00a, + 0xa438, 0xce08, 0xa438, 0xf008, 0xa438, 0xce08, 0xa438, 0xf006, + 0xa438, 0xce08, 0xa438, 0xf004, 0xa438, 0xce08, 0xa438, 0xf002, + 0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xd705, + 0xa438, 0x39cc, 0xa438, 0x91f3, 0xa438, 0x1800, 0xa438, 0x96fe, + 0xa438, 0xd75e, 0xa438, 0x60f4, 0xa438, 0x9b08, 0xa438, 0x9920, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8403, 0xa438, 0x9503, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8208, 0xa438, 0x8404, + 0xa438, 0x9503, 0xa438, 0xcda1, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, + 0xa438, 0xd704, 0xa438, 0x5f37, 0xa438, 0x8b08, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8204, 0xa438, 0x9503, 0xa438, 0xd707, + 0xa438, 0x6536, 0xa438, 0x0c07, 0xa438, 0x0b00, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c03, 0xa438, 0x0200, 0xa438, 0x9503, + 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, + 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce0b, + 0xa438, 0xf00a, 0xa438, 0xce0b, 0xa438, 0xf008, 0xa438, 0xce0a, + 0xa438, 0xf006, 0xa438, 0xce0a, 0xa438, 0xf004, 0xa438, 0xce09, + 0xa438, 0xf002, 0xa438, 0xce09, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa204, 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0xab08, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, 0xa438, 0xd704, + 0xa438, 0x5f37, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8204, + 0xa438, 0x9503, 0xa438, 0x8b08, 0xa438, 0xd414, 0xa438, 0xd202, + 0xa438, 0xb308, 0xa438, 0xb302, 0xa438, 0xb301, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xaf02, 0xa438, 0x9503, 0xa438, 0xcdaf, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x99a0, + 0xa438, 0x1000, 0xa438, 0x99a4, 0xa438, 0xd701, 0xa438, 0x5f21, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8f02, 0xa438, 0x9503, + 0xa438, 0xd400, 0xa438, 0xb302, 0xa438, 0xd200, 0xa438, 0xd704, + 0xa438, 0x67b4, 0xa438, 0x1000, 0xa438, 0x9cc6, 0xa438, 0xbb08, + 0xa438, 0x9a10, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce08, 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, + 0xa438, 0xce08, 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, + 0xa438, 0xce08, 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xa302, + 0xa438, 0xa480, 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, + 0xa438, 0x407d, 0xa438, 0x8644, 0xa438, 0xf003, 0xa438, 0x0c44, + 0xa438, 0x0604, 0xa438, 0xd700, 0xa438, 0x6047, 0xa438, 0xa682, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8230, 0xa438, 0x8410, + 0xa438, 0x9503, 0xa438, 0xd707, 0xa438, 0x404e, 0xa438, 0x8a10, + 0xa438, 0xcda9, 0xa438, 0xd110, 0xa438, 0xd040, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x99a0, 0xa438, 0x1000, + 0xa438, 0x99a4, 0xa438, 0xd700, 0xa438, 0x5f3a, 0xa438, 0x1800, + 0xa438, 0x91f3, 0xa438, 0xba10, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c97, 0xa438, 0x0086, 0xa438, 0xa060, 0xa438, 0xa101, + 0xa438, 0xa408, 0xa438, 0xa240, 0xa438, 0x9503, 0xa438, 0xcda2, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa020, 0xa438, 0xa110, + 0xa438, 0xa011, 0xa438, 0x9503, 0xa438, 0xd17a, 0xa438, 0xd046, + 0xa438, 0xcda3, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, 0xa438, 0xd700, + 0xa438, 0x5f3a, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c83, + 0xa438, 0x0282, 0xa438, 0x9503, 0xa438, 0x0c07, 0xa438, 0x0b02, + 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, + 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce11, + 0xa438, 0xf00a, 0xa438, 0xce11, 0xa438, 0xf008, 0xa438, 0xce10, + 0xa438, 0xf006, 0xa438, 0xce10, 0xa438, 0xf004, 0xa438, 0xce0f, + 0xa438, 0xf002, 0xa438, 0xce0f, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa204, 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0xab08, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, 0xa438, 0xd704, + 0xa438, 0x3ebc, 0xa438, 0x98b7, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8284, 0xa438, 0x9503, 0xa438, 0x8b08, 0xa438, 0xcda4, + 0xa438, 0xd704, 0xa438, 0x6419, 0xa438, 0x3ad1, 0xa438, 0x9969, + 0xa438, 0xd705, 0xa438, 0x60dc, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa402, 0xa438, 0x9503, 0xa438, 0xf012, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8402, 0xa438, 0x9503, 0xa438, 0xd705, + 0xa438, 0x60db, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa401, + 0xa438, 0x9503, 0xa438, 0xf007, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c21, 0xa438, 0x0420, 0xa438, 0x9503, 0xa438, 0x9a10, + 0xa438, 0xd705, 0xa438, 0x34a4, 0xa438, 0x9886, 0xa438, 0x1800, + 0xa438, 0x9969, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c10, + 0xa438, 0x0010, 0xa438, 0x8410, 0xa438, 0xa250, 0xa438, 0x9503, + 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, 0xa438, 0x407d, + 0xa438, 0x8644, 0xa438, 0xf003, 0xa438, 0x0c44, 0xa438, 0x0604, + 0xa438, 0xd700, 0xa438, 0x40e7, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c38, 0xa438, 0x0d28, 0xa438, 0x9503, 0xa438, 0xf007, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0cf8, 0xa438, 0x0d90, + 0xa438, 0x9503, 0xa438, 0xa682, 0xa438, 0xd1f1, 0xa438, 0xd046, + 0xa438, 0xcda5, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, 0xa438, 0xd704, + 0xa438, 0x60ef, 0xa438, 0xd706, 0xa438, 0x40b1, 0xa438, 0xd700, + 0xa438, 0x5eba, 0xa438, 0x1800, 0xa438, 0x9925, 0xa438, 0xd162, + 0xa438, 0xd055, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, 0xa438, 0xd700, + 0xa438, 0x5f3b, 0xa438, 0xd704, 0xa438, 0x692f, 0xa438, 0xd706, + 0xa438, 0x48f1, 0xa438, 0x1800, 0xa438, 0x9907, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf00a, + 0xa438, 0xce08, 0xa438, 0xf008, 0xa438, 0xce08, 0xa438, 0xf006, + 0xa438, 0xce08, 0xa438, 0xf004, 0xa438, 0xce08, 0xa438, 0xf002, + 0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xa00a, + 0xa438, 0xd179, 0xa438, 0xd047, 0xa438, 0xcda6, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x99a0, 0xa438, 0x1000, + 0xa438, 0x99a4, 0xa438, 0xd704, 0xa438, 0x60ef, 0xa438, 0xd706, + 0xa438, 0x40b1, 0xa438, 0xd700, 0xa438, 0x5eba, 0xa438, 0x1800, + 0xa438, 0x995b, 0xa438, 0xd162, 0xa438, 0xd055, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x99a0, 0xa438, 0x1000, + 0xa438, 0x99a4, 0xa438, 0xd700, 0xa438, 0x5f3b, 0xa438, 0xd704, + 0xa438, 0x626f, 0xa438, 0xd706, 0xa438, 0x4231, 0xa438, 0x1800, + 0xa438, 0x993d, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0ccf, + 0xa438, 0x0b46, 0xa438, 0x0c07, 0xa438, 0x0c03, 0xa438, 0x9503, + 0xa438, 0xa1a0, 0xa438, 0xa480, 0xa438, 0xb920, 0xa438, 0x9a10, + 0xa438, 0xcda7, 0xa438, 0x1800, 0xa438, 0x91f3, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8091, 0xa438, 0x8101, 0xa438, 0x8250, + 0xa438, 0x0c28, 0xa438, 0x0420, 0xa438, 0xd700, 0xa438, 0x4087, + 0xa438, 0x0c38, 0xa438, 0x0d18, 0xa438, 0xf003, 0xa438, 0x0cf8, + 0xa438, 0x0d58, 0xa438, 0x9503, 0xa438, 0x8106, 0xa438, 0xcda8, + 0xa438, 0x1800, 0xa438, 0x91f3, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8090, 0xa438, 0x8101, 0xa438, 0x8248, 0xa438, 0x9503, + 0xa438, 0x1000, 0xa438, 0x9cb7, 0xa438, 0x1000, 0xa438, 0x9c90, + 0xa438, 0x8106, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0ccf, + 0xa438, 0x0b03, 0xa438, 0x8c07, 0xa438, 0xd700, 0xa438, 0x4087, + 0xa438, 0x0c38, 0xa438, 0x0d18, 0xa438, 0xf003, 0xa438, 0x0cf8, + 0xa438, 0x0d98, 0xa438, 0x8408, 0xa438, 0x9503, 0xa438, 0x9c01, + 0xa438, 0xa1a0, 0xa438, 0xa302, 0xa438, 0xd707, 0xa438, 0x404e, + 0xa438, 0x8a10, 0xa438, 0x1000, 0xa438, 0x9cc6, 0xa438, 0xcdab, + 0xa438, 0x1800, 0xa438, 0x923e, 0xa438, 0xd71f, 0xa438, 0x2e71, + 0xa438, 0x925e, 0xa438, 0x0800, 0xa438, 0xd704, 0xa438, 0x2739, + 0xa438, 0x96ac, 0xa438, 0x0800, 0xa438, 0xd71f, 0xa438, 0x62ae, + 0xa438, 0xd706, 0xa438, 0x6400, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd705, 0xa438, 0x61d9, 0xa438, 0xd704, 0xa438, 0x4187, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa601, 0xa438, 0x9503, + 0xa438, 0x800a, 0xa438, 0x1000, 0xa438, 0x9d31, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0x1800, 0xa438, 0x96ac, 0xa438, 0x0800, + 0xa438, 0xcd99, 0xa438, 0xa70c, 0xa438, 0x881a, 0xa438, 0x8010, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8683, 0xa438, 0x9503, + 0xa438, 0xd701, 0xa438, 0x4050, 0xa438, 0xab20, 0xa438, 0x1800, + 0xa438, 0x925e, 0xa438, 0x8010, 0xa438, 0xa704, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa60c, 0xa438, 0xa603, 0xa438, 0x9503, + 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8302, 0xa438, 0x8480, + 0xa438, 0x8604, 0xa438, 0x8602, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8702, 0xa438, 0x9503, 0xa438, 0xd701, 0xa438, 0x4050, + 0xa438, 0xab20, 0xa438, 0xcd96, 0xa438, 0xa901, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x7ce0, 0xa438, 0xd706, + 0xa438, 0x2109, 0xa438, 0x9a75, 0xa438, 0xd71f, 0xa438, 0x7aae, + 0xa438, 0xd704, 0xa438, 0x7ea7, 0xa438, 0xd706, 0xa438, 0x5e62, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8f40, 0xa438, 0x9503, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa610, 0xa438, 0x9503, + 0xa438, 0xa708, 0xa438, 0x881a, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0ccf, 0xa438, 0x0b03, 0xa438, 0x8cc7, 0xa438, 0x9503, + 0xa438, 0x8106, 0xa438, 0xd700, 0xa438, 0x60c7, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xcd5a, 0xa438, 0x9503, 0xa438, 0xf006, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c3f, 0xa438, 0x0d19, + 0xa438, 0x9503, 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xa302, + 0xa438, 0xa480, 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, + 0xa438, 0x407d, 0xa438, 0x8604, 0xa438, 0xf002, 0xa438, 0xa604, + 0xa438, 0xd700, 0xa438, 0x60c7, 0xa438, 0xa602, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa702, 0xa438, 0x9503, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8680, 0xa438, 0x9503, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xc500, 0xa438, 0x9503, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x72ce, 0xa438, 0xd704, + 0xa438, 0x5f65, 0xa438, 0xd705, 0xa438, 0x4378, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce1d, 0xa438, 0xf00b, + 0xa438, 0xce1d, 0xa438, 0xf009, 0xa438, 0xce1d, 0xa438, 0xf007, + 0xa438, 0xce1d, 0xa438, 0xf005, 0xa438, 0xce1c, 0xa438, 0xf003, + 0xa438, 0xce1c, 0xa438, 0xf001, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8640, 0xa438, 0x9503, + 0xa438, 0xf01a, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce05, 0xa438, 0xf00b, 0xa438, 0xce05, 0xa438, 0xf009, + 0xa438, 0xce04, 0xa438, 0xf007, 0xa438, 0xce04, 0xa438, 0xf005, + 0xa438, 0xce04, 0xa438, 0xf003, 0xa438, 0xce04, 0xa438, 0xf001, + 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa640, 0xa438, 0x9503, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8602, 0xa438, 0x8601, 0xa438, 0x9503, 0xa438, 0x8640, + 0xa438, 0x1000, 0xa438, 0x9cc6, 0xa438, 0xd703, 0xa438, 0x6131, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xaf80, 0xa438, 0x0c38, + 0xa438, 0x0d30, 0xa438, 0x9503, 0xa438, 0xd193, 0xa438, 0xd067, + 0xa438, 0xcd98, 0xa438, 0x1800, 0xa438, 0x91f3, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa608, 0xa438, 0x9503, 0xa438, 0xa708, + 0xa438, 0xd701, 0xa438, 0x6056, 0xa438, 0x8704, 0xa438, 0x1000, + 0xa438, 0x9d31, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xa81a, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8603, 0xa438, 0x9503, + 0xa438, 0xcd97, 0xa438, 0x1800, 0xa438, 0x96fe, 0xa438, 0xd706, + 0xa438, 0x4103, 0xa438, 0xd705, 0xa438, 0x407f, 0xa438, 0x1800, + 0xa438, 0x9b77, 0xa438, 0xd105, 0xa438, 0xd056, 0xa438, 0xf03f, + 0xa438, 0xd705, 0xa438, 0x3fa7, 0xa438, 0x9a9a, 0xa438, 0xd705, + 0xa438, 0x605a, 0xa438, 0xf092, 0xa438, 0x1800, 0xa438, 0x9b1e, + 0xa438, 0xd704, 0xa438, 0x4146, 0xa438, 0x800a, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d04, 0xa438, 0x9503, + 0xa438, 0x0c06, 0xa438, 0x0102, 0xa438, 0xf010, 0xa438, 0xa00a, + 0xa438, 0x8106, 0xa438, 0xd700, 0xa438, 0x60e7, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d02, 0xa438, 0x9503, + 0xa438, 0xf006, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c07, + 0xa438, 0x0d01, 0xa438, 0x9503, 0xa438, 0xd705, 0xa438, 0x61d4, + 0xa438, 0xd704, 0xa438, 0x609f, 0xa438, 0x6170, 0xa438, 0x2d71, + 0xa438, 0x9ac3, 0xa438, 0xd702, 0xa438, 0x60d7, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0cf0, 0xa438, 0x05a0, 0xa438, 0x9503, + 0xa438, 0xf0fc, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa108, + 0xa438, 0xa204, 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, + 0xa438, 0xd707, 0xa438, 0x5f35, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa410, 0xa438, 0x9503, 0xa438, 0x800a, 0xa438, 0x81a0, + 0xa438, 0x8302, 0xa438, 0x8480, 0xa438, 0x8642, 0xa438, 0x1000, + 0xa438, 0x9d44, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8108, + 0xa438, 0x9503, 0xa438, 0x8c40, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8750, 0xa438, 0x9503, 0xa438, 0xaa10, 0xa438, 0xd706, + 0xa438, 0x4063, 0xa438, 0x8604, 0xa438, 0xf022, 0xa438, 0xa604, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8220, 0xa438, 0x8301, + 0xa438, 0x8420, 0xa438, 0x8108, 0xa438, 0x0c03, 0xa438, 0x0201, + 0xa438, 0x9503, 0xa438, 0x0c07, 0xa438, 0x0b01, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce0e, 0xa438, 0xf00a, + 0xa438, 0xce0e, 0xa438, 0xf008, 0xa438, 0xce0d, 0xa438, 0xf006, + 0xa438, 0xce0d, 0xa438, 0xf004, 0xa438, 0xce0c, 0xa438, 0xf002, + 0xa438, 0xce0c, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xab08, + 0xa438, 0xcda0, 0xa438, 0xd706, 0xa438, 0x4043, 0xa438, 0xf0b1, + 0xa438, 0xd705, 0xa438, 0x411e, 0xa438, 0xd162, 0xa438, 0xd045, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa008, 0xa438, 0x9503, + 0xa438, 0xf0a8, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8008, + 0xa438, 0x9503, 0xa438, 0xd1c4, 0xa438, 0xd055, 0xa438, 0xf0a1, + 0xa438, 0xd700, 0xa438, 0x605a, 0xa438, 0xf09e, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8008, 0xa438, 0x8210, 0xa438, 0xa210, + 0xa438, 0x9503, 0xa438, 0xd162, 0xa438, 0xd055, 0xa438, 0xd704, + 0xa438, 0x608f, 0xa438, 0xd706, 0xa438, 0x4051, 0xa438, 0xf04a, + 0xa438, 0xd705, 0xa438, 0x415e, 0xa438, 0xd705, 0xa438, 0x6112, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c06, 0xa438, 0x0002, + 0xa438, 0xa110, 0xa438, 0xa305, 0xa438, 0x9503, 0xa438, 0xd705, + 0xa438, 0x40de, 0xa438, 0xd706, 0xa438, 0x6771, 0xa438, 0xd700, + 0xa438, 0x60bb, 0xa438, 0xf07f, 0xa438, 0xd700, 0xa438, 0x661b, + 0xa438, 0xf07c, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8090, + 0xa438, 0x8101, 0xa438, 0x0ccf, 0xa438, 0x0b03, 0xa438, 0x8c07, + 0xa438, 0xd700, 0xa438, 0x4087, 0xa438, 0x0c38, 0xa438, 0x0d18, + 0xa438, 0xf003, 0xa438, 0x0cf8, 0xa438, 0x0d58, 0xa438, 0x8270, + 0xa438, 0x8408, 0xa438, 0x8301, 0xa438, 0x9503, 0xa438, 0x8106, + 0xa438, 0x9c01, 0xa438, 0xd705, 0xa438, 0x40b5, 0xa438, 0x1000, + 0xa438, 0x9cb7, 0xa438, 0xd601, 0xa438, 0xd628, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8208, 0xa438, 0x9503, 0xa438, 0xcdac, + 0xa438, 0xd1c4, 0xa438, 0xd054, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, + 0xa438, 0xd700, 0xa438, 0x5f3b, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa230, 0xa438, 0x9503, 0xa438, 0xf04e, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa404, 0xa438, 0x8230, 0xa438, 0x9503, + 0xa438, 0xf048, 0xa438, 0xd700, 0xa438, 0x48db, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8204, 0xa438, 0x8410, 0xa438, 0x9503, + 0xa438, 0x8b08, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce08, 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, + 0xa438, 0xce08, 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, + 0xa438, 0xce08, 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xa480, + 0xa438, 0xd705, 0xa438, 0x611e, 0xa438, 0xa302, 0xa438, 0x1000, + 0xa438, 0x9cc6, 0xa438, 0xd707, 0xa438, 0x404e, 0xa438, 0x8a10, + 0xa438, 0xf006, 0xa438, 0x8302, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8301, 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x6047, + 0xa438, 0xa682, 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, + 0xa438, 0x407d, 0xa438, 0x8644, 0xa438, 0xf003, 0xa438, 0x0c44, + 0xa438, 0x0604, 0xa438, 0xd701, 0xa438, 0x40d0, 0xa438, 0xac40, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa750, 0xa438, 0x9503, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa220, 0xa438, 0x8210, + 0xa438, 0xa210, 0xa438, 0xa620, 0xa438, 0x9503, 0xa438, 0xcdaa, + 0xa438, 0x0800, 0xa438, 0xd202, 0xa438, 0xb309, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fa1, 0xa438, 0xd400, + 0xa438, 0xb302, 0xa438, 0xd200, 0xa438, 0x0800, 0xa438, 0xd71f, + 0xa438, 0x6261, 0xa438, 0xcdff, 0xa438, 0xd705, 0xa438, 0x41d5, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c1f, 0xa438, 0x0e12, + 0xa438, 0x9503, 0xa438, 0xd70c, 0xa438, 0x5ff3, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8e1f, 0xa438, 0x9503, 0xa438, 0xd70c, + 0xa438, 0x7f73, 0xa438, 0x1800, 0xa438, 0x001c, 0xa438, 0x0800, + 0xa438, 0xab10, 0xa438, 0xd71f, 0xa438, 0x210c, 0xa438, 0x001c, + 0xa438, 0xd701, 0xa438, 0x5f98, 0xa438, 0x8b10, 0xa438, 0x0800, + 0xa438, 0xd71f, 0xa438, 0x210c, 0xa438, 0x001c, 0xa438, 0xd701, + 0xa438, 0x5f99, 0xa438, 0x0800, 0xa438, 0xa110, 0xa438, 0x1000, + 0xa438, 0x9be6, 0xa438, 0x8110, 0xa438, 0xa140, 0xa438, 0x1000, + 0xa438, 0x9be6, 0xa438, 0x8140, 0xa438, 0xa004, 0xa438, 0x1000, + 0xa438, 0x9be6, 0xa438, 0x8004, 0xa438, 0xa001, 0xa438, 0x1000, + 0xa438, 0x9be6, 0xa438, 0x8001, 0xa438, 0xa020, 0xa438, 0x1000, + 0xa438, 0x9be6, 0xa438, 0x8020, 0xa438, 0xa101, 0xa438, 0x0800, + 0xa438, 0xd202, 0xa438, 0x0c09, 0xa438, 0x1301, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fa1, 0xa438, 0xb302, + 0xa438, 0xd200, 0xa438, 0x0800, 0xa438, 0xd204, 0xa438, 0x0c09, + 0xa438, 0x1301, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, + 0xa438, 0x5fa2, 0xa438, 0xb302, 0xa438, 0xd200, 0xa438, 0x0800, + 0xa438, 0xd208, 0xa438, 0x0c09, 0xa438, 0x1301, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fa3, 0xa438, 0xb302, + 0xa438, 0xd200, 0xa438, 0x0800, 0xa438, 0xd210, 0xa438, 0x0c09, + 0xa438, 0x1301, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, + 0xa438, 0x5fa4, 0xa438, 0xb302, 0xa438, 0xd200, 0xa438, 0x0800, + 0xa438, 0xac01, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd707, + 0xa438, 0x5fad, 0xa438, 0x0800, 0xa438, 0xac04, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd707, 0xa438, 0x5fab, 0xa438, 0x0800, + 0xa438, 0xd71f, 0xa438, 0x4354, 0xa438, 0x8810, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xb920, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, + 0xa438, 0x9920, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x6085, 0xa438, 0xd71f, 0xa438, 0x7e34, 0xa438, 0xfffa, + 0xa438, 0xb820, 0xa438, 0xa810, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0x0800, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa308, 0xa438, 0x9503, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fb3, + 0xa438, 0x0800, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8308, + 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, + 0xa438, 0x7fb3, 0xa438, 0x0800, 0xa438, 0xc000, 0xa438, 0xc100, + 0xa438, 0xc200, 0xa438, 0xc300, 0xa438, 0xc400, 0xa438, 0xc500, + 0xa438, 0xc600, 0xa438, 0xc700, 0xa438, 0xc828, 0xa438, 0xc904, + 0xa438, 0xca00, 0xa438, 0xcb00, 0xa438, 0xcc00, 0xa438, 0xce00, + 0xa438, 0xcf00, 0xa438, 0xd000, 0xa438, 0xd100, 0xa438, 0xd200, + 0xa438, 0xd300, 0xa438, 0xd400, 0xa438, 0xd500, 0xa438, 0xd700, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xc000, 0xa438, 0xc100, + 0xa438, 0xc200, 0xa438, 0xc300, 0xa438, 0xc400, 0xa438, 0xc500, + 0xa438, 0xc600, 0xa438, 0xa63c, 0xa438, 0xc700, 0xa438, 0xc800, + 0xa438, 0xc900, 0xa438, 0xca00, 0xa438, 0xcb00, 0xa438, 0xcc00, + 0xa438, 0xcd00, 0xa438, 0xce00, 0xa438, 0xcf00, 0xa438, 0x9503, + 0xa438, 0xcd00, 0xa438, 0x0800, 0xa438, 0xd601, 0xa438, 0xd608, + 0xa438, 0xd610, 0xa438, 0xd618, 0xa438, 0xd620, 0xa438, 0xd628, + 0xa438, 0xd630, 0xa438, 0xd638, 0xa438, 0x0800, 0xa438, 0x8b0f, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8097, 0xa438, 0x8109, + 0xa438, 0x82ff, 0xa438, 0x843f, 0xa438, 0x9503, 0xa438, 0x9920, + 0xa438, 0x9b08, 0xa438, 0x9a10, 0xa438, 0xd705, 0xa438, 0x4235, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c1f, 0xa438, 0x0e12, + 0xa438, 0x8440, 0xa438, 0x9503, 0xa438, 0xd70c, 0xa438, 0x5ff3, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8e1f, 0xa438, 0x9503, + 0xa438, 0xd70c, 0xa438, 0x7f73, 0xa438, 0xd601, 0xa438, 0xd628, + 0xa438, 0x0800, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c1f, + 0xa438, 0x0e12, 0xa438, 0x8440, 0xa438, 0x9503, 0xa438, 0xd70c, + 0xa438, 0x5ff3, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8e1f, + 0xa438, 0x9503, 0xa438, 0xd70c, 0xa438, 0x7f73, 0xa438, 0x0800, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x9ccf, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c70, 0xa438, 0x0350, 0xa438, 0x9503, + 0xa438, 0xf00e, 0xa438, 0xd702, 0xa438, 0x40f9, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c70, 0xa438, 0x0340, 0xa438, 0x9503, + 0xa438, 0xf006, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c70, + 0xa438, 0x0340, 0xa438, 0x9503, 0xa438, 0x0800, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x9ce6, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c70, 0xa438, 0x0370, 0xa438, 0x9503, 0xa438, 0xf00e, + 0xa438, 0xd702, 0xa438, 0x40f9, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c70, 0xa438, 0x0350, 0xa438, 0x9503, 0xa438, 0xf006, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c70, 0xa438, 0x0350, + 0xa438, 0x9503, 0xa438, 0x0800, 0xa438, 0x8910, 0xa438, 0xd704, + 0xa438, 0x61a8, 0xa438, 0xd702, 0xa438, 0x60d6, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xaf40, 0xa438, 0x9503, 0xa438, 0xf00a, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8f40, 0xa438, 0x9503, + 0xa438, 0xf005, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8f40, + 0xa438, 0x9503, 0xa438, 0x0800, 0xa438, 0x8910, 0xa438, 0xd705, + 0xa438, 0x4059, 0xa438, 0xa910, 0xa438, 0xd704, 0xa438, 0x6068, + 0xa438, 0x0000, 0xa438, 0xf002, 0xa438, 0x0000, 0xa438, 0x8910, + 0xa438, 0x0800, 0xa438, 0xd703, 0xa438, 0x6080, 0xa438, 0x6121, + 0xa438, 0x61c2, 0xa438, 0x6263, 0xa438, 0xd707, 0xa438, 0x4070, + 0xa438, 0xce98, 0xa438, 0xf015, 0xa438, 0xce94, 0xa438, 0xf013, + 0xa438, 0xd707, 0xa438, 0x4070, 0xa438, 0xce99, 0xa438, 0xf00f, + 0xa438, 0xce95, 0xa438, 0xf00d, 0xa438, 0xd707, 0xa438, 0x4070, + 0xa438, 0xce9a, 0xa438, 0xf009, 0xa438, 0xce96, 0xa438, 0xf007, + 0xa438, 0xd707, 0xa438, 0x4070, 0xa438, 0xce9b, 0xa438, 0xf003, + 0xa438, 0xce97, 0xa438, 0xf001, 0xa438, 0x0800, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce00, 0xa438, 0xf00a, + 0xa438, 0xce00, 0xa438, 0xf008, 0xa438, 0xce00, 0xa438, 0xf006, + 0xa438, 0xce00, 0xa438, 0xf004, 0xa438, 0xce00, 0xa438, 0xf002, + 0xa438, 0xce00, 0xa438, 0x0800, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8702, 0xa438, 0x9503, 0xa438, 0xd101, 0xa438, 0xd040, + 0xa438, 0xd700, 0xa438, 0x5ffa, 0xa438, 0x8680, 0xa438, 0xd700, + 0xa438, 0x60a7, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa702, + 0xa438, 0x9503, 0xa438, 0x0800, 0xa438, 0xcdfe, 0xa438, 0xa708, + 0xa438, 0xa2fc, 0xa438, 0xba20, 0xa438, 0xa980, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, + 0xa438, 0x9920, 0xa438, 0x0ca0, 0xa438, 0x0480, 0xa438, 0xd706, + 0xa438, 0x5fe7, 0xa438, 0x84a0, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x9920, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fa5, + 0xa438, 0xb820, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x7f85, 0xa438, 0x9820, 0xa438, 0xa810, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf00a, + 0xa438, 0xce08, 0xa438, 0xf008, 0xa438, 0xce08, 0xa438, 0xf006, + 0xa438, 0xce08, 0xa438, 0xf004, 0xa438, 0xce08, 0xa438, 0xf002, + 0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xa120, + 0xa438, 0xa4a0, 0xa438, 0xa980, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd706, 0xa438, 0x5faf, 0xa438, 0x1800, 0xa438, 0x9de8, + 0xa438, 0xcdfd, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce08, 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, + 0xa438, 0xce08, 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, + 0xa438, 0xce08, 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xa980, 0xa438, 0xa810, 0xa438, 0xa00a, + 0xa438, 0xa1a0, 0xa438, 0xa312, 0xa438, 0xa4a0, 0xa438, 0xa604, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5faf, + 0xa438, 0x0c30, 0xa438, 0x0320, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd704, 0xa438, 0x5fa4, 0xa438, 0x800a, 0xa438, 0x81a0, + 0xa438, 0x8312, 0xa438, 0x84a0, 0xa438, 0x8604, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, + 0xa438, 0x9920, 0xa438, 0x8340, 0xa438, 0xa801, 0xa438, 0xa980, + 0xa438, 0x8240, 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xa302, + 0xa438, 0xa4a0, 0xa438, 0xa604, 0xa438, 0xd1c8, 0xa438, 0xd045, + 0xa438, 0xd700, 0xa438, 0x5ffa, 0xa438, 0xd706, 0xa438, 0x5faf, + 0xa438, 0xd40a, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd416, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x800a, 0xa438, 0x81a0, + 0xa438, 0x8302, 0xa438, 0x8480, 0xa438, 0xa604, 0xa438, 0xa801, + 0xa438, 0x8818, 0xa438, 0xbb10, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x5fb5, 0xa438, 0x9b10, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb5, 0xa438, 0x1800, + 0xa438, 0x915e, 0xa438, 0x1000, 0xa438, 0x1c45, 0xa438, 0x1000, + 0xa438, 0x9c64, 0xa438, 0xac10, 0xa438, 0x1800, 0xa438, 0x0021, + 0xa436, 0xA10E, 0xa438, 0xffff, 0xa436, 0xA10C, 0xa438, 0xffff, + 0xa436, 0xA10A, 0xa438, 0xffff, 0xa436, 0xA108, 0xa438, 0xffff, + 0xa436, 0xA106, 0xa438, 0xffff, 0xa436, 0xA104, 0xa438, 0xffff, + 0xa436, 0xA102, 0xa438, 0x001c, 0xa436, 0xA100, 0xa438, 0x0073, + 0xa436, 0xA110, 0xa438, 0x0003, 0xa436, 0xA016, 0xa438, 0x0020, + 0xa436, 0xA012, 0xa438, 0x1ff8, 0xa436, 0xA014, 0xa438, 0xb904, + 0xa438, 0xd18a, 0xa438, 0xd17a, 0xa438, 0x9c10, 0xa438, 0x0000, + 0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000, 0xa436, 0xA164, + 0xa438, 0x1CAC, 0xa436, 0xA166, 0xa438, 0x0666, 0xa436, 0xA168, + 0xa438, 0x0669, 0xa436, 0xA16A, 0xa438, 0x001f, 0xa436, 0xA16C, + 0xa438, 0x3fff, 0xa436, 0xA16E, 0xa438, 0x3fff, 0xa436, 0xA170, + 0xa438, 0x3fff, 0xa436, 0xA172, 0xa438, 0x3fff, 0xa436, 0xA162, + 0xa438, 0x000f, 0xa436, 0xb87c, 0xa438, 0x8a23, 0xa436, 0xb87e, + 0xa438, 0xaf8a, 0xa438, 0x3baf, 0xa438, 0x8a4c, 0xa438, 0xaf8a, + 0xa438, 0x5caf, 0xa438, 0x8a62, 0xa438, 0xaf8a, 0xa438, 0x68af, + 0xa438, 0x8a9e, 0xa438, 0xaf8b, 0xa438, 0x4daf, 0xa438, 0x8ba8, + 0xa438, 0xe080, 0xa438, 0x15ad, 0xa438, 0x2505, 0xa438, 0xd101, + 0xa438, 0xaf58, 0xa438, 0xcc02, 0xa438, 0x1e44, 0xa438, 0xaf59, + 0xa438, 0x05ee, 0xa438, 0x88af, 0xa438, 0x0102, 0xa438, 0x7eb6, + 0xa438, 0xad2a, 0xa438, 0x03af, 0xa438, 0x5b41, 0xa438, 0xaf5b, + 0xa438, 0x4a02, 0xa438, 0x8c4c, 0xa438, 0xaf69, 0xa438, 0x4f02, + 0xa438, 0x8d51, 0xa438, 0xaf65, 0xa438, 0xaf02, 0xa438, 0x7b73, + 0xa438, 0xbf7a, 0xa438, 0xa302, 0xa438, 0x7b73, 0xa438, 0xbf8e, + 0xa438, 0x5b02, 0xa438, 0x7b73, 0xa438, 0xbf8e, 0xa438, 0x5e02, + 0xa438, 0x7b73, 0xa438, 0xbf8e, 0xa438, 0x6102, 0xa438, 0x7b73, + 0xa438, 0xbf7a, 0xa438, 0xa602, 0xa438, 0x7b73, 0xa438, 0xbf8e, + 0xa438, 0x6402, 0xa438, 0x7b73, 0xa438, 0xbf8e, 0xa438, 0x6702, + 0xa438, 0x7b73, 0xa438, 0xbf8e, 0xa438, 0x6a02, 0xa438, 0x7b73, + 0xa438, 0xaf64, 0xa438, 0xcfe0, 0xa438, 0x898a, 0xa438, 0xe189, + 0xa438, 0x8bc4, 0xa438, 0xef67, 0xa438, 0xe089, 0xa438, 0x8ce1, + 0xa438, 0x898e, 0xa438, 0x1b01, 0xa438, 0xef20, 0xa438, 0xad27, + 0xa438, 0x04c7, 0xa438, 0x78ff, 0xa438, 0x109e, 0xa438, 0x050d, + 0xa438, 0x8180, 0xa438, 0xaef9, 0xa438, 0xac37, 0xa438, 0x09cf, + 0xa438, 0x027a, 0xa438, 0xcdac, 0xa438, 0x5030, 0xa438, 0xae13, + 0xa438, 0xe089, 0xa438, 0x8ae1, 0xa438, 0x898b, 0xa438, 0xef74, + 0xa438, 0xef46, 0xa438, 0xce02, 0xa438, 0x7acd, 0xa438, 0xef64, + 0xa438, 0xac50, 0xa438, 0x1be0, 0xa438, 0x8991, 0xa438, 0x3802, + 0xa438, 0xac27, 0xa438, 0x13ef, 0xa438, 0x46e4, 0xa438, 0x898a, + 0xa438, 0xe589, 0xa438, 0x8b3b, 0xa438, 0x02e7, 0xa438, 0x898d, + 0xa438, 0xe289, 0xa438, 0x8ee6, 0xa438, 0x898c, 0xa438, 0x1f44, + 0xa438, 0xe189, 0xa438, 0x68e2, 0xa438, 0x898e, 0xa438, 0x3a09, + 0xa438, 0xef32, 0xa438, 0xad37, 0xa438, 0x06c6, 0xa438, 0xef64, + 0xa438, 0x7aff, 0xa438, 0x129e, 0xa438, 0x050d, 0xa438, 0x6182, + 0xa438, 0xaef9, 0xa438, 0xac3f, 0xa438, 0x04ef, 0xa438, 0x74ae, + 0xa438, 0x03ef, 0xa438, 0x76ce, 0xa438, 0x027a, 0xa438, 0xcdad, + 0xa438, 0x501e, 0xa438, 0xe089, 0xa438, 0x9138, 0xa438, 0x02ac, + 0xa438, 0x2709, 0xa438, 0xe089, 0xa438, 0x8f10, 0xa438, 0xe489, + 0xa438, 0x8fae, 0xa438, 0x0de0, 0xa438, 0x8991, 0xa438, 0x2801, + 0xa438, 0xe189, 0xa438, 0x921e, 0xa438, 0x10e5, 0xa438, 0x8992, + 0xa438, 0xe089, 0xa438, 0x9110, 0xa438, 0xe489, 0xa438, 0x91af, + 0xa438, 0x68c5, 0xa438, 0xcffb, 0xa438, 0xee89, 0xa438, 0x6700, + 0xa438, 0xe389, 0xa438, 0x671f, 0xa438, 0x77d8, 0xa438, 0xa000, + 0xa438, 0x1213, 0xa438, 0xe789, 0xa438, 0x67d2, 0xa438, 0x031a, + 0xa438, 0x92e1, 0xa438, 0x8966, 0xa438, 0x1b13, 0xa438, 0x9fed, + 0xa438, 0xaf8b, 0xa438, 0x87a0, 0xa438, 0x040b, 0xa438, 0xbcbf, + 0xa438, 0x7ab2, 0xa438, 0x027d, 0xa438, 0x5abd, 0xa438, 0xac28, + 0xa438, 0xe0d8, 0xa438, 0x1f11, 0xa438, 0x0267, 0xa438, 0xc3d7, + 0xa438, 0x0001, 0xa438, 0xaed5, 0xa438, 0xad50, 0xa438, 0x02ae, + 0xa438, 0x12e0, 0xa438, 0x8987, 0xa438, 0xad20, 0xa438, 0x06ee, + 0xa438, 0x8987, 0xa438, 0x00ae, 0xa438, 0x06ee, 0xa438, 0x8989, + 0xa438, 0x00ae, 0xa438, 0x06e0, 0xa438, 0x895c, 0xa438, 0xe489, + 0xa438, 0x89ff, 0xa438, 0xaf6b, 0xa438, 0x3cd4, 0xa438, 0x000f, + 0xa438, 0xbf79, 0xa438, 0x3202, 0xa438, 0x7b9f, 0xa438, 0xd400, + 0xa438, 0x04bf, 0xa438, 0x5075, 0xa438, 0xd700, 0xa438, 0x0802, + 0xa438, 0x7da0, 0xa438, 0xbf8e, 0xa438, 0x6d02, 0xa438, 0x7b73, + 0xa438, 0xd402, 0xa438, 0x00bf, 0xa438, 0x8e70, 0xa438, 0x027b, + 0xa438, 0x9fd4, 0xa438, 0x000f, 0xa438, 0xbf8e, 0xa438, 0x7302, + 0xa438, 0x7b9f, 0xa438, 0xbf8e, 0xa438, 0x7302, 0xa438, 0x7b73, + 0xa438, 0xd41b, 0xa438, 0x3ebf, 0xa438, 0x8e6d, 0xa438, 0x027b, + 0xa438, 0x9fd4, 0xa438, 0x00dc, 0xa438, 0xbf8e, 0xa438, 0x7002, + 0xa438, 0x7b9f, 0xa438, 0xd400, 0xa438, 0x0fbf, 0xa438, 0x8e73, + 0xa438, 0x027b, 0xa438, 0x9fbf, 0xa438, 0x8e73, 0xa438, 0x027b, + 0xa438, 0x73d4, 0xa438, 0x1bbe, 0xa438, 0xbf8e, 0xa438, 0x6d02, + 0xa438, 0x7b9f, 0xa438, 0xd400, 0xa438, 0xdcbf, 0xa438, 0x8e70, + 0xa438, 0x027b, 0xa438, 0x9fd4, 0xa438, 0x000f, 0xa438, 0xbf8e, + 0xa438, 0x7302, 0xa438, 0x7b9f, 0xa438, 0xbf8e, 0xa438, 0x7302, + 0xa438, 0x7b73, 0xa438, 0xd700, 0xa438, 0x86b7, 0xa438, 0xfed4, + 0xa438, 0x003e, 0xa438, 0xbf8e, 0xa438, 0x6d02, 0xa438, 0x7b9f, + 0xa438, 0xd400, 0xa438, 0xdcbf, 0xa438, 0x8e70, 0xa438, 0x027b, + 0xa438, 0x9fd4, 0xa438, 0x000f, 0xa438, 0xbf8e, 0xa438, 0x7302, + 0xa438, 0x7b9f, 0xa438, 0xbf8e, 0xa438, 0x7302, 0xa438, 0x7b73, + 0xa438, 0xbf79, 0xa438, 0x3202, 0xa438, 0x7b73, 0xa438, 0xaf51, + 0xa438, 0x4ff8, 0xa438, 0xf9fa, 0xa438, 0xcefb, 0xa438, 0xef79, + 0xa438, 0xfbe0, 0xa438, 0x8967, 0xa438, 0xef10, 0xa438, 0x4803, + 0xa438, 0xbf89, 0xa438, 0x721a, 0xa438, 0x90ef, 0xa438, 0x79bf, + 0xa438, 0x8982, 0xa438, 0x1a91, 0xa438, 0xec01, 0xa438, 0x0702, + 0xa438, 0x7eb6, 0xa438, 0xad29, 0xa438, 0x10e0, 0xa438, 0x898f, + 0xa438, 0x3804, 0xa438, 0x9f06, 0xa438, 0xef97, 0xa438, 0x1919, + 0xa438, 0xec02, 0xa438, 0xaf8d, 0xa438, 0x48ef, 0xa438, 0x97ec, + 0xa438, 0x04e0, 0xa438, 0x898f, 0xa438, 0x380c, 0xa438, 0x9e3d, + 0xa438, 0xe089, 0xa438, 0x8d38, 0xa438, 0x03ac, 0xa438, 0x273d, + 0xa438, 0xe089, 0xa438, 0x8d38, 0xa438, 0x039e, 0xa438, 0x3f07, + 0xa438, 0xec03, 0xa438, 0x07e0, 0xa438, 0x898d, 0xa438, 0x3804, + 0xa438, 0x9e53, 0xa438, 0x19ec, 0xa438, 0x02e0, 0xa438, 0x898d, + 0xa438, 0x3807, 0xa438, 0xac27, 0xa438, 0x6089, 0xa438, 0xec03, + 0xa438, 0xe089, 0xa438, 0x8d38, 0xa438, 0x079e, 0xa438, 0x5807, + 0xa438, 0xec04, 0xa438, 0x07e0, 0xa438, 0x898d, 0xa438, 0x3808, + 0xa438, 0x9e61, 0xa438, 0xaf8d, 0xa438, 0x4819, 0xa438, 0xec03, + 0xa438, 0x19ec, 0xa438, 0x02ae, 0xa438, 0x7819, 0xa438, 0xec03, + 0xa438, 0x07ec, 0xa438, 0x0207, 0xa438, 0xae6f, 0xa438, 0xe089, + 0xa438, 0x5da0, 0xa438, 0x010e, 0xa438, 0xe089, 0xa438, 0x92a0, + 0xa438, 0x0308, 0xa438, 0x1f55, 0xa438, 0x07ec, 0xa438, 0x0307, + 0xa438, 0xae55, 0xa438, 0x07ec, 0xa438, 0x0207, 0xa438, 0xd103, + 0xa438, 0x026b, 0xa438, 0x74ae, 0xa438, 0x50e0, 0xa438, 0x895d, + 0xa438, 0xa001, 0xa438, 0x0be0, 0xa438, 0x8992, 0xa438, 0xad21, + 0xa438, 0x05d5, 0xa438, 0x010f, 0xa438, 0xae39, 0xa438, 0xd102, + 0xa438, 0x026b, 0xa438, 0x74ae, 0xa438, 0x38ae, 0xa438, 0x36e0, + 0xa438, 0x895d, 0xa438, 0xa001, 0xa438, 0x1fe0, 0xa438, 0x8992, + 0xa438, 0xa003, 0xa438, 0x191f, 0xa438, 0x5507, 0xa438, 0xec04, + 0xa438, 0x07ae, 0xa438, 0x17e0, 0xa438, 0x895d, 0xa438, 0xa001, + 0xa438, 0x0be0, 0xa438, 0x8992, 0xa438, 0xad21, 0xa438, 0x05d5, + 0xa438, 0x010f, 0xa438, 0xae06, 0xa438, 0xef97, 0xa438, 0xec00, + 0xa438, 0xae0b, 0xa438, 0x0266, 0xa438, 0xa7ae, 0xa438, 0x0602, + 0xa438, 0x66a7, 0xa438, 0x19ec, 0xa438, 0x02ff, 0xa438, 0xef97, + 0xa438, 0xffc6, 0xa438, 0xfefd, 0xa438, 0xfc04, 0xa438, 0xf8f9, + 0xa438, 0xfaef, 0xa438, 0x69fb, 0xa438, 0xcfd3, 0xa438, 0x00e7, + 0xa438, 0x8967, 0xa438, 0xbf89, 0xa438, 0x6e1a, 0xa438, 0x93bc, + 0xa438, 0xdaad, 0xa438, 0x302d, 0xa438, 0xbf76, 0xa438, 0x111f, + 0xa438, 0x4402, 0xa438, 0x7d66, 0xa438, 0x0266, 0xa438, 0xfcbf, + 0xa438, 0x8982, 0xa438, 0xe189, 0xa438, 0x67a1, 0xa438, 0x0001, + 0xa438, 0x191f, 0xa438, 0x44d9, 0xa438, 0x81bf, 0xa438, 0x7aa0, + 0xa438, 0x027d, 0xa438, 0x66bf, 0xa438, 0x7611, 0xa438, 0xd400, + 0xa438, 0x0102, 0xa438, 0x7d66, 0xa438, 0xbdae, 0xa438, 0x02ae, + 0xa438, 0xc9da, 0xa438, 0xad31, 0xa438, 0x31bf, 0xa438, 0x761d, + 0xa438, 0x1f44, 0xa438, 0x027d, 0xa438, 0x6602, 0xa438, 0x66fc, + 0xa438, 0xbf89, 0xa438, 0x8219, 0xa438, 0xe189, 0xa438, 0x67a1, + 0xa438, 0x0202, 0xa438, 0xae06, 0xa438, 0xa103, 0xa438, 0x02ae, + 0xa438, 0x0119, 0xa438, 0x1f44, 0xa438, 0xd981, 0xa438, 0xbf7a, + 0xa438, 0xa302, 0xa438, 0x7d66, 0xa438, 0xbf76, 0xa438, 0x1dd4, + 0xa438, 0x0001, 0xa438, 0x027d, 0xa438, 0x66bd, 0xa438, 0xdaad, + 0xa438, 0x322c, 0xa438, 0xbf76, 0xa438, 0x291f, 0xa438, 0x4402, + 0xa438, 0x7d66, 0xa438, 0x0266, 0xa438, 0xfcbf, 0xa438, 0x8982, + 0xa438, 0x1919, 0xa438, 0xe189, 0xa438, 0x67a1, 0xa438, 0x0302, + 0xa438, 0xae01, 0xa438, 0x191f, 0xa438, 0x44d9, 0xa438, 0x81bf, + 0xa438, 0x7aa6, 0xa438, 0x027d, 0xa438, 0x66bf, 0xa438, 0x7629, + 0xa438, 0xd400, 0xa438, 0x0102, 0xa438, 0x7d66, 0xa438, 0x028e, + 0xa438, 0x0f13, 0xa438, 0xe789, 0xa438, 0x67e0, 0xa438, 0x8966, + 0xa438, 0x1b03, 0xa438, 0x9f8b, 0xa438, 0xc7ff, 0xa438, 0xef96, + 0xa438, 0xfefd, 0xa438, 0xfc04, 0xa438, 0xf8f9, 0xa438, 0xfaef, + 0xa438, 0x69fb, 0xa438, 0xcfe3, 0xa438, 0x8967, 0xa438, 0xbf89, + 0xa438, 0x7e1a, 0xa438, 0x93bc, 0xa438, 0xda6a, 0xa438, 0x009e, + 0xa438, 0x2fbf, 0xa438, 0x7653, 0xa438, 0xe389, 0xa438, 0x671f, + 0xa438, 0x4402, 0xa438, 0x7d66, 0xa438, 0x0267, 0xa438, 0x1abd, + 0xa438, 0x1f00, 0xa438, 0xd959, 0xa438, 0x010c, 0xa438, 0x1269, + 0xa438, 0x08a3, 0xa438, 0x0104, 0xa438, 0x6902, 0xa438, 0xae02, + 0xa438, 0x6901, 0xa438, 0xbf7a, 0xa438, 0xb202, 0xa438, 0x7d66, + 0xa438, 0xd101, 0xa438, 0xbf76, 0xa438, 0x5302, 0xa438, 0x7d66, + 0xa438, 0xc7ff, 0xa438, 0xef96, 0xa438, 0xfefd, 0xa438, 0xfc04, + 0xa438, 0x32b1, 0xa438, 0x4a32, 0xa438, 0xb24a, 0xa438, 0x32b3, + 0xa438, 0x4a10, 0xa438, 0xb14a, 0xa438, 0x10b2, 0xa438, 0x4a10, + 0xa438, 0xb34a, 0xa438, 0xf0bd, 0xa438, 0x94f0, 0xa438, 0xbd92, + 0xa438, 0x74bd, 0xa438, 0x9600, 0xa436, 0xb85e, 0xa438, 0x58ca, + 0xa436, 0xb860, 0xa438, 0x5b3d, 0xa436, 0xb862, 0xa438, 0x694c, + 0xa436, 0xb864, 0xa438, 0x65ac, 0xa436, 0xb886, 0xa438, 0x64cc, + 0xa436, 0xb888, 0xa438, 0x683e, 0xa436, 0xb88a, 0xa438, 0x6ae6, + 0xa436, 0xb88c, 0xa438, 0x5119, 0xa436, 0xb838, 0xa438, 0x00ff, + 0xb820, 0x0010, 0xa464, 0x0001, 0xa436, 0x8474, 0xa438, 0x0000, + 0xa436, 0x8608, 0xa438, 0xaf86, 0xa438, 0x20af, 0xa438, 0x865e, + 0xa438, 0xaf86, 0xa438, 0xacaf, 0xa438, 0x86b7, 0xa438, 0xaf88, + 0xa438, 0xbdaf, 0xa438, 0x88d4, 0xa438, 0xaf88, 0xa438, 0xe3af, + 0xa438, 0x893f, 0xa438, 0xbf8a, 0xa438, 0x1102, 0xa438, 0x7589, + 0xa438, 0xbf8a, 0xa438, 0x1702, 0xa438, 0x756a, 0xa438, 0xbf8a, + 0xa438, 0x1d02, 0xa438, 0x756a, 0xa438, 0xbf8a, 0xa438, 0x1402, + 0xa438, 0x7589, 0xa438, 0xbf8a, 0xa438, 0x1a02, 0xa438, 0x756a, + 0xa438, 0xbf8a, 0xa438, 0x2002, 0xa438, 0x756a, 0xa438, 0xa200, + 0xa438, 0x08bf, 0xa438, 0x6e23, 0xa438, 0x0275, 0xa438, 0x3eae, + 0xa438, 0x0ca2, 0xa438, 0x0609, 0xa438, 0xe08f, 0xa438, 0x7dad, + 0xa438, 0x2003, 0xa438, 0xaf0f, 0xa438, 0xc9af, 0xa438, 0x0fd5, + 0xa438, 0xe084, 0xa438, 0x68a0, 0xa438, 0x0005, 0xa438, 0x0264, + 0xa438, 0xe6ae, 0xa438, 0x0ba0, 0xa438, 0x0105, 0xa438, 0x0265, + 0xa438, 0xb4ae, 0xa438, 0x0302, 0xa438, 0x8677, 0xa438, 0xaf64, + 0xa438, 0xe1f8, 0xa438, 0xf9e0, 0xa438, 0x8469, 0xa438, 0xe184, + 0xa438, 0x6a14, 0xa438, 0xe484, 0xa438, 0x69e5, 0xa438, 0x846a, + 0xa438, 0xe283, 0xa438, 0xade3, 0xa438, 0x83ae, 0xa438, 0x1b45, + 0xa438, 0x9f11, 0xa438, 0xee84, 0xa438, 0x6900, 0xa438, 0xee84, + 0xa438, 0x6a00, 0xa438, 0xee84, 0xa438, 0x6800, 0xa438, 0x0264, + 0xa438, 0xe6ae, 0xa438, 0x08e0, 0xa438, 0x8043, 0xa438, 0xf626, + 0xa438, 0xe480, 0xa438, 0x43fd, 0xa438, 0xfc04, 0xa438, 0xe080, + 0xa438, 0x43f6, 0xa438, 0x26e4, 0xa438, 0x8043, 0xa438, 0xaf65, + 0xa438, 0xa8ee, 0xa438, 0x8468, 0xa438, 0x0202, 0xa438, 0x86c9, + 0xa438, 0xe080, 0xa438, 0x43f6, 0xa438, 0x26e4, 0xa438, 0x8043, + 0xa438, 0xaf66, 0xa438, 0x0bf8, 0xa438, 0xf9ef, 0xa438, 0x59f9, + 0xa438, 0xfafb, 0xa438, 0xe18f, 0xa438, 0x7ea1, 0xa438, 0x0003, + 0xa438, 0xaf88, 0xa438, 0x64bf, 0xa438, 0x6c43, 0xa438, 0x0275, + 0xa438, 0x89d0, 0xa438, 0x003c, 0xa438, 0x008a, 0xa438, 0xad27, + 0xa438, 0x03af, 0xa438, 0x86fe, 0xa438, 0xee8f, 0xa438, 0x7e00, + 0xa438, 0xee84, 0xa438, 0x6d0f, 0xa438, 0xd401, 0xa438, 0x28d6, + 0xa438, 0x0010, 0xa438, 0x0288, 0xa438, 0x6caf, 0xa438, 0x8864, + 0xa438, 0xee84, 0xa438, 0x6d00, 0xa438, 0xd600, 0xa438, 0x8f02, + 0xa438, 0x889b, 0xa438, 0xee84, 0xa438, 0x6d01, 0xa438, 0xd600, + 0xa438, 0xc002, 0xa438, 0x886c, 0xa438, 0xee84, 0xa438, 0x6d01, + 0xa438, 0xd600, 0xa438, 0x8f02, 0xa438, 0x889b, 0xa438, 0xee84, + 0xa438, 0x6d02, 0xa438, 0xd600, 0xa438, 0xc002, 0xa438, 0x886c, + 0xa438, 0xee84, 0xa438, 0x6d02, 0xa438, 0xd600, 0xa438, 0x8f02, + 0xa438, 0x889b, 0xa438, 0xee84, 0xa438, 0x6d04, 0xa438, 0xd600, + 0xa438, 0xc002, 0xa438, 0x886c, 0xa438, 0xee84, 0xa438, 0x6d03, + 0xa438, 0xd600, 0xa438, 0x8f02, 0xa438, 0x889b, 0xa438, 0xee84, + 0xa438, 0x6d08, 0xa438, 0xd600, 0xa438, 0xc002, 0xa438, 0x886c, + 0xa438, 0xee84, 0xa438, 0x6d00, 0xa438, 0xd600, 0xa438, 0x9002, + 0xa438, 0x889b, 0xa438, 0xee84, 0xa438, 0x6d01, 0xa438, 0xd600, + 0xa438, 0xc102, 0xa438, 0x886c, 0xa438, 0xee84, 0xa438, 0x6d01, + 0xa438, 0xd600, 0xa438, 0x9002, 0xa438, 0x889b, 0xa438, 0xee84, + 0xa438, 0x6d02, 0xa438, 0xd600, 0xa438, 0xc102, 0xa438, 0x886c, + 0xa438, 0xee84, 0xa438, 0x6d02, 0xa438, 0xd600, 0xa438, 0x9002, + 0xa438, 0x889b, 0xa438, 0xee84, 0xa438, 0x6d04, 0xa438, 0xd600, + 0xa438, 0xc102, 0xa438, 0x886c, 0xa438, 0xee84, 0xa438, 0x6d03, + 0xa438, 0xd600, 0xa438, 0x9002, 0xa438, 0x889b, 0xa438, 0xee84, + 0xa438, 0x6d08, 0xa438, 0xd600, 0xa438, 0xc102, 0xa438, 0x886c, + 0xa438, 0xee84, 0xa438, 0x6d00, 0xa438, 0xd600, 0xa438, 0x9102, + 0xa438, 0x889b, 0xa438, 0xee84, 0xa438, 0x6d01, 0xa438, 0xd600, + 0xa438, 0xc202, 0xa438, 0x886c, 0xa438, 0xee84, 0xa438, 0x6d01, + 0xa438, 0xd600, 0xa438, 0x9102, 0xa438, 0x889b, 0xa438, 0xee84, + 0xa438, 0x6d02, 0xa438, 0xd600, 0xa438, 0xc202, 0xa438, 0x886c, + 0xa438, 0xee84, 0xa438, 0x6d02, 0xa438, 0xd600, 0xa438, 0x9102, + 0xa438, 0x889b, 0xa438, 0xee84, 0xa438, 0x6d04, 0xa438, 0xd600, + 0xa438, 0xc202, 0xa438, 0x886c, 0xa438, 0xee84, 0xa438, 0x6d03, + 0xa438, 0xd600, 0xa438, 0x9102, 0xa438, 0x889b, 0xa438, 0xee84, + 0xa438, 0x6d08, 0xa438, 0xd600, 0xa438, 0xc202, 0xa438, 0x886c, + 0xa438, 0xee84, 0xa438, 0x6d0f, 0xa438, 0xd414, 0xa438, 0x00d6, + 0xa438, 0x000d, 0xa438, 0x0288, 0xa438, 0x6cbf, 0xa438, 0x8a2c, + 0xa438, 0x0275, 0xa438, 0x3ebf, 0xa438, 0x8a2f, 0xa438, 0x0275, + 0xa438, 0x3ebf, 0xa438, 0x8a32, 0xa438, 0x0275, 0xa438, 0x3ebf, + 0xa438, 0x8a35, 0xa438, 0x0275, 0xa438, 0x3ebf, 0xa438, 0x8a38, + 0xa438, 0x0275, 0xa438, 0x3ed4, 0xa438, 0x000f, 0xa438, 0xbf71, + 0xa438, 0x5c02, 0xa438, 0x756a, 0xa438, 0xbf71, 0xa438, 0x5f02, + 0xa438, 0x7589, 0xa438, 0xef31, 0xa438, 0xe783, 0xa438, 0x9fa3, + 0xa438, 0x0ff2, 0xa438, 0xd400, 0xa438, 0x00bf, 0xa438, 0x715c, + 0xa438, 0x0275, 0xa438, 0x6abf, 0xa438, 0x8a2c, 0xa438, 0x0275, + 0xa438, 0x47bf, 0xa438, 0x8a2f, 0xa438, 0x0275, 0xa438, 0x47bf, + 0xa438, 0x8a32, 0xa438, 0x0275, 0xa438, 0x47bf, 0xa438, 0x8a35, + 0xa438, 0x0275, 0xa438, 0x47bf, 0xa438, 0x8a38, 0xa438, 0x0275, + 0xa438, 0x47ee, 0xa438, 0x846d, 0xa438, 0x0fd4, 0xa438, 0x1000, + 0xa438, 0xd600, 0xa438, 0x0d02, 0xa438, 0x886c, 0xa438, 0xfffe, + 0xa438, 0xfdef, 0xa438, 0x95fd, 0xa438, 0xfc04, 0xa438, 0xf8f9, + 0xa438, 0xef59, 0xa438, 0xf9fa, 0xa438, 0xef44, 0xa438, 0xbf73, + 0xa438, 0xbf02, 0xa438, 0x756a, 0xa438, 0xef46, 0xa438, 0xbf73, + 0xa438, 0xc202, 0xa438, 0x756a, 0xa438, 0xe184, 0xa438, 0x6dbf, + 0xa438, 0x8a29, 0xa438, 0x0275, 0xa438, 0x6ad4, 0xa438, 0x0000, + 0xa438, 0xbf8a, 0xa438, 0x2902, 0xa438, 0x756a, 0xa438, 0xfefd, + 0xa438, 0xef95, 0xa438, 0xfdfc, 0xa438, 0x04f9, 0xa438, 0xef59, + 0xa438, 0xf9fa, 0xa438, 0xef46, 0xa438, 0xbf73, 0xa438, 0xc202, + 0xa438, 0x756a, 0xa438, 0xe184, 0xa438, 0x6dbf, 0xa438, 0x8a23, + 0xa438, 0x0275, 0xa438, 0x6abf, 0xa438, 0x8a26, 0xa438, 0x0275, + 0xa438, 0x89fe, 0xa438, 0xfdef, 0xa438, 0x95fd, 0xa438, 0x04e4, + 0xa438, 0x8044, 0xa438, 0xee84, 0xa438, 0x6d0f, 0xa438, 0xd401, + 0xa438, 0x20d6, 0xa438, 0x0010, 0xa438, 0x0288, 0xa438, 0x6cee, + 0xa438, 0x8f7e, 0xa438, 0x01af, 0xa438, 0x6879, 0xa438, 0xee84, + 0xa438, 0x6900, 0xa438, 0xee84, 0xa438, 0x6a00, 0xa438, 0xee8f, + 0xa438, 0x7e00, 0xa438, 0xaf64, 0xa438, 0x8702, 0xa438, 0x6a2a, + 0xa438, 0xe384, 0xa438, 0xf302, 0xa438, 0x6a4c, 0xa438, 0xac28, + 0xa438, 0x08e0, 0xa438, 0x84f6, 0xa438, 0xf722, 0xa438, 0xe484, + 0xa438, 0xf602, 0xa438, 0x6af9, 0xa438, 0xbf73, 0xa438, 0x4d02, + 0xa438, 0x753e, 0xa438, 0xbf73, 0xa438, 0x5002, 0xa438, 0x753e, + 0xa438, 0xbf73, 0xa438, 0x5302, 0xa438, 0x753e, 0xa438, 0xbf73, + 0xa438, 0x5602, 0xa438, 0x753e, 0xa438, 0xd500, 0xa438, 0x0002, + 0xa438, 0x6b1d, 0xa438, 0xbf73, 0xa438, 0x4402, 0xa438, 0x7547, + 0xa438, 0xbf73, 0xa438, 0x4702, 0xa438, 0x7547, 0xa438, 0xbf73, + 0xa438, 0x7702, 0xa438, 0x753e, 0xa438, 0xbf73, 0xa438, 0x4a02, + 0xa438, 0x7547, 0xa438, 0xbf73, 0xa438, 0x5302, 0xa438, 0x753e, + 0xa438, 0xbf73, 0xa438, 0x5602, 0xa438, 0x7547, 0xa438, 0xaf69, + 0xa438, 0x4d02, 0xa438, 0x8948, 0xa438, 0x021e, 0xa438, 0x40af, + 0xa438, 0x1e3f, 0xa438, 0xf8fa, 0xa438, 0xef69, 0xa438, 0xe080, + 0xa438, 0x4fac, 0xa438, 0x2417, 0xa438, 0xe080, 0xa438, 0x44ad, + 0xa438, 0x2417, 0xa438, 0x0289, 0xa438, 0x74e0, 0xa438, 0x8044, + 0xa438, 0xac24, 0xa438, 0x0ebf, 0xa438, 0x8a3b, 0xa438, 0x0275, + 0xa438, 0x47ae, 0xa438, 0x0602, 0xa438, 0x8a06, 0xa438, 0x0289, + 0xa438, 0xfbef, 0xa438, 0x96fe, 0xa438, 0xfc04, 0xa438, 0xf8f9, + 0xa438, 0xfaef, 0xa438, 0x69fa, 0xa438, 0xfbd2, 0xa438, 0x00a2, + 0xa438, 0x0403, 0xa438, 0xaf89, 0xa438, 0xeabf, 0xa438, 0x6af1, + 0xa438, 0x0277, 0xa438, 0x24ef, 0xa438, 0x010d, 0xa438, 0x11d0, + 0xa438, 0x00ef, 0xa438, 0x640c, 0xa438, 0x66ef, 0xa438, 0x12bf, + 0xa438, 0x8a23, 0xa438, 0x0275, 0xa438, 0x6ad3, 0xa438, 0x01a3, + 0xa438, 0x4302, 0xa438, 0xae44, 0xa438, 0x1f00, 0xa438, 0xef13, + 0xa438, 0xbf73, 0xa438, 0xc202, 0xa438, 0x756a, 0xa438, 0xbf8a, + 0xa438, 0x3e02, 0xa438, 0x7589, 0xa438, 0xd100, 0xa438, 0x0d01, + 0xa438, 0x0c01, 0xa438, 0x1a46, 0xa438, 0xbf73, 0xa438, 0xbf02, + 0xa438, 0x756a, 0xa438, 0x1f00, 0xa438, 0xef13, 0xa438, 0xbf73, + 0xa438, 0xc202, 0xa438, 0x756a, 0xa438, 0xd101, 0xa438, 0xef02, + 0xa438, 0x10b0, 0xa438, 0x02ae, 0xa438, 0x0449, 0xa438, 0x02ae, + 0xa438, 0xf8bf, 0xa438, 0x73c5, 0xa438, 0x0275, 0xa438, 0x6abf, + 0xa438, 0x73c5, 0xa438, 0x0275, 0xa438, 0x3e13, 0xa438, 0xaeb7, + 0xa438, 0x12af, 0xa438, 0x897d, 0xa438, 0xd500, 0xa438, 0x0102, + 0xa438, 0x6b1d, 0xa438, 0x0289, 0xa438, 0xfbff, 0xa438, 0xfeef, + 0xa438, 0x96fe, 0xa438, 0xfdfc, 0xa438, 0x04f8, 0xa438, 0xe080, + 0xa438, 0x44f6, 0xa438, 0x24e4, 0xa438, 0x8044, 0xa438, 0xfc04, + 0xa438, 0xf8e0, 0xa438, 0x804f, 0xa438, 0xf624, 0xa438, 0xe480, + 0xa438, 0x4ffc, 0xa438, 0x0455, 0xa438, 0xa6fe, 0xa438, 0x44a6, + 0xa438, 0xfe66, 0xa438, 0xa4b6, 0xa438, 0x55a4, 0xa438, 0xb666, + 0xa438, 0xac0e, 0xa438, 0x55ac, 0xa438, 0x0efe, 0xa438, 0xbda4, + 0xa438, 0xf0bd, 0xa438, 0x9830, 0xa438, 0xbd96, 0xa438, 0xffbd, + 0xa438, 0xdeee, 0xa438, 0xbdde, 0xa438, 0xddbd, 0xa438, 0xdebb, + 0xa438, 0xbdde, 0xa438, 0xaabd, 0xa438, 0xde44, 0xa438, 0xac00, + 0xa438, 0xf0bd, 0xa438, 0x9a00, 0xa436, 0xb818, 0xa438, 0x0fb9, + 0xa436, 0xb81a, 0xa438, 0x64c3, 0xa436, 0xb81c, 0xa438, 0x64f1, + 0xa436, 0xb81e, 0xa438, 0x6607, 0xa436, 0xb850, 0xa438, 0x6876, + 0xa436, 0xb852, 0xa438, 0x647f, 0xa436, 0xb878, 0xa438, 0x68e8, + 0xa436, 0xb884, 0xa438, 0x1e3c, 0xa436, 0xb832, 0xa438, 0x00df, + 0xB82E, 0x0000, 0xa436, 0x8023, 0xa438, 0x0000, 0xB820, 0x0000, + 0xFFFF, 0xFFFF +}; + +static const u16 phy_mcu_ram_code_8127a_1[] = { + 0xa436, 0x8023, 0xa438, 0x6100, 0xa436, 0xB82E, 0xa438, 0x0001, + 0xb820, 0x0090, 0xa436, 0xA016, 0xa438, 0x0000, 0xa436, 0xA012, + 0xa438, 0x0000, 0xa436, 0xA014, 0xa438, 0x1800, 0xa438, 0x8010, + 0xa438, 0x1800, 0xa438, 0x801a, 0xa438, 0x1800, 0xa438, 0x801a, + 0xa438, 0x1800, 0xa438, 0x801a, 0xa438, 0x1800, 0xa438, 0x801a, + 0xa438, 0x1800, 0xa438, 0x801a, 0xa438, 0x1800, 0xa438, 0x801a, + 0xa438, 0x1800, 0xa438, 0x801a, 0xa438, 0xce00, 0xa438, 0x2941, + 0xa438, 0x8017, 0xa438, 0x2c59, 0xa438, 0x8017, 0xa438, 0x1800, + 0xa438, 0x0e11, 0xa438, 0x8aff, 0xa438, 0x1800, 0xa438, 0x0e11, + 0xa436, 0xA026, 0xa438, 0xffff, 0xa436, 0xA024, 0xa438, 0xffff, + 0xa436, 0xA022, 0xa438, 0xffff, 0xa436, 0xA020, 0xa438, 0xffff, + 0xa436, 0xA006, 0xa438, 0xffff, 0xa436, 0xA004, 0xa438, 0xffff, + 0xa436, 0xA002, 0xa438, 0xffff, 0xa436, 0xA000, 0xa438, 0x0e10, + 0xa436, 0xA008, 0xa438, 0x0100, 0xa436, 0xA016, 0xa438, 0x0000, + 0xa436, 0xA012, 0xa438, 0x0ff8, 0xa436, 0xA014, 0xa438, 0x219a, + 0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000, + 0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000, 0xa436, 0xA152, + 0xa438, 0x21a4, 0xa436, 0xA154, 0xa438, 0x3fff, 0xa436, 0xA156, + 0xa438, 0x3fff, 0xa436, 0xA158, 0xa438, 0x3fff, 0xa436, 0xA15A, + 0xa438, 0x3fff, 0xa436, 0xA15C, 0xa438, 0x3fff, 0xa436, 0xA15E, + 0xa438, 0x3fff, 0xa436, 0xA160, 0xa438, 0x3fff, 0xa436, 0xA150, + 0xa438, 0x0001, 0xa436, 0xA016, 0xa438, 0x0010, 0xa436, 0xA012, + 0xa438, 0x0000, 0xa436, 0xA014, 0xa438, 0x1800, 0xa438, 0x8010, + 0xa438, 0x1800, 0xa438, 0x8014, 0xa438, 0x1800, 0xa438, 0x801a, + 0xa438, 0x1800, 0xa438, 0x801e, 0xa438, 0x1800, 0xa438, 0x8026, + 0xa438, 0x1800, 0xa438, 0x802e, 0xa438, 0x1800, 0xa438, 0x8036, + 0xa438, 0x1800, 0xa438, 0x803a, 0xa438, 0xce01, 0xa438, 0x8208, + 0xa438, 0x1800, 0xa438, 0x0028, 0xa438, 0x1000, 0xa438, 0x02c5, + 0xa438, 0x1000, 0xa438, 0x0304, 0xa438, 0x1800, 0xa438, 0x0119, + 0xa438, 0xce01, 0xa438, 0x8208, 0xa438, 0x1800, 0xa438, 0x009e, + 0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0xa50f, 0xa438, 0x8208, + 0xa438, 0xd500, 0xa438, 0xaa0f, 0xa438, 0x1800, 0xa438, 0x015b, + 0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0xa50f, 0xa438, 0x8208, + 0xa438, 0xd500, 0xa438, 0xaa0f, 0xa438, 0x1800, 0xa438, 0x01a9, + 0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0xa50f, 0xa438, 0x8208, + 0xa438, 0xd500, 0xa438, 0xaa0f, 0xa438, 0x1800, 0xa438, 0x01f4, + 0xa438, 0x8208, 0xa438, 0xd500, 0xa438, 0x1800, 0xa438, 0x02a5, + 0xa438, 0xa208, 0xa438, 0xd500, 0xa438, 0x1800, 0xa438, 0x02b8, + 0xa436, 0xA08E, 0xa438, 0x02b7, 0xa436, 0xA08C, 0xa438, 0x02a4, + 0xa436, 0xA08A, 0xa438, 0x01e7, 0xa436, 0xA088, 0xa438, 0x019c, + 0xa436, 0xA086, 0xa438, 0x014e, 0xa436, 0xA084, 0xa438, 0x009d, + 0xa436, 0xA082, 0xa438, 0x0117, 0xa436, 0xA080, 0xa438, 0x0027, + 0xa436, 0xA090, 0xa438, 0x00ff, 0xa436, 0xA016, 0xa438, 0x0020, + 0xa436, 0xA012, 0xa438, 0x0000, 0xa436, 0xA014, 0xa438, 0x1800, + 0xa438, 0x8010, 0xa438, 0x1800, 0xa438, 0x801d, 0xa438, 0x1800, + 0xa438, 0x803b, 0xa438, 0x1800, 0xa438, 0x8087, 0xa438, 0x1800, + 0xa438, 0x808e, 0xa438, 0x1800, 0xa438, 0x809d, 0xa438, 0x1800, + 0xa438, 0x80b7, 0xa438, 0x1800, 0xa438, 0x80c4, 0xa438, 0xd1bc, + 0xa438, 0xd040, 0xa438, 0x1000, 0xa438, 0x1cd2, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd700, 0xa438, 0x273d, 0xa438, 0x801b, + 0xa438, 0x1800, 0xa438, 0x07d1, 0xa438, 0x1800, 0xa438, 0x080e, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8032, 0xa438, 0x33a9, + 0xa438, 0x802a, 0xa438, 0xd705, 0xa438, 0x4084, 0xa438, 0xd1f4, + 0xa438, 0xd048, 0xa438, 0xf013, 0xa438, 0xd1b7, 0xa438, 0xd04b, + 0xa438, 0xf010, 0xa438, 0xd705, 0xa438, 0x4084, 0xa438, 0xd1f4, + 0xa438, 0xd048, 0xa438, 0xf00b, 0xa438, 0xd1b7, 0xa438, 0xd04b, + 0xa438, 0xf008, 0xa438, 0xd705, 0xa438, 0x4084, 0xa438, 0xd1f4, + 0xa438, 0xd048, 0xa438, 0xf003, 0xa438, 0xd1b7, 0xa438, 0xd04b, + 0xa438, 0x1800, 0xa438, 0x14cc, 0xa438, 0xd700, 0xa438, 0x2b59, + 0xa438, 0x803f, 0xa438, 0xf003, 0xa438, 0x1800, 0xa438, 0x118f, + 0xa438, 0x6060, 0xa438, 0x1800, 0xa438, 0x1167, 0xa438, 0xd700, + 0xa438, 0x60c7, 0xa438, 0xd704, 0xa438, 0x609f, 0xa438, 0xd705, + 0xa438, 0x4043, 0xa438, 0xf003, 0xa438, 0x1800, 0xa438, 0x1150, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8702, 0xa438, 0x8011, + 0xa438, 0x9503, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8302, + 0xa438, 0x8480, 0xa438, 0x8686, 0xa438, 0xcde0, 0xa438, 0xd1ff, + 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x1cd2, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd705, 0xa438, 0x417e, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa011, 0xa438, 0x9503, 0xa438, 0xd1c8, + 0xa438, 0xd045, 0xa438, 0x1000, 0xa438, 0x1cd2, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa702, + 0xa438, 0x9503, 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xa480, + 0xa438, 0xa686, 0xa438, 0xd705, 0xa438, 0x605e, 0xa438, 0xa302, + 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8083, + 0xa438, 0x33a9, 0xa438, 0x807f, 0xa438, 0xd178, 0xa438, 0xd04b, + 0xa438, 0x1800, 0xa438, 0x115d, 0xa438, 0xd1c8, 0xa438, 0xd04b, + 0xa438, 0x1800, 0xa438, 0x115d, 0xa438, 0xd1e6, 0xa438, 0xd04b, + 0xa438, 0x1800, 0xa438, 0x115d, 0xa438, 0xd71f, 0xa438, 0x6080, + 0xa438, 0xd704, 0xa438, 0x1800, 0xa438, 0x1bc0, 0xa438, 0x1800, + 0xa438, 0x1bc4, 0xa438, 0x4134, 0xa438, 0xd115, 0xa438, 0xd04f, + 0xa438, 0x1000, 0xa438, 0x1d0b, 0xa438, 0x1000, 0xa438, 0x80ad, + 0xa438, 0x1800, 0xa438, 0x01f2, 0xa438, 0x1000, 0xa438, 0x1d0b, + 0xa438, 0x1000, 0xa438, 0x80ad, 0xa438, 0x1800, 0xa438, 0x01f9, + 0xa438, 0x2969, 0xa438, 0x80a3, 0xa438, 0xd700, 0xa438, 0x606b, + 0xa438, 0xd701, 0xa438, 0x60b4, 0xa438, 0x1000, 0xa438, 0x80ad, + 0xa438, 0x1800, 0xa438, 0x0551, 0xa438, 0xd196, 0xa438, 0xd04d, + 0xa438, 0x1000, 0xa438, 0x80ad, 0xa438, 0x1800, 0xa438, 0x054d, + 0xa438, 0xd208, 0xa438, 0x0c09, 0xa438, 0x1301, 0xa438, 0x1000, + 0xa438, 0x1cd2, 0xa438, 0xd701, 0xa438, 0x5fa3, 0xa438, 0xb302, + 0xa438, 0xd200, 0xa438, 0x0800, 0xa438, 0xd705, 0xa438, 0x6064, + 0xa438, 0x1800, 0xa438, 0x140a, 0xa438, 0x8810, 0xa438, 0xd199, + 0xa438, 0xd04b, 0xa438, 0x1000, 0xa438, 0x1cd2, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0x1800, 0xa438, 0x140a, 0xa436, 0xA10E, + 0xa438, 0xffff, 0xa436, 0xA10C, 0xa438, 0x1352, 0xa436, 0xA10A, + 0xa438, 0x0545, 0xa436, 0xA108, 0xa438, 0x01ed, 0xa436, 0xA106, + 0xa438, 0x1bbf, 0xa436, 0xA104, 0xa438, 0x114b, 0xa436, 0xA102, + 0xa438, 0x14bf, 0xa436, 0xA100, 0xa438, 0x07ce, 0xa436, 0xA110, + 0xa438, 0x007f, 0xa436, 0xA016, 0xa438, 0x0020, 0xa436, 0xA012, + 0xa438, 0x1ff8, 0xa436, 0xA014, 0xa438, 0xd1ce, 0xa438, 0x0000, + 0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000, + 0xa438, 0x0000, 0xa438, 0x0000, 0xa436, 0xA164, 0xa438, 0x07fc, + 0xa436, 0xA166, 0xa438, 0x143d, 0xa436, 0xA168, 0xa438, 0x3fff, + 0xa436, 0xA16A, 0xa438, 0x3fff, 0xa436, 0xA16C, 0xa438, 0x3fff, + 0xa436, 0xA16E, 0xa438, 0x3fff, 0xa436, 0xA170, 0xa438, 0x3fff, + 0xa436, 0xA172, 0xa438, 0x3fff, 0xa436, 0xA162, 0xa438, 0x0003, + 0xa436, 0xb87c, 0xa438, 0x8994, 0xa436, 0xb87e, 0xa438, 0xaf89, + 0xa438, 0xacaf, 0xa438, 0x89e4, 0xa438, 0xaf89, 0xa438, 0xecaf, + 0xa438, 0x8a04, 0xa438, 0xaf8a, 0xa438, 0x2eaf, 0xa438, 0x8a4a, + 0xa438, 0xaf8d, 0xa438, 0x31af, 0xa438, 0x8dc6, 0xa438, 0x1f55, + 0xa438, 0xe18f, 0xa438, 0xe3a1, 0xa438, 0x0007, 0xa438, 0xee86, + 0xa438, 0xe900, 0xa438, 0xaf4f, 0xa438, 0x9ead, 0xa438, 0x281b, + 0xa438, 0xe18f, 0xa438, 0xfcef, 0xa438, 0x71bf, 0xa438, 0x74f6, + 0xa438, 0x027e, 0xa438, 0xd2ef, 0xa438, 0x641c, 0xa438, 0x670d, + 0xa438, 0x67ef, 0xa438, 0x461f, 0xa438, 0x00bf, 0xa438, 0x74f6, + 0xa438, 0x027e, 0xa438, 0xdee1, 0xa438, 0x8fe3, 0xa438, 0x0d11, + 0xa438, 0xe58f, 0xa438, 0xe313, 0xa438, 0xaeca, 0xa438, 0x028d, + 0xa438, 0xd1d3, 0xa438, 0x01af, 0xa438, 0x40d1, 0xa438, 0xbf7a, + 0xa438, 0x6102, 0xa438, 0x7d44, 0xa438, 0xa100, 0xa438, 0x09e0, + 0xa438, 0x8ffa, 0xa438, 0xe18f, 0xa438, 0xfbaf, 0xa438, 0x683d, + 0xa438, 0x027f, 0xa438, 0xa9af, 0xa438, 0x682c, 0xa438, 0xbf8e, + 0xa438, 0x4102, 0xa438, 0x7d44, 0xa438, 0xe58f, 0xa438, 0xecbf, + 0xa438, 0x74cc, 0xa438, 0x027d, 0xa438, 0x44e3, 0xa438, 0x8fed, + 0xa438, 0x0d31, 0xa438, 0xf63f, 0xa438, 0x0d11, 0xa438, 0xf62f, + 0xa438, 0x1b13, 0xa438, 0xad2f, 0xa438, 0x06bf, 0xa438, 0x8e41, + 0xa438, 0x027c, 0xa438, 0xf9d1, 0xa438, 0x01af, 0xa438, 0x5974, + 0xa438, 0xee88, 0xa438, 0x8600, 0xa438, 0xe08f, 0xa438, 0xebad, + 0xa438, 0x200b, 0xa438, 0xe18f, 0xa438, 0xecbf, 0xa438, 0x8e41, + 0xa438, 0x027d, 0xa438, 0x25ae, 0xa438, 0x04ee, 0xa438, 0x8feb, + 0xa438, 0x01af, 0xa438, 0x5945, 0xa438, 0xad28, 0xa438, 0x2ce0, + 0xa438, 0x8fea, 0xa438, 0xa000, 0xa438, 0x0502, 0xa438, 0x8af0, + 0xa438, 0xae1e, 0xa438, 0xa001, 0xa438, 0x0502, 0xa438, 0x8b9f, + 0xa438, 0xae16, 0xa438, 0xa002, 0xa438, 0x0502, 0xa438, 0x8c0f, + 0xa438, 0xae0e, 0xa438, 0xa003, 0xa438, 0x0502, 0xa438, 0x8c95, + 0xa438, 0xae06, 0xa438, 0xa004, 0xa438, 0x0302, 0xa438, 0x8d08, + 0xa438, 0xaf63, 0xa438, 0x8902, 0xa438, 0x8a7f, 0xa438, 0xaf63, + 0xa438, 0x81f8, 0xa438, 0xef49, 0xa438, 0xf8e0, 0xa438, 0x8015, + 0xa438, 0xad21, 0xa438, 0x19bf, 0xa438, 0x7bd8, 0xa438, 0x027c, + 0xa438, 0xf9bf, 0xa438, 0x7bf3, 0xa438, 0x027d, 0xa438, 0x44bf, + 0xa438, 0x7bf6, 0xa438, 0x027c, 0xa438, 0xf902, 0xa438, 0x638e, + 0xa438, 0xee8f, 0xa438, 0xea00, 0xa438, 0xe080, 0xa438, 0x16ad, + 0xa438, 0x233d, 0xa438, 0xbf7b, 0xa438, 0xf302, 0xa438, 0x7d44, + 0xa438, 0xbf7a, 0xa438, 0x9402, 0xa438, 0x7cf9, 0xa438, 0xbf8e, + 0xa438, 0x4402, 0xa438, 0x7cf9, 0xa438, 0xbf7a, 0xa438, 0xa602, + 0xa438, 0x7cf9, 0xa438, 0xbf7a, 0xa438, 0xa302, 0xa438, 0x7cf9, + 0xa438, 0xbf7a, 0xa438, 0xa902, 0xa438, 0x7cf9, 0xa438, 0xbf7a, + 0xa438, 0xac02, 0xa438, 0x7cf9, 0xa438, 0xbf8e, 0xa438, 0x4702, + 0xa438, 0x7cf9, 0xa438, 0xbf8e, 0xa438, 0x4a02, 0xa438, 0x7cf9, + 0xa438, 0x0263, 0xa438, 0x8eee, 0xa438, 0x8fea, 0xa438, 0x00bf, + 0xa438, 0x7c02, 0xa438, 0x027c, 0xa438, 0xf9fc, 0xa438, 0xef94, + 0xa438, 0xfc04, 0xa438, 0xf8f9, 0xa438, 0xfbef, 0xa438, 0x79fb, + 0xa438, 0xe080, 0xa438, 0x15ac, 0xa438, 0x2103, 0xa438, 0xaf8b, + 0xa438, 0x70ee, 0xa438, 0x8888, 0xa438, 0x00ee, 0xa438, 0x888a, + 0xa438, 0x00ee, 0xa438, 0x888b, 0xa438, 0x00bf, 0xa438, 0x7bd8, + 0xa438, 0x027d, 0xa438, 0x02bf, 0xa438, 0x6000, 0xa438, 0xd788, + 0xa438, 0x881f, 0xa438, 0x44d4, 0xa438, 0x000c, 0xa438, 0x0273, + 0xa438, 0x3b02, 0xa438, 0x7fa9, 0xa438, 0xac28, 0xa438, 0x05ac, + 0xa438, 0x290d, 0xa438, 0xae18, 0xa438, 0xe188, 0xa438, 0x98bf, + 0xa438, 0x7be1, 0xa438, 0x027d, 0xa438, 0x25ae, 0xa438, 0x18e1, + 0xa438, 0x8898, 0xa438, 0x0d11, 0xa438, 0xbf7b, 0xa438, 0xe102, + 0xa438, 0x7d25, 0xa438, 0xae0b, 0xa438, 0xe188, 0xa438, 0x980d, + 0xa438, 0x12bf, 0xa438, 0x7be1, 0xa438, 0x027d, 0xa438, 0x25bf, + 0xa438, 0x88a0, 0xa438, 0xda19, 0xa438, 0xdb19, 0xa438, 0xd819, + 0xa438, 0xd91f, 0xa438, 0x77bf, 0xa438, 0x88b1, 0xa438, 0xde19, + 0xa438, 0xdf19, 0xa438, 0xdc19, 0xa438, 0xdd19, 0xa438, 0x17a7, + 0xa438, 0x0004, 0xa438, 0xf302, 0xa438, 0x63cd, 0xa438, 0xee8f, + 0xa438, 0xea01, 0xa438, 0xe080, 0xa438, 0x16ad, 0xa438, 0x2319, + 0xa438, 0xee88, 0xa438, 0x8800, 0xa438, 0xee88, 0xa438, 0x8a00, + 0xa438, 0xee88, 0xa438, 0x8b00, 0xa438, 0xbf8e, 0xa438, 0x4402, + 0xa438, 0x7d02, 0xa438, 0x0263, 0xa438, 0xcdee, 0xa438, 0x8fea, + 0xa438, 0x0102, 0xa438, 0x70de, 0xa438, 0xbf7c, 0xa438, 0x0202, + 0xa438, 0x7d02, 0xa438, 0xffef, 0xa438, 0x97ff, 0xa438, 0xfdfc, + 0xa438, 0x04f8, 0xa438, 0xf9fa, 0xa438, 0xef69, 0xa438, 0xfae0, + 0xa438, 0x888a, 0xa438, 0xe188, 0xa438, 0x8b14, 0xa438, 0xe488, + 0xa438, 0x8ae5, 0xa438, 0x888b, 0xa438, 0xbf88, 0xa438, 0x94d8, + 0xa438, 0x19d9, 0xa438, 0xef64, 0xa438, 0xe088, 0xa438, 0x8ae1, + 0xa438, 0x888b, 0xa438, 0x1b46, 0xa438, 0x9f30, 0xa438, 0x1f44, + 0xa438, 0xe488, 0xa438, 0x8ae5, 0xa438, 0x888b, 0xa438, 0xe080, + 0xa438, 0x15ad, 0xa438, 0x211a, 0xa438, 0x0260, 0xa438, 0xece0, + 0xa438, 0x8016, 0xa438, 0xad23, 0xa438, 0x1602, 0xa438, 0x7c86, + 0xa438, 0xef47, 0xa438, 0xe48f, 0xa438, 0xe9e5, 0xa438, 0x8fe8, + 0xa438, 0xee8f, 0xa438, 0xea02, 0xa438, 0xae0b, 0xa438, 0x028c, + 0xa438, 0x2eae, 0xa438, 0x0602, 0xa438, 0x8bfe, 0xa438, 0x0270, + 0xa438, 0xdefe, 0xa438, 0xef96, 0xa438, 0xfefd, 0xa438, 0xfc04, + 0xa438, 0xf8e1, 0xa438, 0x8888, 0xa438, 0x11e5, 0xa438, 0x8888, + 0xa438, 0xad2a, 0xa438, 0x04ee, 0xa438, 0x8888, 0xa438, 0x00fc, + 0xa438, 0x04f8, 0xa438, 0xfafb, 0xa438, 0xe08f, 0xa438, 0xe9e1, + 0xa438, 0x8fe8, 0xa438, 0xef64, 0xa438, 0x1f00, 0xa438, 0xe18f, + 0xa438, 0xe6ef, 0xa438, 0x7402, 0xa438, 0x7ca1, 0xa438, 0xad50, + 0xa438, 0x0302, 0xa438, 0x8c2e, 0xa438, 0xfffe, 0xa438, 0xfc04, + 0xa438, 0xf8fa, 0xa438, 0xef69, 0xa438, 0xfbbf, 0xa438, 0x7bf3, + 0xa438, 0x027d, 0xa438, 0x44ac, 0xa438, 0x284c, 0xa438, 0x0264, + 0xa438, 0x1cbf, 0xa438, 0x8e47, 0xa438, 0x027d, 0xa438, 0x02bf, + 0xa438, 0x8e4a, 0xa438, 0x027d, 0xa438, 0x02d1, 0xa438, 0x43b1, + 0xa438, 0xfebf, 0xa438, 0x7aa6, 0xa438, 0x027c, 0xa438, 0xf9bf, + 0xa438, 0x7aa3, 0xa438, 0x027c, 0xa438, 0xf9bf, 0xa438, 0x7aa9, + 0xa438, 0x027c, 0xa438, 0xf9bf, 0xa438, 0x7aac, 0xa438, 0x027d, + 0xa438, 0x02d1, 0xa438, 0x80e0, 0xa438, 0x8888, 0xa438, 0x100e, + 0xa438, 0x11b0, 0xa438, 0xfcbf, 0xa438, 0x7a94, 0xa438, 0x027d, + 0xa438, 0x2502, 0xa438, 0x7c86, 0xa438, 0xef47, 0xa438, 0xe48f, + 0xa438, 0xe9e5, 0xa438, 0x8fe8, 0xa438, 0xee8f, 0xa438, 0xea03, + 0xa438, 0xae07, 0xa438, 0xee8f, 0xa438, 0xea01, 0xa438, 0x0270, + 0xa438, 0xdeff, 0xa438, 0xef96, 0xa438, 0xfefc, 0xa438, 0x04f8, + 0xa438, 0xf9fa, 0xa438, 0xfbef, 0xa438, 0x79fb, 0xa438, 0xbf7a, + 0xa438, 0x9402, 0xa438, 0x7d44, 0xa438, 0xef21, 0xa438, 0xbf7a, + 0xa438, 0xb802, 0xa438, 0x7d44, 0xa438, 0x1f21, 0xa438, 0x9e19, + 0xa438, 0xe08f, 0xa438, 0xe9e1, 0xa438, 0x8fe8, 0xa438, 0xef64, + 0xa438, 0x1f00, 0xa438, 0xe18f, 0xa438, 0xe4ef, 0xa438, 0x7402, + 0xa438, 0x7ca1, 0xa438, 0xad50, 0xa438, 0x3dee, 0xa438, 0x8fe7, + 0xa438, 0x01bf, 0xa438, 0x7a94, 0xa438, 0x027c, 0xa438, 0xf9bf, + 0xa438, 0x7aa6, 0xa438, 0x027c, 0xa438, 0xf9bf, 0xa438, 0x7aa3, + 0xa438, 0x027c, 0xa438, 0xf9bf, 0xa438, 0x7aa9, 0xa438, 0x027c, + 0xa438, 0xf9bf, 0xa438, 0x7aac, 0xa438, 0x027d, 0xa438, 0x02bf, + 0xa438, 0x8e47, 0xa438, 0x027c, 0xa438, 0xf9bf, 0xa438, 0x8e4a, + 0xa438, 0x027c, 0xa438, 0xf902, 0xa438, 0x7c86, 0xa438, 0xef47, + 0xa438, 0xe48f, 0xa438, 0xe9e5, 0xa438, 0x8fe8, 0xa438, 0xee8f, + 0xa438, 0xea04, 0xa438, 0xffef, 0xa438, 0x97ff, 0xa438, 0xfefd, + 0xa438, 0xfc04, 0xa438, 0xf8fa, 0xa438, 0xfbe0, 0xa438, 0x8fe9, + 0xa438, 0xe18f, 0xa438, 0xe8ef, 0xa438, 0x641f, 0xa438, 0x00e1, + 0xa438, 0x8fe5, 0xa438, 0xef74, 0xa438, 0x027c, 0xa438, 0xa1ad, + 0xa438, 0x500d, 0xa438, 0x0263, 0xa438, 0x8e02, 0xa438, 0x8bfe, + 0xa438, 0xee8f, 0xa438, 0xea01, 0xa438, 0x0270, 0xa438, 0xdeff, + 0xa438, 0xfefc, 0xa438, 0x04e3, 0xa438, 0x8fd8, 0xa438, 0xe787, + 0xa438, 0x75e4, 0xa438, 0x8fe1, 0xa438, 0xe58f, 0xa438, 0xe2bf, + 0xa438, 0x8fd9, 0xa438, 0xef32, 0xa438, 0x0c31, 0xa438, 0x1a93, + 0xa438, 0xdc19, 0xa438, 0xdd02, 0xa438, 0x7fa9, 0xa438, 0xac2a, + 0xa438, 0x18e0, 0xa438, 0x8fe1, 0xa438, 0xe18f, 0xa438, 0xe2ef, + 0xa438, 0x74e1, 0xa438, 0x8775, 0xa438, 0x1f00, 0xa438, 0xef64, + 0xa438, 0xe18f, 0xa438, 0xd8e5, 0xa438, 0x8775, 0xa438, 0xaf4d, + 0xa438, 0x72bf, 0xa438, 0x7b3c, 0xa438, 0xef32, 0xa438, 0x4b03, + 0xa438, 0x1a93, 0xa438, 0x027d, 0xa438, 0x44ef, 0xa438, 0x64e1, + 0xa438, 0x8fff, 0xa438, 0x1f00, 0xa438, 0xef74, 0xa438, 0x1b67, + 0xa438, 0xac4f, 0xa438, 0xcee0, 0xa438, 0x8ffd, 0xa438, 0xe18f, + 0xa438, 0xfeef, 0xa438, 0x64e0, 0xa438, 0x8fe1, 0xa438, 0xe18f, + 0xa438, 0xe2ef, 0xa438, 0x7402, 0xa438, 0x7c53, 0xa438, 0xac50, + 0xa438, 0x02ae, 0xa438, 0xb6e1, 0xa438, 0x8775, 0xa438, 0x1f00, + 0xa438, 0xef64, 0xa438, 0xe18f, 0xa438, 0xfcef, 0xa438, 0x711c, + 0xa438, 0x670d, 0xa438, 0x67ef, 0xa438, 0x46e5, 0xa438, 0x8775, + 0xa438, 0xef32, 0xa438, 0xd101, 0xa438, 0xa300, 0xa438, 0x02ae, + 0xa438, 0x050c, 0xa438, 0x1183, 0xa438, 0xaef6, 0xa438, 0xe08f, + 0xa438, 0xe31e, 0xa438, 0x10e5, 0xa438, 0x8fe3, 0xa438, 0xae89, + 0xa438, 0xe287, 0xa438, 0x75e6, 0xa438, 0x8fd8, 0xa438, 0x1f22, + 0xa438, 0xaf4d, 0xa438, 0x42f8, 0xa438, 0xf9ef, 0xa438, 0x59fa, + 0xa438, 0xfbbf, 0xa438, 0x8fee, 0xa438, 0x027f, 0xa438, 0xa90d, + 0xa438, 0x1149, 0xa438, 0x041a, 0xa438, 0x91d7, 0xa438, 0x8df3, + 0xa438, 0xd68e, 0xa438, 0x2302, 0xa438, 0x72aa, 0xa438, 0xfffe, + 0xa438, 0xef95, 0xa438, 0xfdfc, 0xa438, 0x0400, 0xa438, 0x7591, + 0xa438, 0x0275, 0xa438, 0x4404, 0xa438, 0x758e, 0xa438, 0x2675, + 0xa438, 0x4100, 0xa438, 0x8e26, 0xa438, 0x028e, 0xa438, 0x2304, + 0xa438, 0x759d, 0xa438, 0x2675, 0xa438, 0x4700, 0xa438, 0x8e32, + 0xa438, 0x028e, 0xa438, 0x2f04, 0xa438, 0x8e2c, 0xa438, 0x268e, + 0xa438, 0x2900, 0xa438, 0x8e3e, 0xa438, 0x028e, 0xa438, 0x3b04, + 0xa438, 0x8e38, 0xa438, 0x268e, 0xa438, 0x35fe, 0xa438, 0xad96, + 0xa438, 0xdcad, 0xa438, 0x96ba, 0xa438, 0xad96, 0xa438, 0x98ad, + 0xa438, 0x9676, 0xa438, 0xad98, 0xa438, 0x54ad, 0xa438, 0x9876, + 0xa438, 0xae38, 0xa438, 0x54ae, 0xa438, 0x38fe, 0xa438, 0xae3a, + 0xa438, 0xdcae, 0xa438, 0x3abb, 0xa438, 0xbf14, 0xa438, 0x99bd, + 0xa438, 0xe0cc, 0xa438, 0xbdc8, 0xa438, 0xddbd, 0xa438, 0xc800, + 0xa436, 0xb85e, 0xa438, 0x4f9a, 0xa436, 0xb860, 0xa438, 0x40cf, + 0xa436, 0xb862, 0xa438, 0x6829, 0xa436, 0xb864, 0xa438, 0x5972, + 0xa436, 0xb886, 0xa438, 0x5941, 0xa436, 0xb888, 0xa438, 0x636b, + 0xa436, 0xb88a, 0xa438, 0x4d6b, 0xa436, 0xb88c, 0xa438, 0x4d40, + 0xa436, 0xb838, 0xa438, 0x00ff, 0xb820, 0x0010, 0xa436, 0x8608, + 0xa438, 0xaf86, 0xa438, 0xdaaf, 0xa438, 0x894c, 0xa438, 0xaf8a, + 0xa438, 0xf8af, 0xa438, 0x8bf3, 0xa438, 0xaf8b, 0xa438, 0xf3af, + 0xa438, 0x8bf3, 0xa438, 0xaf8b, 0xa438, 0xf3af, 0xa438, 0x8bf3, + 0xa438, 0x006f, 0xa438, 0x4a03, 0xa438, 0x6f47, 0xa438, 0x266f, + 0xa438, 0x5900, 0xa438, 0x6f4d, 0xa438, 0x016f, 0xa438, 0x5004, + 0xa438, 0x6f56, 0xa438, 0x056f, 0xa438, 0x5f06, 0xa438, 0x6f5c, + 0xa438, 0x2774, 0xa438, 0x7800, 0xa438, 0x6f68, 0xa438, 0x246f, + 0xa438, 0x6b20, 0xa438, 0x6f6e, 0xa438, 0x206f, 0xa438, 0x7410, + 0xa438, 0x7469, 0xa438, 0x1074, 0xa438, 0x6c10, 0xa438, 0x746f, + 0xa438, 0x1074, 0xa438, 0x7225, 0xa438, 0x8bfc, 0xa438, 0x008c, + 0xa438, 0x0802, 0xa438, 0x8c02, 0xa438, 0x038b, 0xa438, 0xff04, + 0xa438, 0x6eed, 0xa438, 0x278c, 0xa438, 0x0520, 0xa438, 0x74da, + 0xa438, 0x2074, 0xa438, 0xdd20, 0xa438, 0x74e0, 0xa438, 0x0074, + 0xa438, 0xe300, 0xa438, 0x6ef3, 0xa438, 0x006e, 0xa438, 0xf600, + 0xa438, 0x6ef9, 0xa438, 0x006e, 0xa438, 0xfc00, 0xa438, 0x6eff, + 0xa438, 0x006f, 0xa438, 0x0200, 0xa438, 0x6f05, 0xa438, 0x026f, + 0xa438, 0x0802, 0xa438, 0x6f0b, 0xa438, 0x026f, 0xa438, 0x0e02, + 0xa438, 0x6f11, 0xa438, 0x026f, 0xa438, 0x1402, 0xa438, 0x6f17, + 0xa438, 0x226f, 0xa438, 0x1a00, 0xa438, 0x723e, 0xa438, 0x016e, + 0xa438, 0xed24, 0xa438, 0x6f50, 0xa438, 0x0072, 0xa438, 0x4701, + 0xa438, 0x724a, 0xa438, 0x0272, 0xa438, 0x4d23, 0xa438, 0x7250, + 0xa438, 0x1074, 0xa438, 0x6910, 0xa438, 0x746c, 0xa438, 0x1074, + 0xa438, 0x6f00, 0xa438, 0x7472, 0xa438, 0x158c, 0xa438, 0x0b15, + 0xa438, 0x8c0e, 0xa438, 0x158c, 0xa438, 0x1105, 0xa438, 0x8c14, + 0xa438, 0x006f, 0xa438, 0x4a03, 0xa438, 0x6f47, 0xa438, 0x266f, + 0xa438, 0x5900, 0xa438, 0x731f, 0xa438, 0x0273, 0xa438, 0x2203, + 0xa438, 0x8c08, 0xa438, 0xee84, 0xa438, 0x7100, 0xa438, 0x0286, + 0xa438, 0xece0, 0xa438, 0x8043, 0xa438, 0xf626, 0xa438, 0xe480, + 0xa438, 0x43af, 0xa438, 0x6611, 0xa438, 0xf8e0, 0xa438, 0x8012, + 0xa438, 0xac26, 0xa438, 0x03af, 0xa438, 0x86ff, 0xa438, 0x0287, + 0xa438, 0x0102, 0xa438, 0x8906, 0xa438, 0x0289, 0xa438, 0x29fc, + 0xa438, 0x04f8, 0xa438, 0xf9ef, 0xa438, 0x59f9, 0xa438, 0xfaee, + 0xa438, 0x8476, 0xa438, 0x00d6, 0xa438, 0x008f, 0xa438, 0x0266, + 0xa438, 0x53ef, 0xa438, 0x643e, 0xa438, 0x1200, 0xa438, 0xac4f, + 0xa438, 0x08e4, 0xa438, 0x8fe7, 0xa438, 0xe58f, 0xa438, 0xe8ae, + 0xa438, 0x06e0, 0xa438, 0x8fe7, 0xa438, 0xe18f, 0xa438, 0xe8ee, + 0xa438, 0x8476, 0xa438, 0x01d6, 0xa438, 0x00c0, 0xa438, 0x0266, + 0xa438, 0x71ee, 0xa438, 0x8476, 0xa438, 0x00d6, 0xa438, 0x0090, + 0xa438, 0x0266, 0xa438, 0x53ef, 0xa438, 0x643e, 0xa438, 0x1200, + 0xa438, 0xac4f, 0xa438, 0x08e4, 0xa438, 0x8fe9, 0xa438, 0xe58f, + 0xa438, 0xeaae, 0xa438, 0x06e0, 0xa438, 0x8fe9, 0xa438, 0xe18f, + 0xa438, 0xeaee, 0xa438, 0x8476, 0xa438, 0x01d6, 0xa438, 0x00c1, + 0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x8476, 0xa438, 0x00d6, + 0xa438, 0x0091, 0xa438, 0x0266, 0xa438, 0x53ef, 0xa438, 0x643e, + 0xa438, 0x1200, 0xa438, 0xac4f, 0xa438, 0x08e4, 0xa438, 0x8feb, + 0xa438, 0xe58f, 0xa438, 0xecae, 0xa438, 0x06e0, 0xa438, 0x8feb, + 0xa438, 0xe18f, 0xa438, 0xecee, 0xa438, 0x8476, 0xa438, 0x01d6, + 0xa438, 0x00c2, 0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x8476, + 0xa438, 0x01d6, 0xa438, 0x008f, 0xa438, 0x0266, 0xa438, 0x53ef, + 0xa438, 0x643e, 0xa438, 0x1200, 0xa438, 0xac4f, 0xa438, 0x08e4, + 0xa438, 0x8fed, 0xa438, 0xe58f, 0xa438, 0xeeae, 0xa438, 0x06e0, + 0xa438, 0x8fed, 0xa438, 0xe18f, 0xa438, 0xeeee, 0xa438, 0x8476, + 0xa438, 0x02d6, 0xa438, 0x00c0, 0xa438, 0x0266, 0xa438, 0x71ee, + 0xa438, 0x8476, 0xa438, 0x01d6, 0xa438, 0x0090, 0xa438, 0x0266, + 0xa438, 0x53ef, 0xa438, 0x643e, 0xa438, 0x1200, 0xa438, 0xac4f, + 0xa438, 0x08e4, 0xa438, 0x8fef, 0xa438, 0xe58f, 0xa438, 0xf0ae, + 0xa438, 0x06e0, 0xa438, 0x8fef, 0xa438, 0xe18f, 0xa438, 0xf0ee, + 0xa438, 0x8476, 0xa438, 0x02d6, 0xa438, 0x00c1, 0xa438, 0x0266, + 0xa438, 0x71ee, 0xa438, 0x8476, 0xa438, 0x01d6, 0xa438, 0x0091, + 0xa438, 0x0266, 0xa438, 0x53ef, 0xa438, 0x643e, 0xa438, 0x1200, + 0xa438, 0xac4f, 0xa438, 0x08e4, 0xa438, 0x8ff1, 0xa438, 0xe58f, + 0xa438, 0xf2ae, 0xa438, 0x06e0, 0xa438, 0x8ff1, 0xa438, 0xe18f, + 0xa438, 0xf2ee, 0xa438, 0x8476, 0xa438, 0x02d6, 0xa438, 0x00c2, + 0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x8476, 0xa438, 0x02d6, + 0xa438, 0x008f, 0xa438, 0x0266, 0xa438, 0x53ef, 0xa438, 0x643e, + 0xa438, 0x1200, 0xa438, 0xac4f, 0xa438, 0x08e4, 0xa438, 0x8ff3, + 0xa438, 0xe58f, 0xa438, 0xf4ae, 0xa438, 0x06e0, 0xa438, 0x8ff3, + 0xa438, 0xe18f, 0xa438, 0xf4ee, 0xa438, 0x8476, 0xa438, 0x04d6, + 0xa438, 0x00c0, 0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x8476, + 0xa438, 0x02d6, 0xa438, 0x0090, 0xa438, 0x0266, 0xa438, 0x53ef, + 0xa438, 0x643e, 0xa438, 0x1200, 0xa438, 0xac4f, 0xa438, 0x08e4, + 0xa438, 0x8ff5, 0xa438, 0xe58f, 0xa438, 0xf6ae, 0xa438, 0x06e0, + 0xa438, 0x8ff5, 0xa438, 0xe18f, 0xa438, 0xf6ee, 0xa438, 0x8476, + 0xa438, 0x04d6, 0xa438, 0x00c1, 0xa438, 0x0266, 0xa438, 0x71ee, + 0xa438, 0x8476, 0xa438, 0x02d6, 0xa438, 0x0091, 0xa438, 0x0266, + 0xa438, 0x53ef, 0xa438, 0x643e, 0xa438, 0x1200, 0xa438, 0xac4f, + 0xa438, 0x08e4, 0xa438, 0x8ff7, 0xa438, 0xe58f, 0xa438, 0xf8ae, + 0xa438, 0x06e0, 0xa438, 0x8ff7, 0xa438, 0xe18f, 0xa438, 0xf8ee, + 0xa438, 0x8476, 0xa438, 0x04d6, 0xa438, 0x00c2, 0xa438, 0x0266, + 0xa438, 0x71ee, 0xa438, 0x8476, 0xa438, 0x03d6, 0xa438, 0x008f, + 0xa438, 0x0266, 0xa438, 0x53ef, 0xa438, 0x643e, 0xa438, 0x1200, + 0xa438, 0xac4f, 0xa438, 0x08e4, 0xa438, 0x8ff9, 0xa438, 0xe58f, + 0xa438, 0xfaae, 0xa438, 0x06e0, 0xa438, 0x8ff9, 0xa438, 0xe18f, + 0xa438, 0xfaee, 0xa438, 0x8476, 0xa438, 0x08d6, 0xa438, 0x00c0, + 0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x8476, 0xa438, 0x03d6, + 0xa438, 0x0090, 0xa438, 0x0266, 0xa438, 0x53ef, 0xa438, 0x643e, + 0xa438, 0x1200, 0xa438, 0xac4f, 0xa438, 0x08e4, 0xa438, 0x8ffb, + 0xa438, 0xe58f, 0xa438, 0xfcae, 0xa438, 0x06e0, 0xa438, 0x8ffb, + 0xa438, 0xe18f, 0xa438, 0xfcee, 0xa438, 0x8476, 0xa438, 0x08d6, + 0xa438, 0x00c1, 0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x8476, + 0xa438, 0x03d6, 0xa438, 0x0091, 0xa438, 0x0266, 0xa438, 0x53ef, + 0xa438, 0x643e, 0xa438, 0x1200, 0xa438, 0xac4f, 0xa438, 0x08e4, + 0xa438, 0x8ffd, 0xa438, 0xe58f, 0xa438, 0xfeae, 0xa438, 0x06e0, + 0xa438, 0x8ffd, 0xa438, 0xe18f, 0xa438, 0xfeee, 0xa438, 0x8476, + 0xa438, 0x08d6, 0xa438, 0x00c2, 0xa438, 0x0266, 0xa438, 0x71fe, + 0xa438, 0xfdef, 0xa438, 0x95fd, 0xa438, 0xfc04, 0xa438, 0xf8f9, + 0xa438, 0xfad4, 0xa438, 0x0400, 0xa438, 0xd600, 0xa438, 0x0dd3, + 0xa438, 0x0fe7, 0xa438, 0x8476, 0xa438, 0x0266, 0xa438, 0x71d4, + 0xa438, 0x1400, 0xa438, 0xd600, 0xa438, 0x0dd3, 0xa438, 0x0fe7, + 0xa438, 0x8476, 0xa438, 0x0266, 0xa438, 0x71fe, 0xa438, 0xfdfc, + 0xa438, 0x04f8, 0xa438, 0xf9fa, 0xa438, 0xd410, 0xa438, 0x00d6, + 0xa438, 0x000d, 0xa438, 0xd30f, 0xa438, 0xe784, 0xa438, 0x7602, + 0xa438, 0x6671, 0xa438, 0xd400, 0xa438, 0x00d6, 0xa438, 0x000d, + 0xa438, 0xd30f, 0xa438, 0xe784, 0xa438, 0x7602, 0xa438, 0x6671, + 0xa438, 0xfefd, 0xa438, 0xfc04, 0xa438, 0xe080, 0xa438, 0x4fac, + 0xa438, 0x2317, 0xa438, 0xe080, 0xa438, 0x44ad, 0xa438, 0x231a, + 0xa438, 0x0289, 0xa438, 0x75e0, 0xa438, 0x8044, 0xa438, 0xac23, + 0xa438, 0x11bf, 0xa438, 0x6ecf, 0xa438, 0x0276, 0xa438, 0x74ae, + 0xa438, 0x0902, 0xa438, 0x8adb, 0xa438, 0x021f, 0xa438, 0xe702, + 0xa438, 0x1fbb, 0xa438, 0xaf1f, 0xa438, 0x95f8, 0xa438, 0xf9ef, + 0xa438, 0x59f9, 0xa438, 0xfafb, 0xa438, 0xe080, 0xa438, 0x12ac, + 0xa438, 0x2303, 0xa438, 0xaf8a, 0xa438, 0xd0d4, 0xa438, 0x0120, + 0xa438, 0xd600, 0xa438, 0x10d2, 0xa438, 0x0fe6, 0xa438, 0x8476, + 0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x846f, 0xa438, 0x00d4, + 0xa438, 0x000f, 0xa438, 0xbf72, 0xa438, 0x9e02, 0xa438, 0x7697, + 0xa438, 0x0275, 0xa438, 0xbeef, 0xa438, 0x47e4, 0xa438, 0x8474, + 0xa438, 0xe584, 0xa438, 0x75bf, 0xa438, 0x729b, 0xa438, 0x0276, + 0xa438, 0xb6e5, 0xa438, 0x846f, 0xa438, 0xef31, 0xa438, 0xbf6e, + 0xa438, 0x0602, 0xa438, 0x76b6, 0xa438, 0xef64, 0xa438, 0xbf6e, + 0xa438, 0x0902, 0xa438, 0x76b6, 0xa438, 0x1e64, 0xa438, 0xbf6e, + 0xa438, 0x0f02, 0xa438, 0x76b6, 0xa438, 0x1e64, 0xa438, 0xac40, + 0xa438, 0x05a3, 0xa438, 0x0f0c, 0xa438, 0xae26, 0xa438, 0xa303, + 0xa438, 0x02ae, 0xa438, 0x21a3, 0xa438, 0x0c02, 0xa438, 0xae1c, + 0xa438, 0xe084, 0xa438, 0x74e1, 0xa438, 0x8475, 0xa438, 0xef64, + 0xa438, 0xd000, 0xa438, 0xd196, 0xa438, 0xef74, 0xa438, 0x0275, + 0xa438, 0xd9ad, 0xa438, 0x50b7, 0xa438, 0xe083, 0xa438, 0xecf7, + 0xa438, 0x23e4, 0xa438, 0x83ec, 0xa438, 0xbf72, 0xa438, 0x9e02, + 0xa438, 0x766b, 0xa438, 0x0287, 0xa438, 0x0102, 0xa438, 0x8906, + 0xa438, 0xee83, 0xa438, 0xe800, 0xa438, 0xbf72, 0xa438, 0x6b02, + 0xa438, 0x766b, 0xa438, 0xbf72, 0xa438, 0x6e02, 0xa438, 0x766b, + 0xa438, 0xbf72, 0xa438, 0x7102, 0xa438, 0x766b, 0xa438, 0xbf72, + 0xa438, 0x7402, 0xa438, 0x766b, 0xa438, 0xbf72, 0xa438, 0x7702, + 0xa438, 0x766b, 0xa438, 0xbf72, 0xa438, 0x7a02, 0xa438, 0x766b, + 0xa438, 0xd400, 0xa438, 0x0fbf, 0xa438, 0x7295, 0xa438, 0x0276, + 0xa438, 0x97d7, 0xa438, 0x0400, 0xa438, 0xbf6e, 0xa438, 0x0602, + 0xa438, 0x76b6, 0xa438, 0xef64, 0xa438, 0xbf6e, 0xa438, 0x0902, + 0xa438, 0x76b6, 0xa438, 0x1e64, 0xa438, 0xbf6e, 0xa438, 0x0f02, + 0xa438, 0x76b6, 0xa438, 0x1e64, 0xa438, 0xac40, 0xa438, 0x0fbf, + 0xa438, 0x7298, 0xa438, 0x0276, 0xa438, 0xb6e5, 0xa438, 0x83e8, + 0xa438, 0xa10f, 0xa438, 0x28af, 0xa438, 0x8a95, 0xa438, 0xbf8b, + 0xa438, 0xf302, 0xa438, 0x76b6, 0xa438, 0xac28, 0xa438, 0x02ae, + 0xa438, 0x0bbf, 0xa438, 0x8bf9, 0xa438, 0x0276, 0xa438, 0xb6e5, + 0xa438, 0x83e8, 0xa438, 0xae09, 0xa438, 0xbf8b, 0xa438, 0xf602, + 0xa438, 0x76b6, 0xa438, 0xe583, 0xa438, 0xe8a1, 0xa438, 0x0303, + 0xa438, 0xaf8a, 0xa438, 0x95b7, 0xa438, 0xafe2, 0xa438, 0x83ec, + 0xa438, 0xf735, 0xa438, 0xe683, 0xa438, 0xecbf, 0xa438, 0x7295, + 0xa438, 0x0276, 0xa438, 0x6bbf, 0xa438, 0x726b, 0xa438, 0x0276, + 0xa438, 0x74bf, 0xa438, 0x726e, 0xa438, 0x0276, 0xa438, 0x74bf, + 0xa438, 0x7271, 0xa438, 0x0276, 0xa438, 0x74bf, 0xa438, 0x7274, + 0xa438, 0x0276, 0xa438, 0x74bf, 0xa438, 0x7277, 0xa438, 0x0276, + 0xa438, 0x74bf, 0xa438, 0x727a, 0xa438, 0x0276, 0xa438, 0x7402, + 0xa438, 0x8929, 0xa438, 0xd401, 0xa438, 0x28d6, 0xa438, 0x0010, + 0xa438, 0xd20f, 0xa438, 0xe684, 0xa438, 0x7602, 0xa438, 0x6671, + 0xa438, 0x021f, 0xa438, 0xbbff, 0xa438, 0xfefd, 0xa438, 0xef95, + 0xa438, 0xfdfc, 0xa438, 0x04f8, 0xa438, 0xf9ef, 0xa438, 0x59f9, + 0xa438, 0xe080, 0xa438, 0x12ad, 0xa438, 0x230c, 0xa438, 0xbf72, + 0xa438, 0x9e02, 0xa438, 0x766b, 0xa438, 0xbf72, 0xa438, 0x9502, + 0xa438, 0x766b, 0xa438, 0xfdef, 0xa438, 0x95fd, 0xa438, 0xfc04, + 0xa438, 0xbf6e, 0xa438, 0x0602, 0xa438, 0x76b6, 0xa438, 0xef64, + 0xa438, 0xbf6e, 0xa438, 0x0902, 0xa438, 0x76b6, 0xa438, 0x1e64, + 0xa438, 0xbf6e, 0xa438, 0x0f02, 0xa438, 0x76b6, 0xa438, 0x1e64, + 0xa438, 0xac40, 0xa438, 0x0ebf, 0xa438, 0x7298, 0xa438, 0x0276, + 0xa438, 0xb6e5, 0xa438, 0x8478, 0xa438, 0xa10f, 0xa438, 0x26ae, + 0xa438, 0x47bf, 0xa438, 0x8bf3, 0xa438, 0x0276, 0xa438, 0xb6ac, + 0xa438, 0x2802, 0xa438, 0xae0b, 0xa438, 0xbf8b, 0xa438, 0xf902, + 0xa438, 0x76b6, 0xa438, 0xe584, 0xa438, 0x78ae, 0xa438, 0x09bf, + 0xa438, 0x8bf6, 0xa438, 0x0276, 0xa438, 0xb6e5, 0xa438, 0x8478, + 0xa438, 0xa103, 0xa438, 0x02ae, 0xa438, 0x23e0, 0xa438, 0x8474, + 0xa438, 0xe184, 0xa438, 0x75ef, 0xa438, 0x64e0, 0xa438, 0x83fc, + 0xa438, 0xe183, 0xa438, 0xfdef, 0xa438, 0x7402, 0xa438, 0x75d9, + 0xa438, 0xad50, 0xa438, 0x0ae0, 0xa438, 0x83ec, 0xa438, 0xf721, + 0xa438, 0xe483, 0xa438, 0xecae, 0xa438, 0x03af, 0xa438, 0x68e4, + 0xa438, 0xbf72, 0xa438, 0x9502, 0xa438, 0x766b, 0xa438, 0xe083, + 0xa438, 0xebad, 0xa438, 0x2170, 0xa438, 0xbf73, 0xa438, 0x7f02, + 0xa438, 0x766b, 0xa438, 0xd700, 0xa438, 0x64bf, 0xa438, 0x73c4, + 0xa438, 0x0276, 0xa438, 0xb6a4, 0xa438, 0x0000, 0xa438, 0x02ae, + 0xa438, 0x0d87, 0xa438, 0xa700, 0xa438, 0x00ef, 0xa438, 0xe183, + 0xa438, 0xecf7, 0xa438, 0x2ae5, 0xa438, 0x83ec, 0xa438, 0xbf73, + 0xa438, 0xbe02, 0xa438, 0x766b, 0xa438, 0xbf73, 0xa438, 0xb802, + 0xa438, 0x766b, 0xa438, 0xbf73, 0xa438, 0xc102, 0xa438, 0x766b, + 0xa438, 0xbf73, 0xa438, 0xbb02, 0xa438, 0x766b, 0xa438, 0xe084, + 0xa438, 0x9ee1, 0xa438, 0x849f, 0xa438, 0xbf72, 0xa438, 0x7d02, + 0xa438, 0x7697, 0xa438, 0xbf72, 0xa438, 0x8002, 0xa438, 0x7697, + 0xa438, 0xbf72, 0xa438, 0x8302, 0xa438, 0x7697, 0xa438, 0xbf72, + 0xa438, 0x8602, 0xa438, 0x7697, 0xa438, 0xbf72, 0xa438, 0x8902, + 0xa438, 0x7674, 0xa438, 0xbf72, 0xa438, 0x8c02, 0xa438, 0x7674, + 0xa438, 0xbf72, 0xa438, 0x8f02, 0xa438, 0x7674, 0xa438, 0xbf72, + 0xa438, 0x9202, 0xa438, 0x7674, 0xa438, 0xee84, 0xa438, 0x7700, + 0xa438, 0xe080, 0xa438, 0x44f6, 0xa438, 0x21e4, 0xa438, 0x8044, + 0xa438, 0xaf68, 0xa438, 0xe411, 0xa438, 0xd1a4, 0xa438, 0x10bc, + 0xa438, 0x7432, 0xa438, 0xbc74, 0xa438, 0xbbbf, 0xa438, 0x14cc, + 0xa438, 0xbfaa, 0xa438, 0x00bf, 0xa438, 0x9055, 0xa438, 0xbf06, + 0xa438, 0x10bf, 0xa438, 0xb876, 0xa438, 0xbe02, 0xa438, 0x54be, + 0xa438, 0x0232, 0xa438, 0xbe02, 0xa438, 0x10be, 0xa438, 0x0200, + 0xa436, 0x8fe7, 0xa438, 0x1200, 0xa436, 0x8fe9, 0xa438, 0x1200, + 0xa436, 0x8feb, 0xa438, 0x1200, 0xa436, 0x8fed, 0xa438, 0x1200, + 0xa436, 0x8fef, 0xa438, 0x1200, 0xa436, 0x8ff1, 0xa438, 0x1200, + 0xa436, 0x8ff3, 0xa438, 0x1200, 0xa436, 0x8ff5, 0xa438, 0x1200, + 0xa436, 0x8ff7, 0xa438, 0x1200, 0xa436, 0x8ff9, 0xa438, 0x1200, + 0xa436, 0x8ffb, 0xa438, 0x1200, 0xa436, 0x8ffd, 0xa438, 0x1200, + 0xa436, 0xb818, 0xa438, 0x6602, 0xa436, 0xb81a, 0xa438, 0x1f75, + 0xa436, 0xb81c, 0xa438, 0x67eb, 0xa436, 0xb81e, 0xa438, 0xffff, + 0xa436, 0xb850, 0xa438, 0xffff, 0xa436, 0xb852, 0xa438, 0xffff, + 0xa436, 0xb878, 0xa438, 0xffff, 0xa436, 0xb884, 0xa438, 0xffff, + 0xa436, 0xb832, 0xa438, 0x0007, 0xB82E, 0x0000, 0xa436, 0x8023, + 0xa438, 0x0000, 0xB820, 0x0000, 0xFFFF, 0xFFFF +}; + +static const u16 phy_mcu_ram_code_8127a_2[] = { + 0xb892, 0x0000, 0xB88E, 0xc07c, 0xB890, 0x0203, 0xB890, 0x0304, + 0xB890, 0x0405, 0xB890, 0x0607, 0xB890, 0x0809, 0xB890, 0x0B0D, + 0xB890, 0x0F11, 0xB890, 0x1418, 0xB890, 0x1B20, 0xB890, 0x252B, + 0xB890, 0x343E, 0xB890, 0x4854, 0xB890, 0x6203, 0xB890, 0x0304, + 0xB890, 0x0506, 0xB890, 0x080A, 0xB890, 0x0C0E, 0xB890, 0x1216, + 0xB890, 0x1B22, 0xB890, 0x2A34, 0xB890, 0x404F, 0xB890, 0x6171, + 0xB890, 0x7884, 0xB890, 0x9097, 0xB890, 0x0203, 0xB890, 0x0406, + 0xB890, 0x080B, 0xB890, 0x0E13, 0xB890, 0x1820, 0xB890, 0x2A39, + 0xB890, 0x4856, 0xB890, 0xE060, 0xB890, 0xE050, 0xB890, 0xD080, + 0xB890, 0x8070, 0xB890, 0x70A0, 0xB890, 0x1000, 0xB890, 0x60D0, + 0xB890, 0xB010, 0xB890, 0xE0B0, 0xB890, 0x80C0, 0xB890, 0xE000, + 0xB890, 0x2020, 0xB890, 0x1020, 0xB890, 0xE090, 0xB890, 0x80C0, + 0xB890, 0x3020, 0xB890, 0x00E0, 0xB890, 0x40A0, 0xB890, 0xE020, + 0xB890, 0x5060, 0xB890, 0xE0D0, 0xB890, 0xA000, 0xB890, 0x3030, + 0xB890, 0x4070, 0xB890, 0xE0E0, 0xB890, 0xD080, 0xB890, 0xA010, + 0xB890, 0xE040, 0xB890, 0x80B0, 0xB890, 0x50B0, 0xB890, 0x2090, + 0xB820, 0x0000, 0xFFFF, 0xFFFF +}; + +static void +rtl8127_real_set_phy_mcu_8127a_tc_1(struct net_device *dev) +{ + rtl8127_set_phy_mcu_ram_code(dev, + phy_mcu_ram_code_8127a_tc_1, + ARRAY_SIZE(phy_mcu_ram_code_8127a_tc_1)); +} + +static void +rtl8127_set_phy_mcu_8127a_tc(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + rtl8127_set_phy_mcu_patch_request(tp); + + rtl8127_real_set_phy_mcu_8127a_tc_1(dev); + + rtl8127_clear_phy_mcu_patch_request(tp); +} + +static void +rtl8127_real_set_phy_mcu_8127a_1(struct net_device *dev) +{ + rtl8127_set_phy_mcu_ram_code(dev, + phy_mcu_ram_code_8127a_1, + ARRAY_SIZE(phy_mcu_ram_code_8127a_1)); +} + +static void +rtl8127_real_set_phy_mcu_8127a_2(struct net_device *dev) +{ + rtl8127_set_phy_mcu_ram_code(dev, + phy_mcu_ram_code_8127a_2, + ARRAY_SIZE(phy_mcu_ram_code_8127a_2)); +} + +static void +rtl8127_set_phy_mcu_8127a_1(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + rtl8127_set_phy_mcu_patch_request(tp); + + rtl8127_real_set_phy_mcu_8127a_1(dev); + + rtl8127_clear_phy_mcu_patch_request(tp); + + rtl8127_set_phy_mcu_patch_request(tp); + + rtl8127_real_set_phy_mcu_8127a_2(dev); + + rtl8127_clear_phy_mcu_patch_request(tp); +} + +static void +rtl8127_init_hw_phy_mcu(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u8 require_disable_phy_disable_mode = FALSE; + + if (tp->NotWrRamCodeToMicroP == TRUE) + return; + + if (rtl8127_check_hw_phy_mcu_code_ver(dev)) + return; + + if (HW_SUPPORT_CHECK_PHY_DISABLE_MODE(tp) && rtl8127_is_in_phy_disable_mode(dev)) + require_disable_phy_disable_mode = TRUE; + + if (require_disable_phy_disable_mode) + rtl8127_disable_phy_disable_mode(dev); + + switch (tp->mcfg) { + case CFG_METHOD_1: + rtl8127_set_phy_mcu_8127a_tc(dev); + break; + case CFG_METHOD_2: + rtl8127_set_phy_mcu_8127a_1(dev); + break; + default: + break; + } + + if (require_disable_phy_disable_mode) + rtl8127_enable_phy_disable_mode(dev); + + rtl8127_write_hw_phy_mcu_code_ver(dev); + + rtl8127_mdio_write(tp,0x1F, 0x0000); + + tp->HwHasWrRamCodeToMicroP = TRUE; +} +#endif + +static void +rtl8127_enable_phy_aldps(struct rtl8127_private *tp) +{ + //enable aldps + //GPHY OCP 0xA430 bit[2] = 0x1 (en_aldps) + rtl8127_set_eth_phy_ocp_bit(tp, 0xA430, BIT_2); +} + +static void +rtl8127_tgphy_irq_mask_and_ack(struct rtl8127_private *tp) +{ + switch (tp->mcfg) { + case CFG_METHOD_2: + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA4D2, 0x0000); + (void)rtl8127_mdio_direct_read_phy_ocp(tp, 0xA4D4); + break; + default: + break; + } +} + +static void +rtl8127_hw_phy_config_8127a_tc_1(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + rtl8127_set_eth_phy_ocp_bit(tp, 0xA442, BIT_11); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x815E); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xa438, + 0xFF00, + 0x8600); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x8169); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xa438, + 0xFF00, + 0x8600); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x8174); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xa438, + 0xFF00, + 0xA100); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x83BF); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xa438, + 0xFF00, + 0x5A00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x83C5); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xa438, + 0xFF00, + 0x5A00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x83CB); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xa438, + 0xFF00, + 0x8B00); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8238); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xC000, + 0x4000); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x823A); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xA000); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x8148); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xa438, + 0xFF00, + 0x0100); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84AD); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x0C00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84B2); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x0800); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84B7); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x1400); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84BC); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0040); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84C0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00D6); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84BE); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00A0); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84AE); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0C0C); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84B0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0C0C); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xBD7A, 0xAAAA); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xBCE0, 0x6666); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x85FC); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0AAA); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x85FF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0AAA); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xAC32, BIT_3); + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xAC32, BIT_11); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xADDC, + 0x3FFF, + 0x2000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8111); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0F00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80E9); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0F00); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xAEC4, + 0xFF00, + 0x4600); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xAC56, + 0x0007, + 0x0005); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x825B); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0D00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8283); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0334); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8289); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x5600); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xADB8, 0x0190); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xAE3A, + 0x00FF, + 0x0026); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xAE4A, + 0x0FF0, + 0x0150); + + rtl8127_set_eth_phy_ocp_bit(tp, 0xAEC2, BIT_12); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xAE22, 0x0352); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xAEC0, 0x00FA); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8188); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xF500); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8203); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xF500); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x827E); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xF500); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81CB); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8246); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x818A); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xF500); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8205); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xF500); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8280); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xF500); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x83DD); + rtl8127_set_eth_phy_ocp_bit(tp, 0xB87E, BIT_9); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x840B); + rtl8127_set_eth_phy_ocp_bit(tp, 0xB87E, BIT_9); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x83BC); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_10); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x83BE); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_10); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x83C0); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x0700, + 0x0400); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x83C2); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_10); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x83C4); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_10); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x83C6); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x0700, + 0x0400); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84C2); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0xFEFF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84C4); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0003); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84C6); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0116); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84C8); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x6300); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x822A); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x4FFF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81AF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x4067); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8134); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x5069); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x822C); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x1A00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81B1); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x3A00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8136); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x5000); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x810E); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0x9000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8114); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0x9000); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xB63C, BIT_9); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80B4); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xB63B); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80E4); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0700); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80E5); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0x7000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80EA); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0500); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80EB); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0x5000); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8291); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0416); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8015); + rtl8127_set_eth_phy_ocp_bit(tp, 0xB87E, BIT_11); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x895E); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x01A0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8960); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x01A0); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x826B); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xF0AF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81F0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xF0AF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8175); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xF0AF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x826D); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0100); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81F2); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0100); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8177); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0100); + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xAC1C, BIT_8 | BIT_7); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xAC1E, BIT_13 | BIT_12); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xAD96, 0xAAFF); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xAD98, + 0x00FF, + 0x00AA); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xAE38, + 0x3FFF, + 0x2554); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xAE3A, + 0xF000, + 0xA000); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8932); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0900); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x892F); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0900); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x892C); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0900); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x85B6); + rtl8127_set_eth_phy_ocp_bit(tp, 0xB87E, 0xFF00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x85B4); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xFFFF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8905); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0xFF00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8853); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x2800); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x884B); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x3F00); + + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xBDE6, 0x3FFF); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xBDE8, 0x3FFF); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xBF0E, 0x0003); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8156); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x1600); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x80AB); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x7500); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x80C3); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x090D); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x80C6); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0xC600); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x80BF); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x5500); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8096); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x4500); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x809D); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x0200); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x809B); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0xE50A); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8099); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x9906); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x831F); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x5000); + + + rtl8127_set_eth_phy_ocp_bit(tp, 0xB648, BIT_14); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA4E0, BIT_15); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x849A); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0004); + + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xAC1C, 0x0C00); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA42C, BIT_6); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xACBA, 0xFC00); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8122); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0xC000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8123); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0xC000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80FA); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0x2000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x825B); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0xB000); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80D2); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0xD000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80D3); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0xD000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80C8); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0200); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80CA); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0300); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80E2); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x2300); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80A9); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0A00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80AA); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0x5000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80AB); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0xA000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x805A); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0x2000); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8106); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x40CC); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x812C); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x40CC); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8096); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x7500); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x809C); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x6300); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x859E); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x1F00); + + + if (aspm && HW_HAS_WRITE_PHY_MCU_RAM_CODE(tp)) + rtl8127_enable_phy_aldps(tp); +} + +static void +rtl8127_hw_phy_config_8127a_1(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + rtl8127_tgphy_irq_mask_and_ack(tp); + + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA442, BIT_11); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8415); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x9300); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81A3); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x0F00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81AE); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x0F00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81B9); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0xB900); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x83B0); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0x0E00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x83C5); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0x0E00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x83DA); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0x0E00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x83EF); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0x0E00); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8173); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x8620); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8175); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x8671); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x817C); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_13); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8187); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_13); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8192); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_13); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x819D); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_13); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81A8); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA438, BIT_13); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81B3); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA438, BIT_13); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81BE); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_13); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x817D); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0xA600); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8188); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0xA600); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8193); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0xA600); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x819E); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0xA600); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81A9); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x1400); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81B4); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x1400); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81BF); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0xA600); + + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xAEAA, (BIT_5 | BIT_3)); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84F0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x201C); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84F2); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x3117); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xAEC6, 0x0000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xAE20, 0xFFFF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xAECE, 0xFFFF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xAED2, 0xFFFF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xAEC8, 0x0000); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xAED0, BIT_0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xADB8, 0x0150); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8197); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x5000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8231); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x5000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x82CB); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x5000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x82CD); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x5700); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8233); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x5700); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8199); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x5700); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x815A); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0150); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81F4); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0150); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x828E); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0150); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81B1); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x824B); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x82E5); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0000); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84F7); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x2800); + rtl8127_set_eth_phy_ocp_bit(tp, 0xAEC2, BIT_12); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81B3); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xAD00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x824D); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xAD00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x82E7); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xAD00); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xAE4E, + 0x000F, + 0x0001); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x82CE); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0x4000); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84AC); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84AE); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84B0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xF818); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84B2); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x6000); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FFC); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x6008); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FFE); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xF450); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8015); + rtl8127_set_eth_phy_ocp_bit(tp, 0xB87E, BIT_9); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8016); + rtl8127_set_eth_phy_ocp_bit(tp, 0xB87E, BIT_11); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FE6); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x0800); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FE4); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x2114); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8647); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xA7B1); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8649); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xBBCA); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x864B); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xDC00); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8154); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xC000, + 0x4000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8158); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0xC000); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x826C); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xFFFF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x826E); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xFFFF); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8872); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x0E00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8012); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_11); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8012); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_14); + rtl8127_set_eth_phy_ocp_bit(tp, 0xB576, BIT_0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x834A); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x0700); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8217); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x3F00, + 0x2A00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81B1); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x0B00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FED); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x4E00); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8370); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x8671); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8372); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x86C8); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8401); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x86C8); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8403); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x86DA); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8406); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x1800, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8408); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x1800, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x840A); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x1800, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x840C); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x1800, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x840E); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x1800, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8410); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x1800, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8412); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x1800, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8414); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x1800, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8416); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x1800, + 0x1000); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x82BD); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x1F40); + + + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xBFB4, + 0x07FF, + 0x0328); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xBFB6, 0x3E14); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81C4); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x003B); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0086); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00B7); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00DB); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00FE); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00FE); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00FE); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00FE); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00C3); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0078); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0047); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0023); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x88D7); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x01A0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x88D9); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x01A0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FFA); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x002A); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FEE); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xFFDF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FF0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xFFFF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FF2); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0A4A); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FF4); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xAA5A); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FF6); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0A4A); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FF8); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xAA5A); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x88D5); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x0200); + + + rtl8127_set_eth_phy_ocp_bit(tp, 0xA430, BIT_1 | BIT_0); + + + if (aspm && HW_HAS_WRITE_PHY_MCU_RAM_CODE(tp)) + rtl8127_enable_phy_aldps(tp); +} + +static void +rtl8127_hw_phy_config(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned long flags; + + if (tp->resume_not_chg_speed) + return; + + tp->phy_reset_enable(dev); + + if (HW_DASH_SUPPORT_TYPE_3(tp) && tp->HwPkgDet == 0x06) + return; + + spin_lock_irqsave(&tp->phy_lock, flags); + +#ifndef ENABLE_USE_FIRMWARE_FILE + if (!tp->rtl_fw) + rtl8127_init_hw_phy_mcu(dev); +#endif + + switch (tp->mcfg) { + case CFG_METHOD_1: + rtl8127_hw_phy_config_8127a_tc_1(dev); + break; + case CFG_METHOD_2: + rtl8127_hw_phy_config_8127a_1(dev); + break; + default: + break; + } + + //legacy force mode(Chap 22) + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA5B4, BIT_15); + + rtl8127_mdio_write(tp, 0x1F, 0x0000); + + if (HW_HAS_WRITE_PHY_MCU_RAM_CODE(tp)) { + if (tp->eee.eee_enabled) + rtl8127_enable_eee(tp); + else + rtl8127_disable_eee(tp); + } + + spin_unlock_irqrestore(&tp->phy_lock, flags); +} + +static void +rtl8127_up(struct net_device *dev) +{ + rtl8127_hw_init(dev); + rtl8127_hw_reset(dev); + rtl8127_powerup_pll(dev); + rtl8127_hw_ephy_config(dev); + rtl8127_hw_phy_config(dev); + rtl8127_hw_config(dev); +} + +/* +static inline void rtl8127_delete_esd_timer(struct net_device *dev, struct timer_list *timer) +{ + del_timer_sync(timer); +} + +static inline void rtl8127_request_esd_timer(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + struct timer_list *timer = &tp->esd_timer; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) + setup_timer(timer, rtl8127_esd_timer, (unsigned long)dev); +#else + timer_setup(timer, rtl8127_esd_timer, 0); +#endif + mod_timer(timer, jiffies + RTL8127_ESD_TIMEOUT); +} +*/ + +/* +static inline void rtl8127_delete_link_timer(struct net_device *dev, struct timer_list *timer) +{ + del_timer_sync(timer); +} + +static inline void rtl8127_request_link_timer(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + struct timer_list *timer = &tp->link_timer; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) + setup_timer(timer, rtl8127_link_timer, (unsigned long)dev); +#else + timer_setup(timer, rtl8127_link_timer, 0); +#endif + mod_timer(timer, jiffies + RTL8127_LINK_TIMEOUT); +} +*/ + +#ifdef CONFIG_NET_POLL_CONTROLLER +/* + * Polling 'interrupt' - used by things like netconsole to send skbs + * without having to re-enable interrupts. It's not called while + * the interrupt routine is executing. + */ +static void +rtl8127_netpoll(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i; + for (i = 0; i < tp->irq_nvecs; i++) { + struct r8127_irq *irq = &tp->irq_tbl[i]; + struct r8127_napi *r8127napi = &tp->r8127napi[i]; + + disable_irq(irq->vector); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + irq->handler(irq->vector, r8127napi); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) + irq->handler(irq->vector, r8127napi, NULL); +#else + irq->handler(irq->vector, r8127napi); +#endif + + enable_irq(irq->vector); + } +} +#endif //CONFIG_NET_POLL_CONTROLLER + +static void +rtl8127_setup_interrupt_mask(struct rtl8127_private *tp) +{ + int i; + + if (tp->HwCurrIsrVer == 6) { + tp->intr_mask = ISRIMR_V6_LINKCHG | ISRIMR_V6_TOK_Q0; + if (tp->num_tx_rings > 1) + tp->intr_mask |= ISRIMR_V6_TOK_Q1; + for (i = 0; i < tp->num_rx_rings; i++) + tp->intr_mask |= ISRIMR_V6_ROK_Q0 << i; + } else if (tp->HwCurrIsrVer == 5) { + tp->intr_mask = ISRIMR_V5_LINKCHG | ISRIMR_V5_TOK_Q0; + if (tp->num_tx_rings > 1) + tp->intr_mask |= ISRIMR_V5_TOK_Q1; + for (i = 0; i < tp->num_rx_rings; i++) + tp->intr_mask |= ISRIMR_V5_ROK_Q0 << i; + } else if (tp->HwCurrIsrVer == 4) { + tp->intr_mask = ISRIMR_V4_LINKCHG; + for (i = 0; i < tp->num_rx_rings; i++) + tp->intr_mask |= ISRIMR_V4_ROK_Q0 << i; + } else if (tp->HwCurrIsrVer == 3) { + tp->intr_mask = ISRIMR_V2_LINKCHG; + for (i = 0; i < max(tp->num_tx_rings, tp->num_rx_rings); i++) + tp->intr_mask |= ISRIMR_V2_ROK_Q0 << i; + } else if (tp->HwCurrIsrVer == 2) { + tp->intr_mask = ISRIMR_V2_LINKCHG | ISRIMR_TOK_Q0; + if (tp->num_tx_rings > 1) + tp->intr_mask |= ISRIMR_TOK_Q1; + + for (i = 0; i < tp->num_rx_rings; i++) + tp->intr_mask |= ISRIMR_V2_ROK_Q0 << i; + } else { + tp->intr_mask = LinkChg | RxDescUnavail | TxOK | RxOK | SWInt; + tp->timer_intr_mask = LinkChg | PCSTimeout; + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) { + if (HW_DASH_SUPPORT_TYPE_3(tp)) { + tp->timer_intr_mask |= (ISRIMR_DASH_INTR_EN | ISRIMR_DASH_INTR_CMAC_RESET); + tp->intr_mask |= (ISRIMR_DASH_INTR_EN | ISRIMR_DASH_INTR_CMAC_RESET); + } + } +#endif + } +} + +static void +rtl8127_setup_mqs_reg(struct rtl8127_private *tp) +{ + u16 hw_clo_ptr0_reg, sw_tail_ptr0_reg; + u16 reg_len; + int i; + + //tx + tp->tx_ring[0].tdsar_reg = TxDescStartAddrLow; + for (i = 1; i < tp->HwSuppNumTxQueues; i++) + tp->tx_ring[i].tdsar_reg = (u16)(TNPDS_Q1_LOW_8125 + (i - 1) * 8); + + switch (tp->HwSuppTxNoCloseVer) { + case 4: + case 5: + hw_clo_ptr0_reg = HW_CLO_PTR0_8126; + sw_tail_ptr0_reg = SW_TAIL_PTR0_8126; + reg_len = 4; + break; + case 6: + hw_clo_ptr0_reg = HW_CLO_PTR0_8125BP; + sw_tail_ptr0_reg = SW_TAIL_PTR0_8125BP; + reg_len = 8; + break; + default: + hw_clo_ptr0_reg = HW_CLO_PTR0_8125; + sw_tail_ptr0_reg = SW_TAIL_PTR0_8125; + reg_len = 4; + break; + } + + for (i = 0; i < tp->HwSuppNumTxQueues; i++) { + tp->tx_ring[i].hw_clo_ptr_reg = (u16)(hw_clo_ptr0_reg + i * reg_len); + tp->tx_ring[i].sw_tail_ptr_reg = (u16)(sw_tail_ptr0_reg + i * reg_len); + } + + //rx + tp->rx_ring[0].rdsar_reg = RxDescAddrLow; + for (i = 1; i < tp->HwSuppNumRxQueues; i++) + tp->rx_ring[i].rdsar_reg = (u16)(RDSAR_Q1_LOW_8125 + (i - 1) * 8); + + tp->isr_reg[0] = ISR0_8125; + for (i = 1; i < tp->hw_supp_irq_nvecs; i++) + tp->isr_reg[i] = (u16)(ISR1_8125 + (i - 1) * 4); + + tp->imr_reg[0] = IMR0_8125; + for (i = 1; i < tp->hw_supp_irq_nvecs; i++) + tp->imr_reg[i] = (u16)(IMR1_8125 + (i - 1) * 4); +} + +static void +rtl8127_init_software_variable(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + struct pci_dev *pdev = tp->pci_dev; + +#ifdef ENABLE_LIB_SUPPORT + tp->ring_lib_enabled = 1; +#endif + + switch (tp->mcfg) { + default: + tp->HwSuppDashVer = 0; + break; + } + tp->AllowAccessDashOcp = rtl8127_is_allow_access_dash_ocp(tp); + + tp->HwPkgDet = rtl8127_mac_ocp_read(tp, 0xDC00); + tp->HwPkgDet = (tp->HwPkgDet >> 3) & 0x07; + + if (HW_DASH_SUPPORT_TYPE_3(tp) && tp->HwPkgDet == 0x06) + eee_enable = 0; + + tp->HwSuppNowIsOobVer = 1; + + tp->HwPcieSNOffset = 0x168; + +#ifdef ENABLE_REALWOW_SUPPORT + rtl8127_get_realwow_hw_version(dev); +#endif //ENABLE_REALWOW_SUPPORT + + if (HW_DASH_SUPPORT_DASH(tp) && rtl8127_check_dash(tp)) + tp->DASH = 1; + else + tp->DASH = 0; + + if (tp->DASH) { + if (HW_DASH_SUPPORT_TYPE_3(tp)) { + u64 CmacMemPhysAddress; + void __iomem *cmac_ioaddr = NULL; + + //map CMAC IO space + CmacMemPhysAddress = rtl8127_csi_other_fun_read(tp, 0, 0x18); + if (!(CmacMemPhysAddress & BIT_0)) { + if (CmacMemPhysAddress & BIT_2) + CmacMemPhysAddress |= (u64)rtl8127_csi_other_fun_read(tp, 0, 0x1C) << 32; + + CmacMemPhysAddress &= 0xFFFFFFF0; + /* ioremap MMIO region */ + cmac_ioaddr = ioremap(CmacMemPhysAddress, R8127_REGS_SIZE); + } + + if (cmac_ioaddr == NULL) { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_probe(tp)) + dev_err(&pdev->dev, "cannot remap CMAC MMIO, aborting\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + } + + if (cmac_ioaddr == NULL) + tp->DASH = 0; + else + tp->mapped_cmac_ioaddr = cmac_ioaddr; + } + + eee_enable = 0; + } + + if (HW_DASH_SUPPORT_TYPE_3(tp)) + tp->cmac_ioaddr = tp->mapped_cmac_ioaddr; + + if (aspm) { + tp->org_pci_offset_99 = rtl8127_csi_fun0_read_byte(tp, 0x99); + tp->org_pci_offset_99 &= ~(BIT_5|BIT_6); + + tp->org_pci_offset_180 = rtl8127_csi_fun0_read_byte(tp, 0x22c); + } + + pci_read_config_byte(pdev, 0x80, &tp->org_pci_offset_80); + pci_read_config_byte(pdev, 0x81, &tp->org_pci_offset_81); + + tp->use_timer_interrupt = TRUE; + + tp->HwSuppMaxPhyLinkSpeed = 10000; + + if (timer_count == 0 || tp->mcfg == CFG_METHOD_DEFAULT) + tp->use_timer_interrupt = FALSE; + + tp->ShortPacketSwChecksum = TRUE; + tp->UseSwPaddingShortPkt = TRUE; + + tp->HwSuppMagicPktVer = WAKEUP_MAGIC_PACKET_V3; + + tp->HwSuppLinkChgWakeUpVer = 3; + + tp->HwSuppD0SpeedUpVer = 2; + + tp->HwSuppCheckPhyDisableModeVer = 3; + + tp->HwSuppTxNoCloseVer = 6; + + switch (tp->HwSuppTxNoCloseVer) { + case 5: + case 6: + tp->MaxTxDescPtrMask = MAX_TX_NO_CLOSE_DESC_PTR_MASK_V4; + break; + case 4: + tp->MaxTxDescPtrMask = MAX_TX_NO_CLOSE_DESC_PTR_MASK_V3; + break; + case 3: + tp->MaxTxDescPtrMask = MAX_TX_NO_CLOSE_DESC_PTR_MASK_V2; + break; + default: + tx_no_close_enable = 0; + break; + } + + if (tp->HwSuppTxNoCloseVer > 0 && tx_no_close_enable == 1) + tp->EnableTxNoClose = TRUE; + + switch (tp->mcfg) { + case CFG_METHOD_1: + tp->sw_ram_code_ver = NIC_RAMCODE_VERSION_CFG_METHOD_1; + break; + case CFG_METHOD_2: + tp->sw_ram_code_ver = NIC_RAMCODE_VERSION_CFG_METHOD_2; + break; + default: + break; + } + + if (tp->HwIcVerUnknown) { + tp->NotWrRamCodeToMicroP = TRUE; + tp->NotWrMcuPatchCode = TRUE; + } + + tp->HwSuppMacMcuVer = 2; + + tp->MacMcuPageSize = RTL8127_MAC_MCU_PAGE_SIZE; + + tp->HwSuppNumTxQueues = 2; + tp->HwSuppNumRxQueues = 4; + + //init interrupt + tp->HwSuppIsrVer = 6; + + tp->HwCurrIsrVer = tp->HwSuppIsrVer; + if (tp->HwCurrIsrVer > 1) { + if (!(tp->features & RTL_FEATURE_MSIX) || + tp->irq_nvecs < tp->min_irq_nvecs) + tp->HwCurrIsrVer = 1; + } + + tp->num_tx_rings = 1; +#ifdef ENABLE_MULTIPLE_TX_QUEUE +#ifndef ENABLE_LIB_SUPPORT + tp->num_tx_rings = tp->HwSuppNumTxQueues; +#endif +#endif + if (tp->HwCurrIsrVer < 2 || + (tp->HwCurrIsrVer == 2 && tp->irq_nvecs < 19)) + tp->num_tx_rings = 1; + + //RSS + tp->HwSuppRssVer = 5; + tp->HwSuppIndirTblEntries = 128; + + tp->num_rx_rings = 1; +#ifdef ENABLE_RSS_SUPPORT +#ifdef ENABLE_LIB_SUPPORT + if (tp->HwSuppRssVer > 0) + tp->EnableRss = 1; +#else + if (tp->HwSuppRssVer > 0 && tp->HwCurrIsrVer > 1) { + u8 rss_queue_num = netif_get_num_default_rss_queues(); + tp->num_rx_rings = (tp->HwSuppNumRxQueues > rss_queue_num)? + rss_queue_num : tp->HwSuppNumRxQueues; + + if (!(tp->num_rx_rings >= 2 && tp->irq_nvecs >= tp->num_rx_rings)) + tp->num_rx_rings = 1; + + if (tp->num_rx_rings >= 2) + tp->EnableRss = 1; + } +#endif +#endif + + //interrupt mask + rtl8127_setup_interrupt_mask(tp); + + rtl8127_setup_mqs_reg(tp); + + rtl8127_set_ring_size(tp, NUM_RX_DESC, NUM_TX_DESC); + + tp->HwSuppPtpVer = 2; +#ifdef ENABLE_PTP_SUPPORT + if (tp->HwSuppPtpVer > 0) + tp->EnablePtp = 1; +#endif + + tp->HwSuppIntMitiVer = 6; + + tp->HwSuppTcamVer = 2; + + tp->TcamNotValidReg = TCAM_NOTVALID_ADDR_V2; + tp->TcamValidReg = TCAM_VALID_ADDR_V2; + tp->TcamMaAddrcOffset = TCAM_MAC_ADDR_V2; + tp->TcamVlanTagOffset = TCAM_VLAN_TAG_V2; + + tp->HwSuppExtendTallyCounterVer = 1; + + timer_count_v2 = (timer_count / 0x200); + + tp->HwSuppRxDescType = RX_DESC_RING_TYPE_4; + + tp->InitRxDescType = RX_DESC_RING_TYPE_1; + tp->RxDescLength = RX_DESC_LEN_TYPE_1; + switch (tp->HwSuppRxDescType) { + case RX_DESC_RING_TYPE_3: + if (tp->EnableRss) { + tp->InitRxDescType = RX_DESC_RING_TYPE_3; + tp->RxDescLength = RX_DESC_LEN_TYPE_3; + } + break; + case RX_DESC_RING_TYPE_4: + if (tp->EnableRss) { + tp->InitRxDescType = RX_DESC_RING_TYPE_4; + tp->RxDescLength = RX_DESC_LEN_TYPE_4; + } + break; + } + + tp->rtl8127_rx_config = rtl_chip_info[tp->chipset].RCR_Cfg; + if (tp->InitRxDescType == RX_DESC_RING_TYPE_3) + tp->rtl8127_rx_config |= EnableRxDescV3; + else if (tp->InitRxDescType == RX_DESC_RING_TYPE_4) + tp->rtl8127_rx_config &= ~EnableRxDescV4_1; + + tp->NicCustLedValue = RTL_R16(tp, CustomLED); + + tp->wol_opts = rtl8127_get_hw_wol(tp); + tp->wol_enabled = (tp->wol_opts) ? WOL_ENABLED : WOL_DISABLED; + + rtl8127_set_link_option(tp, autoneg_mode, speed_mode, duplex_mode, + rtl8127_fc_full); + + tp->max_jumbo_frame_size = rtl_chip_info[tp->chipset].jumbo_frame_sz; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + /* MTU range: 60 - hw-specific max */ + dev->min_mtu = ETH_MIN_MTU; + dev->max_mtu = tp->max_jumbo_frame_size; +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + + if (tp->mcfg != CFG_METHOD_DEFAULT) { + struct ethtool_keee *eee = &tp->eee; + + eee->eee_enabled = eee_enable; +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0) + eee->supported = SUPPORTED_100baseT_Full | + SUPPORTED_1000baseT_Full | + SUPPORTED_2500baseX_Full; + eee->advertised = mmd_eee_adv_to_ethtool_adv_t(MDIO_EEE_1000T | MDIO_EEE_100TX); + eee->advertised |= SUPPORTED_2500baseX_Full; +#else + linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, eee->supported); + linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT, eee->supported); + linkmode_set_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, eee->supported); + linkmode_set_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, eee->supported); + linkmode_set_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, eee->supported); + linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, eee->advertised); + linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT, eee->advertised); + linkmode_set_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, eee->advertised); + linkmode_set_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, eee->advertised); + linkmode_set_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, eee->advertised); +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0) */ + eee->tx_lpi_enabled = eee_enable; + eee->tx_lpi_timer = dev->mtu + ETH_HLEN + 0x20; + } + +#ifdef ENABLE_RSS_SUPPORT + if (tp->EnableRss) + rtl8127_init_rss(tp); +#endif +} + +static void +rtl8127_release_board(struct pci_dev *pdev, + struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + void __iomem *ioaddr = tp->mmio_addr; + + rtl8127_rar_set(tp, tp->org_mac_addr); + tp->wol_enabled = WOL_DISABLED; + + if (!tp->DASH) + rtl8127_phy_power_down(dev); + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) + FreeAllocatedDashShareMemory(dev); +#endif + + if (tp->mapped_cmac_ioaddr != NULL) + iounmap(tp->mapped_cmac_ioaddr); + + iounmap(ioaddr); + pci_release_regions(pdev); + pci_clear_mwi(pdev); + pci_disable_device(pdev); + free_netdev(dev); +} + +static void +rtl8127_hw_address_set(struct net_device *dev, u8 mac_addr[MAC_ADDR_LEN]) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0) + eth_hw_addr_set(dev, mac_addr); +#else + memcpy(dev->dev_addr, mac_addr, MAC_ADDR_LEN); +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0) +} + +static int +rtl8127_get_mac_address(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i; + u8 mac_addr[MAC_ADDR_LEN]; + + for (i = 0; i < MAC_ADDR_LEN; i++) + mac_addr[i] = RTL_R8(tp, MAC0 + i); + + *(u32*)&mac_addr[0] = RTL_R32(tp, BACKUP_ADDR0_8125); + *(u16*)&mac_addr[4] = RTL_R16(tp, BACKUP_ADDR1_8125); + + if (!is_valid_ether_addr(mac_addr)) { + netif_err(tp, probe, dev, "Invalid ether addr %pM\n", + mac_addr); + eth_random_addr(mac_addr); + dev->addr_assign_type = NET_ADDR_RANDOM; + netif_info(tp, probe, dev, "Random ether addr %pM\n", + mac_addr); + tp->random_mac = 1; + } + + rtl8127_hw_address_set(dev, mac_addr); + rtl8127_rar_set(tp, mac_addr); + + /* keep the original MAC address */ + memcpy(tp->org_mac_addr, dev->dev_addr, MAC_ADDR_LEN); +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,13) + memcpy(dev->perm_addr, dev->dev_addr, MAC_ADDR_LEN); +#endif + return 0; +} + +/** + * rtl8127_set_mac_address - Change the Ethernet Address of the NIC + * @dev: network interface device structure + * @p: pointer to an address structure + * + * Return 0 on success, negative on failure + **/ +static int +rtl8127_set_mac_address(struct net_device *dev, + void *p) +{ + struct rtl8127_private *tp = netdev_priv(dev); + struct sockaddr *addr = p; + + if (!is_valid_ether_addr(addr->sa_data)) + return -EADDRNOTAVAIL; + + rtl8127_hw_address_set(dev, addr->sa_data); + + rtl8127_rar_set(tp, dev->dev_addr); + + return 0; +} + +/****************************************************************************** + * rtl8127_rar_set - Puts an ethernet address into a receive address register. + * + * tp - The private data structure for driver + * addr - Address to put into receive address register + *****************************************************************************/ +void +rtl8127_rar_set(struct rtl8127_private *tp, + const u8 *addr) +{ + uint32_t rar_low = 0; + uint32_t rar_high = 0; + + rar_low = ((uint32_t) addr[0] | + ((uint32_t) addr[1] << 8) | + ((uint32_t) addr[2] << 16) | + ((uint32_t) addr[3] << 24)); + + rar_high = ((uint32_t) addr[4] | + ((uint32_t) addr[5] << 8)); + + rtl8127_enable_cfg9346_write(tp); + RTL_W32(tp, MAC0, rar_low); + RTL_W32(tp, MAC4, rar_high); + + rtl8127_disable_cfg9346_write(tp); +} + +#ifdef ETHTOOL_OPS_COMPAT +static int ethtool_get_settings(struct net_device *dev, void *useraddr) +{ + struct ethtool_cmd cmd = { ETHTOOL_GSET }; + int err; + + if (!ethtool_ops->get_settings) + return -EOPNOTSUPP; + + err = ethtool_ops->get_settings(dev, &cmd); + if (err < 0) + return err; + + if (copy_to_user(useraddr, &cmd, sizeof(cmd))) + return -EFAULT; + return 0; +} + +static int ethtool_set_settings(struct net_device *dev, void *useraddr) +{ + struct ethtool_cmd cmd; + + if (!ethtool_ops->set_settings) + return -EOPNOTSUPP; + + if (copy_from_user(&cmd, useraddr, sizeof(cmd))) + return -EFAULT; + + return ethtool_ops->set_settings(dev, &cmd); +} + +static int ethtool_get_drvinfo(struct net_device *dev, void *useraddr) +{ + struct ethtool_drvinfo info; + struct ethtool_ops *ops = ethtool_ops; + + if (!ops->get_drvinfo) + return -EOPNOTSUPP; + + memset(&info, 0, sizeof(info)); + info.cmd = ETHTOOL_GDRVINFO; + ops->get_drvinfo(dev, &info); + + if (ops->self_test_count) + info.testinfo_len = ops->self_test_count(dev); + if (ops->get_stats_count) + info.n_stats = ops->get_stats_count(dev); + if (ops->get_regs_len) + info.regdump_len = ops->get_regs_len(dev); + if (ops->get_eeprom_len) + info.eedump_len = ops->get_eeprom_len(dev); + + if (copy_to_user(useraddr, &info, sizeof(info))) + return -EFAULT; + return 0; +} + +static int ethtool_get_regs(struct net_device *dev, char *useraddr) +{ + struct ethtool_regs regs; + struct ethtool_ops *ops = ethtool_ops; + void *regbuf; + int reglen, ret; + + if (!ops->get_regs || !ops->get_regs_len) + return -EOPNOTSUPP; + + if (copy_from_user(®s, useraddr, sizeof(regs))) + return -EFAULT; + + reglen = ops->get_regs_len(dev); + if (regs.len > reglen) + regs.len = reglen; + + regbuf = kmalloc(reglen, GFP_USER); + if (!regbuf) + return -ENOMEM; + + ops->get_regs(dev, ®s, regbuf); + + ret = -EFAULT; + if (copy_to_user(useraddr, ®s, sizeof(regs))) + goto out; + useraddr += offsetof(struct ethtool_regs, data); + if (copy_to_user(useraddr, regbuf, reglen)) + goto out; + ret = 0; + +out: + kfree(regbuf); + return ret; +} + +static int ethtool_get_wol(struct net_device *dev, char *useraddr) +{ + struct ethtool_wolinfo wol = { ETHTOOL_GWOL }; + + if (!ethtool_ops->get_wol) + return -EOPNOTSUPP; + + ethtool_ops->get_wol(dev, &wol); + + if (copy_to_user(useraddr, &wol, sizeof(wol))) + return -EFAULT; + return 0; +} + +static int ethtool_set_wol(struct net_device *dev, char *useraddr) +{ + struct ethtool_wolinfo wol; + + if (!ethtool_ops->set_wol) + return -EOPNOTSUPP; + + if (copy_from_user(&wol, useraddr, sizeof(wol))) + return -EFAULT; + + return ethtool_ops->set_wol(dev, &wol); +} + +static int ethtool_get_msglevel(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata = { ETHTOOL_GMSGLVL }; + + if (!ethtool_ops->get_msglevel) + return -EOPNOTSUPP; + + edata.data = ethtool_ops->get_msglevel(dev); + + if (copy_to_user(useraddr, &edata, sizeof(edata))) + return -EFAULT; + return 0; +} + +static int ethtool_set_msglevel(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata; + + if (!ethtool_ops->set_msglevel) + return -EOPNOTSUPP; + + if (copy_from_user(&edata, useraddr, sizeof(edata))) + return -EFAULT; + + ethtool_ops->set_msglevel(dev, edata.data); + return 0; +} + +static int ethtool_nway_reset(struct net_device *dev) +{ + if (!ethtool_ops->nway_reset) + return -EOPNOTSUPP; + + return ethtool_ops->nway_reset(dev); +} + +static int ethtool_get_link(struct net_device *dev, void *useraddr) +{ + struct ethtool_value edata = { ETHTOOL_GLINK }; + + if (!ethtool_ops->get_link) + return -EOPNOTSUPP; + + edata.data = ethtool_ops->get_link(dev); + + if (copy_to_user(useraddr, &edata, sizeof(edata))) + return -EFAULT; + return 0; +} + +static int ethtool_get_eeprom(struct net_device *dev, void *useraddr) +{ + struct ethtool_eeprom eeprom; + struct ethtool_ops *ops = ethtool_ops; + u8 *data; + int ret; + + if (!ops->get_eeprom || !ops->get_eeprom_len) + return -EOPNOTSUPP; + + if (copy_from_user(&eeprom, useraddr, sizeof(eeprom))) + return -EFAULT; + + /* Check for wrap and zero */ + if (eeprom.offset + eeprom.len <= eeprom.offset) + return -EINVAL; + + /* Check for exceeding total eeprom len */ + if (eeprom.offset + eeprom.len > ops->get_eeprom_len(dev)) + return -EINVAL; + + data = kmalloc(eeprom.len, GFP_USER); + if (!data) + return -ENOMEM; + + ret = -EFAULT; + if (copy_from_user(data, useraddr + sizeof(eeprom), eeprom.len)) + goto out; + + ret = ops->get_eeprom(dev, &eeprom, data); + if (ret) + goto out; + + ret = -EFAULT; + if (copy_to_user(useraddr, &eeprom, sizeof(eeprom))) + goto out; + if (copy_to_user(useraddr + sizeof(eeprom), data, eeprom.len)) + goto out; + ret = 0; + +out: + kfree(data); + return ret; +} + +static int ethtool_set_eeprom(struct net_device *dev, void *useraddr) +{ + struct ethtool_eeprom eeprom; + struct ethtool_ops *ops = ethtool_ops; + u8 *data; + int ret; + + if (!ops->set_eeprom || !ops->get_eeprom_len) + return -EOPNOTSUPP; + + if (copy_from_user(&eeprom, useraddr, sizeof(eeprom))) + return -EFAULT; + + /* Check for wrap and zero */ + if (eeprom.offset + eeprom.len <= eeprom.offset) + return -EINVAL; + + /* Check for exceeding total eeprom len */ + if (eeprom.offset + eeprom.len > ops->get_eeprom_len(dev)) + return -EINVAL; + + data = kmalloc(eeprom.len, GFP_USER); + if (!data) + return -ENOMEM; + + ret = -EFAULT; + if (copy_from_user(data, useraddr + sizeof(eeprom), eeprom.len)) + goto out; + + ret = ops->set_eeprom(dev, &eeprom, data); + if (ret) + goto out; + + if (copy_to_user(useraddr + sizeof(eeprom), data, eeprom.len)) + ret = -EFAULT; + +out: + kfree(data); + return ret; +} + +static int ethtool_get_coalesce(struct net_device *dev, void *useraddr) +{ + struct ethtool_coalesce coalesce = { ETHTOOL_GCOALESCE }; + + if (!ethtool_ops->get_coalesce) + return -EOPNOTSUPP; + + ethtool_ops->get_coalesce(dev, &coalesce); + + if (copy_to_user(useraddr, &coalesce, sizeof(coalesce))) + return -EFAULT; + return 0; +} + +static int ethtool_set_coalesce(struct net_device *dev, void *useraddr) +{ + struct ethtool_coalesce coalesce; + + if (!ethtool_ops->get_coalesce) + return -EOPNOTSUPP; + + if (copy_from_user(&coalesce, useraddr, sizeof(coalesce))) + return -EFAULT; + + return ethtool_ops->set_coalesce(dev, &coalesce); +} + +static int ethtool_get_ringparam(struct net_device *dev, void *useraddr) +{ + struct ethtool_ringparam ringparam = { ETHTOOL_GRINGPARAM }; + + if (!ethtool_ops->get_ringparam) + return -EOPNOTSUPP; + + ethtool_ops->get_ringparam(dev, &ringparam); + + if (copy_to_user(useraddr, &ringparam, sizeof(ringparam))) + return -EFAULT; + return 0; +} + +static int ethtool_set_ringparam(struct net_device *dev, void *useraddr) +{ + struct ethtool_ringparam ringparam; + + if (!ethtool_ops->get_ringparam) + return -EOPNOTSUPP; + + if (copy_from_user(&ringparam, useraddr, sizeof(ringparam))) + return -EFAULT; + + return ethtool_ops->set_ringparam(dev, &ringparam); +} + +static int ethtool_get_pauseparam(struct net_device *dev, void *useraddr) +{ + struct ethtool_pauseparam pauseparam = { ETHTOOL_GPAUSEPARAM }; + + if (!ethtool_ops->get_pauseparam) + return -EOPNOTSUPP; + + ethtool_ops->get_pauseparam(dev, &pauseparam); + + if (copy_to_user(useraddr, &pauseparam, sizeof(pauseparam))) + return -EFAULT; + return 0; +} + +static int ethtool_set_pauseparam(struct net_device *dev, void *useraddr) +{ + struct ethtool_pauseparam pauseparam; + + if (!ethtool_ops->get_pauseparam) + return -EOPNOTSUPP; + + if (copy_from_user(&pauseparam, useraddr, sizeof(pauseparam))) + return -EFAULT; + + return ethtool_ops->set_pauseparam(dev, &pauseparam); +} + +static int ethtool_get_rx_csum(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata = { ETHTOOL_GRXCSUM }; + + if (!ethtool_ops->get_rx_csum) + return -EOPNOTSUPP; + + edata.data = ethtool_ops->get_rx_csum(dev); + + if (copy_to_user(useraddr, &edata, sizeof(edata))) + return -EFAULT; + return 0; +} + +static int ethtool_set_rx_csum(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata; + + if (!ethtool_ops->set_rx_csum) + return -EOPNOTSUPP; + + if (copy_from_user(&edata, useraddr, sizeof(edata))) + return -EFAULT; + + ethtool_ops->set_rx_csum(dev, edata.data); + return 0; +} + +static int ethtool_get_tx_csum(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata = { ETHTOOL_GTXCSUM }; + + if (!ethtool_ops->get_tx_csum) + return -EOPNOTSUPP; + + edata.data = ethtool_ops->get_tx_csum(dev); + + if (copy_to_user(useraddr, &edata, sizeof(edata))) + return -EFAULT; + return 0; +} + +static int ethtool_set_tx_csum(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata; + + if (!ethtool_ops->set_tx_csum) + return -EOPNOTSUPP; + + if (copy_from_user(&edata, useraddr, sizeof(edata))) + return -EFAULT; + + return ethtool_ops->set_tx_csum(dev, edata.data); +} + +static int ethtool_get_sg(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata = { ETHTOOL_GSG }; + + if (!ethtool_ops->get_sg) + return -EOPNOTSUPP; + + edata.data = ethtool_ops->get_sg(dev); + + if (copy_to_user(useraddr, &edata, sizeof(edata))) + return -EFAULT; + return 0; +} + +static int ethtool_set_sg(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata; + + if (!ethtool_ops->set_sg) + return -EOPNOTSUPP; + + if (copy_from_user(&edata, useraddr, sizeof(edata))) + return -EFAULT; + + return ethtool_ops->set_sg(dev, edata.data); +} + +static int ethtool_get_tso(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata = { ETHTOOL_GTSO }; + + if (!ethtool_ops->get_tso) + return -EOPNOTSUPP; + + edata.data = ethtool_ops->get_tso(dev); + + if (copy_to_user(useraddr, &edata, sizeof(edata))) + return -EFAULT; + return 0; +} + +static int ethtool_set_tso(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata; + + if (!ethtool_ops->set_tso) + return -EOPNOTSUPP; + + if (copy_from_user(&edata, useraddr, sizeof(edata))) + return -EFAULT; + + return ethtool_ops->set_tso(dev, edata.data); +} + +static int ethtool_self_test(struct net_device *dev, char *useraddr) +{ + struct ethtool_test test; + struct ethtool_ops *ops = ethtool_ops; + u64 *data; + int ret; + + if (!ops->self_test || !ops->self_test_count) + return -EOPNOTSUPP; + + if (copy_from_user(&test, useraddr, sizeof(test))) + return -EFAULT; + + test.len = ops->self_test_count(dev); + data = kmalloc(test.len * sizeof(u64), GFP_USER); + if (!data) + return -ENOMEM; + + ops->self_test(dev, &test, data); + + ret = -EFAULT; + if (copy_to_user(useraddr, &test, sizeof(test))) + goto out; + useraddr += sizeof(test); + if (copy_to_user(useraddr, data, test.len * sizeof(u64))) + goto out; + ret = 0; + +out: + kfree(data); + return ret; +} + +static int ethtool_get_strings(struct net_device *dev, void *useraddr) +{ + struct ethtool_gstrings gstrings; + struct ethtool_ops *ops = ethtool_ops; + u8 *data; + int ret; + + if (!ops->get_strings) + return -EOPNOTSUPP; + + if (copy_from_user(&gstrings, useraddr, sizeof(gstrings))) + return -EFAULT; + + switch (gstrings.string_set) { + case ETH_SS_TEST: + if (!ops->self_test_count) + return -EOPNOTSUPP; + gstrings.len = ops->self_test_count(dev); + break; + case ETH_SS_STATS: + if (!ops->get_stats_count) + return -EOPNOTSUPP; + gstrings.len = ops->get_stats_count(dev); + break; + default: + return -EINVAL; + } + + data = kmalloc(gstrings.len * ETH_GSTRING_LEN, GFP_USER); + if (!data) + return -ENOMEM; + + ops->get_strings(dev, gstrings.string_set, data); + + ret = -EFAULT; + if (copy_to_user(useraddr, &gstrings, sizeof(gstrings))) + goto out; + useraddr += sizeof(gstrings); + if (copy_to_user(useraddr, data, gstrings.len * ETH_GSTRING_LEN)) + goto out; + ret = 0; + +out: + kfree(data); + return ret; +} + +static int ethtool_phys_id(struct net_device *dev, void *useraddr) +{ + struct ethtool_value id; + + if (!ethtool_ops->phys_id) + return -EOPNOTSUPP; + + if (copy_from_user(&id, useraddr, sizeof(id))) + return -EFAULT; + + return ethtool_ops->phys_id(dev, id.data); +} + +static int ethtool_get_stats(struct net_device *dev, void *useraddr) +{ + struct ethtool_stats stats; + struct ethtool_ops *ops = ethtool_ops; + u64 *data; + int ret; + + if (!ops->get_ethtool_stats || !ops->get_stats_count) + return -EOPNOTSUPP; + + if (copy_from_user(&stats, useraddr, sizeof(stats))) + return -EFAULT; + + stats.n_stats = ops->get_stats_count(dev); + data = kmalloc(stats.n_stats * sizeof(u64), GFP_USER); + if (!data) + return -ENOMEM; + + ops->get_ethtool_stats(dev, &stats, data); + + ret = -EFAULT; + if (copy_to_user(useraddr, &stats, sizeof(stats))) + goto out; + useraddr += sizeof(stats); + if (copy_to_user(useraddr, data, stats.n_stats * sizeof(u64))) + goto out; + ret = 0; + +out: + kfree(data); + return ret; +} + +static int ethtool_ioctl(struct ifreq *ifr) +{ + struct net_device *dev = __dev_get_by_name(ifr->ifr_name); + void *useraddr = (void *) ifr->ifr_data; + u32 ethcmd; + + /* + * XXX: This can be pushed down into the ethtool_* handlers that + * need it. Keep existing behaviour for the moment. + */ + if (!capable(CAP_NET_ADMIN)) + return -EPERM; + + if (!dev || !netif_device_present(dev)) + return -ENODEV; + + if (copy_from_user(ðcmd, useraddr, sizeof (ethcmd))) + return -EFAULT; + + switch (ethcmd) { + case ETHTOOL_GSET: + return ethtool_get_settings(dev, useraddr); + case ETHTOOL_SSET: + return ethtool_set_settings(dev, useraddr); + case ETHTOOL_GDRVINFO: + return ethtool_get_drvinfo(dev, useraddr); + case ETHTOOL_GREGS: + return ethtool_get_regs(dev, useraddr); + case ETHTOOL_GWOL: + return ethtool_get_wol(dev, useraddr); + case ETHTOOL_SWOL: + return ethtool_set_wol(dev, useraddr); + case ETHTOOL_GMSGLVL: + return ethtool_get_msglevel(dev, useraddr); + case ETHTOOL_SMSGLVL: + return ethtool_set_msglevel(dev, useraddr); + case ETHTOOL_NWAY_RST: + return ethtool_nway_reset(dev); + case ETHTOOL_GLINK: + return ethtool_get_link(dev, useraddr); + case ETHTOOL_GEEPROM: + return ethtool_get_eeprom(dev, useraddr); + case ETHTOOL_SEEPROM: + return ethtool_set_eeprom(dev, useraddr); + case ETHTOOL_GCOALESCE: + return ethtool_get_coalesce(dev, useraddr); + case ETHTOOL_SCOALESCE: + return ethtool_set_coalesce(dev, useraddr); + case ETHTOOL_GRINGPARAM: + return ethtool_get_ringparam(dev, useraddr); + case ETHTOOL_SRINGPARAM: + return ethtool_set_ringparam(dev, useraddr); + case ETHTOOL_GPAUSEPARAM: + return ethtool_get_pauseparam(dev, useraddr); + case ETHTOOL_SPAUSEPARAM: + return ethtool_set_pauseparam(dev, useraddr); + case ETHTOOL_GRXCSUM: + return ethtool_get_rx_csum(dev, useraddr); + case ETHTOOL_SRXCSUM: + return ethtool_set_rx_csum(dev, useraddr); + case ETHTOOL_GTXCSUM: + return ethtool_get_tx_csum(dev, useraddr); + case ETHTOOL_STXCSUM: + return ethtool_set_tx_csum(dev, useraddr); + case ETHTOOL_GSG: + return ethtool_get_sg(dev, useraddr); + case ETHTOOL_SSG: + return ethtool_set_sg(dev, useraddr); + case ETHTOOL_GTSO: + return ethtool_get_tso(dev, useraddr); + case ETHTOOL_STSO: + return ethtool_set_tso(dev, useraddr); + case ETHTOOL_TEST: + return ethtool_self_test(dev, useraddr); + case ETHTOOL_GSTRINGS: + return ethtool_get_strings(dev, useraddr); + case ETHTOOL_PHYS_ID: + return ethtool_phys_id(dev, useraddr); + case ETHTOOL_GSTATS: + return ethtool_get_stats(dev, useraddr); + default: + return -EOPNOTSUPP; + } + + return -EOPNOTSUPP; +} +#endif //ETHTOOL_OPS_COMPAT + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,15,0) +static int rtl8127_siocdevprivate(struct net_device *dev, struct ifreq *ifr, + void __user *data, int cmd) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int ret = 0; + + switch (cmd) { +#ifdef ENABLE_DASH_SUPPORT + case SIOCDEVPRIVATE_RTLDASH: + if (!netif_running(dev)) { + ret = -ENODEV; + break; + } + if (!capable(CAP_NET_ADMIN)) { + ret = -EPERM; + break; + } + + ret = rtl8127_dash_ioctl(dev, ifr); + break; +#endif + +#ifdef ENABLE_REALWOW_SUPPORT + case SIOCDEVPRIVATE_RTLREALWOW: + if (!netif_running(dev)) { + ret = -ENODEV; + break; + } + + ret = rtl8127_realwow_ioctl(dev, ifr); + break; +#endif + + case SIOCRTLTOOL: + if (!capable(CAP_NET_ADMIN)) { + ret = -EPERM; + break; + } + + ret = rtl8127_tool_ioctl(tp, ifr); + break; + + default: + ret = -EOPNOTSUPP; + } + + return ret; +} +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,15,0) + +static int +rtl8127_do_ioctl(struct net_device *dev, + struct ifreq *ifr, + int cmd) +{ + struct rtl8127_private *tp = netdev_priv(dev); + struct mii_ioctl_data *data = if_mii(ifr); + int ret = 0; + + switch (cmd) { + case SIOCGMIIPHY: + data->phy_id = 32; /* Internal PHY */ + break; + + case SIOCGMIIREG: + rtl8127_mdio_write(tp, 0x1F, 0x0000); + data->val_out = rtl8127_mdio_read(tp, data->reg_num); + break; + + case SIOCSMIIREG: + if (!capable(CAP_NET_ADMIN)) + return -EPERM; + rtl8127_mdio_write(tp, 0x1F, 0x0000); + rtl8127_mdio_write(tp, data->reg_num, data->val_in); + break; + +#ifdef ETHTOOL_OPS_COMPAT + case SIOCETHTOOL: + ret = ethtool_ioctl(ifr); + break; +#endif + +#ifdef ENABLE_PTP_SUPPORT + case SIOCSHWTSTAMP: + case SIOCGHWTSTAMP: + if (tp->EnablePtp) + ret = rtl8127_ptp_ioctl(dev, ifr, cmd); + else + ret = -EOPNOTSUPP; + break; +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,15,0) +#ifdef ENABLE_DASH_SUPPORT + case SIOCDEVPRIVATE_RTLDASH: + if (!netif_running(dev)) { + ret = -ENODEV; + break; + } + if (!capable(CAP_NET_ADMIN)) { + ret = -EPERM; + break; + } + + ret = rtl8127_dash_ioctl(dev, ifr); + break; +#endif + +#ifdef ENABLE_REALWOW_SUPPORT + case SIOCDEVPRIVATE_RTLREALWOW: + if (!netif_running(dev)) { + ret = -ENODEV; + break; + } + + if (!capable(CAP_NET_ADMIN)) { + ret = -EPERM; + break; + } + + ret = rtl8127_realwow_ioctl(dev, ifr); + break; +#endif + + case SIOCRTLTOOL: + if (!capable(CAP_NET_ADMIN)) { + ret = -EPERM; + break; + } + + ret = rtl8127_tool_ioctl(tp, ifr); + break; +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(5,15,0) + + default: + ret = -EOPNOTSUPP; + break; + } + + return ret; +} + +static void +rtl8127_phy_power_up(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned long flags; + + if (rtl8127_is_in_phy_disable_mode(dev)) + return; + + spin_lock_irqsave(&tp->phy_lock, flags); + + rtl8127_mdio_write(tp, 0x1F, 0x0000); + rtl8127_mdio_write(tp, MII_BMCR, BMCR_ANENABLE); + + //wait ups resume (phy state 3) + rtl8127_wait_phy_ups_resume(dev, 3); + + spin_unlock_irqrestore(&tp->phy_lock, flags); +} + +static void +rtl8127_phy_power_down(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned long flags; + + spin_lock_irqsave(&tp->phy_lock, flags); + rtl8127_mdio_write(tp, 0x1F, 0x0000); + rtl8127_mdio_write(tp, MII_BMCR, BMCR_ANENABLE | BMCR_PDOWN); + spin_unlock_irqrestore(&tp->phy_lock, flags); +} + +static int __devinit +rtl8127_init_board(struct pci_dev *pdev, + struct net_device **dev_out, + void __iomem **ioaddr_out) +{ + void __iomem *ioaddr; + struct net_device *dev; + struct rtl8127_private *tp; + int rc = -ENOMEM, i, pm_cap; + + assert(ioaddr_out != NULL); + + /* dev zeroed in alloc_etherdev */ + dev = alloc_etherdev_mq(sizeof (*tp), R8127_MAX_QUEUES); + if (dev == NULL) { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_drv(&debug)) + dev_err(&pdev->dev, "unable to alloc new ethernet\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + goto err_out; + } + + SET_MODULE_OWNER(dev); + SET_NETDEV_DEV(dev, &pdev->dev); + tp = netdev_priv(dev); + tp->dev = dev; + tp->pci_dev = pdev; + tp->msg_enable = netif_msg_init(debug.msg_enable, R8127_MSG_DEFAULT); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) + if (!aspm) + pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1 | + PCIE_LINK_STATE_CLKPM); +#endif + + /* enable device (incl. PCI PM wakeup and hotplug setup) */ + rc = pci_enable_device(pdev); + if (rc < 0) { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_probe(tp)) + dev_err(&pdev->dev, "enable failure\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + goto err_out_free_dev; + } + + if (pci_set_mwi(pdev) < 0) { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_drv(&debug)) + dev_info(&pdev->dev, "Mem-Wr-Inval unavailable.\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + } + + /* save power state before pci_enable_device overwrites it */ + pm_cap = pci_find_capability(pdev, PCI_CAP_ID_PM); + if (pm_cap) { + u16 pwr_command; + + pci_read_config_word(pdev, pm_cap + PCI_PM_CTRL, &pwr_command); + } else { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_probe(tp)) + dev_err(&pdev->dev, "PowerManagement capability not found.\n"); +#else + printk("PowerManagement capability not found.\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + + } + + /* make sure PCI base addr 1 is MMIO */ + if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_probe(tp)) + dev_err(&pdev->dev, "region #1 not an MMIO resource, aborting\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + rc = -ENODEV; + goto err_out_mwi; + } + /* check for weird/broken PCI region reporting */ + if (pci_resource_len(pdev, 2) < R8127_REGS_SIZE) { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_probe(tp)) + dev_err(&pdev->dev, "Invalid PCI region size(s), aborting\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + rc = -ENODEV; + goto err_out_mwi; + } + + rc = pci_request_regions(pdev, MODULENAME); + if (rc < 0) { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_probe(tp)) + dev_err(&pdev->dev, "could not request regions.\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + goto err_out_mwi; + } + + if ((sizeof(dma_addr_t) > 4) && + use_dac && + !dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) && + !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) { + dev->features |= NETIF_F_HIGHDMA; + } else { + rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); + if (rc < 0) { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_probe(tp)) + dev_err(&pdev->dev, "DMA configuration failed.\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + goto err_out_free_res; + } + } + + /* ioremap MMIO region */ + ioaddr = ioremap(pci_resource_start(pdev, 2), pci_resource_len(pdev, 2)); + if (ioaddr == NULL) { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_probe(tp)) + dev_err(&pdev->dev, "cannot remap MMIO, aborting\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + rc = -EIO; + goto err_out_free_res; + } + + tp->mmio_addr = ioaddr; + + /* Identify chip attached to board */ + rtl8127_get_mac_version(tp); + + rtl8127_print_mac_version(tp); + + for (i = ARRAY_SIZE(rtl_chip_info) - 1; i >= 0; i--) { + if (tp->mcfg == rtl_chip_info[i].mcfg) + break; + } + + if (i < 0) { + /* Unknown chip: assume array element #0, original RTL-8125 */ +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_probe(tp)) + dev_printk(KERN_DEBUG, &pdev->dev, "unknown chip version, assuming %s\n", rtl_chip_info[0].name); +#else + printk("Realtek unknown chip version, assuming %s\n", rtl_chip_info[0].name); +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) + i++; + } + + tp->chipset = i; + + *ioaddr_out = ioaddr; + *dev_out = dev; +out: + return rc; + +err_out_free_res: + pci_release_regions(pdev); +err_out_mwi: + pci_clear_mwi(pdev); + pci_disable_device(pdev); +err_out_free_dev: + free_netdev(dev); +err_out: + *ioaddr_out = NULL; + *dev_out = NULL; + goto out; +} + +static void +rtl8127_esd_checker(struct rtl8127_private *tp) +{ + struct net_device *dev = tp->dev; + struct pci_dev *pdev = tp->pci_dev; + u8 cmd; + u16 io_base_l; + u16 mem_base_l; + u16 mem_base_h; + u8 ilr; + u16 resv_0x1c_h; + u16 resv_0x1c_l; + u16 resv_0x20_l; + u16 resv_0x20_h; + u16 resv_0x24_l; + u16 resv_0x24_h; + u16 resv_0x2c_h; + u16 resv_0x2c_l; + u32 pci_sn_l; + u32 pci_sn_h; + + if (unlikely(tp->rtk_enable_diag)) + goto exit; + + tp->esd_flag = 0; + + pci_read_config_byte(pdev, PCI_COMMAND, &cmd); + if (cmd != tp->pci_cfg_space.cmd) { + printk(KERN_ERR "%s: cmd = 0x%02x, should be 0x%02x \n.", dev->name, cmd, tp->pci_cfg_space.cmd); + pci_write_config_byte(pdev, PCI_COMMAND, tp->pci_cfg_space.cmd); + tp->esd_flag |= BIT_0; + + pci_read_config_byte(pdev, PCI_COMMAND, &cmd); + if (cmd == 0xff) { + printk(KERN_ERR "%s: pci link is down \n.", dev->name); + goto exit; + } + } + + pci_read_config_word(pdev, PCI_BASE_ADDRESS_0, &io_base_l); + if (io_base_l != tp->pci_cfg_space.io_base_l) { + printk(KERN_ERR "%s: io_base_l = 0x%04x, should be 0x%04x \n.", dev->name, io_base_l, tp->pci_cfg_space.io_base_l); + pci_write_config_word(pdev, PCI_BASE_ADDRESS_0, tp->pci_cfg_space.io_base_l); + tp->esd_flag |= BIT_1; + } + + pci_read_config_word(pdev, PCI_BASE_ADDRESS_2, &mem_base_l); + if (mem_base_l != tp->pci_cfg_space.mem_base_l) { + printk(KERN_ERR "%s: mem_base_l = 0x%04x, should be 0x%04x \n.", dev->name, mem_base_l, tp->pci_cfg_space.mem_base_l); + pci_write_config_word(pdev, PCI_BASE_ADDRESS_2, tp->pci_cfg_space.mem_base_l); + tp->esd_flag |= BIT_2; + } + + pci_read_config_word(pdev, PCI_BASE_ADDRESS_2 + 2, &mem_base_h); + if (mem_base_h!= tp->pci_cfg_space.mem_base_h) { + printk(KERN_ERR "%s: mem_base_h = 0x%04x, should be 0x%04x \n.", dev->name, mem_base_h, tp->pci_cfg_space.mem_base_h); + pci_write_config_word(pdev, PCI_BASE_ADDRESS_2 + 2, tp->pci_cfg_space.mem_base_h); + tp->esd_flag |= BIT_3; + } + + pci_read_config_word(pdev, PCI_BASE_ADDRESS_3, &resv_0x1c_l); + if (resv_0x1c_l != tp->pci_cfg_space.resv_0x1c_l) { + printk(KERN_ERR "%s: resv_0x1c_l = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x1c_l, tp->pci_cfg_space.resv_0x1c_l); + pci_write_config_word(pdev, PCI_BASE_ADDRESS_3, tp->pci_cfg_space.resv_0x1c_l); + tp->esd_flag |= BIT_4; + } + + pci_read_config_word(pdev, PCI_BASE_ADDRESS_3 + 2, &resv_0x1c_h); + if (resv_0x1c_h != tp->pci_cfg_space.resv_0x1c_h) { + printk(KERN_ERR "%s: resv_0x1c_h = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x1c_h, tp->pci_cfg_space.resv_0x1c_h); + pci_write_config_word(pdev, PCI_BASE_ADDRESS_3 + 2, tp->pci_cfg_space.resv_0x1c_h); + tp->esd_flag |= BIT_5; + } + + pci_read_config_word(pdev, PCI_BASE_ADDRESS_4, &resv_0x20_l); + if (resv_0x20_l != tp->pci_cfg_space.resv_0x20_l) { + printk(KERN_ERR "%s: resv_0x20_l = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x20_l, tp->pci_cfg_space.resv_0x20_l); + pci_write_config_word(pdev, PCI_BASE_ADDRESS_4, tp->pci_cfg_space.resv_0x20_l); + tp->esd_flag |= BIT_6; + } + + pci_read_config_word(pdev, PCI_BASE_ADDRESS_4 + 2, &resv_0x20_h); + if (resv_0x20_h != tp->pci_cfg_space.resv_0x20_h) { + printk(KERN_ERR "%s: resv_0x20_h = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x20_h, tp->pci_cfg_space.resv_0x20_h); + pci_write_config_word(pdev, PCI_BASE_ADDRESS_4 + 2, tp->pci_cfg_space.resv_0x20_h); + tp->esd_flag |= BIT_7; + } + + pci_read_config_word(pdev, PCI_BASE_ADDRESS_5, &resv_0x24_l); + if (resv_0x24_l != tp->pci_cfg_space.resv_0x24_l) { + printk(KERN_ERR "%s: resv_0x24_l = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x24_l, tp->pci_cfg_space.resv_0x24_l); + pci_write_config_word(pdev, PCI_BASE_ADDRESS_5, tp->pci_cfg_space.resv_0x24_l); + tp->esd_flag |= BIT_8; + } + + pci_read_config_word(pdev, PCI_BASE_ADDRESS_5 + 2, &resv_0x24_h); + if (resv_0x24_h != tp->pci_cfg_space.resv_0x24_h) { + printk(KERN_ERR "%s: resv_0x24_h = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x24_h, tp->pci_cfg_space.resv_0x24_h); + pci_write_config_word(pdev, PCI_BASE_ADDRESS_5 + 2, tp->pci_cfg_space.resv_0x24_h); + tp->esd_flag |= BIT_9; + } + + pci_read_config_byte(pdev, PCI_INTERRUPT_LINE, &ilr); + if (ilr != tp->pci_cfg_space.ilr) { + printk(KERN_ERR "%s: ilr = 0x%02x, should be 0x%02x \n.", dev->name, ilr, tp->pci_cfg_space.ilr); + pci_write_config_byte(pdev, PCI_INTERRUPT_LINE, tp->pci_cfg_space.ilr); + tp->esd_flag |= BIT_10; + } + + pci_read_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID, &resv_0x2c_l); + if (resv_0x2c_l != tp->pci_cfg_space.resv_0x2c_l) { + printk(KERN_ERR "%s: resv_0x2c_l = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x2c_l, tp->pci_cfg_space.resv_0x2c_l); + pci_write_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID, tp->pci_cfg_space.resv_0x2c_l); + tp->esd_flag |= BIT_11; + } + + pci_read_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID + 2, &resv_0x2c_h); + if (resv_0x2c_h != tp->pci_cfg_space.resv_0x2c_h) { + printk(KERN_ERR "%s: resv_0x2c_h = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x2c_h, tp->pci_cfg_space.resv_0x2c_h); + pci_write_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID + 2, tp->pci_cfg_space.resv_0x2c_h); + tp->esd_flag |= BIT_12; + } + + if (tp->HwPcieSNOffset > 0) { + pci_sn_l = rtl8127_csi_read(tp, tp->HwPcieSNOffset); + if (pci_sn_l != tp->pci_cfg_space.pci_sn_l) { + printk(KERN_ERR "%s: pci_sn_l = 0x%08x, should be 0x%08x \n.", dev->name, pci_sn_l, tp->pci_cfg_space.pci_sn_l); + rtl8127_csi_write(tp, tp->HwPcieSNOffset, tp->pci_cfg_space.pci_sn_l); + tp->esd_flag |= BIT_13; + } + + pci_sn_h = rtl8127_csi_read(tp, tp->HwPcieSNOffset + 4); + if (pci_sn_h != tp->pci_cfg_space.pci_sn_h) { + printk(KERN_ERR "%s: pci_sn_h = 0x%08x, should be 0x%08x \n.", dev->name, pci_sn_h, tp->pci_cfg_space.pci_sn_h); + rtl8127_csi_write(tp, tp->HwPcieSNOffset + 4, tp->pci_cfg_space.pci_sn_h); + tp->esd_flag |= BIT_14; + } + } + + if (tp->esd_flag != 0) { + printk(KERN_ERR "%s: esd_flag = 0x%04x\n.\n", dev->name, tp->esd_flag); + netif_carrier_off(dev); + netif_tx_disable(dev); + rtl8127_hw_reset(dev); + rtl8127_tx_clear(tp); + rtl8127_rx_clear(tp); + rtl8127_init_ring(dev); + rtl8127_up(dev); + rtl8127_enable_hw_linkchg_interrupt(tp); + rtl8127_set_speed(dev, tp->autoneg, tp->speed, tp->duplex, tp->advertising); + tp->esd_flag = 0; + } +exit: + return; +} +/* +static void +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) +rtl8127_esd_timer(unsigned long __opaque) +#else +rtl8127_esd_timer(struct timer_list *t) +#endif +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) + struct net_device *dev = (struct net_device *)__opaque; + struct rtl8127_private *tp = netdev_priv(dev); + struct timer_list *timer = &tp->esd_timer; +#else + struct rtl8127_private *tp = from_timer(tp, t, esd_timer); + //struct net_device *dev = tp->dev; + struct timer_list *timer = t; +#endif + rtl8127_esd_checker(tp); + + mod_timer(timer, jiffies + timeout); +} +*/ + +/* +static void +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) +rtl8127_link_timer(unsigned long __opaque) +#else +rtl8127_link_timer(struct timer_list *t) +#endif +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) + struct net_device *dev = (struct net_device *)__opaque; + struct rtl8127_private *tp = netdev_priv(dev); + struct timer_list *timer = &tp->link_timer; +#else + struct rtl8127_private *tp = from_timer(tp, t, link_timer); + struct net_device *dev = tp->dev; + struct timer_list *timer = t; +#endif + rtl8127_check_link_status(dev); + + mod_timer(timer, jiffies + RTL8127_LINK_TIMEOUT); +} +*/ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,14,0) +static int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries, + int minvec, int maxvec) +{ + int nvec = maxvec; + int rc; + + if (maxvec < minvec) + return -ERANGE; + + do { + rc = pci_enable_msix(dev, entries, nvec); + if (rc < 0) { + return rc; + } else if (rc > 0) { + if (rc < minvec) + return -ENOSPC; + nvec = rc; + } + } while (rc); + + return nvec; +} +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,14,0) */ + +static int rtl8127_enable_msix(struct rtl8127_private *tp) +{ + int i, nvecs = 0; + struct msix_entry msix_ent[R8127_MAX_MSIX_VEC]; + //struct net_device *dev = tp->dev; + //const int len = sizeof(tp->irq_tbl[0].name); + + for (i = 0; i < R8127_MAX_MSIX_VEC; i++) { + msix_ent[i].entry = i; + msix_ent[i].vector = 0; + } + + nvecs = pci_enable_msix_range(tp->pci_dev, msix_ent, + tp->min_irq_nvecs, tp->max_irq_nvecs); + if (nvecs < 0) + goto out; + + for (i = 0; i < nvecs; i++) { + struct r8127_irq *irq = &tp->irq_tbl[i]; + irq->vector = msix_ent[i].vector; + //snprintf(irq->name, len, "%s-%d", dev->name, i); + //irq->handler = rtl8127_interrupt_msix; + } + +out: + return nvecs; +} + +/* Cfg9346_Unlock assumed. */ +static int rtl8127_try_msi(struct rtl8127_private *tp) +{ + struct pci_dev *pdev = tp->pci_dev; + unsigned int hw_supp_irq_nvecs; + unsigned msi = 0; + int nvecs = 1; + + hw_supp_irq_nvecs = R8127_MAX_MSIX_VEC_8125B; + tp->hw_supp_irq_nvecs = clamp_val(hw_supp_irq_nvecs, 1, + R8127_MAX_MSIX_VEC); + + tp->max_irq_nvecs = tp->hw_supp_irq_nvecs; + tp->min_irq_nvecs = R8127_MIN_MSIX_VEC_8127; +#ifdef DISABLE_MULTI_MSIX_VECTOR + tp->max_irq_nvecs = 1; +#endif + +#if defined(RTL_USE_NEW_INTR_API) + if ((nvecs = pci_alloc_irq_vectors(pdev, tp->min_irq_nvecs, tp->max_irq_nvecs, PCI_IRQ_MSIX)) > 0) + msi |= RTL_FEATURE_MSIX; + else if ((nvecs = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES)) > 0 && + pci_dev_msi_enabled(pdev)) + msi |= RTL_FEATURE_MSI; +#elif LINUX_VERSION_CODE > KERNEL_VERSION(2,6,13) + if ((nvecs = rtl8127_enable_msix(tp)) > 0) + msi |= RTL_FEATURE_MSIX; + else if (!pci_enable_msi(pdev)) + msi |= RTL_FEATURE_MSI; +#endif + if (!(msi & (RTL_FEATURE_MSI | RTL_FEATURE_MSIX))) + dev_info(&pdev->dev, "no MSI/MSI-X. Back to INTx.\n"); + + if (!(msi & RTL_FEATURE_MSIX) || nvecs < 1) + nvecs = 1; + + tp->irq_nvecs = nvecs; + + tp->features |= msi; + + return nvecs; +} + +static void rtl8127_disable_msi(struct pci_dev *pdev, struct rtl8127_private *tp) +{ +#if defined(RTL_USE_NEW_INTR_API) + if (tp->features & (RTL_FEATURE_MSI | RTL_FEATURE_MSIX)) + pci_free_irq_vectors(pdev); +#elif LINUX_VERSION_CODE > KERNEL_VERSION(2,6,13) + if (tp->features & (RTL_FEATURE_MSIX)) + pci_disable_msix(pdev); + else if (tp->features & (RTL_FEATURE_MSI)) + pci_disable_msi(pdev); +#endif + tp->features &= ~(RTL_FEATURE_MSI | RTL_FEATURE_MSIX); +} + +static int rtl8127_get_irq(struct pci_dev *pdev) +{ +#if defined(RTL_USE_NEW_INTR_API) + return pci_irq_vector(pdev, 0); +#else + return pdev->irq; +#endif +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,11,0) +static void +rtl8127_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) +{ + struct rtl8127_private *tp = netdev_priv(dev); + struct rtl8127_counters *counters = tp->tally_vaddr; + dma_addr_t paddr = tp->tally_paddr; + + if (!counters) + return; + + netdev_stats_to_stats64(stats, &dev->stats); + dev_fetch_sw_netstats(stats, dev->tstats); + + /* + * Fetch additional counter values missing in stats collected by driver + * from tally counters. + */ + rtl8127_dump_tally_counter(tp, paddr); + + stats->tx_errors = le64_to_cpu(counters->tx_errors); + stats->collisions = le32_to_cpu(counters->tx_multi_collision); + stats->tx_aborted_errors = le16_to_cpu(counters->tx_aborted); + stats->rx_missed_errors = le16_to_cpu(counters->rx_missed); +} +#else +/** + * rtl8127_get_stats - Get rtl8127 read/write statistics + * @dev: The Ethernet Device to get statistics for + * + * Get TX/RX statistics for rtl8127 + */ +static struct +net_device_stats *rtl8127_get_stats(struct net_device *dev) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) + struct rtl8127_private *tp = netdev_priv(dev); +#endif + return &RTLDEV->stats; +} +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,36) + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29) +static const struct net_device_ops rtl8127_netdev_ops = { + .ndo_open = rtl8127_open, + .ndo_stop = rtl8127_close, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,11,0) + .ndo_get_stats64 = rtl8127_get_stats64, +#else + .ndo_get_stats = rtl8127_get_stats, +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,11,0) + .ndo_start_xmit = rtl8127_start_xmit, + .ndo_tx_timeout = rtl8127_tx_timeout, + .ndo_change_mtu = rtl8127_change_mtu, + .ndo_set_mac_address = rtl8127_set_mac_address, +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,15,0) + .ndo_do_ioctl = rtl8127_do_ioctl, +#else + .ndo_siocdevprivate = rtl8127_siocdevprivate, + .ndo_eth_ioctl = rtl8127_do_ioctl, +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(5,15,0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,1,0) + .ndo_set_multicast_list = rtl8127_set_rx_mode, +#else + .ndo_set_rx_mode = rtl8127_set_rx_mode, +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) +#ifdef CONFIG_R8127_VLAN + .ndo_vlan_rx_register = rtl8127_vlan_rx_register, +#endif +#else + .ndo_fix_features = rtl8127_fix_features, + .ndo_set_features = rtl8127_set_features, +#endif +#ifdef CONFIG_NET_POLL_CONTROLLER + .ndo_poll_controller = rtl8127_netpoll, +#endif +}; +#endif + + +#ifdef CONFIG_R8127_NAPI + +static int rtl8127_poll(napi_ptr napi, napi_budget budget) +{ + struct r8127_napi *r8127napi = RTL_GET_PRIV(napi, struct r8127_napi); + struct rtl8127_private *tp = r8127napi->priv; + RTL_GET_NETDEV(tp) + unsigned int work_to_do = RTL_NAPI_QUOTA(budget, dev); + unsigned int work_done = 0; + int i; + + for (i = 0; i < tp->num_tx_rings; i++) + rtl8127_tx_interrupt(&tp->tx_ring[i], budget); + + for (i = 0; i < tp->num_rx_rings; i++) + work_done += rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[i], budget); + + work_done = min(work_done, work_to_do); + + RTL_NAPI_QUOTA_UPDATE(dev, work_done, budget); + + if (work_done < work_to_do) { +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) + HandleDashInterrupt(tp->dev); +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + if (RTL_NETIF_RX_COMPLETE(dev, napi, work_done) == FALSE) + return RTL_NAPI_RETURN_VALUE; +#else + RTL_NETIF_RX_COMPLETE(dev, napi, work_done); +#endif + /* + * 20040426: the barrier is not strictly required but the + * behavior of the irq handler could be less predictable + * without it. Btw, the lack of flush for the posted pci + * write is safe - FR + */ + smp_wmb(); + + rtl8127_switch_to_timer_interrupt(tp); + } + + return RTL_NAPI_RETURN_VALUE; +} + +static int rtl8127_poll_msix_ring(napi_ptr napi, napi_budget budget) +{ + struct r8127_napi *r8127napi = RTL_GET_PRIV(napi, struct r8127_napi); + struct rtl8127_private *tp = r8127napi->priv; + RTL_GET_NETDEV(tp) + unsigned int work_to_do = RTL_NAPI_QUOTA(budget, dev); + unsigned int work_done = 0; + const int message_id = r8127napi->index; + + if (message_id < tp->num_tx_rings) + rtl8127_tx_interrupt_with_vector(tp, message_id, budget); + + if (message_id < tp->num_rx_rings) + work_done += rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[message_id], budget); + + RTL_NAPI_QUOTA_UPDATE(dev, work_done, budget); + + if (work_done < work_to_do) { +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH && message_id == 0) + HandleDashInterrupt(tp->dev); +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + if (RTL_NETIF_RX_COMPLETE(dev, napi, work_done) == FALSE) + return RTL_NAPI_RETURN_VALUE; +#else + RTL_NETIF_RX_COMPLETE(dev, napi, work_done); +#endif + /* + * 20040426: the barrier is not strictly required but the + * behavior of the irq handler could be less predictable + * without it. Btw, the lack of flush for the posted pci + * write is safe - FR + */ + smp_wmb(); + + rtl8127_enable_hw_interrupt_v2(tp, message_id); + } + + return RTL_NAPI_RETURN_VALUE; +} + +static int rtl8127_poll_msix_tx(napi_ptr napi, napi_budget budget) +{ + struct r8127_napi *r8127napi = RTL_GET_PRIV(napi, struct r8127_napi); + struct rtl8127_private *tp = r8127napi->priv; + RTL_GET_NETDEV(tp) + unsigned int work_to_do = RTL_NAPI_QUOTA(budget, dev); + unsigned int work_done = 0; + const int message_id = r8127napi->index; + + //suppress unused variable + (void)(dev); + + rtl8127_tx_interrupt_with_vector(tp, message_id, budget); + + RTL_NAPI_QUOTA_UPDATE(dev, work_done, budget); + + if (work_done < work_to_do) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + if (RTL_NETIF_RX_COMPLETE(dev, napi, work_done) == FALSE) + return RTL_NAPI_RETURN_VALUE; +#else + RTL_NETIF_RX_COMPLETE(dev, napi, work_done); +#endif + /* + * 20040426: the barrier is not strictly required but the + * behavior of the irq handler could be less predictable + * without it. Btw, the lack of flush for the posted pci + * write is safe - FR + */ + smp_wmb(); + + rtl8127_enable_hw_interrupt_v2(tp, message_id); + } + + return RTL_NAPI_RETURN_VALUE; +} + +static int rtl8127_poll_msix_other(napi_ptr napi, napi_budget budget) +{ + struct r8127_napi *r8127napi = RTL_GET_PRIV(napi, struct r8127_napi); + struct rtl8127_private *tp = r8127napi->priv; + RTL_GET_NETDEV(tp) + unsigned int work_to_do = RTL_NAPI_QUOTA(budget, dev); + const int message_id = r8127napi->index; + + //suppress unused variable + (void)(dev); + (void)(work_to_do); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + RTL_NETIF_RX_COMPLETE(dev, napi, work_to_do); +#else + RTL_NETIF_RX_COMPLETE(dev, napi, work_to_do); +#endif + + rtl8127_enable_hw_interrupt_v2(tp, message_id); + + return 1; +} + +static int rtl8127_poll_msix_rx(napi_ptr napi, napi_budget budget) +{ + struct r8127_napi *r8127napi = RTL_GET_PRIV(napi, struct r8127_napi); + struct rtl8127_private *tp = r8127napi->priv; + RTL_GET_NETDEV(tp) + unsigned int work_to_do = RTL_NAPI_QUOTA(budget, dev); + unsigned int work_done = 0; + const int message_id = r8127napi->index; + + if (message_id < tp->num_rx_rings) + work_done += rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[message_id], budget); + + RTL_NAPI_QUOTA_UPDATE(dev, work_done, budget); + + if (work_done < work_to_do) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + if (RTL_NETIF_RX_COMPLETE(dev, napi, work_done) == FALSE) + return RTL_NAPI_RETURN_VALUE; +#else + RTL_NETIF_RX_COMPLETE(dev, napi, work_done); +#endif + /* + * 20040426: the barrier is not strictly required but the + * behavior of the irq handler could be less predictable + * without it. Btw, the lack of flush for the posted pci + * write is safe - FR + */ + smp_wmb(); + + rtl8127_enable_hw_interrupt_v2(tp, message_id); + } + + return RTL_NAPI_RETURN_VALUE; +} + +void rtl8127_enable_napi(struct rtl8127_private *tp) +{ +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + int i; + + for (i = 0; i < tp->irq_nvecs; i++) + RTL_NAPI_ENABLE(tp->dev, &tp->r8127napi[i].napi); +#endif +} + +static void rtl8127_disable_napi(struct rtl8127_private *tp) +{ +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + int i; + + for (i = 0; i < tp->irq_nvecs; i++) + RTL_NAPI_DISABLE(tp->dev, &tp->r8127napi[i].napi); +#endif +} + +static void rtl8127_del_napi(struct rtl8127_private *tp) +{ +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + int i; + + for (i = 0; i < tp->irq_nvecs; i++) + RTL_NAPI_DEL((&tp->r8127napi[i])); +#endif +} +#endif //CONFIG_R8127_NAPI + +static void rtl8127_init_napi(struct rtl8127_private *tp) +{ + int i; + + for (i=0; iirq_nvecs; i++) { + struct r8127_napi *r8127napi = &tp->r8127napi[i]; +#ifdef CONFIG_R8127_NAPI + int (*poll)(struct napi_struct *, int); + + poll = rtl8127_poll; + if (tp->features & RTL_FEATURE_MSIX) { + switch (tp->HwCurrIsrVer) { + case 6: + if (i < R8127_MAX_RX_QUEUES_VEC_V4) + poll = rtl8127_poll_msix_rx; + else if (i == 8 || i == 9) + poll = rtl8127_poll_msix_tx; + else + poll = rtl8127_poll_msix_other; + break; + case 5: + if (i < R8127_MAX_RX_QUEUES_VEC_V3) + poll = rtl8127_poll_msix_rx; + else if (i == 16 || i == 17) + poll = rtl8127_poll_msix_tx; + else + poll = rtl8127_poll_msix_other; + break; + case 2: + if (i < R8127_MAX_RX_QUEUES_VEC_V3) + poll = rtl8127_poll_msix_rx; + else if (i == 16 || i == 18) + poll = rtl8127_poll_msix_tx; + else + poll = rtl8127_poll_msix_other; + break; + case 3: + case 4: + if (i < R8127_MAX_RX_QUEUES_VEC_V3) + poll = rtl8127_poll_msix_ring; + else + poll = rtl8127_poll_msix_other; + break; + } + } + + RTL_NAPI_CONFIG(tp->dev, r8127napi, poll, R8127_NAPI_WEIGHT); +#endif + + r8127napi->priv = tp; + r8127napi->index = i; + } +} + +static int +rtl8127_set_real_num_queue(struct rtl8127_private *tp) +{ + int retval = 0; + + retval = netif_set_real_num_tx_queues(tp->dev, tp->num_tx_rings); + if (retval < 0) + goto exit; + + retval = netif_set_real_num_rx_queues(tp->dev, tp->num_rx_rings); + if (retval < 0) + goto exit; + +exit: + return retval; +} + +static int __devinit +rtl8127_init_one(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + struct net_device *dev = NULL; + struct rtl8127_private *tp; + void __iomem *ioaddr = NULL; + static int board_idx = -1; + + int rc; + + assert(pdev != NULL); + assert(ent != NULL); + + board_idx++; + + if (netif_msg_drv(&debug)) + printk(KERN_INFO "%s Ethernet controller driver %s loaded\n", + MODULENAME, RTL8127_VERSION); + + rc = rtl8127_init_board(pdev, &dev, &ioaddr); + if (rc) + goto out; + + tp = netdev_priv(dev); + assert(ioaddr != NULL); + + spin_lock_init(&tp->phy_lock); + + tp->set_speed = rtl8127_set_speed_xmii; + tp->get_settings = rtl8127_gset_xmii; + tp->phy_reset_enable = rtl8127_xmii_reset_enable; + tp->phy_reset_pending = rtl8127_xmii_reset_pending; + tp->link_ok = rtl8127_xmii_link_ok; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,11,0) + dev->tstats = devm_netdev_alloc_pcpu_stats(&pdev->dev, + struct pcpu_sw_netstats); + if (!dev->tstats) + goto err_out_1; +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,11,0) + + rc = rtl8127_try_msi(tp); + if (rc < 0) { + dev_err(&pdev->dev, "Can't allocate interrupt\n"); + goto err_out_1; + } + + rtl8127_init_software_variable(dev); + + RTL_NET_DEVICE_OPS(rtl8127_netdev_ops); + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) + SET_ETHTOOL_OPS(dev, &rtl8127_ethtool_ops); +#endif + + dev->watchdog_timeo = RTL8127_TX_TIMEOUT; + dev->irq = rtl8127_get_irq(pdev); + dev->base_addr = (unsigned long) ioaddr; + + rtl8127_init_napi(tp); + +#ifdef CONFIG_R8127_VLAN + if (tp->mcfg != CFG_METHOD_DEFAULT) { + dev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX; +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) + dev->vlan_rx_kill_vid = rtl8127_vlan_rx_kill_vid; +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) + } +#endif + + /* There has been a number of reports that using SG/TSO results in + * tx timeouts. However for a lot of people SG/TSO works fine. + * Therefore disable both features by default, but allow users to + * enable them. Use at own risk! + */ + tp->cp_cmd |= RTL_R16(tp, CPlusCmd); + if (tp->mcfg != CFG_METHOD_DEFAULT) { + dev->features |= NETIF_F_IP_CSUM; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + tp->cp_cmd |= RxChkSum; +#else + dev->features |= NETIF_F_RXCSUM; + dev->features |= NETIF_F_SG | NETIF_F_TSO; + dev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO | + NETIF_F_RXCSUM | NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX; + dev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO | + NETIF_F_HIGHDMA; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,15,0) + dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(3,15,0) + dev->hw_features |= NETIF_F_RXALL; + dev->hw_features |= NETIF_F_RXFCS; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,22) + dev->hw_features |= NETIF_F_IPV6_CSUM | NETIF_F_TSO6; + dev->features |= NETIF_F_IPV6_CSUM; + dev->features |= NETIF_F_TSO6; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,19,0) + netif_set_tso_max_size(dev, LSO_64K); + netif_set_tso_max_segs(dev, NIC_MAX_PHYS_BUF_COUNT_LSO2); +#else //LINUX_VERSION_CODE >= KERNEL_VERSION(5,19,0) + netif_set_gso_max_size(dev, LSO_64K); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,18,0) + dev->gso_max_segs = NIC_MAX_PHYS_BUF_COUNT_LSO2; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,7,0) + dev->gso_min_segs = NIC_MIN_PHYS_BUF_COUNT; +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,7,0) +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(3,18,0) +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,19,0) + +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,22) +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + +#ifdef ENABLE_RSS_SUPPORT + if (tp->EnableRss) { + dev->hw_features |= NETIF_F_RXHASH; + dev->features |= NETIF_F_RXHASH; + } +#endif + } + + netdev_sw_irq_coalesce_default_on(dev); + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) + AllocateDashShareMemory(dev); +#endif + +#ifdef ENABLE_LIB_SUPPORT + BLOCKING_INIT_NOTIFIER_HEAD(&tp->lib_nh); +#endif + rtl8127_init_all_schedule_work(tp); + + rc = rtl8127_set_real_num_queue(tp); + if (rc < 0) + goto err_out; + + rtl8127_exit_oob(dev); + + rtl8127_powerup_pll(dev); + + rtl8127_hw_init(dev); + + rtl8127_hw_reset(dev); + + /* Get production from EEPROM */ + rtl8127_eeprom_type(tp); + + if (tp->eeprom_type == EEPROM_TYPE_93C46 || tp->eeprom_type == EEPROM_TYPE_93C56) + rtl8127_set_eeprom_sel_low(tp); + + rtl8127_get_mac_address(dev); + + tp->fw_name = rtl_chip_fw_infos[tp->mcfg].fw_name; + + tp->tally_vaddr = dma_alloc_coherent(&pdev->dev, sizeof(*tp->tally_vaddr), + &tp->tally_paddr, GFP_KERNEL); + if (!tp->tally_vaddr) { + rc = -ENOMEM; + goto err_out; + } + + rtl8127_tally_counter_clear(tp); + + pci_set_drvdata(pdev, dev); + + rc = register_netdev(dev); + if (rc) + goto err_out; + + printk(KERN_INFO "%s: This product is covered by one or more of the following patents: US6,570,884, US6,115,776, and US6,327,625.\n", MODULENAME); + + rtl8127_disable_rxdvgate(dev); + + device_set_wakeup_enable(&pdev->dev, tp->wol_enabled); + + netif_carrier_off(dev); + +#ifdef ENABLE_R8127_SYSFS + rtl8127_sysfs_init(dev); +#endif /* ENABLE_R8127_SYSFS */ + + printk("%s", GPL_CLAIM); + +out: + return rc; + +err_out: + if (tp->tally_vaddr != NULL) { + dma_free_coherent(&pdev->dev, sizeof(*tp->tally_vaddr), tp->tally_vaddr, + tp->tally_paddr); + + tp->tally_vaddr = NULL; + } +#ifdef CONFIG_R8127_NAPI + rtl8127_del_napi(tp); +#endif + rtl8127_disable_msi(pdev, tp); + +err_out_1: + rtl8127_release_board(pdev, dev); + + goto out; +} + +static void __devexit +rtl8127_remove_one(struct pci_dev *pdev) +{ + struct net_device *dev = pci_get_drvdata(pdev); + struct rtl8127_private *tp = netdev_priv(dev); + + assert(dev != NULL); + assert(tp != NULL); + + set_bit(R8127_FLAG_DOWN, tp->task_flags); + + rtl8127_cancel_all_schedule_work(tp); + +#ifdef CONFIG_R8127_NAPI + rtl8127_del_napi(tp); +#endif + if (HW_DASH_SUPPORT_DASH(tp)) + rtl8127_driver_stop(tp); + + rtl8127_disable_pci_offset_180(tp); + +#ifdef ENABLE_R8127_SYSFS + rtl8127_sysfs_remove(dev); +#endif //ENABLE_R8127_SYSFS + + unregister_netdev(dev); + rtl8127_disable_msi(pdev, tp); +#ifdef ENABLE_R8127_PROCFS + rtl8127_proc_remove(dev); +#endif + if (tp->tally_vaddr != NULL) { + dma_free_coherent(&pdev->dev, sizeof(*tp->tally_vaddr), tp->tally_vaddr, tp->tally_paddr); + tp->tally_vaddr = NULL; + } + + rtl8127_release_board(pdev, dev); + +#ifdef ENABLE_USE_FIRMWARE_FILE + rtl8127_release_firmware(tp); +#endif + + pci_set_drvdata(pdev, NULL); +} + +#ifdef ENABLE_PAGE_REUSE +static inline unsigned int rtl8127_rx_page_order(unsigned rx_buf_sz, unsigned page_size) +{ + unsigned truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + + SKB_DATA_ALIGN(rx_buf_sz + R8127_RX_ALIGN); + + return get_order(truesize * 2); +} +#endif //ENABLE_PAGE_REUSE + +static void +rtl8127_set_rxbufsize(struct rtl8127_private *tp, + struct net_device *dev) +{ + unsigned int mtu = dev->mtu; + + tp->rms = (mtu > ETH_DATA_LEN) ? + mtu + ETH_HLEN + RT_VALN_HLEN + ETH_FCS_LEN: + RX_BUF_SIZE; + tp->rx_buf_sz = tp->rms; +#ifdef ENABLE_RX_PACKET_FRAGMENT + tp->rx_buf_sz = SKB_DATA_ALIGN(RX_BUF_SIZE); +#endif //ENABLE_RX_PACKET_FRAGMENT +#ifdef ENABLE_PAGE_REUSE + tp->rx_buf_page_order = rtl8127_rx_page_order(tp->rx_buf_sz, PAGE_SIZE); + tp->rx_buf_page_size = rtl8127_rx_page_size(tp->rx_buf_page_order); +#endif //ENABLE_PAGE_REUSE +} + +static void +rtl8127_set_rms(struct rtl8127_private *tp, u16 rms) +{ + RTL_W16(tp, RxMaxSize, rms | AcceppVlanPhys); +} + +static void rtl8127_free_irq(struct rtl8127_private *tp) +{ + int i; + + for (i=0; iirq_nvecs; i++) { + struct r8127_irq *irq = &tp->irq_tbl[i]; + struct r8127_napi *r8127napi = &tp->r8127napi[i]; + + if (irq->requested) { + irq->requested = 0; +#if defined(RTL_USE_NEW_INTR_API) + pci_free_irq(tp->pci_dev, i, r8127napi); +#else + free_irq(irq->vector, r8127napi); +#endif + } + } +} + +static int rtl8127_alloc_irq(struct rtl8127_private *tp) +{ + struct net_device *dev = tp->dev; + int rc = 0; + struct r8127_irq *irq; + struct r8127_napi *r8127napi; + int i = 0; + const int len = sizeof(tp->irq_tbl[0].name); + +#if defined(RTL_USE_NEW_INTR_API) + for (i=0; iirq_nvecs; i++) { + irq = &tp->irq_tbl[i]; + if (tp->features & RTL_FEATURE_MSIX && + tp->HwCurrIsrVer > 1) + irq->handler = rtl8127_interrupt_msix; + else + irq->handler = rtl8127_interrupt; + + r8127napi = &tp->r8127napi[i]; + snprintf(irq->name, len, "%s-%d", dev->name, i); + rc = pci_request_irq(tp->pci_dev, i, irq->handler, NULL, r8127napi, + irq->name); + if (rc) + break; + + irq->vector = pci_irq_vector(tp->pci_dev, i); + irq->requested = 1; + } +#else + unsigned long irq_flags = 0; +#ifdef ENABLE_LIB_SUPPORT + irq_flags |= IRQF_NO_SUSPEND; +#endif + if (tp->features & RTL_FEATURE_MSIX && + tp->HwCurrIsrVer > 1) { + for (i=0; iirq_nvecs; i++) { + irq = &tp->irq_tbl[i]; + irq->handler = rtl8127_interrupt_msix; + r8127napi = &tp->r8127napi[i]; + snprintf(irq->name, len, "%s-%d", dev->name, i); + rc = request_irq(irq->vector, irq->handler, irq_flags, irq->name, r8127napi); + + if (rc) + break; + + irq->requested = 1; + } + } else { + irq = &tp->irq_tbl[0]; + irq->handler = rtl8127_interrupt; + r8127napi = &tp->r8127napi[0]; + snprintf(irq->name, len, "%s-0", dev->name); + if (!(tp->features & RTL_FEATURE_MSIX)) + irq->vector = dev->irq; + irq_flags |= (tp->features & (RTL_FEATURE_MSI | RTL_FEATURE_MSIX)) ? 0 : SA_SHIRQ; + rc = request_irq(irq->vector, irq->handler, irq_flags, irq->name, r8127napi); + + if (rc == 0) + irq->requested = 1; + } +#endif + if (rc) + rtl8127_free_irq(tp); + + return rc; +} + +static int rtl8127_alloc_tx_desc(struct rtl8127_private *tp) +{ + struct rtl8127_tx_ring *ring; + struct pci_dev *pdev = tp->pci_dev; + int i; + + for (i = 0; i < tp->num_tx_rings; i++) { + ring = &tp->tx_ring[i]; + ring->TxDescAllocSize = (ring->num_tx_desc + 1) * sizeof(struct TxDesc); + ring->TxDescArray = dma_alloc_coherent(&pdev->dev, + ring->TxDescAllocSize, + &ring->TxPhyAddr, + GFP_KERNEL); + + if (!ring->TxDescArray) + return -1; + } + + return 0; +} + +static int rtl8127_alloc_rx_desc(struct rtl8127_private *tp) +{ + struct rtl8127_rx_ring *ring; + struct pci_dev *pdev = tp->pci_dev; + int i; + + for (i = 0; i < tp->num_rx_rings; i++) { + ring = &tp->rx_ring[i]; + ring->RxDescAllocSize = (ring->num_rx_desc + 1) * tp->RxDescLength; + ring->RxDescArray = dma_alloc_coherent(&pdev->dev, + ring->RxDescAllocSize, + &ring->RxPhyAddr, + GFP_KERNEL); + + if (!ring->RxDescArray) + return -1; + } + + return 0; +} + +static void rtl8127_free_tx_desc(struct rtl8127_private *tp) +{ + struct rtl8127_tx_ring *ring; + struct pci_dev *pdev = tp->pci_dev; + int i; + + for (i = 0; i < tp->num_tx_rings; i++) { + ring = &tp->tx_ring[i]; + if (ring->TxDescArray) { + dma_free_coherent(&pdev->dev, + ring->TxDescAllocSize, + ring->TxDescArray, + ring->TxPhyAddr); + ring->TxDescArray = NULL; + } + } +} + +static void rtl8127_free_rx_desc(struct rtl8127_private *tp) +{ + struct rtl8127_rx_ring *ring; + struct pci_dev *pdev = tp->pci_dev; + int i; + + for (i = 0; i < tp->num_rx_rings; i++) { + ring = &tp->rx_ring[i]; + if (ring->RxDescArray) { + dma_free_coherent(&pdev->dev, + ring->RxDescAllocSize, + ring->RxDescArray, + ring->RxPhyAddr); + ring->RxDescArray = NULL; + } + } +} + +static void rtl8127_free_alloc_resources(struct rtl8127_private *tp) +{ + rtl8127_free_rx_desc(tp); + + rtl8127_free_tx_desc(tp); +} + +#ifdef ENABLE_USE_FIRMWARE_FILE +static void rtl8127_request_firmware(struct rtl8127_private *tp) +{ + struct rtl8127_fw *rtl_fw; + + /* firmware loaded already or no firmware available */ + if (tp->rtl_fw || !tp->fw_name) + return; + + rtl_fw = kzalloc(sizeof(*rtl_fw), GFP_KERNEL); + if (!rtl_fw) + return; + + rtl_fw->phy_write = rtl8127_mdio_write; + rtl_fw->phy_read = rtl8127_mdio_read; + rtl_fw->mac_mcu_write = mac_mcu_write; + rtl_fw->mac_mcu_read = mac_mcu_read; + rtl_fw->fw_name = tp->fw_name; + rtl_fw->dev = tp_to_dev(tp); + + if (rtl8127_fw_request_firmware(rtl_fw)) + kfree(rtl_fw); + else + tp->rtl_fw = rtl_fw; +} +#endif + +int rtl8127_open(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int retval; + + retval = -ENOMEM; + +#ifdef ENABLE_R8127_PROCFS + rtl8127_proc_init(dev); +#endif + rtl8127_set_rxbufsize(tp, dev); + /* + * Rx and Tx descriptors needs 256 bytes alignment. + * pci_alloc_consistent provides more. + */ + if (rtl8127_alloc_tx_desc(tp) < 0 || rtl8127_alloc_rx_desc(tp) < 0) + goto err_free_all_allocated_mem; + + retval = rtl8127_init_ring(dev); + if (retval < 0) + goto err_free_all_allocated_mem; + + retval = rtl8127_alloc_irq(tp); + if (retval < 0) + goto err_free_all_allocated_mem; + + if (netif_msg_probe(tp)) { + printk(KERN_INFO "%s: 0x%lx, " + "%2.2x:%2.2x:%2.2x:%2.2x:%2.2x:%2.2x, " + "IRQ %d\n", + dev->name, + dev->base_addr, + dev->dev_addr[0], dev->dev_addr[1], + dev->dev_addr[2], dev->dev_addr[3], + dev->dev_addr[4], dev->dev_addr[5], dev->irq); + } + +#ifdef ENABLE_USE_FIRMWARE_FILE + rtl8127_request_firmware(tp); +#endif + pci_set_master(tp->pci_dev); + +#ifdef CONFIG_R8127_NAPI + rtl8127_enable_napi(tp); +#endif + + rtl8127_exit_oob(dev); + + rtl8127_up(dev); + +#ifdef ENABLE_PTP_SUPPORT + if (tp->EnablePtp) + rtl8127_ptp_init(tp); +#endif + clear_bit(R8127_FLAG_DOWN, tp->task_flags); + + if (tp->resume_not_chg_speed) + _rtl8127_check_link_status(dev, R8127_LINK_STATE_UNKNOWN); + else + rtl8127_set_speed(dev, tp->autoneg, tp->speed, tp->duplex, tp->advertising); + + if (tp->esd_flag == 0) { + //rtl8127_request_esd_timer(dev); + + rtl8127_schedule_esd_work(tp); + } + + //rtl8127_request_link_timer(dev); + + rtl8127_enable_hw_linkchg_interrupt(tp); + +out: + + return retval; + +err_free_all_allocated_mem: + rtl8127_free_alloc_resources(tp); + + goto out; +} + +static void +_rtl8127_set_l1_l0s_entry_latency(struct rtl8127_private *tp, u8 setting) +{ + u32 csi_tmp; + u32 temp; + + temp = setting & 0x3f; + temp <<= 24; + /*set PCI configuration space offset 0x70F to setting*/ + /*When the register offset of PCI configuration space larger than 0xff, use CSI to access it.*/ + + csi_tmp = rtl8127_csi_read(tp, 0x70c) & 0xc0ffffff; + rtl8127_csi_write(tp, 0x70c, csi_tmp | temp); +} + +static void +rtl8127_set_l1_l0s_entry_latency(struct rtl8127_private *tp) +{ + _rtl8127_set_l1_l0s_entry_latency(tp, 0x27); +} + +static void +_rtl8127_set_mrrs(struct rtl8127_private *tp, u8 setting) +{ + //Set PCI configuration space offset 0x79 to setting + + struct pci_dev *pdev = tp->pci_dev; + u8 device_control; + + pci_read_config_byte(pdev, 0x79, &device_control); + device_control &= ~0x70; + device_control |= setting; + pci_write_config_byte(pdev, 0x79, device_control); +} + +static void +rtl8127_set_mrrs(struct rtl8127_private *tp) +{ + if (hwoptimize & HW_PATCH_SOC_LAN) + return; + + _rtl8127_set_mrrs(tp, 0x40); +} + +static void +rtl8127_disable_l1_timeout(struct rtl8127_private *tp) +{ + rtl8127_csi_write(tp, 0x890, rtl8127_csi_read(tp, 0x890) & ~BIT(0)); +} + +void +rtl8127_hw_set_rx_packet_filter(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u32 mc_filter[2]; /* Multicast hash filter */ + int rx_mode; + u32 tmp = 0; + + if (dev->flags & IFF_PROMISC) { + /* Unconditionally log net taps. */ + if (netif_msg_link(tp)) + printk(KERN_NOTICE "%s: Promiscuous mode enabled.\n", + dev->name); + + rx_mode = + AcceptBroadcast | AcceptMulticast | AcceptMyPhys | + AcceptAllPhys; + mc_filter[1] = mc_filter[0] = 0xffffffff; + } else if (dev->flags & IFF_ALLMULTI) { + /* accept all multicasts. */ + rx_mode = AcceptBroadcast | AcceptMulticast | AcceptMyPhys; + mc_filter[1] = mc_filter[0] = 0xffffffff; + } else { +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,35) + struct dev_mc_list *mclist; + unsigned int i; + + rx_mode = AcceptBroadcast | AcceptMyPhys; + mc_filter[1] = mc_filter[0] = 0; + for (i = 0, mclist = dev->mc_list; mclist && i < dev->mc_count; + i++, mclist = mclist->next) { + int bit_nr = ether_crc(ETH_ALEN, mclist->dmi_addr) >> 26; + mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 31); + rx_mode |= AcceptMulticast; + } +#else + struct netdev_hw_addr *ha; + + rx_mode = AcceptBroadcast | AcceptMyPhys; + mc_filter[1] = mc_filter[0] = 0; + netdev_for_each_mc_addr(ha, dev) { + int bit_nr = ether_crc(ETH_ALEN, ha->addr) >> 26; + mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 31); + rx_mode |= AcceptMulticast; + } +#endif + } + + if (dev->features & NETIF_F_RXALL) + rx_mode |= (AcceptErr | AcceptRunt); + + tmp = mc_filter[0]; + mc_filter[0] = swab32(mc_filter[1]); + mc_filter[1] = swab32(tmp); + + tmp = tp->rtl8127_rx_config | rx_mode | (RTL_R32(tp, RxConfig) & rtl_chip_info[tp->chipset].RxConfigMask); + + RTL_W32(tp, RxConfig, tmp); + RTL_W32(tp, MAR0 + 0, mc_filter[0]); + RTL_W32(tp, MAR0 + 4, mc_filter[1]); +} + +static void +rtl8127_set_rx_mode(struct net_device *dev) +{ + rtl8127_hw_set_rx_packet_filter(dev); +} + +void +rtl8127_set_rx_q_num(struct rtl8127_private *tp, + unsigned int num_rx_queues) +{ + u16 q_ctrl; + u16 rx_q_num; + + rx_q_num = (u16)ilog2(num_rx_queues); + rx_q_num &= (BIT_0 | BIT_1 | BIT_2); + rx_q_num <<= 2; + q_ctrl = RTL_R16(tp, Q_NUM_CTRL_8125); + q_ctrl &= ~(BIT_2 | BIT_3 | BIT_4); + q_ctrl |= rx_q_num; + RTL_W16(tp, Q_NUM_CTRL_8125, q_ctrl); +} + +void +rtl8127_set_tx_q_num(struct rtl8127_private *tp, + unsigned int num_tx_queues) +{ + u16 mac_ocp_data; + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xE63E); + mac_ocp_data &= ~(BIT_11 | BIT_10); + mac_ocp_data |= ((ilog2(num_tx_queues) & 0x03) << 10); + rtl8127_mac_ocp_write(tp, 0xE63E, mac_ocp_data); +} + +void +rtl8127_enable_mcu(struct rtl8127_private *tp, bool enable) +{ + if (FALSE == HW_SUPPORT_MAC_MCU(tp)) + return; + + if (enable) + rtl8127_set_mac_ocp_bit(tp, 0xC0B4, BIT_0); + else + rtl8127_clear_mac_ocp_bit(tp, 0xC0B4, BIT_0); +} + +static void +rtl8127_clear_tcam_entries(struct rtl8127_private *tp) +{ + if (FALSE == HW_SUPPORT_TCAM(tp)) + return; + + rtl8127_set_mac_ocp_bit(tp, 0xEB54, BIT_0); + fsleep(1); + rtl8127_clear_mac_ocp_bit(tp, 0xEB54, BIT_0); +} + +static u8 +rtl8127_get_l1off_cap_bits(struct rtl8127_private *tp) +{ + u8 l1offCapBits = 0; + + l1offCapBits = (BIT_0 | BIT_1); + l1offCapBits |= (BIT_2 | BIT_3); + + return l1offCapBits; +} + +void +rtl8127_hw_config(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + struct pci_dev *pdev = tp->pci_dev; + u16 mac_ocp_data; + + rtl8127_disable_rx_packet_filter(tp); + + rtl8127_hw_reset(dev); + + rtl8127_enable_cfg9346_write(tp); + rtl8127_enable_force_clkreq(tp, 0); + rtl8127_enable_aspm_clkreq_lock(tp, 0); + + rtl8127_set_eee_lpi_timer(tp); + + //keep magic packet only + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xC0B6); + mac_ocp_data &= BIT_0; + rtl8127_mac_ocp_write(tp, 0xC0B6, mac_ocp_data); + + rtl8127_tally_counter_addr_fill(tp); + + rtl8127_enable_extend_tally_couter(tp); + + rtl8127_desc_addr_fill(tp); + + /* Set DMA burst size and Interframe Gap Time */ + RTL_W32(tp, TxConfig, (TX_DMA_BURST_unlimited << TxDMAShift) | + (InterFrameGap << TxInterFrameGapShift)); + + if (tp->EnableTxNoClose) + RTL_W32(tp, TxConfig, (RTL_R32(tp, TxConfig) | BIT_6)); + + if (enable_double_vlan) + rtl8127_enable_double_vlan(tp); + else + rtl8127_disable_double_vlan(tp); + + rtl8127_set_l1_l0s_entry_latency(tp); + + rtl8127_set_mrrs(tp); + + rtl8127_disable_l1_timeout(tp); + +#ifdef ENABLE_RSS_SUPPORT + rtl8127_config_rss(tp); +#else + RTL_W32(tp, RSS_CTRL_8125, 0x00); +#endif + rtl8127_set_rx_q_num(tp, rtl8127_tot_rx_rings(tp)); + + RTL_W8(tp, Config1, RTL_R8(tp, Config1) & ~0x10); + + rtl8127_mac_ocp_write(tp, 0xC140, 0xFFFF); + rtl8127_mac_ocp_write(tp, 0xC142, 0xFFFF); + + //new tx desc format + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xEB58); + mac_ocp_data &= ~(BIT_0 | BIT_1); + mac_ocp_data |= (BIT_0); + rtl8127_mac_ocp_write(tp, 0xEB58, mac_ocp_data); + + if (tp->EnableTxNoClose) + RTL_W8(tp, 0x20E4, RTL_R8(tp, 0x20E4) | BIT_2); + else + RTL_W8(tp, 0x20E4, RTL_R8(tp, 0x20E4) & ~BIT_2); + + if (tp->HwSuppRxDescType == RX_DESC_RING_TYPE_4) { + if (tp->InitRxDescType == RX_DESC_RING_TYPE_4) + RTL_W8(tp, 0xd8, RTL_R8(tp, 0xd8) | + EnableRxDescV4_0); + else + RTL_W8(tp, 0xd8, RTL_R8(tp, 0xd8) & + ~EnableRxDescV4_0); + } + + if (tp->mcfg == CFG_METHOD_2) { + rtl8127_clear_mac_ocp_bit(tp, 0xE00C, BIT_12); + + rtl8127_clear_mac_ocp_bit(tp, 0xC0C2, BIT_6); + } + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xE614); + mac_ocp_data &= ~(BIT_11 | BIT_10 | BIT_9 | BIT_8); + mac_ocp_data |= (15 << 8); + rtl8127_mac_ocp_write(tp, 0xE614, mac_ocp_data); + + rtl8127_set_tx_q_num(tp, rtl8127_tot_tx_rings(tp)); + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xE63E); + mac_ocp_data &= ~(BIT_5 | BIT_4); + mac_ocp_data |= ((0x02 & 0x03) << 4); + rtl8127_mac_ocp_write(tp, 0xE63E, mac_ocp_data); + + rtl8127_enable_mcu(tp, 0); + rtl8127_enable_mcu(tp, 1); + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xC0B4); + mac_ocp_data |= (BIT_3 | BIT_2); + rtl8127_mac_ocp_write(tp, 0xC0B4, mac_ocp_data); + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xEB6A); + mac_ocp_data &= ~(BIT_7 | BIT_6 | BIT_5 | BIT_4 | BIT_3 | BIT_2 | BIT_1 | BIT_0); + mac_ocp_data |= (BIT_5 | BIT_4 | BIT_1 | BIT_0); + rtl8127_mac_ocp_write(tp, 0xEB6A, mac_ocp_data); + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xEB50); + mac_ocp_data &= ~(BIT_9 | BIT_8 | BIT_7 | BIT_6 | BIT_5); + mac_ocp_data |= (BIT_6); + rtl8127_mac_ocp_write(tp, 0xEB50, mac_ocp_data); + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xE056); + mac_ocp_data &= ~(BIT_7 | BIT_6 | BIT_5 | BIT_4); + //mac_ocp_data |= (BIT_4 | BIT_5); + rtl8127_mac_ocp_write(tp, 0xE056, mac_ocp_data); + + RTL_W8(tp, TDFNR, 0x10); + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xE040); + mac_ocp_data &= ~(BIT_12); + rtl8127_mac_ocp_write(tp, 0xE040, mac_ocp_data); + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xEA1C); + mac_ocp_data &= ~(BIT_1 | BIT_0); + mac_ocp_data |= (BIT_0); + rtl8127_mac_ocp_write(tp, 0xEA1C, mac_ocp_data); + + rtl8127_mac_ocp_write(tp, 0xE0C0, 0x4000); + + rtl8127_set_mac_ocp_bit(tp, 0xE052, (BIT_6 | BIT_5)); + rtl8127_clear_mac_ocp_bit(tp, 0xE052, BIT_3 | BIT_7); + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xD430); + mac_ocp_data &= ~(BIT_11 | BIT_10 | BIT_9 | BIT_8 | BIT_7 | BIT_6 | BIT_5 | BIT_4 | BIT_3 | BIT_2 | BIT_1 | BIT_0); + mac_ocp_data |= 0x45F; + rtl8127_mac_ocp_write(tp, 0xD430, mac_ocp_data); + + //rtl8127_mac_ocp_write(tp, 0xE0C0, 0x4F87); + if (!tp->DASH) + RTL_W8(tp, 0xD0, RTL_R8(tp, 0xD0) | BIT_6 | BIT_7); + else + RTL_W8(tp, 0xD0, RTL_R8(tp, 0xD0) & ~(BIT_6 | BIT_7)); + + rtl8127_disable_eee_plus(tp); + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xEA1C); + mac_ocp_data &= ~(BIT_2); + mac_ocp_data &= ~(BIT_9 | BIT_8); + rtl8127_mac_ocp_write(tp, 0xEA1C, mac_ocp_data); + + rtl8127_clear_tcam_entries(tp); + + RTL_W16(tp, 0x1880, RTL_R16(tp, 0x1880) & ~(BIT_4 | BIT_5)); + + rtl8127_clear_set_mac_ocp_bit(tp, 0xD40C, 0xE038, 0x8020); + + /* csum offload command for RTL8125 */ + tp->tx_tcp_csum_cmd = TxTCPCS_C; + tp->tx_udp_csum_cmd = TxUDPCS_C; + tp->tx_ip_csum_cmd = TxIPCS_C; + tp->tx_ipv6_csum_cmd = TxIPV6F_C; + + /* config interrupt type for RTL8125B */ + if (tp->HwSuppIsrVer > 1) + rtl8127_hw_set_interrupt_type(tp, tp->HwCurrIsrVer); + + //other hw parameters + rtl8127_hw_clear_timer_int(dev); + + rtl8127_hw_clear_int_miti(dev); + + if (tp->use_timer_interrupt && + (tp->HwCurrIsrVer > 1) && + (tp->HwSuppIntMitiVer > 3) && + (tp->features & RTL_FEATURE_MSIX)) { + int i; + for (i = 0; i < tp->irq_nvecs; i++) + rtl8127_hw_set_timer_int(tp, i, timer_count_v2); + } + + rtl8127_enable_exit_l1_mask(tp); + + rtl8127_mac_ocp_write(tp, 0xE098, 0xC302); + + if (aspm && (tp->org_pci_offset_99 & (BIT_2 | BIT_5 | BIT_6))) + rtl8127_init_pci_offset_99(tp); + else + rtl8127_disable_pci_offset_99(tp); + + if (aspm && (tp->org_pci_offset_180 & rtl8127_get_l1off_cap_bits(tp))) + rtl8127_init_pci_offset_180(tp); + else + rtl8127_disable_pci_offset_180(tp); + + tp->cp_cmd &= ~(EnableBist | Macdbgo_oe | Force_halfdup | + Force_rxflow_en | Force_txflow_en | Cxpl_dbg_sel | + ASF | Macdbgo_sel); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + RTL_W16(tp, CPlusCmd, tp->cp_cmd); +#else + rtl8127_hw_set_features(dev, dev->features); +#endif + rtl8127_set_rms(tp, tp->rms); + + rtl8127_disable_rxdvgate(dev); + + if (!tp->pci_cfg_is_read) { + pci_read_config_byte(pdev, PCI_COMMAND, &tp->pci_cfg_space.cmd); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_0, &tp->pci_cfg_space.io_base_l); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_0 + 2, &tp->pci_cfg_space.io_base_h); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_2, &tp->pci_cfg_space.mem_base_l); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_2 + 2, &tp->pci_cfg_space.mem_base_h); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_3, &tp->pci_cfg_space.resv_0x1c_l); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_3 + 2, &tp->pci_cfg_space.resv_0x1c_h); + pci_read_config_byte(pdev, PCI_INTERRUPT_LINE, &tp->pci_cfg_space.ilr); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_4, &tp->pci_cfg_space.resv_0x20_l); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_4 + 2, &tp->pci_cfg_space.resv_0x20_h); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_5, &tp->pci_cfg_space.resv_0x24_l); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_5 + 2, &tp->pci_cfg_space.resv_0x24_h); + pci_read_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID, &tp->pci_cfg_space.resv_0x2c_l); + pci_read_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID + 2, &tp->pci_cfg_space.resv_0x2c_h); + if (tp->HwPcieSNOffset > 0) { + tp->pci_cfg_space.pci_sn_l = rtl8127_csi_read(tp, tp->HwPcieSNOffset); + tp->pci_cfg_space.pci_sn_h = rtl8127_csi_read(tp, tp->HwPcieSNOffset + 4); + } + + tp->pci_cfg_is_read = 1; + } + + /* Set Rx packet filter */ + rtl8127_hw_set_rx_packet_filter(dev); + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH && !tp->dash_printer_enabled) + NICChkTypeEnableDashInterrupt(tp); +#endif + + rtl8127_enable_aspm_clkreq_lock(tp, aspm ? 1 : 0); + + rtl8127_disable_cfg9346_write(tp); + + fsleep(10); +} + +void +rtl8127_hw_start(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + +#ifdef ENABLE_LIB_SUPPORT + rtl8127_init_lib_ring(tp); +#endif + + RTL_W8(tp, ChipCmd, CmdTxEnb | CmdRxEnb); + + rtl8127_enable_hw_interrupt(tp); + + rtl8127_lib_reset_complete(tp); +} + +static int +rtl8127_change_mtu(struct net_device *dev, + int new_mtu) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int ret = 0; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,10,0) + if (new_mtu < ETH_MIN_MTU) + return -EINVAL; + else if (new_mtu > tp->max_jumbo_frame_size) + new_mtu = tp->max_jumbo_frame_size; +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,10,0) + + dev->mtu = new_mtu; + + tp->eee.tx_lpi_timer = dev->mtu + ETH_HLEN + 0x20; + + if (!netif_running(dev)) + goto out; + + rtl8127_down(dev); + + rtl8127_set_rxbufsize(tp, dev); + + ret = rtl8127_init_ring(dev); + + if (ret < 0) + goto err_out; + +#ifdef CONFIG_R8127_NAPI + rtl8127_enable_napi(tp); +#endif//CONFIG_R8127_NAPI + + if (tp->link_ok(dev)) + rtl8127_link_on_patch(dev); + else + rtl8127_link_down_patch(dev); + + //mod_timer(&tp->esd_timer, jiffies + RTL8127_ESD_TIMEOUT); + //mod_timer(&tp->link_timer, jiffies + RTL8127_LINK_TIMEOUT); +out: +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0) + netdev_update_features(dev); +#endif + +err_out: + return ret; +} + +static inline void +rtl8127_set_desc_dma_addr(struct rtl8127_private *tp, + struct RxDesc *desc, + dma_addr_t mapping) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + ((struct RxDescV3 *)desc)->addr = cpu_to_le64(mapping); + break; + case RX_DESC_RING_TYPE_4: + ((struct RxDescV4 *)desc)->addr = cpu_to_le64(mapping); + break; + default: + desc->addr = cpu_to_le64(mapping); + break; + } +} + +static inline void +rtl8127_mark_to_asic_v1(struct RxDesc *desc, + u32 rx_buf_sz) +{ + u32 eor = le32_to_cpu(desc->opts1) & RingEnd; + + WRITE_ONCE(desc->opts1, cpu_to_le32(DescOwn | eor | rx_buf_sz)); +} + +static inline void +rtl8127_mark_to_asic_v3(struct RxDescV3 *descv3, + u32 rx_buf_sz) +{ + u32 eor = le32_to_cpu(descv3->RxDescNormalDDWord4.opts1) & RingEnd; + + WRITE_ONCE(descv3->RxDescNormalDDWord4.opts1, cpu_to_le32(DescOwn | eor | rx_buf_sz)); +} + +static inline void +rtl8127_mark_to_asic_v4(struct RxDescV4 *descv4, + u32 rx_buf_sz) +{ + u32 eor = le32_to_cpu(descv4->RxDescNormalDDWord2.opts1) & RingEnd; + + WRITE_ONCE(descv4->RxDescNormalDDWord2.opts1, cpu_to_le32(DescOwn | eor | rx_buf_sz)); +} + +void +rtl8127_mark_to_asic(struct rtl8127_private *tp, + struct RxDesc *desc, + u32 rx_buf_sz) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + rtl8127_mark_to_asic_v3((struct RxDescV3 *)desc, rx_buf_sz); + break; + case RX_DESC_RING_TYPE_4: + rtl8127_mark_to_asic_v4((struct RxDescV4 *)desc, rx_buf_sz); + break; + default: + rtl8127_mark_to_asic_v1(desc, rx_buf_sz); + break; + } +} + +static inline void +rtl8127_map_to_asic(struct rtl8127_private *tp, + struct rtl8127_rx_ring *ring, + struct RxDesc *desc, + dma_addr_t mapping, + u32 rx_buf_sz, + const u32 cur_rx) +{ + ring->RxDescPhyAddr[cur_rx] = mapping; + rtl8127_set_desc_dma_addr(tp, desc, mapping); + wmb(); + rtl8127_mark_to_asic(tp, desc, rx_buf_sz); +} + +#ifdef ENABLE_PAGE_REUSE + +static int +rtl8127_alloc_rx_page(struct rtl8127_private *tp, struct rtl8127_rx_ring *ring, + struct rtl8127_rx_buffer *rxb) +{ + struct page *page; + dma_addr_t dma; + unsigned int order = tp->rx_buf_page_order; + + //get free page + page = dev_alloc_pages(order); + + if (unlikely(!page)) + return -ENOMEM; + + dma = dma_map_page_attrs(&tp->pci_dev->dev, page, 0, + tp->rx_buf_page_size, + DMA_FROM_DEVICE, + (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)); + + if (unlikely(dma_mapping_error(&tp->pci_dev->dev, dma))) { + __free_pages(page, order); + return -ENOMEM; + } + + rxb->page = page; + rxb->data = page_address(page); + rxb->page_offset = ring->rx_offset; + rxb->dma = dma; + + //after page alloc, page refcount already = 1 + + return 0; +} + +static void +rtl8127_free_rx_page(struct rtl8127_private *tp, struct rtl8127_rx_buffer *rxb) +{ + if (!rxb->page) + return; + + dma_unmap_page_attrs(&tp->pci_dev->dev, rxb->dma, + tp->rx_buf_page_size, + DMA_FROM_DEVICE, + (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)); + __free_pages(rxb->page, tp->rx_buf_page_order); + rxb->page = NULL; +} + +static void +_rtl8127_rx_clear(struct rtl8127_private *tp, struct rtl8127_rx_ring *ring) +{ + int i; + struct rtl8127_rx_buffer *rxb; + + for (i = 0; i < ring->num_rx_desc; i++) { + rxb = &ring->rx_buffer[i]; + if (rxb->skb) { + dev_kfree_skb(rxb->skb); + rxb->skb = NULL; + } + rtl8127_free_rx_page(tp, rxb); + } +} + +static u32 +rtl8127_rx_fill(struct rtl8127_private *tp, + struct rtl8127_rx_ring *ring, + struct net_device *dev, + u32 start, + u32 end, + u8 in_intr) +{ + u32 cur; + struct rtl8127_rx_buffer *rxb; + + for (cur = start; end - cur > 0; cur++) { + int ret, i = cur % ring->num_rx_desc; + + rxb = &ring->rx_buffer[i]; + if (rxb->page) + continue; + + ret = rtl8127_alloc_rx_page(tp, ring, rxb); + if (ret) + break; + + dma_sync_single_range_for_device(tp_to_dev(tp), + rxb->dma, + rxb->page_offset, + tp->rx_buf_sz, + DMA_FROM_DEVICE); + + rtl8127_map_to_asic(tp, ring, + rtl8127_get_rxdesc(tp, ring->RxDescArray, i), + rxb->dma + rxb->page_offset, + tp->rx_buf_sz, i); + } + return cur - start; +} + +#else //ENABLE_PAGE_REUSE + +static void +rtl8127_free_rx_skb(struct rtl8127_private *tp, + struct rtl8127_rx_ring *ring, + struct sk_buff **sk_buff, + struct RxDesc *desc, + const u32 cur_rx) +{ + struct pci_dev *pdev = tp->pci_dev; + + dma_unmap_single(&pdev->dev, ring->RxDescPhyAddr[cur_rx], tp->rx_buf_sz, + DMA_FROM_DEVICE); + dev_kfree_skb(*sk_buff); + *sk_buff = NULL; + rtl8127_make_unusable_by_asic(tp, desc); +} + +static int +rtl8127_alloc_rx_skb(struct rtl8127_private *tp, + struct rtl8127_rx_ring *ring, + struct sk_buff **sk_buff, + struct RxDesc *desc, + int rx_buf_sz, + const u32 cur_rx, + u8 in_intr) +{ + struct sk_buff *skb; + dma_addr_t mapping; + int ret = 0; + + if (in_intr) + skb = RTL_ALLOC_SKB_INTR(&tp->r8127napi[ring->index].napi, rx_buf_sz + R8127_RX_ALIGN); + else + skb = dev_alloc_skb(rx_buf_sz + R8127_RX_ALIGN); + + if (unlikely(!skb)) + goto err_out; + + if (!in_intr || !R8127_USE_NAPI_ALLOC_SKB) + skb_reserve(skb, R8127_RX_ALIGN); + + mapping = dma_map_single(tp_to_dev(tp), skb->data, rx_buf_sz, + DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(tp_to_dev(tp), mapping))) { + if (unlikely(net_ratelimit())) + netif_err(tp, drv, tp->dev, "Failed to map RX DMA!\n"); + goto err_out; + } + + *sk_buff = skb; + rtl8127_map_to_asic(tp, ring, desc, mapping, rx_buf_sz, cur_rx); +out: + return ret; + +err_out: + if (skb) + dev_kfree_skb(skb); + ret = -ENOMEM; + rtl8127_make_unusable_by_asic(tp, desc); + goto out; +} + +static void +_rtl8127_rx_clear(struct rtl8127_private *tp, struct rtl8127_rx_ring *ring) +{ + int i; + + for (i = 0; i < ring->num_rx_desc; i++) { + if (ring->Rx_skbuff[i]) { + rtl8127_free_rx_skb(tp, + ring, + ring->Rx_skbuff + i, + rtl8127_get_rxdesc(tp, ring->RxDescArray, i), + i); + ring->Rx_skbuff[i] = NULL; + } + } +} + +static u32 +rtl8127_rx_fill(struct rtl8127_private *tp, + struct rtl8127_rx_ring *ring, + struct net_device *dev, + u32 start, + u32 end, + u8 in_intr) +{ + u32 cur; + + for (cur = start; end - cur > 0; cur++) { + int ret, i = cur % ring->num_rx_desc; + + if (ring->Rx_skbuff[i]) + continue; + + ret = rtl8127_alloc_rx_skb(tp, + ring, + ring->Rx_skbuff + i, + rtl8127_get_rxdesc(tp, ring->RxDescArray, i), + tp->rx_buf_sz, + i, + in_intr); + if (ret < 0) + break; + } + return cur - start; +} + +#endif //ENABLE_PAGE_REUSE + +void +rtl8127_rx_clear(struct rtl8127_private *tp) +{ + int i; + + for (i = 0; i < tp->num_rx_rings; i++) { + struct rtl8127_rx_ring *ring = &tp->rx_ring[i]; + + _rtl8127_rx_clear(tp, ring); + } +} + +static void +rtl8127_mark_as_last_descriptor_v1(struct RxDesc *desc) +{ + desc->opts1 |= cpu_to_le32(RingEnd); +} + +static void +rtl8127_mark_as_last_descriptor_v3(struct RxDescV3 *descv3) +{ + descv3->RxDescNormalDDWord4.opts1 |= cpu_to_le32(RingEnd); +} + +static void +rtl8127_mark_as_last_descriptor_v4(struct RxDescV4 *descv4) +{ + descv4->RxDescNormalDDWord2.opts1 |= cpu_to_le32(RingEnd); +} + +void +rtl8127_mark_as_last_descriptor(struct rtl8127_private *tp, + struct RxDesc *desc) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + rtl8127_mark_as_last_descriptor_v3((struct RxDescV3 *)desc); + break; + case RX_DESC_RING_TYPE_4: + rtl8127_mark_as_last_descriptor_v4((struct RxDescV4 *)desc); + break; + default: + rtl8127_mark_as_last_descriptor_v1(desc); + break; + } +} + +static void +rtl8127_desc_addr_fill(struct rtl8127_private *tp) +{ + int i; + + for (i = 0; i < tp->num_tx_rings; i++) { + struct rtl8127_tx_ring *ring = &tp->tx_ring[i]; + RTL_W32(tp, ring->tdsar_reg, ((u64)ring->TxPhyAddr & DMA_BIT_MASK(32))); + RTL_W32(tp, ring->tdsar_reg + 4, ((u64)ring->TxPhyAddr >> 32)); + } + + for (i = 0; i < tp->num_rx_rings; i++) { + struct rtl8127_rx_ring *ring = &tp->rx_ring[i]; + RTL_W32(tp, ring->rdsar_reg, ((u64)ring->RxPhyAddr & DMA_BIT_MASK(32))); + RTL_W32(tp, ring->rdsar_reg + 4, ((u64)ring->RxPhyAddr >> 32)); + } +} + +static void +rtl8127_tx_desc_init(struct rtl8127_private *tp) +{ + int i = 0; + + for (i = 0; i < tp->num_tx_rings; i++) { + struct rtl8127_tx_ring *ring = &tp->tx_ring[i]; + memset(ring->TxDescArray, 0x0, ring->TxDescAllocSize); + + ring->TxDescArray[ring->num_tx_desc - 1].opts1 = cpu_to_le32(RingEnd); + } +} + +static void +rtl8127_rx_desc_init(struct rtl8127_private *tp) +{ + int i; + + for (i = 0; i < tp->num_rx_rings; i++) { + struct rtl8127_rx_ring *ring = &tp->rx_ring[i]; + memset(ring->RxDescArray, 0x0, ring->RxDescAllocSize); + } +} + +int +rtl8127_init_ring(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i; + + rtl8127_init_ring_indexes(tp); + + rtl8127_tx_desc_init(tp); + rtl8127_rx_desc_init(tp); + + for (i = 0; i < tp->num_tx_rings; i++) { + struct rtl8127_tx_ring *ring = &tp->tx_ring[i]; + memset(ring->tx_skb, 0x0, sizeof(ring->tx_skb)); + } + + for (i = 0; i < tp->num_rx_rings; i++) { + struct rtl8127_rx_ring *ring = &tp->rx_ring[i]; +#ifdef ENABLE_PAGE_REUSE + ring->rx_offset = R8127_RX_ALIGN; +#else + memset(ring->Rx_skbuff, 0x0, sizeof(ring->Rx_skbuff)); +#endif //ENABLE_PAGE_REUSE + if (rtl8127_rx_fill(tp, ring, dev, 0, ring->num_rx_desc, 0) != ring->num_rx_desc) + goto err_out; + + rtl8127_mark_as_last_descriptor(tp, rtl8127_get_rxdesc(tp, ring->RxDescArray, ring->num_rx_desc - 1)); + } + + return 0; + +err_out: + rtl8127_rx_clear(tp); + return -ENOMEM; +} + +static void +rtl8127_unmap_tx_skb(struct pci_dev *pdev, + struct ring_info *tx_skb, + struct TxDesc *desc) +{ + unsigned int len = tx_skb->len; + + dma_unmap_single(&pdev->dev, le64_to_cpu(desc->addr), len, DMA_TO_DEVICE); + + desc->opts1 = cpu_to_le32(RTK_MAGIC_DEBUG_VALUE); + desc->opts2 = 0x00; + desc->addr = RTL8127_MAGIC_NUMBER; + tx_skb->len = 0; +} + +static void +rtl8127_tx_clear_range(struct rtl8127_private *tp, + struct rtl8127_tx_ring *ring, + u32 start, + unsigned int n) +{ + unsigned int i; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,22) + struct net_device *dev = tp->dev; +#endif + + for (i = 0; i < n; i++) { + unsigned int entry = (start + i) % ring->num_tx_desc; + struct ring_info *tx_skb = ring->tx_skb + entry; + unsigned int len = tx_skb->len; + + if (len) { + struct sk_buff *skb = tx_skb->skb; + + rtl8127_unmap_tx_skb(tp->pci_dev, tx_skb, + ring->TxDescArray + entry); + if (skb) { + RTLDEV->stats.tx_dropped++; + dev_kfree_skb_any(skb); + tx_skb->skb = NULL; + } + } + } +} + +void +rtl8127_tx_clear(struct rtl8127_private *tp) +{ + int i; + + for (i = 0; i < tp->num_tx_rings; i++) { + struct rtl8127_tx_ring *ring = &tp->tx_ring[i]; + rtl8127_tx_clear_range(tp, ring, ring->dirty_tx, ring->num_tx_desc); + ring->cur_tx = ring->dirty_tx = 0; + } +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) +static void rtl8127_schedule_reset_work(struct rtl8127_private *tp) +{ +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + set_bit(R8127_FLAG_TASK_RESET_PENDING, tp->task_flags); + schedule_delayed_work(&tp->reset_task, 4); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) +} + +static void rtl8127_schedule_esd_work(struct rtl8127_private *tp) +{ +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + set_bit(R8127_FLAG_TASK_ESD_CHECK_PENDING, tp->task_flags); + schedule_delayed_work(&tp->esd_task, RTL8127_ESD_TIMEOUT); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) +} + +static void rtl8127_schedule_linkchg_work(struct rtl8127_private *tp) +{ +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + set_bit(R8127_FLAG_TASK_LINKCHG_CHECK_PENDING, tp->task_flags); + schedule_delayed_work(&tp->linkchg_task, 4); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) +} + +#define rtl8127_cancel_schedule_reset_work(a) +#define rtl8127_cancel_schedule_esd_work(a) +#define rtl8127_cancel_schedule_linkchg_work(a) + +#else +static void rtl8127_schedule_reset_work(struct rtl8127_private *tp) +{ + set_bit(R8127_FLAG_TASK_RESET_PENDING, tp->task_flags); + schedule_delayed_work(&tp->reset_task, 4); +} + +static void rtl8127_cancel_schedule_reset_work(struct rtl8127_private *tp) +{ + struct work_struct *work = &tp->reset_task.work; + + if (!work->func) + return; + + cancel_delayed_work_sync(&tp->reset_task); +} + +static void rtl8127_schedule_esd_work(struct rtl8127_private *tp) +{ + set_bit(R8127_FLAG_TASK_ESD_CHECK_PENDING, tp->task_flags); + schedule_delayed_work(&tp->esd_task, RTL8127_ESD_TIMEOUT); +} + +static void rtl8127_cancel_schedule_esd_work(struct rtl8127_private *tp) +{ + struct work_struct *work = &tp->esd_task.work; + + if (!work->func) + return; + + cancel_delayed_work_sync(&tp->esd_task); +} + +static void rtl8127_schedule_linkchg_work(struct rtl8127_private *tp) +{ + set_bit(R8127_FLAG_TASK_LINKCHG_CHECK_PENDING, tp->task_flags); + schedule_delayed_work(&tp->linkchg_task, 4); +} + +static void rtl8127_cancel_schedule_linkchg_work(struct rtl8127_private *tp) +{ + struct work_struct *work = &tp->linkchg_task.work; + + if (!work->func) + return; + + cancel_delayed_work_sync(&tp->linkchg_task); +} +#endif + +static void rtl8127_init_all_schedule_work(struct rtl8127_private *tp) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) + INIT_WORK(&tp->reset_task, rtl8127_reset_task, dev); + INIT_WORK(&tp->esd_task, rtl8127_esd_task, dev); + INIT_WORK(&tp->linkchg_task, rtl8127_linkchg_task, dev); +#else + INIT_DELAYED_WORK(&tp->reset_task, rtl8127_reset_task); + INIT_DELAYED_WORK(&tp->esd_task, rtl8127_esd_task); + INIT_DELAYED_WORK(&tp->linkchg_task, rtl8127_linkchg_task); +#endif +} + +static void rtl8127_cancel_all_schedule_work(struct rtl8127_private *tp) +{ + rtl8127_cancel_schedule_reset_work(tp); + rtl8127_cancel_schedule_esd_work(tp); + rtl8127_cancel_schedule_linkchg_work(tp); +} + +static void +rtl8127_wait_for_irq_complete(struct rtl8127_private *tp) +{ + if (tp->features & RTL_FEATURE_MSIX) { + int i; + for (i = 0; i < tp->irq_nvecs; i++) + synchronize_irq(tp->irq_tbl[i].vector); + } else { + synchronize_irq(tp->dev->irq); + } +} + +void +_rtl8127_wait_for_quiescence(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + /* Wait for any pending NAPI task to complete */ +#ifdef CONFIG_R8127_NAPI + rtl8127_disable_napi(tp); +#endif//CONFIG_R8127_NAPI + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,5,67) + /* Give a racing hard_start_xmit a few cycles to complete. */ + synchronize_net(); +#endif + + rtl8127_irq_mask_and_ack(tp); + + rtl8127_wait_for_irq_complete(tp); +} + +static void +rtl8127_wait_for_quiescence(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + //suppress unused variable + (void)(tp); + + _rtl8127_wait_for_quiescence(dev); + +#ifdef CONFIG_R8127_NAPI + rtl8127_enable_napi(tp); +#endif//CONFIG_R8127_NAPI +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) +static void rtl8127_reset_task(void *_data) +{ + struct net_device *dev = _data; + struct rtl8127_private *tp = netdev_priv(dev); +#else +static void rtl8127_reset_task(struct work_struct *work) +{ + struct rtl8127_private *tp = + container_of(work, struct rtl8127_private, reset_task.work); + struct net_device *dev = tp->dev; +#endif + int i; + + rtnl_lock(); + + if (!netif_running(dev) || + test_bit(R8127_FLAG_DOWN, tp->task_flags) || + !test_and_clear_bit(R8127_FLAG_TASK_RESET_PENDING, tp->task_flags)) + goto out_unlock; + + netdev_err(dev, "Device reseting!\n"); + + netif_carrier_off(dev); + netif_tx_disable(dev); + _rtl8127_wait_for_quiescence(dev); + rtl8127_hw_reset(dev); + + rtl8127_tx_clear(tp); + + rtl8127_init_ring_indexes(tp); + + rtl8127_tx_desc_init(tp); + for (i = 0; i < tp->num_rx_rings; i++) { + struct rtl8127_rx_ring *ring; + u32 entry; + + ring = &tp->rx_ring[i]; + for (entry = 0; entry < ring->num_rx_desc; entry++) { + struct RxDesc *desc; + + desc = rtl8127_get_rxdesc(tp, ring->RxDescArray, entry); + rtl8127_mark_to_asic(tp, desc, tp->rx_buf_sz); + } + } + +#ifdef ENABLE_PTP_SUPPORT + rtl8127_ptp_reset(tp); +#endif + +#ifdef CONFIG_R8127_NAPI + rtl8127_enable_napi(tp); +#endif //CONFIG_R8127_NAPI + + if (tp->resume_not_chg_speed) { + _rtl8127_check_link_status(dev, R8127_LINK_STATE_UNKNOWN); + + tp->resume_not_chg_speed = 0; + } else { + rtl8127_enable_hw_linkchg_interrupt(tp); + + rtl8127_set_speed(dev, tp->autoneg, tp->speed, tp->duplex, tp->advertising); + } + +out_unlock: + rtnl_unlock(); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) +static void rtl8127_esd_task(void *_data) +{ + struct net_device *dev = _data; + struct rtl8127_private *tp = netdev_priv(dev); +#else +static void rtl8127_esd_task(struct work_struct *work) +{ + struct rtl8127_private *tp = + container_of(work, struct rtl8127_private, esd_task.work); + struct net_device *dev = tp->dev; +#endif + rtnl_lock(); + + if (!netif_running(dev) || + test_bit(R8127_FLAG_DOWN, tp->task_flags) || + !test_and_clear_bit(R8127_FLAG_TASK_ESD_CHECK_PENDING, tp->task_flags)) + goto out_unlock; + + rtl8127_esd_checker(tp); + + rtl8127_schedule_esd_work(tp); + +out_unlock: + rtnl_unlock(); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) +static void rtl8127_linkchg_task(void *_data) +{ + struct net_device *dev = _data; + //struct rtl8127_private *tp = netdev_priv(dev); +#else +static void rtl8127_linkchg_task(struct work_struct *work) +{ + struct rtl8127_private *tp = + container_of(work, struct rtl8127_private, linkchg_task.work); + struct net_device *dev = tp->dev; +#endif + rtnl_lock(); + + if (!netif_running(dev) || + test_bit(R8127_FLAG_DOWN, tp->task_flags) || + !test_and_clear_bit(R8127_FLAG_TASK_LINKCHG_CHECK_PENDING, tp->task_flags)) + goto out_unlock; + + rtl8127_check_link_status(dev); + +out_unlock: + rtnl_unlock(); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0) +static void +rtl8127_tx_timeout(struct net_device *dev, unsigned int txqueue) +#else +static void +rtl8127_tx_timeout(struct net_device *dev) +#endif +{ + struct rtl8127_private *tp = netdev_priv(dev); + + netdev_err(dev, "Transmit timeout reset Device!\n"); + + /* Let's wait a bit while any (async) irq lands on */ + rtl8127_schedule_reset_work(tp); +} + +static u32 +rtl8127_get_txd_opts1(struct rtl8127_tx_ring *ring, + u32 opts1, + u32 len, + unsigned int entry) +{ + u32 status = opts1 | len; + + if (entry == ring->num_tx_desc - 1) + status |= RingEnd; + + return status; +} + +static int +rtl8127_xmit_frags(struct rtl8127_private *tp, + struct rtl8127_tx_ring *ring, + struct sk_buff *skb, + const u32 *opts) +{ + struct skb_shared_info *info = skb_shinfo(skb); + unsigned int cur_frag, entry; + struct TxDesc *txd = NULL; + const unsigned char nr_frags = info->nr_frags; + unsigned long PktLenCnt = 0; + bool LsoPatchEnabled = FALSE; + + entry = ring->cur_tx; + for (cur_frag = 0; cur_frag < nr_frags; cur_frag++) { + skb_frag_t *frag = info->frags + cur_frag; + dma_addr_t mapping; + u32 status, len; + void *addr; + + entry = (entry + 1) % ring->num_tx_desc; + + txd = ring->TxDescArray + entry; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,2,0) + len = frag->size; + addr = ((void *) page_address(frag->page)) + frag->page_offset; +#else + len = skb_frag_size(frag); + addr = skb_frag_address(frag); +#endif + mapping = dma_map_single(tp_to_dev(tp), addr, len, DMA_TO_DEVICE); + + if (unlikely(dma_mapping_error(tp_to_dev(tp), mapping))) { + if (unlikely(net_ratelimit())) + netif_err(tp, drv, tp->dev, + "Failed to map TX fragments DMA!\n"); + goto err_out; + } + + /* anti gcc 2.95.3 bugware (sic) */ + status = rtl8127_get_txd_opts1(ring, opts[0], len, entry); + if (cur_frag == (nr_frags - 1) || LsoPatchEnabled == TRUE) + status |= LastFrag; + + txd->addr = cpu_to_le64(mapping); + + ring->tx_skb[entry].len = len; + + txd->opts2 = cpu_to_le32(opts[1]); + wmb(); + txd->opts1 = cpu_to_le32(status); + + PktLenCnt += len; + } + + return cur_frag; + +err_out: + rtl8127_tx_clear_range(tp, ring, ring->cur_tx + 1, cur_frag); + return -EIO; +} + +static inline +__be16 get_protocol(struct sk_buff *skb) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,37) + return vlan_get_protocol(skb); +#else + __be16 protocol; + + if (skb->protocol == htons(ETH_P_8021Q)) + protocol = vlan_eth_hdr(skb)->h_vlan_encapsulated_proto; + else + protocol = skb->protocol; + + return protocol; +#endif +} + +static inline +u8 rtl8127_get_l4_protocol(struct sk_buff *skb) +{ + int no = skb_network_offset(skb); + struct ipv6hdr *i6h, _i6h; + struct iphdr *ih, _ih; + u8 ip_protocol = IPPROTO_RAW; + + switch (get_protocol(skb)) { + case __constant_htons(ETH_P_IP): + ih = skb_header_pointer(skb, no, sizeof(_ih), &_ih); + if (ih) + ip_protocol = ih->protocol; + break; + case __constant_htons(ETH_P_IPV6): + i6h = skb_header_pointer(skb, no, sizeof(_i6h), &_i6h); + if (i6h) + ip_protocol = i6h->nexthdr; + break; + } + + return ip_protocol; +} + +static bool rtl8127_skb_pad_with_len(struct sk_buff *skb, unsigned int len) +{ + if (skb_padto(skb, len)) + return false; + skb_put(skb, len - skb->len); + return true; +} + +static bool rtl8127_skb_pad(struct sk_buff *skb) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,19,0) + return rtl8127_skb_pad_with_len(skb, ETH_ZLEN); +#else + return !eth_skb_pad(skb); +#endif +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0) +/* msdn_giant_send_check() + * According to the document of microsoft, the TCP Pseudo Header excludes the + * packet length for IPv6 TCP large packets. + */ +static int msdn_giant_send_check(struct sk_buff *skb) +{ + const struct ipv6hdr *ipv6h; + struct tcphdr *th; + int ret; + + ret = skb_cow_head(skb, 0); + if (ret) + return ret; + + ipv6h = ipv6_hdr(skb); + th = tcp_hdr(skb); + + th->check = 0; + th->check = ~tcp_v6_check(0, &ipv6h->saddr, &ipv6h->daddr, 0); + + return ret; +} +#endif + +static bool rtl8127_require_pad_ptp_pkt(struct rtl8127_private *tp) +{ + return false; +} + +#define MIN_PATCH_LEN (47) +static u32 +rtl8127_get_patch_pad_len(struct rtl8127_private *tp, + struct sk_buff *skb) +{ + u32 pad_len = 0; + int trans_data_len; + u32 hdr_len; + u32 pkt_len = skb->len; + u8 ip_protocol; + bool has_trans = skb_transport_header_was_set(skb); + + if (!rtl8127_require_pad_ptp_pkt(tp)) + goto no_padding; + + if (!(has_trans && (pkt_len < 175))) //128 + MIN_PATCH_LEN + goto no_padding; + + ip_protocol = rtl8127_get_l4_protocol(skb); + if (!(ip_protocol == IPPROTO_TCP || ip_protocol == IPPROTO_UDP)) + goto no_padding; + + trans_data_len = pkt_len - + (skb->transport_header - + skb_headroom(skb)); + if (ip_protocol == IPPROTO_UDP) { + if (trans_data_len > 3 && trans_data_len < MIN_PATCH_LEN) { + u16 dest_port = 0; + + skb_copy_bits(skb, skb->transport_header - skb_headroom(skb) + 2, &dest_port, 2); + dest_port = ntohs(dest_port); + + if (dest_port == 0x13f || + dest_port == 0x140) { + pad_len = MIN_PATCH_LEN - trans_data_len; + goto out; + } + } + } + + hdr_len = 0; + if (ip_protocol == IPPROTO_TCP) + hdr_len = 20; + else if (ip_protocol == IPPROTO_UDP) + hdr_len = 8; + if (trans_data_len < hdr_len) + pad_len = hdr_len - trans_data_len; + +out: + if ((pkt_len + pad_len) < ETH_ZLEN) + pad_len = ETH_ZLEN - pkt_len; + + return pad_len; + +no_padding: + + return 0; +} + +static bool +rtl8127_tso_csum(struct sk_buff *skb, + struct net_device *dev, + u32 *opts, + unsigned int *bytecount, + unsigned short *gso_segs) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned long large_send = 0; + u32 csum_cmd = 0; + u8 sw_calc_csum = false; + u8 check_patch_required = true; + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (dev->features & (NETIF_F_TSO | NETIF_F_TSO6)) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) + u32 mss = skb_shinfo(skb)->tso_size; +#else + u32 mss = skb_shinfo(skb)->gso_size; +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) + + /* TCP Segmentation Offload (or TCP Large Send) */ + if (mss) { + union { + struct iphdr *v4; + struct ipv6hdr *v6; + unsigned char *hdr; + } ip; + union { + struct tcphdr *tcp; + struct udphdr *udp; + unsigned char *hdr; + } l4; + u32 l4_offset, hdr_len; + + ip.hdr = skb_network_header(skb); + l4.hdr = skb_checksum_start(skb); + + l4_offset = skb_transport_offset(skb); + assert((l4_offset%2) == 0); + switch (get_protocol(skb)) { + case __constant_htons(ETH_P_IP): + if (l4_offset <= GTTCPHO_MAX) { + opts[0] |= GiantSendv4; + opts[0] |= l4_offset << GTTCPHO_SHIFT; + opts[1] |= min(mss, MSS_MAX) << 18; + large_send = 1; + } + break; + case __constant_htons(ETH_P_IPV6): +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0) + if (msdn_giant_send_check(skb)) + return false; +#endif + if (l4_offset <= GTTCPHO_MAX) { + opts[0] |= GiantSendv6; + opts[0] |= l4_offset << GTTCPHO_SHIFT; + opts[1] |= min(mss, MSS_MAX) << 18; + large_send = 1; + } + break; + default: + if (unlikely(net_ratelimit())) + dprintk("tso proto=%x!\n", skb->protocol); + break; + } + + if (large_send == 0) + return false; + + + /* compute length of segmentation header */ + hdr_len = (l4.tcp->doff * 4) + l4_offset; + /* update gso size and bytecount with header size */ + *gso_segs = skb_shinfo(skb)->gso_segs; + *bytecount += (*gso_segs - 1) * hdr_len; + + return true; + } + } +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + + if (skb->ip_summed == CHECKSUM_PARTIAL) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) + const struct iphdr *ip = skb->nh.iph; + + if (dev->features & NETIF_F_IP_CSUM) { + if (ip->protocol == IPPROTO_TCP) + csum_cmd = tp->tx_ip_csum_cmd | tp->tx_tcp_csum_cmd; + else if (ip->protocol == IPPROTO_UDP) + csum_cmd = tp->tx_ip_csum_cmd | tp->tx_udp_csum_cmd; + else if (ip->protocol == IPPROTO_IP) + csum_cmd = tp->tx_ip_csum_cmd; + } +#else + u8 ip_protocol = IPPROTO_RAW; + + switch (get_protocol(skb)) { + case __constant_htons(ETH_P_IP): + if (dev->features & NETIF_F_IP_CSUM) { + ip_protocol = ip_hdr(skb)->protocol; + csum_cmd = tp->tx_ip_csum_cmd; + } + break; + case __constant_htons(ETH_P_IPV6): + if (dev->features & NETIF_F_IPV6_CSUM) { + if (skb_transport_offset(skb) > 0 && skb_transport_offset(skb) <= TCPHO_MAX) { + ip_protocol = ipv6_hdr(skb)->nexthdr; + csum_cmd = tp->tx_ipv6_csum_cmd; + csum_cmd |= skb_transport_offset(skb) << TCPHO_SHIFT; + } + } + break; + default: + if (unlikely(net_ratelimit())) + dprintk("checksum_partial proto=%x!\n", skb->protocol); + break; + } + + if (ip_protocol == IPPROTO_TCP) + csum_cmd |= tp->tx_tcp_csum_cmd; + else if (ip_protocol == IPPROTO_UDP) + csum_cmd |= tp->tx_udp_csum_cmd; +#endif + if (csum_cmd == 0) { + sw_calc_csum = true; +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + WARN_ON(1); /* we need a WARN() */ +#endif + } + + if (ip_protocol == IPPROTO_TCP) + check_patch_required = false; + } + + if (check_patch_required) { + u32 pad_len = rtl8127_get_patch_pad_len(tp, skb); + + if (pad_len > 0) { + if (!rtl8127_skb_pad_with_len(skb, skb->len + pad_len)) + return false; + + if (csum_cmd != 0) + sw_calc_csum = true; + } + } + + if (skb->len < ETH_ZLEN) { + if (tp->UseSwPaddingShortPkt || + (tp->ShortPacketSwChecksum && csum_cmd != 0)) { + if (!rtl8127_skb_pad(skb)) + return false; + + if (csum_cmd != 0) + sw_calc_csum = true; + } + } + + if (sw_calc_csum) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,10) && LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,7) + skb_checksum_help(&skb, 0); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) && LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,10) + skb_checksum_help(skb, 0); +#else + skb_checksum_help(skb); +#endif + } else + opts[1] |= csum_cmd; + + return true; +} + +static bool rtl8127_tx_slots_avail(struct rtl8127_private *tp, + struct rtl8127_tx_ring *ring) +{ + unsigned int slots_avail = READ_ONCE(ring->dirty_tx) + ring->num_tx_desc + - READ_ONCE(ring->cur_tx); + + /* A skbuff with nr_frags needs nr_frags+1 entries in the tx queue */ + return slots_avail > MAX_SKB_FRAGS; +} + +static inline u32 +rtl8127_fast_mod_mask(const u32 input, const u32 mask) +{ + return input > mask ? input & mask : input; +} + +static void rtl8127_doorbell(struct rtl8127_private *tp, + struct rtl8127_tx_ring *ring) +{ + if (tp->EnableTxNoClose) { + if (tp->HwSuppTxNoCloseVer > 3) + RTL_W32(tp, ring->sw_tail_ptr_reg, ring->cur_tx); + else + RTL_W16(tp, ring->sw_tail_ptr_reg, ring->cur_tx); + } else { + /* set polling bit */ + RTL_W32(tp, TPPOLL_8125, BIT(ring->index)); + } +} + +static netdev_tx_t +rtl8127_start_xmit(struct sk_buff *skb, + struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned int bytecount; + unsigned short gso_segs; + struct ring_info *last; + unsigned int last_entry; + unsigned int entry; + struct TxDesc *txd; + dma_addr_t mapping; + u32 len; + u32 opts[2]; + netdev_tx_t ret = NETDEV_TX_OK; + int frags; + u8 EnableTxNoClose = tp->EnableTxNoClose; + const u16 queue_mapping = skb_get_queue_mapping(skb); + struct rtl8127_tx_ring *ring; + bool stop_queue; + + assert(queue_mapping < tp->num_tx_rings); + + ring = &tp->tx_ring[queue_mapping]; + + if (unlikely(!rtl8127_tx_slots_avail(tp, ring))) { + if (netif_msg_drv(tp)) { + printk(KERN_ERR + "%s: BUG! Tx Ring[%d] full when queue awake!\n", + dev->name, + queue_mapping); + } + goto err_stop; + } + + entry = ring->cur_tx % ring->num_tx_desc; + txd = ring->TxDescArray + entry; + + if (!EnableTxNoClose) { + if (unlikely(le32_to_cpu(txd->opts1) & DescOwn)) { + if (netif_msg_drv(tp)) { + printk(KERN_ERR + "%s: BUG! Tx Desc is own by hardware!\n", + dev->name); + } + goto err_stop; + } + } + + bytecount = skb->len; + gso_segs = 1; + + opts[0] = DescOwn; + opts[1] = rtl8127_tx_vlan_tag(tp, skb); + + if (unlikely(!rtl8127_tso_csum(skb, dev, opts, &bytecount, &gso_segs))) + goto err_dma_0; + + frags = rtl8127_xmit_frags(tp, ring, skb, opts); + if (unlikely(frags < 0)) + goto err_dma_0; + if (frags) { + len = skb_headlen(skb); + opts[0] |= FirstFrag; + } else { + len = skb->len; + opts[0] |= FirstFrag | LastFrag; + } + + opts[0] = rtl8127_get_txd_opts1(ring, opts[0], len, entry); + mapping = dma_map_single(tp_to_dev(tp), skb->data, len, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(tp_to_dev(tp), mapping))) { + if (unlikely(net_ratelimit())) + netif_err(tp, drv, dev, "Failed to map TX DMA!\n"); + goto err_dma_1; + } + +#ifdef ENABLE_PTP_SUPPORT + if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { + if (!test_and_set_bit_lock(__RTL8127_PTP_TX_IN_PROGRESS, &tp->state)) { + if (tp->hwtstamp_config.tx_type == HWTSTAMP_TX_ON && + !tp->ptp_tx_skb) { + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + + tp->ptp_tx_skb = skb_get(skb); + tp->ptp_tx_start = jiffies; + schedule_work(&tp->ptp_tx_work); + } else + tp->tx_hwtstamp_skipped++; + } + } +#endif + /* set first fragment's length */ + ring->tx_skb[entry].len = len; + + /* set skb to last fragment */ + last_entry = (entry + frags) % ring->num_tx_desc; + last = &ring->tx_skb[last_entry]; + last->skb = skb; + last->gso_segs = gso_segs; + last->bytecount = bytecount; + + txd->addr = cpu_to_le64(mapping); + txd->opts2 = cpu_to_le32(opts[1]); + wmb(); + txd->opts1 = cpu_to_le32(opts[0]); + + netdev_tx_sent_queue(txring_txq(ring), bytecount); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) + dev->trans_start = jiffies; +#else + skb_tx_timestamp(skb); +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) + + /* rtl_tx needs to see descriptor changes before updated tp->cur_tx */ + smp_wmb(); + + WRITE_ONCE(ring->cur_tx, ring->cur_tx + frags + 1); + + stop_queue = !rtl8127_tx_slots_avail(tp, ring); + if (unlikely(stop_queue)) { + /* Avoid wrongly optimistic queue wake-up: rtl_tx thread must + * not miss a ring update when it notices a stopped queue. + */ + smp_wmb(); + netif_stop_subqueue(dev, queue_mapping); + } + + if (netif_xmit_stopped(txring_txq(ring)) || !netdev_xmit_more()) + rtl8127_doorbell(tp, ring); + + if (unlikely(stop_queue)) { + /* Sync with rtl_tx: + * - publish queue status and cur_tx ring index (write barrier) + * - refresh dirty_tx ring index (read barrier). + * May the current thread have a pessimistic view of the ring + * status and forget to wake up queue, a racing rtl_tx thread + * can't. + */ + smp_mb(); + if (rtl8127_tx_slots_avail(tp, ring)) + netif_start_subqueue(dev, queue_mapping); + } +out: + return ret; +err_dma_1: + rtl8127_tx_clear_range(tp, ring, ring->cur_tx + 1, frags); +err_dma_0: + RTLDEV->stats.tx_dropped++; + dev_kfree_skb_any(skb); + ret = NETDEV_TX_OK; + goto out; +err_stop: + netif_stop_subqueue(dev, queue_mapping); + ret = NETDEV_TX_BUSY; + RTLDEV->stats.tx_dropped++; + goto out; +} + +/* recycle tx no close desc*/ +static int +rtl8127_tx_interrupt_noclose(struct rtl8127_tx_ring *ring, int budget) +{ + unsigned int total_bytes = 0, total_packets = 0; + struct rtl8127_private *tp = ring->priv; + struct net_device *dev = tp->dev; + unsigned int dirty_tx, tx_left; + unsigned int tx_desc_closed; + unsigned int count = 0; + + dirty_tx = ring->dirty_tx; + ring->NextHwDesCloPtr = rtl8127_get_hw_clo_ptr(ring); + tx_desc_closed = rtl8127_fast_mod_mask(ring->NextHwDesCloPtr - + ring->BeginHwDesCloPtr, + tp->MaxTxDescPtrMask); + tx_left = min((READ_ONCE(ring->cur_tx) - dirty_tx), tx_desc_closed); + ring->BeginHwDesCloPtr += tx_left; + + while (tx_left > 0) { + unsigned int entry = dirty_tx % ring->num_tx_desc; + struct ring_info *tx_skb = ring->tx_skb + entry; + + rtl8127_unmap_tx_skb(tp->pci_dev, + tx_skb, + ring->TxDescArray + entry); + + if (tx_skb->skb != NULL) { + /* update the statistics for this packet */ + total_bytes += tx_skb->bytecount; + total_packets += tx_skb->gso_segs; + + RTL_NAPI_CONSUME_SKB_ANY(tx_skb->skb, budget); + tx_skb->skb = NULL; + } + dirty_tx++; + tx_left--; + } + + if (total_packets) { + netdev_tx_completed_queue(txring_txq(ring), + total_packets, total_bytes); + + RTLDEV->stats.tx_bytes += total_bytes; + RTLDEV->stats.tx_packets+= total_packets; + } + + if (ring->dirty_tx != dirty_tx) { + count = dirty_tx - ring->dirty_tx; + WRITE_ONCE(ring->dirty_tx, dirty_tx); + smp_wmb(); + if (__netif_subqueue_stopped(dev, ring->index) && + (rtl8127_tx_slots_avail(tp, ring))) { + netif_start_subqueue(dev, ring->index); + } + } + + return count; +} + +/* recycle tx close desc*/ +static int +rtl8127_tx_interrupt_close(struct rtl8127_tx_ring *ring, int budget) +{ + unsigned int total_bytes = 0, total_packets = 0; + struct rtl8127_private *tp = ring->priv; + struct net_device *dev = tp->dev; + unsigned int dirty_tx, tx_left; + unsigned int count = 0; + + dirty_tx = ring->dirty_tx; + tx_left = READ_ONCE(ring->cur_tx) - dirty_tx; + + while (tx_left > 0) { + unsigned int entry = dirty_tx % ring->num_tx_desc; + struct ring_info *tx_skb = ring->tx_skb + entry; + + if (le32_to_cpu(READ_ONCE(ring->TxDescArray[entry].opts1)) & DescOwn) + break; + + rtl8127_unmap_tx_skb(tp->pci_dev, + tx_skb, + ring->TxDescArray + entry); + + if (tx_skb->skb != NULL) { + /* update the statistics for this packet */ + total_bytes += tx_skb->bytecount; + total_packets += tx_skb->gso_segs; + + RTL_NAPI_CONSUME_SKB_ANY(tx_skb->skb, budget); + tx_skb->skb = NULL; + } + dirty_tx++; + tx_left--; + } + + if (total_packets) { + netdev_tx_completed_queue(txring_txq(ring), + total_packets, total_bytes); + + RTLDEV->stats.tx_bytes += total_bytes; + RTLDEV->stats.tx_packets+= total_packets; + } + + if (ring->dirty_tx != dirty_tx) { + count = dirty_tx - ring->dirty_tx; + WRITE_ONCE(ring->dirty_tx, dirty_tx); + smp_wmb(); + if (__netif_subqueue_stopped(dev, ring->index) && + (rtl8127_tx_slots_avail(tp, ring))) { + netif_start_subqueue(dev, ring->index); + } + + if (READ_ONCE(ring->cur_tx) != dirty_tx) + rtl8127_doorbell(tp, ring); + } + + return count; +} + +static int +rtl8127_tx_interrupt(struct rtl8127_tx_ring *ring, int budget) +{ + struct rtl8127_private *tp = ring->priv; + + if (tp->EnableTxNoClose) + return rtl8127_tx_interrupt_noclose(ring, budget); + else + return rtl8127_tx_interrupt_close(ring, budget); +} + +static int +rtl8127_tx_interrupt_with_vector(struct rtl8127_private *tp, + const int message_id, + int budget) +{ + int count = 0; + + switch (tp->HwCurrIsrVer) { + case 3: + case 4: + if (message_id < tp->num_tx_rings) + count += rtl8127_tx_interrupt(&tp->tx_ring[message_id], budget); + break; + case 5: + if (message_id == 16) + count += rtl8127_tx_interrupt(&tp->tx_ring[0], budget); +#ifdef ENABLE_MULTIPLE_TX_QUEUE + else if (message_id == 17 && tp->num_tx_rings > 1) + count += rtl8127_tx_interrupt(&tp->tx_ring[1], budget); +#endif + break; + case 6: + if (message_id == 8) + count += rtl8127_tx_interrupt(&tp->tx_ring[0], budget); +#ifdef ENABLE_MULTIPLE_TX_QUEUE + else if (message_id == 9 && tp->num_tx_rings > 1) + count += rtl8127_tx_interrupt(&tp->tx_ring[1], budget); +#endif + break; + default: + if (message_id == 16) + count += rtl8127_tx_interrupt(&tp->tx_ring[0], budget); +#ifdef ENABLE_MULTIPLE_TX_QUEUE + else if (message_id == 18 && tp->num_tx_rings > 1) + count += rtl8127_tx_interrupt(&tp->tx_ring[1], budget); +#endif + break; + } + + return count; +} + +static inline int +rtl8127_fragmented_frame(struct rtl8127_private *tp, u32 status) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + return (status & (FirstFrag_V3 | LastFrag_V3)) != (FirstFrag_V3 | LastFrag_V3); + case RX_DESC_RING_TYPE_4: + return (status & (FirstFrag_V4 | LastFrag_V4)) != (FirstFrag_V4 | LastFrag_V4); + default: + return (status & (FirstFrag | LastFrag)) != (FirstFrag | LastFrag); + } +} + +static inline int +rtl8127_is_non_eop(struct rtl8127_private *tp, u32 status) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + return !(status & LastFrag_V3); + case RX_DESC_RING_TYPE_4: + return !(status & LastFrag_V4); + default: + return !(status & LastFrag); + } +} + +static inline int +rtl8127_rx_desc_type(u32 status) +{ + return ((status >> 26) & 0x0F); +} + +static inline void +rtl8127_rx_v1_csum(struct rtl8127_private *tp, + struct sk_buff *skb, + struct RxDesc *desc) +{ + u32 opts1 = le32_to_cpu(desc->opts1); + + if (((opts1 & RxTCPT) && !(opts1 & RxTCPF)) || + ((opts1 & RxUDPT) && !(opts1 & RxUDPF))) + skb->ip_summed = CHECKSUM_UNNECESSARY; + else + skb_checksum_none_assert(skb); +} + +static inline void +rtl8127_rx_v3_csum(struct rtl8127_private *tp, + struct sk_buff *skb, + struct RxDescV3 *descv3) +{ + u32 opts2 = le32_to_cpu(descv3->RxDescNormalDDWord4.opts2); + + /* rx csum offload for RTL8125 */ + if (((opts2 & RxTCPT_v3) && !(opts2 & RxTCPF_v3)) || + ((opts2 & RxUDPT_v3) && !(opts2 & RxUDPF_v3))) + skb->ip_summed = CHECKSUM_UNNECESSARY; + else + skb_checksum_none_assert(skb); +} + +static inline void +rtl8127_rx_v4_csum(struct rtl8127_private *tp, + struct sk_buff *skb, + struct RxDescV4 *descv4) +{ + u32 opts1 = le32_to_cpu(descv4->RxDescNormalDDWord2.opts1); + + /* rx csum offload for RTL8125 */ + if (((opts1 & RxTCPT_v4) && !(opts1 & RxTCPF_v4)) || + ((opts1 & RxUDPT_v4) && !(opts1 & RxUDPF_v4))) + skb->ip_summed = CHECKSUM_UNNECESSARY; + else + skb_checksum_none_assert(skb); +} + +static inline void +rtl8127_rx_csum(struct rtl8127_private *tp, + struct sk_buff *skb, + struct RxDesc *desc) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + rtl8127_rx_v3_csum(tp, skb, (struct RxDescV3 *)desc); + break; + case RX_DESC_RING_TYPE_4: + rtl8127_rx_v4_csum(tp, skb, (struct RxDescV4 *)desc); + break; + default: + rtl8127_rx_v1_csum(tp, skb, desc); + break; + } +} + +/* +static inline int +rtl8127_try_rx_copy(struct rtl8127_private *tp, + struct rtl8127_rx_ring *ring, + struct sk_buff **sk_buff, + int pkt_size, + struct RxDesc *desc, + int rx_buf_sz) +{ + int ret = -1; + + struct sk_buff *skb; + + skb = RTL_ALLOC_SKB_INTR(&tp->r8127napi[ring->index].napi, pkt_size + R8127_RX_ALIGN); + if (skb) { + u8 *data; + + data = sk_buff[0]->data; + if (!R8127_USE_NAPI_ALLOC_SKB) + skb_reserve(skb, R8127_RX_ALIGN); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,37) + prefetch(data - R8127_RX_ALIGN); +#endif + eth_copy_and_sum(skb, data, pkt_size, 0); + *sk_buff = skb; + rtl8127_mark_to_asic(tp, desc, rx_buf_sz); + ret = 0; + } + + return ret; +} +*/ + +static inline void +rtl8127_rx_skb(struct rtl8127_private *tp, + struct sk_buff *skb, + u32 ring_index) +{ +#ifdef CONFIG_R8127_NAPI +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29) + netif_receive_skb(skb); +#else + napi_gro_receive(&tp->r8127napi[ring_index].napi, skb); +#endif +#else + netif_rx(skb); +#endif +} + +static int +rtl8127_check_rx_desc_error(struct net_device *dev, + struct rtl8127_private *tp, + u32 status) +{ + int ret = 0; + + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + if (unlikely(status & RxRES_V3)) { + if (status & (RxRWT_V3 | RxRUNT_V3)) + RTLDEV->stats.rx_length_errors++; + if (status & RxCRC_V3) + RTLDEV->stats.rx_crc_errors++; + + ret = -1; + } + break; + case RX_DESC_RING_TYPE_4: + if (unlikely(status & RxRES_V4)) { + if (status & RxRUNT_V4) + RTLDEV->stats.rx_length_errors++; + if (status & RxCRC_V4) + RTLDEV->stats.rx_crc_errors++; + + ret = -1; + } + break; + default: + if (unlikely(status & RxRES)) { + if (status & (RxRWT | RxRUNT)) + RTLDEV->stats.rx_length_errors++; + if (status & RxCRC) + RTLDEV->stats.rx_crc_errors++; + + ret = -1; + } + break; + } + + return ret; +} + +#ifdef ENABLE_PAGE_REUSE + +static inline bool +rtl8127_reuse_rx_ok(struct page *page) +{ + /* avoid re-using remote pages */ + if (!dev_page_is_reusable(page)) { + //printk(KERN_INFO "r8127 page pfmemalloc, can't reuse!\n"); + return false; + } + /* if we are only owner of page we can reuse it */ + if (unlikely(page_ref_count(page) != 1)) { + //printk(KERN_INFO "r8127 page refcnt %d, can't reuse!\n", page_ref_count(page)); + return false; + } + + return true; +} + +static void +rtl8127_reuse_rx_buffer(struct rtl8127_private *tp, struct rtl8127_rx_ring *ring, u32 cur_rx, struct rtl8127_rx_buffer *rxb) +{ + struct page *page = rxb->page; + + u32 dirty_rx = ring->dirty_rx; + u32 entry = dirty_rx % ring->num_rx_desc; + struct rtl8127_rx_buffer *nrxb = &ring->rx_buffer[entry]; + + u32 noffset; + + //the page gonna be shared by us and kernel, keep page ref = 2 + page_ref_inc(page); + + //flip the buffer in page to use next + noffset = rxb->page_offset ^ (tp->rx_buf_page_size / 2); //one page, two buffer, ping-pong + + nrxb->dma = rxb->dma; + nrxb->page_offset = noffset; + nrxb->data = rxb->data; + + if (cur_rx != dirty_rx) { + //move the buffer to other slot + nrxb->page = page; + rxb->page = NULL; + } +} + +static void rtl8127_put_rx_buffer(struct rtl8127_private *tp, + struct rtl8127_rx_ring *ring, + u32 cur_rx, + struct rtl8127_rx_buffer *rxb) +{ + struct rtl8127_rx_buffer *nrxb; + struct page *page = rxb->page; + u32 entry; + + entry = ring->dirty_rx % ring->num_rx_desc; + nrxb = &ring->rx_buffer[entry]; + if (likely(rtl8127_reuse_rx_ok(page))) { + /* hand second half of page back to the ring */ + rtl8127_reuse_rx_buffer(tp, ring, cur_rx, rxb); + } else { + tp->page_reuse_fail_cnt++; + + dma_unmap_page_attrs(&tp->pci_dev->dev, rxb->dma, + tp->rx_buf_page_size, + DMA_FROM_DEVICE, + (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)); + //the page ref is kept 1, uniquely owned by kernel now + rxb->page = NULL; + + return; + } + + dma_sync_single_range_for_device(tp_to_dev(tp), + nrxb->dma, + nrxb->page_offset, + tp->rx_buf_sz, + DMA_FROM_DEVICE); + + rtl8127_map_to_asic(tp, ring, + rtl8127_get_rxdesc(tp, ring->RxDescArray, entry), + nrxb->dma + nrxb->page_offset, + tp->rx_buf_sz, entry); + + ring->dirty_rx++; +} + +#endif //ENABLE_PAGE_REUSE + +static int +rtl8127_rx_interrupt(struct net_device *dev, + struct rtl8127_private *tp, + struct rtl8127_rx_ring *ring, + napi_budget budget) +{ + unsigned int cur_rx, rx_left; + unsigned int delta, count = 0; + unsigned int entry; + struct RxDesc *desc; + struct sk_buff *skb; + u32 status; + u32 rx_quota; + u32 ring_index = ring->index; +#ifdef ENABLE_PAGE_REUSE + struct rtl8127_rx_buffer *rxb; +#else //ENABLE_PAGE_REUSE + u64 rx_buf_phy_addr; +#endif //ENABLE_PAGE_REUSE + unsigned int total_rx_multicast_packets = 0; + unsigned int total_rx_bytes = 0, total_rx_packets = 0; + + assert(dev != NULL); + assert(tp != NULL); + + if (ring->RxDescArray == NULL) + goto rx_out; + + rx_quota = RTL_RX_QUOTA(budget); + cur_rx = ring->cur_rx; + rx_left = ring->num_rx_desc + ring->dirty_rx - cur_rx; + rx_left = rtl8127_rx_quota(rx_left, (u32)rx_quota); + + for (; rx_left > 0; rx_left--, cur_rx++) { +#ifndef ENABLE_PAGE_REUSE + const void *rx_buf; +#endif //!ENABLE_PAGE_REUSE + u32 pkt_size; + + entry = cur_rx % ring->num_rx_desc; + desc = rtl8127_get_rxdesc(tp, ring->RxDescArray, entry); + status = le32_to_cpu(rtl8127_rx_desc_opts1(tp, desc)); + if (status & DescOwn) { + RTL_R8(tp, tp->imr_reg[0]); + status = le32_to_cpu(rtl8127_rx_desc_opts1(tp, desc)); + if (status & DescOwn) + break; + } + + rmb(); + + if (unlikely(rtl8127_check_rx_desc_error(dev, tp, status) < 0)) { + if (netif_msg_rx_err(tp)) { + printk(KERN_INFO + "%s: Rx ERROR. status = %08x\n", + dev->name, status); + } + + RTLDEV->stats.rx_errors++; + + if (!(dev->features & NETIF_F_RXALL)) + goto release_descriptor; + } + pkt_size = status & 0x00003fff; + if (likely(!(dev->features & NETIF_F_RXFCS))) { +#ifdef ENABLE_RX_PACKET_FRAGMENT + if (rtl8127_is_non_eop(tp, status) && + pkt_size == tp->rx_buf_sz) { + struct RxDesc *desc_next; + unsigned int entry_next; + int pkt_size_next; + u32 status_next; + + entry_next = (cur_rx + 1) % ring->num_rx_desc; + desc_next = rtl8127_get_rxdesc(tp, ring->RxDescArray, entry_next); + status_next = le32_to_cpu(rtl8127_rx_desc_opts1(tp, desc_next)); + if (!(status_next & DescOwn)) { + pkt_size_next = status_next & 0x00003fff; + if (pkt_size_next < ETH_FCS_LEN) + pkt_size -= (ETH_FCS_LEN - pkt_size_next); + } + } +#endif //ENABLE_RX_PACKET_FRAGMENT + if (!rtl8127_is_non_eop(tp, status)) { + if (pkt_size < ETH_FCS_LEN) { +#ifdef ENABLE_RX_PACKET_FRAGMENT + pkt_size = 0; +#else + goto drop_packet; +#endif //ENABLE_RX_PACKET_FRAGMENT + } else + pkt_size -= ETH_FCS_LEN; + } + } + + if (unlikely(pkt_size > tp->rx_buf_sz)) + goto drop_packet; + +#if !defined(ENABLE_RX_PACKET_FRAGMENT) || !defined(ENABLE_PAGE_REUSE) + /* + * The driver does not support incoming fragmented + * frames. They are seen as a symptom of over-mtu + * sized frames. + */ + if (unlikely(rtl8127_fragmented_frame(tp, status))) + goto drop_packet; +#endif //!ENABLE_RX_PACKET_FRAGMENT || !ENABLE_PAGE_REUSE + +#ifdef ENABLE_PAGE_REUSE + rxb = &ring->rx_buffer[entry]; + skb = rxb->skb; + rxb->skb = NULL; + if (!skb) { + skb = RTL_BUILD_SKB_INTR(rxb->data + rxb->page_offset - ring->rx_offset, tp->rx_buf_page_size / 2); + if (!skb) { + //netdev_err(tp->dev, "Failed to allocate RX skb!\n"); + goto drop_packet; + } + + skb->dev = dev; + if (!R8127_USE_NAPI_ALLOC_SKB) + skb_reserve(skb, R8127_RX_ALIGN); + skb_put(skb, pkt_size); +#ifdef ENABLE_RSS_SUPPORT + rtl8127_rx_hash(tp, desc, skb); +#endif + rtl8127_rx_csum(tp, skb, desc); + } else + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rxb->page, + rxb->page_offset, pkt_size, tp->rx_buf_page_size / 2); + //recycle desc + rtl8127_put_rx_buffer(tp, ring, cur_rx, rxb); + + dma_sync_single_range_for_cpu(tp_to_dev(tp), + rxb->dma, + rxb->page_offset, + tp->rx_buf_sz, + DMA_FROM_DEVICE); +#else //ENABLE_PAGE_REUSE + skb = RTL_ALLOC_SKB_INTR(&tp->r8127napi[ring->index].napi, pkt_size + R8127_RX_ALIGN); + if (!skb) { + //netdev_err(tp->dev, "Failed to allocate RX skb!\n"); + goto drop_packet; + } + + skb->dev = dev; + if (!R8127_USE_NAPI_ALLOC_SKB) + skb_reserve(skb, R8127_RX_ALIGN); + skb_put(skb, pkt_size); + + rx_buf_phy_addr = ring->RxDescPhyAddr[entry]; + dma_sync_single_for_cpu(tp_to_dev(tp), + rx_buf_phy_addr, tp->rx_buf_sz, + DMA_FROM_DEVICE); + rx_buf = ring->Rx_skbuff[entry]->data; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,37) + prefetch(rx_buf - R8127_RX_ALIGN); +#endif + eth_copy_and_sum(skb, rx_buf, pkt_size, 0); + + dma_sync_single_for_device(tp_to_dev(tp), rx_buf_phy_addr, + tp->rx_buf_sz, DMA_FROM_DEVICE); +#endif //ENABLE_PAGE_REUSE + +#ifdef ENABLE_PTP_SUPPORT + if (tp->flags & RTL_FLAG_RX_HWTSTAMP_ENABLED) + rtl8127_rx_ptp_timestamp(tp, skb); +#endif // ENABLE_PTP_SUPPORT + +#ifdef ENABLE_RX_PACKET_FRAGMENT + if (rtl8127_is_non_eop(tp, status)) { + unsigned int entry_next; + entry_next = (entry + 1) % ring->num_rx_desc; + rxb = &ring->rx_buffer[entry_next]; + rxb->skb = skb; + continue; + } +#endif //ENABLE_RX_PACKET_FRAGMENT + +#ifndef ENABLE_PAGE_REUSE +#ifdef ENABLE_RSS_SUPPORT + rtl8127_rx_hash(tp, desc, skb); +#endif + rtl8127_rx_csum(tp, skb, desc); +#endif /* !ENABLE_PAGE_REUSE */ + + skb->protocol = eth_type_trans(skb, dev); + + total_rx_bytes += skb->len; + + if (skb->pkt_type == PACKET_MULTICAST) + total_rx_multicast_packets++; + + if (rtl8127_rx_vlan_skb(tp, desc, skb) < 0) + rtl8127_rx_skb(tp, skb, ring_index); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,11,0) + dev->last_rx = jiffies; +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,11,0) + total_rx_packets++; + +#ifdef ENABLE_PAGE_REUSE + rxb->skb = NULL; + continue; +#endif + +release_descriptor: + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + case RX_DESC_RING_TYPE_4: + rtl8127_set_desc_dma_addr(tp, desc, + ring->RxDescPhyAddr[entry]); + wmb(); + break; + } + rtl8127_mark_to_asic(tp, desc, tp->rx_buf_sz); + continue; +drop_packet: + RTLDEV->stats.rx_dropped++; + RTLDEV->stats.rx_length_errors++; + goto release_descriptor; + } + + count = cur_rx - ring->cur_rx; + ring->cur_rx = cur_rx; + + delta = rtl8127_rx_fill(tp, ring, dev, ring->dirty_rx, ring->cur_rx, 1); + if (!delta && count && netif_msg_intr(tp)) + printk(KERN_INFO "%s: no Rx buffer allocated\n", dev->name); + ring->dirty_rx += delta; + + RTLDEV->stats.rx_bytes += total_rx_bytes; + RTLDEV->stats.rx_packets += total_rx_packets; + RTLDEV->stats.multicast += total_rx_multicast_packets; + + /* + * FIXME: until there is periodic timer to try and refill the ring, + * a temporary shortage may definitely kill the Rx process. + * - disable the asic to try and avoid an overflow and kick it again + * after refill ? + * - how do others driver handle this condition (Uh oh...). + */ + if ((ring->dirty_rx + ring->num_rx_desc == ring->cur_rx) && netif_msg_intr(tp)) + printk(KERN_EMERG "%s: Rx buffers exhausted\n", dev->name); + +rx_out: + return total_rx_packets; +} + +static bool +rtl8127_linkchg_interrupt(struct rtl8127_private *tp, u32 status) +{ + switch (tp->HwCurrIsrVer) { + case 2: + case 3: + return status & ISRIMR_V2_LINKCHG; + case 4: + return status & ISRIMR_V4_LINKCHG; + case 5: + return status & ISRIMR_V5_LINKCHG; + case 6: + return status & ISRIMR_V6_LINKCHG; + default: + return status & LinkChg; + } +} + +static u32 +rtl8127_get_linkchg_message_id(struct rtl8127_private *tp) +{ + switch (tp->HwCurrIsrVer) { + case 4: + case 6: + return 29; + case 5: + return 18; + default: + return 21; + } +} + +/* + *The interrupt handler does all of the Rx thread work and cleans up after + *the Tx thread. + */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) +static irqreturn_t rtl8127_interrupt(int irq, void *dev_instance, struct pt_regs *regs) +#else +static irqreturn_t rtl8127_interrupt(int irq, void *dev_instance) +#endif +{ + struct r8127_napi *r8127napi = dev_instance; + struct rtl8127_private *tp = r8127napi->priv; + struct net_device *dev = tp->dev; + u32 status; + int handled = 0; + + do { + status = RTL_R32(tp, tp->isr_reg[0]); + + if (!(tp->features & (RTL_FEATURE_MSI | RTL_FEATURE_MSIX))) { + /* hotplug/major error/no more work/shared irq */ + if (!status) + break; + + if (status == 0xFFFFFFFF) + break; + + if (!(status & (tp->intr_mask | tp->timer_intr_mask))) + break; + } + + handled = 1; + +#if defined(RTL_USE_NEW_INTR_API) + if (!tp->irq_tbl[0].requested) + break; +#endif + rtl8127_disable_hw_interrupt(tp); + + RTL_W32(tp, tp->isr_reg[0], status&~RxFIFOOver); + + if (rtl8127_linkchg_interrupt(tp, status)) + rtl8127_schedule_linkchg_work(tp); + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) { + if (HW_DASH_SUPPORT_TYPE_3(tp)) { + u8 DashIntType2Status; + + if (status & ISRIMR_DASH_INTR_CMAC_RESET) + tp->CmacResetIntr = TRUE; + + DashIntType2Status = RTL_CMAC_R8(tp, CMAC_IBISR0); + if (DashIntType2Status & ISRIMR_DASH_TYPE2_ROK) + tp->RcvFwDashOkEvt = TRUE; + if (DashIntType2Status & ISRIMR_DASH_TYPE2_TOK) + tp->SendFwHostOkEvt = TRUE; + if (DashIntType2Status & ISRIMR_DASH_TYPE2_RX_DISABLE_IDLE) + tp->DashFwDisableRx = TRUE; + + RTL_CMAC_W8(tp, CMAC_IBISR0, DashIntType2Status); + } + } +#endif + +#ifdef CONFIG_R8127_NAPI + if (status & tp->intr_mask || tp->keep_intr_cnt-- > 0) { + if (status & tp->intr_mask) + tp->keep_intr_cnt = RTK_KEEP_INTERRUPT_COUNT; + + if (likely(RTL_NETIF_RX_SCHEDULE_PREP(dev, &tp->r8127napi[0].napi))) + __RTL_NETIF_RX_SCHEDULE(dev, &tp->r8127napi[0].napi); + else if (netif_msg_intr(tp)) + printk(KERN_INFO "%s: interrupt %04x in poll\n", + dev->name, status); + } else { + tp->keep_intr_cnt = RTK_KEEP_INTERRUPT_COUNT; + rtl8127_switch_to_hw_interrupt(tp); + } +#else + if (status & tp->intr_mask || tp->keep_intr_cnt-- > 0) { + u32 budget = ~(u32)0; + int i; + + if (status & tp->intr_mask) + tp->keep_intr_cnt = RTK_KEEP_INTERRUPT_COUNT; + + for (i = 0; i < tp->num_tx_rings; i++) + rtl8127_tx_interrupt(&tp->tx_ring[i], ~(u32)0); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) + rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[0], &budget); +#else + rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[0], budget); +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) { + struct net_device *dev = tp->dev; + + HandleDashInterrupt(dev); + } +#endif + + rtl8127_switch_to_timer_interrupt(tp); + } else { + tp->keep_intr_cnt = RTK_KEEP_INTERRUPT_COUNT; + rtl8127_switch_to_hw_interrupt(tp); + } +#endif + } while (false); + + return IRQ_RETVAL(handled); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) +static irqreturn_t rtl8127_interrupt_msix(int irq, void *dev_instance, struct pt_regs *regs) +#else +static irqreturn_t rtl8127_interrupt_msix(int irq, void *dev_instance) +#endif +{ + struct r8127_napi *r8127napi = dev_instance; + struct rtl8127_private *tp = r8127napi->priv; + struct net_device *dev = tp->dev; + int message_id = r8127napi->index; +#ifndef CONFIG_R8127_NAPI + u32 budget = ~(u32)0; +#endif + + do { +#if defined(RTL_USE_NEW_INTR_API) + if (!tp->irq_tbl[message_id].requested) + break; +#endif + //link change + if (message_id == rtl8127_get_linkchg_message_id(tp)) { + rtl8127_disable_hw_interrupt_v2(tp, message_id); + rtl8127_clear_hw_isr_v2(tp, message_id); + rtl8127_schedule_linkchg_work(tp); + break; + } + +#ifdef CONFIG_R8127_NAPI + if (likely(RTL_NETIF_RX_SCHEDULE_PREP(dev, &r8127napi->napi))) { + rtl8127_disable_hw_interrupt_v2(tp, message_id); + __RTL_NETIF_RX_SCHEDULE(dev, &r8127napi->napi); + } else if (netif_msg_intr(tp)) + printk(KERN_INFO "%s: interrupt message id %d in poll_msix\n", + dev->name, message_id); + rtl8127_clear_hw_isr_v2(tp, message_id); +#else + rtl8127_disable_hw_interrupt_v2(tp, message_id); + + rtl8127_clear_hw_isr_v2(tp, message_id); + + rtl8127_tx_interrupt_with_vector(tp, message_id, ~(u32)0); + + if (message_id < tp->num_rx_rings) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) + rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[message_id], &budget); +#else + rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[message_id], budget); +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) + } + + rtl8127_enable_hw_interrupt_v2(tp, message_id); +#endif + + } while (false); + + return IRQ_HANDLED; +} + +static void rtl8127_down(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + //rtl8127_delete_esd_timer(dev, &tp->esd_timer); + + //rtl8127_delete_link_timer(dev, &tp->link_timer); + + netif_carrier_off(dev); + + netif_tx_disable(dev); + + _rtl8127_wait_for_quiescence(dev); + + rtl8127_hw_reset(dev); + + rtl8127_tx_clear(tp); + + rtl8127_rx_clear(tp); +} + +static int rtl8127_resource_freed(struct rtl8127_private *tp) +{ + int i; + + for (i = 0; i < tp->num_tx_rings; i++) + if (tp->tx_ring[i].TxDescArray) + return 0; + + for (i = 0; i < tp->num_rx_rings; i++) + if (tp->rx_ring[i].RxDescArray) + return 0; + + return 1; +} + +int rtl8127_close(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (!rtl8127_resource_freed(tp)) { + set_bit(R8127_FLAG_DOWN, tp->task_flags); + + rtl8127_down(dev); + + pci_clear_master(tp->pci_dev); + +#ifdef ENABLE_PTP_SUPPORT + rtl8127_ptp_stop(tp); +#endif + rtl8127_hw_d3_para(dev); + + rtl8127_powerdown_pll(dev, 0); + + rtl8127_free_irq(tp); + + rtl8127_free_alloc_resources(tp); + } else { + rtl8127_hw_d3_para(dev); + + rtl8127_powerdown_pll(dev, 0); + } + + return 0; +} + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,11) +static void rtl8127_shutdown(struct pci_dev *pdev) +{ + struct net_device *dev = pci_get_drvdata(pdev); + struct rtl8127_private *tp = netdev_priv(dev); + + rtnl_lock(); + + if (HW_DASH_SUPPORT_DASH(tp)) + rtl8127_driver_stop(tp); + + rtl8127_disable_pci_offset_180(tp); + + if (s5_keep_curr_mac == 0 && tp->random_mac == 0) + rtl8127_rar_set(tp, tp->org_mac_addr); + + if (s5wol == 0) + tp->wol_enabled = WOL_DISABLED; + + rtl8127_close(dev); + rtl8127_disable_msi(pdev, tp); + + rtnl_unlock(); + + if (system_state == SYSTEM_POWER_OFF) { + pci_clear_master(tp->pci_dev); + pci_wake_from_d3(pdev, tp->wol_enabled); + pci_set_power_state(pdev, PCI_D3hot); + } +} +#endif + +#ifdef CONFIG_PM + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,11) +static int +rtl8127_suspend(struct pci_dev *pdev, u32 state) +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29) +static int +rtl8127_suspend(struct device *device) +#else +static int +rtl8127_suspend(struct pci_dev *pdev, pm_message_t state) +#endif +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29) + struct pci_dev *pdev = to_pci_dev(device); + struct net_device *dev = pci_get_drvdata(pdev); +#else + struct net_device *dev = pci_get_drvdata(pdev); +#endif + struct rtl8127_private *tp = netdev_priv(dev); +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,10) + u32 pci_pm_state = pci_choose_state(pdev, state); +#endif + rtnl_lock(); + + if (!netif_running(dev)) + goto out; + + set_bit(R8127_FLAG_DOWN, tp->task_flags); + + netif_carrier_off(dev); + + netif_tx_disable(dev); + + netif_device_detach(dev); + +#ifdef ENABLE_PTP_SUPPORT + rtl8127_ptp_suspend(tp); +#endif + rtl8127_hw_reset(dev); + + pci_clear_master(pdev); + + rtl8127_hw_d3_para(dev); + + rtl8127_powerdown_pll(dev, 1); +out: + if (HW_DASH_SUPPORT_DASH(tp)) + rtl8127_driver_stop(tp); + + rtnl_unlock(); + + pci_disable_device(pdev); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,10) + pci_save_state(pdev, &pci_pm_state); +#else + pci_save_state(pdev); +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29) + pci_enable_wake(pdev, pci_choose_state(pdev, state), tp->wol_enabled); +#endif + + pci_prepare_to_sleep(pdev); + + return 0; +} + +static int +rtl8127_hw_d3_not_power_off(struct net_device *dev) +{ + return rtl8127_check_hw_phy_mcu_code_ver(dev); +} + +static int rtl8127_wait_phy_nway_complete_sleep(struct rtl8127_private *tp) +{ + int i, val; + + for (i = 0; i < 30; i++) { + val = rtl8127_mdio_read(tp, MII_BMSR) & BMSR_ANEGCOMPLETE; + if (val) + return 0; + + fsleep(100000); + } + + return -1; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29) +static int +rtl8127_resume(struct pci_dev *pdev) +#else +static int +rtl8127_resume(struct device *device) +#endif +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29) + struct pci_dev *pdev = to_pci_dev(device); + struct net_device *dev = pci_get_drvdata(pdev); +#else + struct net_device *dev = pci_get_drvdata(pdev); +#endif + struct rtl8127_private *tp = netdev_priv(dev); +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,10) + u32 pci_pm_state = PCI_D0; +#endif + u32 err; + + rtnl_lock(); + + err = pci_enable_device(pdev); + if (err) { + dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); + goto out_unlock; + } +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,10) + pci_restore_state(pdev, &pci_pm_state); +#else + pci_restore_state(pdev); +#endif + pci_enable_wake(pdev, PCI_D0, 0); + + /* restore last modified mac address */ + rtl8127_rar_set(tp, dev->dev_addr); + + tp->resume_not_chg_speed = 0; + if (tp->check_keep_link_speed && + //tp->link_ok(dev) && + rtl8127_hw_d3_not_power_off(dev) && + rtl8127_wait_phy_nway_complete_sleep(tp) == 0) + tp->resume_not_chg_speed = 1; + + if (!netif_running(dev)) + goto out_unlock; + + pci_set_master(pdev); + + rtl8127_exit_oob(dev); + + rtl8127_up(dev); + + clear_bit(R8127_FLAG_DOWN, tp->task_flags); + + rtl8127_schedule_reset_work(tp); + + rtl8127_schedule_esd_work(tp); + + //mod_timer(&tp->esd_timer, jiffies + RTL8127_ESD_TIMEOUT); + //mod_timer(&tp->link_timer, jiffies + RTL8127_LINK_TIMEOUT); +out_unlock: + netif_device_attach(dev); + + rtnl_unlock(); + + return err; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29) + +static struct dev_pm_ops rtl8127_pm_ops = { + .suspend = rtl8127_suspend, + .resume = rtl8127_resume, + .freeze = rtl8127_suspend, + .thaw = rtl8127_resume, + .poweroff = rtl8127_suspend, + .restore = rtl8127_resume, +}; + +#define RTL8127_PM_OPS (&rtl8127_pm_ops) + +#endif + +#else /* !CONFIG_PM */ + +#define RTL8127_PM_OPS NULL + +#endif /* CONFIG_PM */ + +static struct pci_driver rtl8127_pci_driver = { + .name = MODULENAME, + .id_table = rtl8127_pci_tbl, + .probe = rtl8127_init_one, + .remove = __devexit_p(rtl8127_remove_one), +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,11) + .shutdown = rtl8127_shutdown, +#endif +#ifdef CONFIG_PM +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29) + .suspend = rtl8127_suspend, + .resume = rtl8127_resume, +#else + .driver.pm = RTL8127_PM_OPS, +#endif +#endif +}; + +static int __init +rtl8127_init_module(void) +{ + int ret = 0; +#ifdef ENABLE_R8127_PROCFS + rtl8127_proc_module_init(); +#endif + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + + ret = pci_register_driver(&rtl8127_pci_driver); +#else + ret = pci_module_init(&rtl8127_pci_driver); +#endif + + return ret; +} + +static void __exit +rtl8127_cleanup_module(void) +{ + pci_unregister_driver(&rtl8127_pci_driver); + +#ifdef ENABLE_R8127_PROCFS + if (rtl8127_proc) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) + remove_proc_subtree(MODULENAME, init_net.proc_net); +#else +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,32) + remove_proc_entry(MODULENAME, init_net.proc_net); +#else + remove_proc_entry(MODULENAME, proc_net); +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,32) +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) + rtl8127_proc = NULL; + } +#endif +} + +module_init(rtl8127_init_module); +module_exit(rtl8127_cleanup_module); diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127_ptp.c b/drivers/net/ethernet/realtek/r8127/src/r8127_ptp.c new file mode 100755 index 0000000000000..f3fd421625c0c --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/src/r8127_ptp.c @@ -0,0 +1,944 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "r8127.h" +#include "r8127_ptp.h" + +static void rtl8127_wait_clkadj_ready(struct rtl8127_private *tp) +{ + int i; + + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) + if (!(rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CLK_CFG_8126) & CLKADJ_MODE_SET)) + break; +} + +static void rtl8127_set_clkadj_mode(struct rtl8127_private *tp, u16 cmd) +{ + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + PTP_CLK_CFG_8126, + BIT_3 | BIT_2 | BIT_1, + CLKADJ_MODE_SET | cmd); + + rtl8127_wait_clkadj_ready(tp); +} + +static int _rtl8127_phc_gettime(struct rtl8127_private *tp, struct timespec64 *ts64) +{ + unsigned long flags; + + spin_lock_irqsave(&tp->phy_lock, flags); + + //Direct Read + rtl8127_set_clkadj_mode(tp, DIRECT_READ); + + /* nanoseconds */ + //Ns[29:16] E414[13:0] + ts64->tv_nsec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_NS_HI_8126) & 0x3fff; + ts64->tv_nsec <<= 16; + //Ns[15:0] E412[15:0] + ts64->tv_nsec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_NS_LO_8126); + + + /* seconds */ + //S[47:32] E41A[15:0] + ts64->tv_sec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_S_HI_8126); + ts64->tv_sec <<= 16; + //S[31:16] E418[15:0] + ts64->tv_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_S_MI_8126); + ts64->tv_sec <<= 16; + //S[15:0] E416[15:0] + ts64->tv_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_S_LO_8126); + + spin_unlock_irqrestore(&tp->phy_lock, flags); + + return 0; +} + +static int _rtl8127_phc_settime(struct rtl8127_private *tp, const struct timespec64 *ts64) +{ + unsigned long flags; + + spin_lock_irqsave(&tp->phy_lock, flags); + + /* nanoseconds */ + //Ns[15:0] E412[15:0] + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_NS_LO_8126, ts64->tv_nsec); + //Ns[29:16] E414[13:0] + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_NS_HI_8126, (ts64->tv_nsec & 0x3fff0000) >> 16); + + /* seconds */ + //S[15:0] E416[15:0] + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_S_LO_8126, ts64->tv_sec); + //S[31:16] E418[15:0] + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_S_MI_8126, (ts64->tv_sec >> 16)); + //S[47:32] E41A[15:0] + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_S_HI_8126, (ts64->tv_sec >> 32)); + + //Direct Write + rtl8127_set_clkadj_mode(tp, DIRECT_WRITE); + + spin_unlock_irqrestore(&tp->phy_lock, flags); + + return 0; +} + +static int _rtl8127_phc_adjtime(struct rtl8127_private *tp, s64 delta) +{ + unsigned long flags; + struct timespec64 d; + bool negative; + u64 tohw; + u32 nsec; + u64 sec; + + if (delta < 0) { + negative = true; + tohw = -delta; + } else { + negative = false; + tohw = delta; + } + + d = ns_to_timespec64(tohw); + + nsec = d.tv_nsec; + sec = d.tv_sec; + + nsec &= 0x3fffffff; + sec &= 0x0000ffffffffffff; + + spin_lock_irqsave(&tp->phy_lock, flags); + + /* nanoseconds */ + //Ns[15:0] E412[15:0] + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_NS_LO_8126, nsec); + //Ns[29:16] E414[13:0] + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_NS_HI_8126, (nsec >> 16)); + + /* seconds */ + //S[15:0] E416[15:0] + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_S_LO_8126, sec); + //S[31:16] E418[15:0] + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_S_MI_8126, (sec >> 16)); + //S[47:32] E41A[15:0] + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_S_HI_8126, (sec >> 32)); + + if (negative) + rtl8127_set_clkadj_mode(tp, DECREMENT_STEP); + else + rtl8127_set_clkadj_mode(tp, INCREMENT_STEP); + + spin_unlock_irqrestore(&tp->phy_lock, flags); + + return 0; +} + +static int rtl8127_phc_adjtime(struct ptp_clock_info *ptp, s64 delta) +{ + struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info); + int ret; + + //netif_info(tp, drv, tp->dev, "phc adjust time\n"); + + ret = _rtl8127_phc_adjtime(tp, delta); + + return ret; +} + +/* + * delta = delta * 10^6 ppm = delta * 10^9 ppb (in this equation ppm and ppb are not variable) + * + * in adjfreq ppb is a variable + * ppb = delta * 10^9 + * delta = ppb / 10^9 + * rate_value = |delta| * 2^32 = |ppb| / 10^9 * 2^32 = (|ppb| << 32) / 10^9 + */ +static int _rtl8127_phc_adjfreq(struct ptp_clock_info *ptp, s32 ppb) +{ + struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info); + unsigned long flags; + u32 rate_value; + + if (ppb < 0) { + rate_value = ((u64)-ppb << 32) / 1000000000; + rate_value = ~rate_value + 1; + } else + rate_value = ((u64)ppb << 32) / 1000000000; + + spin_lock_irqsave(&tp->phy_lock, flags); + + /* nanoseconds */ + //Ns[15:0] E412[15:0] + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_NS_LO_8126, rate_value); + //Ns[22:16] E414[13:0] + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_NS_HI_8126, (rate_value & 0x003f0000) >> 16); + + rtl8127_set_clkadj_mode(tp, RATE_WRITE); + + spin_unlock_irqrestore(&tp->phy_lock, flags); + + return 0; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,2,0) +static int rtl8127_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) +{ + s32 ppb = scaled_ppm_to_ppb(scaled_ppm); + + if (ppb > ptp->max_adj || ppb < -ptp->max_adj) + return -EINVAL; + + _rtl8127_phc_adjfreq(ptp, ppb); + + return 0; +} + +#else +static int rtl8127_phc_adjfreq(struct ptp_clock_info *ptp, s32 delta) +{ + //struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info); + + //netif_info(tp, drv, tp->dev, "phc adjust freq\n"); + + if (delta > ptp->max_adj || delta < -ptp->max_adj) + return -EINVAL; + + _rtl8127_phc_adjfreq(ptp, delta); + + return 0; +} +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6,2,0) */ + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0) +static int rtl8127_phc_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts64, + struct ptp_system_timestamp *sts) +{ + struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info); + int ret; + + //netif_info(tp, drv, tp->dev, "phc get ts\n"); + + ptp_read_system_prets(sts); + ret = _rtl8127_phc_gettime(tp, ts64); + ptp_read_system_postts(sts); + + return ret; +} +#else +static int rtl8127_phc_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts64) +{ + struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info); + int ret; + + //netif_info(tp, drv, tp->dev, "phc get ts\n"); + + ret = _rtl8127_phc_gettime(tp, ts64); + + return ret; +} +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0) */ + +static int rtl8127_phc_settime(struct ptp_clock_info *ptp, + const struct timespec64 *ts64) +{ + struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info); + int ret; + + //netif_info(tp, drv, tp->dev, "phc set ts\n"); + + ret = _rtl8127_phc_settime(tp, ts64); + + return ret; +} + +static void _rtl8127_phc_enable(struct ptp_clock_info *ptp, + struct ptp_clock_request *rq, int on) +{ + struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info); + unsigned long flags; + u16 phy_ocp_data; + + if (on) { + tp->pps_enable = 1; + rtl8127_clear_mac_ocp_bit(tp, 0xDC00, BIT_6); + rtl8127_clear_mac_ocp_bit(tp, 0xDC20, BIT_1); + + spin_lock_irqsave(&tp->phy_lock, flags); + + /* Set periodic pulse 1pps */ + /* E432[8:0] = 0x017d */ + phy_ocp_data = rtl8127_mdio_direct_read_phy_ocp(tp, 0xE432); + phy_ocp_data &= 0xFE00; + phy_ocp_data |= 0x017d; + rtl8127_mdio_direct_write_phy_ocp(tp, 0xE432, phy_ocp_data); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xE434, 0x7840); + + /* E436[8:0] = 0xbe */ + phy_ocp_data = rtl8127_mdio_direct_read_phy_ocp(tp, 0xE436); + phy_ocp_data &= 0xFE00; + phy_ocp_data |= 0xbe; + rtl8127_mdio_direct_write_phy_ocp(tp, 0xE436, phy_ocp_data); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xE438, 0xbc20); + + spin_unlock_irqrestore(&tp->phy_lock, flags); + + /* start hrtimer */ + hrtimer_start(&tp->pps_timer, 1000000000, HRTIMER_MODE_REL); + } else + tp->pps_enable = 0; +} + +static int rtl8127_phc_enable(struct ptp_clock_info *ptp, + struct ptp_clock_request *rq, int on) +{ + switch (rq->type) { + case PTP_CLK_REQ_PPS: + _rtl8127_phc_enable(ptp, rq, on); + return 0; + default: + return -EOPNOTSUPP; + } +} + +static void rtl8127_ptp_enable_config(struct rtl8127_private *tp) +{ + if (tp->syncE_en) + rtl8127_set_eth_phy_ocp_bit(tp, PTP_SYNCE_CTL, BIT_0); + else + rtl8127_clear_eth_phy_ocp_bit(tp, PTP_SYNCE_CTL, BIT_0); + + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CTL, PTP_CTL_TYPE_3 | BIT_12); + + rtl8127_set_eth_phy_ocp_bit(tp, 0xA640, BIT_15); +} + +int rtl8127_get_ts_info(struct net_device *netdev, + struct ethtool_ts_info *info) +{ + struct rtl8127_private *tp = netdev_priv(netdev); + + /* we always support timestamping disabled */ + info->rx_filters = BIT(HWTSTAMP_FILTER_NONE); + + if (tp->HwSuppPtpVer == 0) + return ethtool_op_get_ts_info(netdev, info); + + info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | + SOF_TIMESTAMPING_RX_SOFTWARE | + SOF_TIMESTAMPING_SOFTWARE | + SOF_TIMESTAMPING_TX_HARDWARE | + SOF_TIMESTAMPING_RX_HARDWARE | + SOF_TIMESTAMPING_RAW_HARDWARE; + + if (tp->ptp_clock) + info->phc_index = ptp_clock_index(tp->ptp_clock); + else + info->phc_index = -1; + + info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON); + + info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) | + BIT(HWTSTAMP_FILTER_PTP_V2_EVENT) | + BIT(HWTSTAMP_FILTER_PTP_V2_L4_EVENT) | + BIT(HWTSTAMP_FILTER_PTP_V2_SYNC) | + BIT(HWTSTAMP_FILTER_PTP_V2_L4_SYNC) | + BIT(HWTSTAMP_FILTER_PTP_V2_DELAY_REQ) | + BIT(HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ); + + return 0; +} + +static const struct ptp_clock_info rtl_ptp_clock_info = { + .owner = THIS_MODULE, + .n_alarm = 0, + .n_ext_ts = 0, + .n_per_out = 0, + .n_pins = 0, + .pps = 1, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,2,0) + .adjfine = rtl8127_ptp_adjfine, +#else + .adjfreq = rtl8127_phc_adjfreq, +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6,2,0) */ + .adjtime = rtl8127_phc_adjtime, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0) + .gettimex64 = rtl8127_phc_gettime, +#else + .gettime64 = rtl8127_phc_gettime, +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0) */ + + .settime64 = rtl8127_phc_settime, + .enable = rtl8127_phc_enable, +}; + +static u16 rtl8127_ptp_get_tx_msgtype(struct rtl8127_private *tp) +{ + u16 tx_ts_ready = 0; + int i; + + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { + tx_ts_ready = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_STA) & 0xF000; + if (tx_ts_ready) + break; + } + + switch (tx_ts_ready) { + case TX_TS_PDLYRSP_RDY: + return PTP_MSGTYPE_PDELAY_RESP; + case TX_TS_PDLYREQ_RDY: + return PTP_MSGTYPE_PDELAY_REQ; + case TX_TS_DLYREQ_RDY: + return PTP_MSGTYPE_DELAY_REQ; + case TX_TS_SYNC_RDY: + default: + return PTP_MSGTYPE_SYNC; + } +} + +/* +static u16 rtl8127_ptp_get_rx_msgtype(struct rtl8127_private *tp) +{ + u16 rx_ts_ready = 0; + int i; + + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { + rx_ts_ready = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_STA) & 0x0F00; + if (rx_ts_ready) + break; + } + + switch (rx_ts_ready) { + case RX_TS_PDLYRSP_RDY: + return PTP_MSGTYPE_PDELAY_RESP; + case RX_TS_PDLYREQ_RDY: + return PTP_MSGTYPE_PDELAY_REQ; + case RX_TS_DLYREQ_RDY: + return PTP_MSGTYPE_DELAY_REQ; + case RX_TS_SYNC_RDY: + default: + return PTP_MSGTYPE_SYNC; + } +} +*/ + +static void rtl8127_wait_trx_ts_ready(struct rtl8127_private *tp) +{ + int i; + + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) + if (!(rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_STA) & TRX_TS_RD)) + break; +} + +static void rtl8127_set_trx_ts_cmd(struct rtl8127_private *tp, u16 cmd) +{ + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + PTP_TRX_TS_STA, + TRXTS_SEL | BIT_3 | BIT_2, + TRX_TS_RD | cmd); + + rtl8127_wait_trx_ts_ready(tp); +} + +static void rtl8127_ptp_egresstime(struct rtl8127_private *tp, struct timespec64 *ts64) +{ + u16 msgtype; + + msgtype = rtl8127_ptp_get_tx_msgtype(tp); + + msgtype <<= 2; + + rtl8127_set_trx_ts_cmd(tp, (msgtype | BIT_4)); + + /* nanoseconds */ + //Ns[29:16] E448[13:0] + ts64->tv_nsec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_NS_HI) & 0x3fff; + ts64->tv_nsec <<= 16; + //Ns[15:0] E446[15:0] + ts64->tv_nsec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_NS_LO); + + /* seconds */ + //S[47:32] E44E[15:0] + ts64->tv_sec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_S_HI); + ts64->tv_sec <<= 16; + //S[31:16] E44C[15:0] + ts64->tv_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_S_MI); + ts64->tv_sec <<= 16; + //S[15:0] E44A[15:0] + ts64->tv_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_S_LO); +} + +static void rtl8127_ptp_ingresstime(struct rtl8127_private *tp, struct timespec64 *ts64, u8 type) +{ + u16 msgtype; + + switch (type) { + case PTP_MSGTYPE_PDELAY_RESP: + case PTP_MSGTYPE_PDELAY_REQ: + case PTP_MSGTYPE_DELAY_REQ: + case PTP_MSGTYPE_SYNC: + msgtype = type << 2; + break; + default: + return; + } + + rtl8127_set_trx_ts_cmd(tp, (TRXTS_SEL | msgtype | BIT_4)); + + /* nanoseconds */ + //Ns[29:16] E448[13:0] + ts64->tv_nsec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_NS_HI) & 0x3fff; + ts64->tv_nsec <<= 16; + //Ns[15:0] E446[15:0] + ts64->tv_nsec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_NS_LO); + + /* seconds */ + //S[47:32] E44E[15:0] + ts64->tv_sec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_S_HI); + ts64->tv_sec <<= 16; + //S[31:16] E44C[15:0] + ts64->tv_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_S_MI); + ts64->tv_sec <<= 16; + //S[15:0] E44A[15:0] + ts64->tv_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_S_LO); +} + +static void rtl8127_ptp_tx_hwtstamp(struct rtl8127_private *tp) +{ + struct sk_buff *skb = tp->ptp_tx_skb; + struct skb_shared_hwtstamps shhwtstamps = { 0 }; + struct timespec64 ts64; + + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_INSR, TX_TX_INTR); + + rtl8127_ptp_egresstime(tp, &ts64); + + /* Upper 32 bits contain s, lower 32 bits contain ns. */ + shhwtstamps.hwtstamp = ktime_set(ts64.tv_sec, + ts64.tv_nsec); + + /* Clear the lock early before calling skb_tstamp_tx so that + * applications are not woken up before the lock bit is clear. We use + * a copy of the skb pointer to ensure other threads can't change it + * while we're notifying the stack. + */ + tp->ptp_tx_skb = NULL; + clear_bit_unlock(__RTL8127_PTP_TX_IN_PROGRESS, &tp->state); + + /* Notify the stack and free the skb after we've unlocked */ + skb_tstamp_tx(skb, &shhwtstamps); + dev_kfree_skb_any(skb); +} + +#define RTL8127_PTP_TX_TIMEOUT (HZ * 15) +static void rtl8127_ptp_tx_work(struct work_struct *work) +{ + struct rtl8127_private *tp = container_of(work, struct rtl8127_private, + ptp_tx_work); + unsigned long flags; + + if (!tp->ptp_tx_skb) + return; + + if (time_is_before_jiffies(tp->ptp_tx_start + + RTL8127_PTP_TX_TIMEOUT)) { + dev_kfree_skb_any(tp->ptp_tx_skb); + tp->ptp_tx_skb = NULL; + clear_bit_unlock(__RTL8127_PTP_TX_IN_PROGRESS, &tp->state); + tp->tx_hwtstamp_timeouts++; + /* Clear the tx valid bit in TSYNCTXCTL register to enable + * interrupt + */ + spin_lock_irqsave(&tp->phy_lock, flags); + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_INSR, TX_TX_INTR); + spin_unlock_irqrestore(&tp->phy_lock, flags); + return; + } + + spin_lock_irqsave(&tp->phy_lock, flags); + if (rtl8127_mdio_direct_read_phy_ocp(tp, PTP_INSR) & TX_TX_INTR) { + rtl8127_ptp_tx_hwtstamp(tp); + spin_unlock_irqrestore(&tp->phy_lock, flags); + } else { + spin_unlock_irqrestore(&tp->phy_lock, flags); + /* reschedule to check later */ + schedule_work(&tp->ptp_tx_work); + } +} + +static int rtl8127_hwtstamp_enable(struct rtl8127_private *tp, bool enable) +{ + unsigned long flags; + + spin_lock_irqsave(&tp->phy_lock, flags); + + if (enable) { + //trx timestamp interrupt enable + rtl8127_set_eth_phy_ocp_bit(tp, PTP_INER, BIT_2 | BIT_3); + + //set isr clear mode + rtl8127_set_eth_phy_ocp_bit(tp, PTP_GEN_CFG, BIT_0); + + //clear ptp isr + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_INSR, 0xFFFF); + + //enable ptp + rtl8127_ptp_enable_config(tp); + + //rtl8127_set_local_time(tp); + } else { + /* trx timestamp interrupt disable */ + rtl8127_clear_eth_phy_ocp_bit(tp, PTP_INER, BIT_2 | BIT_3); + + /* disable ptp */ + rtl8127_clear_eth_phy_ocp_bit(tp, PTP_SYNCE_CTL, BIT_0); + rtl8127_clear_eth_phy_ocp_bit(tp, PTP_CTL, BIT_0); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA640, BIT_15); + } + + spin_unlock_irqrestore(&tp->phy_lock, flags); + + return 0; +} + +void rtl8127_set_local_time(struct rtl8127_private *tp) +{ + struct timespec64 ts64; + //set system time + ktime_get_real_ts64(&ts64); + _rtl8127_phc_settime(tp, &ts64); +} + +static long rtl8127_ptp_create_clock(struct rtl8127_private *tp) +{ + struct net_device *netdev = tp->dev; + long err; + + if (!IS_ERR_OR_NULL(tp->ptp_clock)) + return 0; + + if (tp->HwSuppPtpVer == 0) { + tp->ptp_clock = NULL; + return -EOPNOTSUPP; + } + + tp->ptp_clock_info = rtl_ptp_clock_info; + tp->ptp_clock_info.max_adj = 488281;//0x1FFFFF * 10^9 / 2^32 + + snprintf(tp->ptp_clock_info.name, sizeof(tp->ptp_clock_info.name), + "%pm", tp->dev->dev_addr); + tp->ptp_clock = ptp_clock_register(&tp->ptp_clock_info, &tp->pci_dev->dev); + if (IS_ERR(tp->ptp_clock)) { + err = PTR_ERR(tp->ptp_clock); + tp->ptp_clock = NULL; + netif_err(tp, drv, tp->dev, "ptp_clock_register failed\n"); + return err; + } else + netif_info(tp, drv, tp->dev, "registered PHC device on %s\n", netdev->name); + + return 0; +} + +static enum hrtimer_restart +rtl8127_hrtimer_for_pps(struct hrtimer *timer) { + struct rtl8127_private *tp = container_of(timer, struct rtl8127_private, pps_timer); + u16 tai_cfg = BIT_8 | BIT_3 | BIT_1 | BIT_0; + s64 pps_sec; + + if (tp->pps_enable) + { + unsigned long flags; + + spin_lock_irqsave(&tp->phy_lock, flags); + + //Direct Read + rtl8127_set_clkadj_mode(tp, DIRECT_READ); + + pps_sec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_S_HI_8126); + pps_sec <<= 16; + pps_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_S_MI_8126); + pps_sec <<= 16; + pps_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_S_LO_8126); + pps_sec++; + + //E42A[15:0] + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_TAI_TS_S_LO, pps_sec & 0xffff); + //E42C[31:16] + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_TAI_TS_S_HI, (pps_sec & 0xffff0000) >> 16); + //Periodic Tai start + rtl8127_mdio_direct_write_phy_ocp(tp, PTP_TAI_CFG, tai_cfg); + + spin_unlock_irqrestore(&tp->phy_lock, flags); + + hrtimer_forward_now(&tp->pps_timer, 1000000000); //rekick + return HRTIMER_RESTART; + } else + return HRTIMER_NORESTART; +} + +void rtl8127_ptp_reset(struct rtl8127_private *tp) +{ + if (!tp->ptp_clock) + return; + + netif_info(tp, drv, tp->dev, "reset PHC clock\n"); + + rtl8127_hwtstamp_enable(tp, false); +} + +void rtl8127_ptp_init(struct rtl8127_private *tp) +{ + /* obtain a PTP device, or re-use an existing device */ + if (rtl8127_ptp_create_clock(tp)) + return; + + /* we have a clock so we can initialize work now */ + INIT_WORK(&tp->ptp_tx_work, rtl8127_ptp_tx_work); + + /* init a hrtimer for pps */ + tp->pps_enable = 0; + hrtimer_init(&tp->pps_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + tp->pps_timer.function = rtl8127_hrtimer_for_pps; + + /* reset the PTP related hardware bits */ + rtl8127_ptp_reset(tp); + + return; +} + +void rtl8127_ptp_suspend(struct rtl8127_private *tp) +{ + if (!tp->ptp_clock) + return; + + netif_info(tp, drv, tp->dev, "suspend PHC clock\n"); + + rtl8127_hwtstamp_enable(tp, false); + + /* ensure that we cancel any pending PTP Tx work item in progress */ + cancel_work_sync(&tp->ptp_tx_work); + + hrtimer_cancel(&tp->pps_timer); +} + +void rtl8127_ptp_stop(struct rtl8127_private *tp) +{ + struct net_device *netdev = tp->dev; + + netif_info(tp, drv, tp->dev, "stop PHC clock\n"); + + /* first, suspend PTP activity */ + rtl8127_ptp_suspend(tp); + + /* disable the PTP clock device */ + if (tp->ptp_clock) { + ptp_clock_unregister(tp->ptp_clock); + tp->ptp_clock = NULL; + netif_info(tp, drv, tp->dev, "removed PHC on %s\n", + netdev->name); + } +} + +static int rtl8127_set_tstamp(struct net_device *netdev, struct ifreq *ifr) +{ + struct rtl8127_private *tp = netdev_priv(netdev); + struct hwtstamp_config config; + bool hwtstamp = 0; + + //netif_info(tp, drv, tp->dev, "ptp set ts\n"); + + if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) + return -EFAULT; + + if (config.flags) + return -EINVAL; + + switch (config.tx_type) { + case HWTSTAMP_TX_ON: + hwtstamp = 1; + break; + case HWTSTAMP_TX_OFF: + break; + case HWTSTAMP_TX_ONESTEP_SYNC: + default: + return -ERANGE; + } + + switch (config.rx_filter) { + case HWTSTAMP_FILTER_PTP_V2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: + config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; + hwtstamp = 1; + tp->flags |= RTL_FLAG_RX_HWTSTAMP_ENABLED; + break; + case HWTSTAMP_FILTER_NONE: + tp->flags &= ~RTL_FLAG_RX_HWTSTAMP_ENABLED; + break; + default: + tp->flags &= ~RTL_FLAG_RX_HWTSTAMP_ENABLED; + return -ERANGE; + } + + if (tp->hwtstamp_config.tx_type != config.tx_type || + tp->hwtstamp_config.rx_filter != config.rx_filter) { + tp->hwtstamp_config = config; + + rtl8127_hwtstamp_enable(tp, hwtstamp); + } + + return copy_to_user(ifr->ifr_data, &config, + sizeof(config)) ? -EFAULT : 0; +} + +static int rtl8127_get_tstamp(struct net_device *netdev, struct ifreq *ifr) +{ + struct rtl8127_private *tp = netdev_priv(netdev); + + //netif_info(tp, drv, tp->dev, "ptp get ts\n"); + + return copy_to_user(ifr->ifr_data, &tp->hwtstamp_config, + sizeof(tp->hwtstamp_config)) ? -EFAULT : 0; +} + +int rtl8127_ptp_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) +{ + int ret; + + //netif_info(tp, drv, tp->dev, "ptp ioctl\n"); + + switch (cmd) { +#ifdef ENABLE_PTP_SUPPORT + case SIOCSHWTSTAMP: + ret = rtl8127_set_tstamp(netdev, ifr); + break; + case SIOCGHWTSTAMP: + ret = rtl8127_get_tstamp(netdev, ifr); + break; +#endif + default: + ret = -EOPNOTSUPP; + break; + } + + return ret; +} + +static void rtl8127_rx_ptp_pktstamp(struct rtl8127_private *tp, struct sk_buff *skb, u8 type) +{ + struct timespec64 ts64; + unsigned long flags; + + spin_lock_irqsave(&tp->phy_lock, flags); + + rtl8127_ptp_ingresstime(tp, &ts64, type); + + spin_unlock_irqrestore(&tp->phy_lock, flags); + + skb_hwtstamps(skb)->hwtstamp = ktime_set(ts64.tv_sec, ts64.tv_nsec); + + return; +} + +void rtl8127_rx_ptp_timestamp(struct rtl8127_private *tp, struct sk_buff *skb) +{ + unsigned int ptp_class; + struct ptp_header *hdr; + u8 msgtype; + + ptp_class = ptp_classify_raw(skb); + if (ptp_class == PTP_CLASS_NONE) + return; + + skb_reset_mac_header(skb); + hdr = ptp_parse_header(skb, ptp_class); + if (unlikely(!hdr)) + return; + + msgtype = ptp_get_msgtype(hdr, ptp_class); + rtl8127_rx_ptp_pktstamp(tp, skb, msgtype); + + return; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0) +struct ptp_header *ptp_parse_header(struct sk_buff *skb, unsigned int type) +{ + u8 *ptr = skb_mac_header(skb); + + if (type & PTP_CLASS_VLAN) + //ptr += VLAN_HLEN; + ptr += 4; + + switch (type & PTP_CLASS_PMASK) { + case PTP_CLASS_IPV4: + ptr += IPV4_HLEN(ptr) + UDP_HLEN; + break; + case PTP_CLASS_IPV6: + ptr += IP6_HLEN + UDP_HLEN; + break; + case PTP_CLASS_L2: + break; + default: + return NULL; + } + + ptr += ETH_HLEN; + + /* Ensure that the entire header is present in this packet. */ + if (ptr + sizeof(struct ptp_header) > skb->data + skb->len) + return NULL; + + return (struct ptp_header *)ptr; +} +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0) */ diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127_ptp.h b/drivers/net/ethernet/realtek/r8127/src/r8127_ptp.h new file mode 100755 index 0000000000000..e96afafd70a85 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/src/r8127_ptp.h @@ -0,0 +1,202 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#ifndef _LINUX_R8127_PTP_H +#define _LINUX_R8127_PTP_H + +#include +#include +#include +#include +#include + +#ifndef PTP_CLASS_NONE +#define PTP_CLASS_NONE 0x00 +#endif + +#ifndef PTP_MSGTYPE_SYNC +#define PTP_MSGTYPE_SYNC 0x0 +#endif +#ifndef PTP_MSGTYPE_DELAY_REQ +#define PTP_MSGTYPE_DELAY_REQ 0x1 +#endif +#ifndef PTP_MSGTYPE_PDELAY_REQ +#define PTP_MSGTYPE_PDELAY_REQ 0x2 +#endif +#ifndef PTP_MSGTYPE_PDELAY_RESP +#define PTP_MSGTYPE_PDELAY_RESP 0x3 +#endif + +struct rtl8127_ptp_info { + s64 time_sec; + u32 time_ns; + u16 ts_info; +}; + +#ifndef _STRUCT_TIMESPEC +#define _STRUCT_TIMESPEC +struct timespec { + __kernel_old_time_t tv_sec; /* seconds */ + long tv_nsec; /* nanoseconds */ +}; +#endif + +enum PTP_CMD_TYPE { + PTP_CMD_SET_LOCAL_TIME = 0, + PTP_CMD_DRIFT_LOCAL_TIME, + PTP_CMD_LATCHED_LOCAL_TIME, +}; + +enum PTP_CLKADJ_MOD_TYPE { + NO_FUNCTION = 0, + CLKADJ_MODE_SET = 1, + RESERVED = 2, + DIRECT_READ = 4, + DIRECT_WRITE = 6, + INCREMENT_STEP = 8, + DECREMENT_STEP = 10, + RATE_READ = 12, + RATE_WRITE = 14, +}; + +enum PTP_INSR_TYPE { + EVENT_CAP_INTR = (1 << 0), + TRIG_GEN_INTR = (1 << 1), + RX_TS_INTR = (1 << 2), + TX_TX_INTR = (1 << 3), +}; + +enum PTP_TRX_TS_STA_REG { + TRX_TS_RD = (1 << 0), + TRXTS_SEL = (1 << 1), + RX_TS_PDLYRSP_RDY = (1 << 8), + RX_TS_PDLYREQ_RDY = (1 << 9), + RX_TS_DLYREQ_RDY = (1 << 10), + RX_TS_SYNC_RDY = (1 << 11), + TX_TS_PDLYRSP_RDY = (1 << 12), + TX_TS_PDLYREQ_RDY = (1 << 13), + TX_TS_DLYREQ_RDY = (1 << 14), + TX_TS_SYNC_RDY = (1 << 15), +}; + +#define PTP_CTL_TYPE_0 (0xF3F) +#define PTP_CTL_TYPE_1 (0x2FF) +#define PTP_CTL_TYPE_2 (0x0FF) +#define PTP_CTL_TYPE_3 (0x03F) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0) +struct clock_identity { + u8 id[8]; +} __packed; + +struct port_identity { + struct clock_identity clock_identity; + __be16 port_number; +} __packed; + +struct ptp_header { + u8 tsmt; /* transportSpecific | messageType */ + u8 ver; /* reserved | versionPTP */ + __be16 message_length; + u8 domain_number; + u8 reserved1; + u8 flag_field[2]; + __be64 correction; + __be32 reserved2; + struct port_identity source_port_identity; + __be16 sequence_id; + u8 control; + u8 log_message_interval; +} __packed; + +/** + * ptp_parse_header - Get pointer to the PTP v2 header + * @skb: packet buffer + * @type: type of the packet (see ptp_classify_raw()) + * + * This function takes care of the VLAN, UDP, IPv4 and IPv6 headers. The length + * is checked. + * + * Note, internally skb_mac_header() is used. Make sure that the @skb is + * initialized accordingly. + * + * Return: Pointer to the ptp v2 header or NULL if not found + */ +struct ptp_header *ptp_parse_header(struct sk_buff *skb, unsigned int type); + +/** + * ptp_get_msgtype - Extract ptp message type from given header + * @hdr: ptp header + * @type: type of the packet (see ptp_classify_raw()) + * + * This function returns the message type for a given ptp header. It takes care + * of the different ptp header versions (v1 or v2). + * + * Return: The message type + */ +static inline u8 ptp_get_msgtype(const struct ptp_header *hdr, + unsigned int type) +{ + u8 msgtype; + + if (unlikely(type & PTP_CLASS_V1)) { + /* msg type is located at the control field for ptp v1 */ + msgtype = hdr->control; + } else { + msgtype = hdr->tsmt & 0x0f; + } + + return msgtype; +} + +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0) */ + +struct rtl8127_private; +struct RxDescV3; + +int rtl8127_get_ts_info(struct net_device *netdev, + struct ethtool_ts_info *info); + +void rtl8127_ptp_reset(struct rtl8127_private *tp); +void rtl8127_ptp_init(struct rtl8127_private *tp); +void rtl8127_ptp_suspend(struct rtl8127_private *tp); +void rtl8127_ptp_stop(struct rtl8127_private *tp); + +int rtl8127_ptp_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd); + +void rtl8127_rx_ptp_timestamp(struct rtl8127_private *tp, struct sk_buff *skb); + +void rtl8127_set_local_time(struct rtl8127_private *tp); + +#endif /* _LINUX_R8127_PTP_H */ diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127_realwow.h b/drivers/net/ethernet/realtek/r8127/src/r8127_realwow.h new file mode 100755 index 0000000000000..a869b6c532b33 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/src/r8127_realwow.h @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#ifndef _LINUX_R8127_REALWOW_H +#define _LINUX_R8127_REALWOW_H + +#define SIOCDEVPRIVATE_RTLREALWOW SIOCDEVPRIVATE+3 + +#define MAX_RealWoW_KCP_SIZE (100) +#define MAX_RealWoW_Payload (64) + +#define KA_TX_PACKET_SIZE (100) +#define KA_WAKEUP_PATTERN_SIZE (120) + +//HwSuppKeepAliveOffloadVer +#define HW_SUPPORT_KCP_OFFLOAD(_M) ((_M)->HwSuppKCPOffloadVer > 0) + +enum rtl_realwow_cmd { + + RTL_REALWOW_SET_KCP_DISABLE=0, + RTL_REALWOW_SET_KCP_INFO, + RTL_REALWOW_SET_KCP_CONTENT, + + RTL_REALWOW_SET_KCP_ACKPKTINFO, + RTL_REALWOW_SET_KCP_WPINFO, + RTL_REALWOW_SET_KCPDHCP_TIMEOUT, + + RTLT_REALWOW_COMMAND_INVALID +}; + +struct rtl_realwow_ioctl_struct { + __u32 cmd; + __u32 offset; + __u32 len; + union { + __u32 data; + void *data_buffer; + }; +}; + +typedef struct _MP_KCPInfo { + u8 DIPv4[4]; + u8 MacID[6]; + u16 UdpPort[2]; + u8 PKTLEN[2]; + + u16 ackLostCnt; + u8 KCP_WakePattern[MAX_RealWoW_Payload]; + u8 KCP_AckPacket[MAX_RealWoW_Payload]; + u32 KCP_interval; + u8 KCP_WakePattern_Len; + u8 KCP_AckPacket_Len; + u8 KCP_TxPacket[2][KA_TX_PACKET_SIZE]; +} MP_KCP_INFO, *PMP_KCP_INFO; + +typedef struct _KCPInfo { + u32 nId; // = id + u8 DIPv4[4]; + u8 MacID[6]; + u16 UdpPort; + u16 PKTLEN; +} KCPInfo, *PKCPInfo; + +typedef struct _KCPContent { + u32 id; // = id + u32 mSec; // = msec + u32 size; // =size + u8 bPacket[MAX_RealWoW_KCP_SIZE]; // put packet here +} KCPContent, *PKCPContent; + +typedef struct _RealWoWAckPktInfo { + u16 ackLostCnt; + u16 patterntSize; + u8 pattern[MAX_RealWoW_Payload]; +} RealWoWAckPktInfo,*PRealWoWAckPktInfo; + +typedef struct _RealWoWWPInfo { + u16 patterntSize; + u8 pattern[MAX_RealWoW_Payload]; +} RealWoWWPInfo,*PRealWoWWPInfo; + +int rtl8127_realwow_ioctl(struct net_device *dev, struct ifreq *ifr); +void rtl8127_realwow_hw_init(struct net_device *dev); +void rtl8127_get_realwow_hw_version(struct net_device *dev); +void rtl8127_set_realwow_d3_para(struct net_device *dev); + +#endif /* _LINUX_R8127_REALWOW_H */ diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127_rss.c b/drivers/net/ethernet/realtek/r8127/src/r8127_rss.c new file mode 100755 index 0000000000000..e364621910052 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/src/r8127_rss.c @@ -0,0 +1,583 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#include +#include "r8127.h" + +enum rtl8127_rss_register_content { + /* RSS */ + RSS_CTRL_TCP_IPV4_SUPP = (1 << 0), + RSS_CTRL_IPV4_SUPP = (1 << 1), + RSS_CTRL_TCP_IPV6_SUPP = (1 << 2), + RSS_CTRL_IPV6_SUPP = (1 << 3), + RSS_CTRL_IPV6_EXT_SUPP = (1 << 4), + RSS_CTRL_TCP_IPV6_EXT_SUPP = (1 << 5), + RSS_HALF_SUPP = (1 << 7), + RSS_CTRL_UDP_IPV4_SUPP = (1 << 11), + RSS_CTRL_UDP_IPV6_SUPP = (1 << 12), + RSS_CTRL_UDP_IPV6_EXT_SUPP = (1 << 13), + RSS_QUAD_CPU_EN = (1 << 16), + RSS_HQ_Q_SUP_R = (1 << 31), +}; + +static int rtl8127_get_rss_hash_opts(struct rtl8127_private *tp, + struct ethtool_rxnfc *cmd) +{ + cmd->data = 0; + + /* Report default options for RSS */ + switch (cmd->flow_type) { + case TCP_V4_FLOW: + cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3; + fallthrough; + case UDP_V4_FLOW: + if (tp->rss_flags & RTL_8125_RSS_FLAG_HASH_UDP_IPV4) + cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3; + fallthrough; + case IPV4_FLOW: + cmd->data |= RXH_IP_SRC | RXH_IP_DST; + break; + case TCP_V6_FLOW: + cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3; + fallthrough; + case UDP_V6_FLOW: + if (tp->rss_flags & RTL_8125_RSS_FLAG_HASH_UDP_IPV6) + cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3; + fallthrough; + case IPV6_FLOW: + cmd->data |= RXH_IP_SRC | RXH_IP_DST; + break; + default: + return -EINVAL; + } + + return 0; +} + +int rtl8127_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd, + u32 *rule_locs) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int ret = -EOPNOTSUPP; + + if (!(dev->features & NETIF_F_RXHASH)) + return ret; + + switch (cmd->cmd) { + case ETHTOOL_GRXRINGS: + cmd->data = rtl8127_tot_rx_rings(tp); + ret = 0; + break; + case ETHTOOL_GRXFH: + ret = rtl8127_get_rss_hash_opts(tp, cmd); + break; + default: + break; + } + + return ret; +} + +u32 rtl8127_rss_indir_tbl_entries(struct rtl8127_private *tp) +{ + return tp->HwSuppIndirTblEntries; +} + +#define RSS_MASK_BITS_OFFSET (8) +#define RSS_CPU_NUM_OFFSET (16) +#define RTL8127_UDP_RSS_FLAGS (RTL_8125_RSS_FLAG_HASH_UDP_IPV4 | \ + RTL_8125_RSS_FLAG_HASH_UDP_IPV6) +static int _rtl8127_set_rss_hash_opt(struct rtl8127_private *tp) +{ + u32 rss_flags = tp->rss_flags; + u32 hash_mask_len; + u32 rss_ctrl; + + rss_ctrl = ilog2(rtl8127_tot_rx_rings(tp)); + rss_ctrl &= (BIT_0 | BIT_1 | BIT_2); + rss_ctrl <<= RSS_CPU_NUM_OFFSET; + + /* Perform hash on these packet types */ + rss_ctrl |= RSS_CTRL_TCP_IPV4_SUPP + | RSS_CTRL_IPV4_SUPP + | RSS_CTRL_IPV6_SUPP + | RSS_CTRL_IPV6_EXT_SUPP + | RSS_CTRL_TCP_IPV6_SUPP + | RSS_CTRL_TCP_IPV6_EXT_SUPP; + + if (rss_flags & RTL_8125_RSS_FLAG_HASH_UDP_IPV4) + rss_ctrl |= RSS_CTRL_UDP_IPV4_SUPP; + + if (rss_flags & RTL_8125_RSS_FLAG_HASH_UDP_IPV6) + rss_ctrl |= RSS_CTRL_UDP_IPV6_SUPP | + RSS_CTRL_UDP_IPV6_EXT_SUPP; + + hash_mask_len = ilog2(rtl8127_rss_indir_tbl_entries(tp)); + hash_mask_len &= (BIT_0 | BIT_1 | BIT_2); + rss_ctrl |= hash_mask_len << RSS_MASK_BITS_OFFSET; + + RTL_W32(tp, RSS_CTRL_8125, rss_ctrl); + + return 0; +} + +static int rtl8127_set_rss_hash_opt(struct rtl8127_private *tp, + struct ethtool_rxnfc *nfc) +{ + u32 rss_flags = tp->rss_flags; + + /* + * RSS does not support anything other than hashing + * to queues on src and dst IPs and ports + */ + if (nfc->data & ~(RXH_IP_SRC | RXH_IP_DST | + RXH_L4_B_0_1 | RXH_L4_B_2_3)) + return -EINVAL; + + switch (nfc->flow_type) { + case TCP_V4_FLOW: + case TCP_V6_FLOW: + if (!(nfc->data & RXH_IP_SRC) || + !(nfc->data & RXH_IP_DST) || + !(nfc->data & RXH_L4_B_0_1) || + !(nfc->data & RXH_L4_B_2_3)) + return -EINVAL; + break; + case UDP_V4_FLOW: + if (!(nfc->data & RXH_IP_SRC) || + !(nfc->data & RXH_IP_DST)) + return -EINVAL; + switch (nfc->data & (RXH_L4_B_0_1 | RXH_L4_B_2_3)) { + case 0: + rss_flags &= ~RTL_8125_RSS_FLAG_HASH_UDP_IPV4; + break; + case (RXH_L4_B_0_1 | RXH_L4_B_2_3): + rss_flags |= RTL_8125_RSS_FLAG_HASH_UDP_IPV4; + break; + default: + return -EINVAL; + } + break; + case UDP_V6_FLOW: + if (!(nfc->data & RXH_IP_SRC) || + !(nfc->data & RXH_IP_DST)) + return -EINVAL; + switch (nfc->data & (RXH_L4_B_0_1 | RXH_L4_B_2_3)) { + case 0: + rss_flags &= ~RTL_8125_RSS_FLAG_HASH_UDP_IPV6; + break; + case (RXH_L4_B_0_1 | RXH_L4_B_2_3): + rss_flags |= RTL_8125_RSS_FLAG_HASH_UDP_IPV6; + break; + default: + return -EINVAL; + } + break; + case SCTP_V4_FLOW: + case AH_ESP_V4_FLOW: + case AH_V4_FLOW: + case ESP_V4_FLOW: + case SCTP_V6_FLOW: + case AH_ESP_V6_FLOW: + case AH_V6_FLOW: + case ESP_V6_FLOW: + case IP_USER_FLOW: + case ETHER_FLOW: + /* RSS is not supported for these protocols */ + if (nfc->data) { + netif_err(tp, drv, tp->dev, "Command parameters not supported\n"); + return -EINVAL; + } + return 0; + break; + default: + return -EINVAL; + } + + /* if we changed something we need to update flags */ + if (rss_flags != tp->rss_flags) { + u32 rss_ctrl = RTL_R32(tp, RSS_CTRL_8125); + + if ((rss_flags & RTL8127_UDP_RSS_FLAGS) && + !(tp->rss_flags & RTL8127_UDP_RSS_FLAGS)) + netdev_warn(tp->dev, + "enabling UDP RSS: fragmented packets may " + "arrive out of order to the stack above\n"); + + tp->rss_flags = rss_flags; + + /* Perform hash on these packet types */ + rss_ctrl |= RSS_CTRL_TCP_IPV4_SUPP + | RSS_CTRL_IPV4_SUPP + | RSS_CTRL_IPV6_SUPP + | RSS_CTRL_IPV6_EXT_SUPP + | RSS_CTRL_TCP_IPV6_SUPP + | RSS_CTRL_TCP_IPV6_EXT_SUPP; + + rss_ctrl &= ~(RSS_CTRL_UDP_IPV4_SUPP | + RSS_CTRL_UDP_IPV6_SUPP | + RSS_CTRL_UDP_IPV6_EXT_SUPP); + + if (rss_flags & RTL_8125_RSS_FLAG_HASH_UDP_IPV4) + rss_ctrl |= RSS_CTRL_UDP_IPV4_SUPP; + + if (rss_flags & RTL_8125_RSS_FLAG_HASH_UDP_IPV6) + rss_ctrl |= RSS_CTRL_UDP_IPV6_SUPP | + RSS_CTRL_UDP_IPV6_EXT_SUPP; + + RTL_W32(tp, RSS_CTRL_8125, rss_ctrl); + } + + return 0; +} + +int rtl8127_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int ret = -EOPNOTSUPP; + + if (!(dev->features & NETIF_F_RXHASH)) + return ret; + + switch (cmd->cmd) { + case ETHTOOL_SRXFH: + ret = rtl8127_set_rss_hash_opt(tp, cmd); + break; + default: + break; + } + + return ret; +} + +static u32 _rtl8127_get_rxfh_key_size(struct rtl8127_private *tp) +{ + return sizeof(tp->rss_key); +} + +u32 rtl8127_get_rxfh_key_size(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (!(dev->features & NETIF_F_RXHASH)) + return 0; + + return _rtl8127_get_rxfh_key_size(tp); +} + +u32 rtl8127_rss_indir_size(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (!(dev->features & NETIF_F_RXHASH)) + return 0; + + return rtl8127_rss_indir_tbl_entries(tp); +} + +static void rtl8127_get_reta(struct rtl8127_private *tp, u32 *indir) +{ + int i, reta_size = rtl8127_rss_indir_tbl_entries(tp); + + for (i = 0; i < reta_size; i++) + indir[i] = tp->rss_indir_tbl[i]; +} + +static u32 rtl8127_rss_key_reg(struct rtl8127_private *tp) +{ + return RSS_KEY_8125; +} + +static u32 rtl8127_rss_indir_tbl_reg(struct rtl8127_private *tp) +{ + return RSS_INDIRECTION_TBL_8125_V2; +} + +static void rtl8127_store_reta(struct rtl8127_private *tp) +{ + u16 indir_tbl_reg = rtl8127_rss_indir_tbl_reg(tp); + u32 i, reta_entries = rtl8127_rss_indir_tbl_entries(tp); + u32 reta = 0; + u8 *indir_tbl = tp->rss_indir_tbl; + + /* Write redirection table to HW */ + for (i = 0; i < reta_entries; i++) { + reta |= indir_tbl[i] << (i & 0x3) * 8; + if ((i & 3) == 3) { + RTL_W32(tp, indir_tbl_reg, reta); + + indir_tbl_reg += 4; + reta = 0; + } + } +} + +static void rtl8127_store_rss_key(struct rtl8127_private *tp) +{ + const u16 rss_key_reg = rtl8127_rss_key_reg(tp); + u32 i, rss_key_size = _rtl8127_get_rxfh_key_size(tp); + u32 *rss_key = (u32*)tp->rss_key; + + /* Write redirection table to HW */ + for (i = 0; i < rss_key_size; i+=4) + RTL_W32(tp, rss_key_reg + i, *rss_key++); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,8,0) +int rtl8127_get_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (!(dev->features & NETIF_F_RXHASH)) + return -EOPNOTSUPP; + + rxfh->hfunc = ETH_RSS_HASH_TOP; + + if (rxfh->indir) + rtl8127_get_reta(tp, rxfh->indir); + + if (rxfh->key) + memcpy(rxfh->key, tp->rss_key, RTL8127_RSS_KEY_SIZE); + + return 0; +} + +int rtl8127_set_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i; + u32 reta_entries = rtl8127_rss_indir_tbl_entries(tp); + + /* We require at least one supported parameter to be changed and no + * change in any of the unsupported parameters + */ + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && rxfh->hfunc != ETH_RSS_HASH_TOP) + return -EOPNOTSUPP; + + /* Fill out the redirection table */ + if (rxfh->indir) { + int max_queues = tp->num_rx_rings; + + /* Verify user input. */ + for (i = 0; i < reta_entries; i++) + if (rxfh->indir[i] >= max_queues) + return -EINVAL; + + for (i = 0; i < reta_entries; i++) + tp->rss_indir_tbl[i] = rxfh->indir[i]; + } + + /* Fill out the rss hash key */ + if (rxfh->key) + memcpy(tp->rss_key, rxfh->key, RTL8127_RSS_KEY_SIZE); + + rtl8127_store_reta(tp); + + rtl8127_store_rss_key(tp); + + return 0; +} +#else +int rtl8127_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, + u8 *hfunc) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (!(dev->features & NETIF_F_RXHASH)) + return -EOPNOTSUPP; + + if (hfunc) + *hfunc = ETH_RSS_HASH_TOP; + + if (indir) + rtl8127_get_reta(tp, indir); + + if (key) + memcpy(key, tp->rss_key, RTL8127_RSS_KEY_SIZE); + + return 0; +} + +int rtl8127_set_rxfh(struct net_device *dev, const u32 *indir, + const u8 *key, const u8 hfunc) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i; + u32 reta_entries = rtl8127_rss_indir_tbl_entries(tp); + + /* We require at least one supported parameter to be changed and no + * change in any of the unsupported parameters + */ + if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + return -EOPNOTSUPP; + + /* Fill out the redirection table */ + if (indir) { + int max_queues = tp->num_rx_rings; + + /* Verify user input. */ + for (i = 0; i < reta_entries; i++) + if (indir[i] >= max_queues) + return -EINVAL; + + for (i = 0; i < reta_entries; i++) + tp->rss_indir_tbl[i] = indir[i]; + } + + /* Fill out the rss hash key */ + if (key) + memcpy(tp->rss_key, key, RTL8127_RSS_KEY_SIZE); + + rtl8127_store_reta(tp); + + rtl8127_store_rss_key(tp); + + return 0; +} +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6,8,0) */ + +static u32 rtl8127_get_rx_desc_hash(struct rtl8127_private *tp, + struct RxDesc *desc) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + return le32_to_cpu(((struct RxDescV3 *)desc)->RxDescNormalDDWord2.RSSResult); + case RX_DESC_RING_TYPE_4: + return le32_to_cpu(((struct RxDescV4 *)desc)->RxDescNormalDDWord1.RSSResult); + default: + return 0; + } +} + +#define RXS_8125B_RSS_UDP BIT(9) +#define RXS_8125_RSS_IPV4 BIT(10) +#define RXS_8125_RSS_IPV6 BIT(12) +#define RXS_8125_RSS_TCP BIT(13) +#define RTL8127_RXS_RSS_L3_TYPE_MASK (RXS_8125_RSS_IPV4 | RXS_8125_RSS_IPV6) +#define RTL8127_RXS_RSS_L4_TYPE_MASK (RXS_8125_RSS_TCP | RXS_8125B_RSS_UDP) + +#define RXS_8125B_RSS_UDP_V4 BIT(27) +#define RXS_8125_RSS_IPV4_V4 BIT(28) +#define RXS_8125_RSS_IPV6_V4 BIT(29) +#define RXS_8125_RSS_TCP_V4 BIT(30) +#define RTL8127_RXS_RSS_L3_TYPE_MASK_V4 (RXS_8125_RSS_IPV4_V4 | RXS_8125_RSS_IPV6_V4) +#define RTL8127_RXS_RSS_L4_TYPE_MASK_V4 (RXS_8125_RSS_TCP_V4 | RXS_8125B_RSS_UDP_V4) +static void rtl8127_rx_hash_v3(struct rtl8127_private *tp, + struct RxDescV3 *descv3, + struct sk_buff *skb) +{ + u16 rss_header_info; + + if (!(tp->dev->features & NETIF_F_RXHASH)) + return; + + rss_header_info = le16_to_cpu(descv3->RxDescNormalDDWord2.HeaderInfo); + + if (!(rss_header_info & RTL8127_RXS_RSS_L3_TYPE_MASK)) + return; + + skb_set_hash(skb, rtl8127_get_rx_desc_hash(tp, (struct RxDesc *)descv3), + (RTL8127_RXS_RSS_L4_TYPE_MASK & rss_header_info) ? + PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); +} + +static void rtl8127_rx_hash_v4(struct rtl8127_private *tp, + struct RxDescV4 *descv4, + struct sk_buff *skb) +{ + u32 rss_header_info; + + if (!(tp->dev->features & NETIF_F_RXHASH)) + return; + + rss_header_info = le32_to_cpu(descv4->RxDescNormalDDWord1.RSSInfo); + + if (!(rss_header_info & RTL8127_RXS_RSS_L3_TYPE_MASK_V4)) + return; + + skb_set_hash(skb, rtl8127_get_rx_desc_hash(tp, (struct RxDesc *)descv4), + (RTL8127_RXS_RSS_L4_TYPE_MASK_V4 & rss_header_info) ? + PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); +} + +void rtl8127_rx_hash(struct rtl8127_private *tp, + struct RxDesc *desc, + struct sk_buff *skb) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + rtl8127_rx_hash_v3(tp, (struct RxDescV3 *)desc, skb); + break; + case RX_DESC_RING_TYPE_4: + rtl8127_rx_hash_v4(tp, (struct RxDescV4 *)desc, skb); + break; + default: + return; + } +} + +void rtl8127_disable_rss(struct rtl8127_private *tp) +{ + RTL_W32(tp, RSS_CTRL_8125, 0x00); +} + +void _rtl8127_config_rss(struct rtl8127_private *tp) +{ + _rtl8127_set_rss_hash_opt(tp); + + rtl8127_store_reta(tp); + + rtl8127_store_rss_key(tp); +} + +void rtl8127_config_rss(struct rtl8127_private *tp) +{ + if (!tp->EnableRss) { + rtl8127_disable_rss(tp); + return; + } + + _rtl8127_config_rss(tp); +} + +void rtl8127_init_rss(struct rtl8127_private *tp) +{ + int i; + + for (i = 0; i < rtl8127_rss_indir_tbl_entries(tp); i++) + tp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, tp->num_rx_rings); + + netdev_rss_key_fill(tp->rss_key, RTL8127_RSS_KEY_SIZE); +} diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127_rss.h b/drivers/net/ethernet/realtek/r8127/src/r8127_rss.h new file mode 100755 index 0000000000000..8e92bb830c969 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/src/r8127_rss.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#ifndef _LINUX_R8127_RSS_H +#define _LINUX_R8127_RSS_H + +#include +#include + +#define RTL8127_RSS_KEY_SIZE 40 /* size of RSS Hash Key in bytes */ +#define RTL8127_MAX_INDIRECTION_TABLE_ENTRIES 128 + +enum rtl8127_rss_flag { + RTL_8125_RSS_FLAG_HASH_UDP_IPV4 = (1 << 0), + RTL_8125_RSS_FLAG_HASH_UDP_IPV6 = (1 << 1), +}; + +struct rtl8127_private; +struct RxDesc; + +int rtl8127_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd, + u32 *rule_locs); +int rtl8127_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd); +u32 rtl8127_get_rxfh_key_size(struct net_device *netdev); +u32 rtl8127_rss_indir_size(struct net_device *netdev); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,8,0) +int rtl8127_get_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh); +int rtl8127_set_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack); +#else +int rtl8127_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, + u8 *hfunc); +int rtl8127_set_rxfh(struct net_device *netdev, const u32 *indir, + const u8 *key, const u8 hfunc); +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6,8,0) */ +void rtl8127_rx_hash(struct rtl8127_private *tp, + struct RxDesc *desc, + struct sk_buff *skb); +void _rtl8127_config_rss(struct rtl8127_private *tp); +void rtl8127_config_rss(struct rtl8127_private *tp); +void rtl8127_init_rss(struct rtl8127_private *tp); +u32 rtl8127_rss_indir_tbl_entries(struct rtl8127_private *tp); +void rtl8127_disable_rss(struct rtl8127_private *tp); + +#endif /* _LINUX_R8127_RSS_H */ diff --git a/drivers/net/ethernet/realtek/r8127/src/rtl_eeprom.c b/drivers/net/ethernet/realtek/r8127/src/rtl_eeprom.c new file mode 100755 index 0000000000000..c95bbf0d19537 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/src/rtl_eeprom.c @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#include +#include +#include +#include +#include +#include + +#include + +#include "r8127.h" +#include "rtl_eeprom.h" + +//------------------------------------------------------------------- +//rtl8127_eeprom_type(): +// tell the eeprom type +//return value: +// 0: the eeprom type is 93C46 +// 1: the eeprom type is 93C56 or 93C66 +//------------------------------------------------------------------- +void rtl8127_eeprom_type(struct rtl8127_private *tp) +{ + u16 magic = 0; + + if (tp->mcfg == CFG_METHOD_DEFAULT) + goto out_no_eeprom; + + if(RTL_R8(tp, 0xD2)&0x04) { + //not support + //tp->eeprom_type = EEPROM_TWSI; + //tp->eeprom_len = 256; + goto out_no_eeprom; + } else if(RTL_R32(tp, RxConfig) & RxCfg_9356SEL) { + tp->eeprom_type = EEPROM_TYPE_93C56; + tp->eeprom_len = 256; + } else { + tp->eeprom_type = EEPROM_TYPE_93C46; + tp->eeprom_len = 128; + } + + magic = rtl8127_eeprom_read_sc(tp, 0); + +out_no_eeprom: + if ((magic != 0x8129) && (magic != 0x8128)) { + tp->eeprom_type = EEPROM_TYPE_NONE; + tp->eeprom_len = 0; + } +} + +void rtl8127_eeprom_cleanup(struct rtl8127_private *tp) +{ + u8 x; + + x = RTL_R8(tp, Cfg9346); + x &= ~(Cfg9346_EEDI | Cfg9346_EECS); + + RTL_W8(tp, Cfg9346, x); + + rtl8127_raise_clock(tp, &x); + rtl8127_lower_clock(tp, &x); +} + +static int rtl8127_eeprom_cmd_done(struct rtl8127_private *tp) +{ + u8 x; + int i; + + rtl8127_stand_by(tp); + + for (i = 0; i < 50000; i++) { + x = RTL_R8(tp, Cfg9346); + + if (x & Cfg9346_EEDO) { + fsleep(RTL_CLOCK_RATE * 2 * 3); + return 0; + } + fsleep(1); + } + + return -1; +} + +//------------------------------------------------------------------- +//rtl8127_eeprom_read_sc(): +// read one word from eeprom +//------------------------------------------------------------------- +u16 rtl8127_eeprom_read_sc(struct rtl8127_private *tp, u16 reg) +{ + int addr_sz = 6; + u8 x; + u16 data; + + if(tp->eeprom_type == EEPROM_TYPE_NONE) + return -1; + + if (tp->eeprom_type==EEPROM_TYPE_93C46) + addr_sz = 6; + else if (tp->eeprom_type==EEPROM_TYPE_93C56) + addr_sz = 8; + + x = Cfg9346_EEM1 | Cfg9346_EECS; + RTL_W8(tp, Cfg9346, x); + + rtl8127_shift_out_bits(tp, RTL_EEPROM_READ_OPCODE, 3); + rtl8127_shift_out_bits(tp, reg, addr_sz); + + data = rtl8127_shift_in_bits(tp); + + rtl8127_eeprom_cleanup(tp); + + RTL_W8(tp, Cfg9346, 0); + + return data; +} + +//------------------------------------------------------------------- +//rtl8127_eeprom_write_sc(): +// write one word to a specific address in the eeprom +//------------------------------------------------------------------- +void rtl8127_eeprom_write_sc(struct rtl8127_private *tp, u16 reg, u16 data) +{ + u8 x; + int addr_sz = 6; + int w_dummy_addr = 4; + + if(tp->eeprom_type == EEPROM_TYPE_NONE) + return; + + if (tp->eeprom_type==EEPROM_TYPE_93C46) { + addr_sz = 6; + w_dummy_addr = 4; + } else if (tp->eeprom_type==EEPROM_TYPE_93C56) { + addr_sz = 8; + w_dummy_addr = 6; + } + + x = Cfg9346_EEM1 | Cfg9346_EECS; + RTL_W8(tp, Cfg9346, x); + + rtl8127_shift_out_bits(tp, RTL_EEPROM_EWEN_OPCODE, 5); + rtl8127_shift_out_bits(tp, reg, w_dummy_addr); + rtl8127_stand_by(tp); + + rtl8127_shift_out_bits(tp, RTL_EEPROM_ERASE_OPCODE, 3); + rtl8127_shift_out_bits(tp, reg, addr_sz); + if (rtl8127_eeprom_cmd_done(tp) < 0) + return; + rtl8127_stand_by(tp); + + rtl8127_shift_out_bits(tp, RTL_EEPROM_WRITE_OPCODE, 3); + rtl8127_shift_out_bits(tp, reg, addr_sz); + rtl8127_shift_out_bits(tp, data, 16); + if (rtl8127_eeprom_cmd_done(tp) < 0) + return; + rtl8127_stand_by(tp); + + rtl8127_shift_out_bits(tp, RTL_EEPROM_EWDS_OPCODE, 5); + rtl8127_shift_out_bits(tp, reg, w_dummy_addr); + + rtl8127_eeprom_cleanup(tp); + RTL_W8(tp, Cfg9346, 0); +} + +void rtl8127_raise_clock(struct rtl8127_private *tp, u8 *x) +{ + *x = *x | Cfg9346_EESK; + RTL_W8(tp, Cfg9346, *x); + fsleep(RTL_CLOCK_RATE); +} + +void rtl8127_lower_clock(struct rtl8127_private *tp, u8 *x) +{ + + *x = *x & ~Cfg9346_EESK; + RTL_W8(tp, Cfg9346, *x); + fsleep(RTL_CLOCK_RATE); +} + +void rtl8127_shift_out_bits(struct rtl8127_private *tp, int data, int count) +{ + u8 x; + int mask; + + mask = 0x01 << (count - 1); + x = RTL_R8(tp, Cfg9346); + x &= ~(Cfg9346_EEDI | Cfg9346_EEDO); + + do { + if (data & mask) + x |= Cfg9346_EEDI; + else + x &= ~Cfg9346_EEDI; + + RTL_W8(tp, Cfg9346, x); + fsleep(RTL_CLOCK_RATE); + rtl8127_raise_clock(tp, &x); + rtl8127_lower_clock(tp, &x); + mask = mask >> 1; + } while(mask); + + x &= ~Cfg9346_EEDI; + RTL_W8(tp, Cfg9346, x); +} + +u16 rtl8127_shift_in_bits(struct rtl8127_private *tp) +{ + u8 x; + u16 d, i; + + x = RTL_R8(tp, Cfg9346); + x &= ~(Cfg9346_EEDI | Cfg9346_EEDO); + + d = 0; + + for (i = 0; i < 16; i++) { + d = d << 1; + rtl8127_raise_clock(tp, &x); + + x = RTL_R8(tp, Cfg9346); + x &= ~Cfg9346_EEDI; + + if (x & Cfg9346_EEDO) + d |= 1; + + rtl8127_lower_clock(tp, &x); + } + + return d; +} + +void rtl8127_stand_by(struct rtl8127_private *tp) +{ + u8 x; + + x = RTL_R8(tp, Cfg9346); + x &= ~(Cfg9346_EECS | Cfg9346_EESK); + RTL_W8(tp, Cfg9346, x); + fsleep(RTL_CLOCK_RATE); + + x |= Cfg9346_EECS; + RTL_W8(tp, Cfg9346, x); +} + +void rtl8127_set_eeprom_sel_low(struct rtl8127_private *tp) +{ + RTL_W8(tp, Cfg9346, Cfg9346_EEM1); + RTL_W8(tp, Cfg9346, Cfg9346_EEM1 | Cfg9346_EESK); + + fsleep(20); + + RTL_W8(tp, Cfg9346, Cfg9346_EEM1); +} diff --git a/drivers/net/ethernet/realtek/r8127/src/rtl_eeprom.h b/drivers/net/ethernet/realtek/r8127/src/rtl_eeprom.h new file mode 100755 index 0000000000000..e4d8c6c3f6765 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/src/rtl_eeprom.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#ifndef _LINUX_RTLEEPROM_H +#define _LINUX_RTLEEPROM_H + +//EEPROM opcodes +#define RTL_EEPROM_READ_OPCODE 06 +#define RTL_EEPROM_WRITE_OPCODE 05 +#define RTL_EEPROM_ERASE_OPCODE 07 +#define RTL_EEPROM_EWEN_OPCODE 19 +#define RTL_EEPROM_EWDS_OPCODE 16 + +#define RTL_CLOCK_RATE 3 + +void rtl8127_eeprom_type(struct rtl8127_private *tp); +void rtl8127_eeprom_cleanup(struct rtl8127_private *tp); +u16 rtl8127_eeprom_read_sc(struct rtl8127_private *tp, u16 reg); +void rtl8127_eeprom_write_sc(struct rtl8127_private *tp, u16 reg, u16 data); +void rtl8127_shift_out_bits(struct rtl8127_private *tp, int data, int count); +u16 rtl8127_shift_in_bits(struct rtl8127_private *tp); +void rtl8127_raise_clock(struct rtl8127_private *tp, u8 *x); +void rtl8127_lower_clock(struct rtl8127_private *tp, u8 *x); +void rtl8127_stand_by(struct rtl8127_private *tp); +void rtl8127_set_eeprom_sel_low(struct rtl8127_private *tp); + +#endif /* _LINUX_RTLEEPROM_H */ diff --git a/drivers/net/ethernet/realtek/r8127/src/rtltool.c b/drivers/net/ethernet/realtek/r8127/src/rtltool.c new file mode 100755 index 0000000000000..ba3a0cfd32420 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/src/rtltool.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "r8127.h" +#include "rtl_eeprom.h" +#include "rtltool.h" + +int rtl8127_tool_ioctl(struct rtl8127_private *tp, struct ifreq *ifr) +{ + struct rtltool_cmd my_cmd; + int ret; + + if (copy_from_user(&my_cmd, ifr->ifr_data, sizeof(my_cmd))) + return -EFAULT; + + ret = 0; + switch (my_cmd.cmd) { + case RTLTOOL_READ_MAC: + if ((my_cmd.offset + my_cmd.len) > pci_resource_len(tp->pci_dev, 2)) { + ret = -EINVAL; + break; + } + + if (my_cmd.len==1) + my_cmd.data = readb(tp->mmio_addr+my_cmd.offset); + else if (my_cmd.len==2) + my_cmd.data = readw(tp->mmio_addr+(my_cmd.offset&~1)); + else if (my_cmd.len==4) + my_cmd.data = readl(tp->mmio_addr+(my_cmd.offset&~3)); + else { + ret = -EOPNOTSUPP; + break; + } + + if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) { + ret = -EFAULT; + break; + } + break; + + case RTLTOOL_WRITE_MAC: + if ((my_cmd.offset + my_cmd.len) > pci_resource_len(tp->pci_dev, 2)) { + ret = -EINVAL; + break; + } + + if (my_cmd.len==1) + writeb(my_cmd.data, tp->mmio_addr+my_cmd.offset); + else if (my_cmd.len==2) + writew(my_cmd.data, tp->mmio_addr+(my_cmd.offset&~1)); + else if (my_cmd.len==4) + writel(my_cmd.data, tp->mmio_addr+(my_cmd.offset&~3)); + else { + ret = -EOPNOTSUPP; + break; + } + + break; + + case RTLTOOL_READ_PHY: + my_cmd.data = rtl8127_mdio_prot_read(tp, my_cmd.offset); + if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) { + ret = -EFAULT; + break; + } + + break; + + case RTLTOOL_WRITE_PHY: + rtl8127_mdio_prot_write(tp, my_cmd.offset, my_cmd.data); + break; + + case RTLTOOL_READ_EPHY: + my_cmd.data = rtl8127_ephy_read(tp, my_cmd.offset); + if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) { + ret = -EFAULT; + break; + } + + break; + + case RTLTOOL_WRITE_EPHY: + rtl8127_ephy_write(tp, my_cmd.offset, my_cmd.data); + break; + + case RTLTOOL_READ_ERI: + my_cmd.data = 0; + if (my_cmd.len==1 || my_cmd.len==2 || my_cmd.len==4) { + my_cmd.data = rtl8127_eri_read(tp, my_cmd.offset, my_cmd.len, ERIAR_ExGMAC); + } else { + ret = -EOPNOTSUPP; + break; + } + + if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) { + ret = -EFAULT; + break; + } + + break; + + case RTLTOOL_WRITE_ERI: + if (my_cmd.len==1 || my_cmd.len==2 || my_cmd.len==4) { + rtl8127_eri_write(tp, my_cmd.offset, my_cmd.len, my_cmd.data, ERIAR_ExGMAC); + } else { + ret = -EOPNOTSUPP; + break; + } + break; + + case RTLTOOL_READ_PCI: + my_cmd.data = 0; + if (my_cmd.len==1) + pci_read_config_byte(tp->pci_dev, my_cmd.offset, + (u8 *)&my_cmd.data); + else if (my_cmd.len==2) + pci_read_config_word(tp->pci_dev, my_cmd.offset, + (u16 *)&my_cmd.data); + else if (my_cmd.len==4) + pci_read_config_dword(tp->pci_dev, my_cmd.offset, + &my_cmd.data); + else { + ret = -EOPNOTSUPP; + break; + } + + if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) { + ret = -EFAULT; + break; + } + break; + + case RTLTOOL_WRITE_PCI: + if (my_cmd.len==1) + pci_write_config_byte(tp->pci_dev, my_cmd.offset, + my_cmd.data); + else if (my_cmd.len==2) + pci_write_config_word(tp->pci_dev, my_cmd.offset, + my_cmd.data); + else if (my_cmd.len==4) + pci_write_config_dword(tp->pci_dev, my_cmd.offset, + my_cmd.data); + else { + ret = -EOPNOTSUPP; + break; + } + + break; + + case RTLTOOL_READ_EEPROM: + my_cmd.data = rtl8127_eeprom_read_sc(tp, my_cmd.offset); + if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) { + ret = -EFAULT; + break; + } + + break; + + case RTLTOOL_WRITE_EEPROM: + rtl8127_eeprom_write_sc(tp, my_cmd.offset, my_cmd.data); + break; + + case RTL_READ_OOB_MAC: + rtl8127_oob_mutex_lock(tp); + my_cmd.data = rtl8127_ocp_read(tp, my_cmd.offset, 4); + rtl8127_oob_mutex_unlock(tp); + if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) { + ret = -EFAULT; + break; + } + break; + + case RTL_WRITE_OOB_MAC: + if (my_cmd.len == 0 || my_cmd.len > 4) + return -EOPNOTSUPP; + + rtl8127_oob_mutex_lock(tp); + rtl8127_ocp_write(tp, my_cmd.offset, my_cmd.len, my_cmd.data); + rtl8127_oob_mutex_unlock(tp); + break; + + case RTL_ENABLE_PCI_DIAG: + tp->rtk_enable_diag = 1; + + dprintk("enable rtk diag\n"); + break; + + case RTL_DISABLE_PCI_DIAG: + tp->rtk_enable_diag = 0; + + dprintk("disable rtk diag\n"); + break; + + case RTL_READ_MAC_OCP: + if (my_cmd.offset % 2) + return -EOPNOTSUPP; + + my_cmd.data = rtl8127_mac_ocp_read(tp, my_cmd.offset); + if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) { + ret = -EFAULT; + break; + } + break; + + case RTL_WRITE_MAC_OCP: + if ((my_cmd.offset % 2) || (my_cmd.len != 2)) + return -EOPNOTSUPP; + + rtl8127_mac_ocp_write(tp, my_cmd.offset, (u16)my_cmd.data); + break; + + case RTL_DIRECT_READ_PHY_OCP: + my_cmd.data = rtl8127_mdio_prot_direct_read_phy_ocp(tp, my_cmd.offset); + if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) { + ret = -EFAULT; + break; + } + + break; + + case RTL_DIRECT_WRITE_PHY_OCP: + rtl8127_mdio_prot_direct_write_phy_ocp(tp, my_cmd.offset, my_cmd.data); + break; + + default: + ret = -EOPNOTSUPP; + break; + } + + return ret; +} diff --git a/drivers/net/ethernet/realtek/r8127/src/rtltool.h b/drivers/net/ethernet/realtek/r8127/src/rtltool.h new file mode 100755 index 0000000000000..7b0ee22707be0 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/src/rtltool.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#ifndef _LINUX_RTLTOOL_H +#define _LINUX_RTLTOOL_H + +#define SIOCRTLTOOL SIOCDEVPRIVATE+1 + +enum rtl_cmd { + RTLTOOL_READ_MAC=0, + RTLTOOL_WRITE_MAC, + RTLTOOL_READ_PHY, + RTLTOOL_WRITE_PHY, + RTLTOOL_READ_EPHY, + RTLTOOL_WRITE_EPHY, + RTLTOOL_READ_ERI, + RTLTOOL_WRITE_ERI, + RTLTOOL_READ_PCI, + RTLTOOL_WRITE_PCI, + RTLTOOL_READ_EEPROM, + RTLTOOL_WRITE_EEPROM, + + RTL_READ_OOB_MAC, + RTL_WRITE_OOB_MAC, + + RTL_ENABLE_PCI_DIAG, + RTL_DISABLE_PCI_DIAG, + + RTL_READ_MAC_OCP, + RTL_WRITE_MAC_OCP, + + RTL_DIRECT_READ_PHY_OCP, + RTL_DIRECT_WRITE_PHY_OCP, + + RTLTOOL_INVALID +}; + +struct rtltool_cmd { + __u32 cmd; + __u32 offset; + __u32 len; + __u32 data; +}; + +enum mode_access { + MODE_NONE=0, + MODE_READ, + MODE_WRITE +}; + +#ifdef __KERNEL__ +int rtl8127_tool_ioctl(struct rtl8127_private *tp, struct ifreq *ifr); +#endif + +#endif /* _LINUX_RTLTOOL_H */ From fb53a36b73fca3a9acc8747eb930337fd102a3eb Mon Sep 17 00:00:00 2001 From: Abhishek Sahu Date: Mon, 28 Apr 2025 15:45:04 +0000 Subject: [PATCH 037/311] NVIDIA: SAUCE: r8127: Remove Realtek r8127 non required files BugLink: https://bugs.launchpad.net/bugs/2109730 These files are not needed to build r8127 as part of kernel source code build, so removed these non required files. Signed-off-by: Abhishek Sahu Acked-by: Matt Ochs Acked-by: Carol L Soto Acked-by: Ian May Acked-by: Jacob Martin Acked-by: Noah Wager Signed-off-by: Ian May (cherry picked from commit 063d338317508dbb456771ea6a7ec0b1f0b9ea6e noble:linux-nvidia-6.11) Signed-off-by: Jacob Martin (cherry picked from commit 712fc60b2f26 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit ee5f3b0a99180394cdd9afb196f2cd9453ffeaca noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/net/ethernet/realtek/r8127/Makefile | 59 ------- drivers/net/ethernet/realtek/r8127/README | 147 ------------------ drivers/net/ethernet/realtek/r8127/autorun.sh | 101 ------------ .../realtek/r8127/src/Makefile_linux24x | 75 --------- 4 files changed, 382 deletions(-) delete mode 100755 drivers/net/ethernet/realtek/r8127/Makefile delete mode 100755 drivers/net/ethernet/realtek/r8127/README delete mode 100755 drivers/net/ethernet/realtek/r8127/autorun.sh delete mode 100755 drivers/net/ethernet/realtek/r8127/src/Makefile_linux24x diff --git a/drivers/net/ethernet/realtek/r8127/Makefile b/drivers/net/ethernet/realtek/r8127/Makefile deleted file mode 100755 index 39e846ad3fc9f..0000000000000 --- a/drivers/net/ethernet/realtek/r8127/Makefile +++ /dev/null @@ -1,59 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -################################################################################ -# -# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet -# controllers with PCI-Express interface. -# -# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; either version 2 of the License, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, see . -# -# Author: -# Realtek NIC software team -# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan -# -################################################################################ - -################################################################################ -# This product is covered by one or more of the following patents: -# US6,570,884, US6,115,776, and US6,327,625. -################################################################################ - -KFLAG := 2$(shell uname -r | sed -ne 's/^2\.[4]\..*/4/p')x - -all: clean modules install - -modules: -ifeq ($(KFLAG),24x) - $(MAKE) -C src/ -f Makefile_linux24x modules -else - $(MAKE) -C src/ modules -endif - -clean: -ifeq ($(KFLAG),24x) - $(MAKE) -C src/ -f Makefile_linux24x clean -else - $(MAKE) -C src/ clean -endif - -install: -ifeq ($(KFLAG),24x) - $(MAKE) -C src/ -f Makefile_linux24x install -else - $(MAKE) -C src/ install -endif - - - diff --git a/drivers/net/ethernet/realtek/r8127/README b/drivers/net/ethernet/realtek/r8127/README deleted file mode 100755 index a2d451d938cab..0000000000000 --- a/drivers/net/ethernet/realtek/r8127/README +++ /dev/null @@ -1,147 +0,0 @@ - - - This is the Linux device driver released for Realtek 5 Gigabit Ethernet controllers with PCI-Express interface. - - - - - Kernel source tree (supported Linux kernel 2.6.x and 2.4.x) - - For linux kernel 2.4.x, this driver supports 2.4.20 and latter. - - Compiler/binutils for kernel compilation - - - Unpack the tarball : - # tar vjxf r8127-11.aaa.bb.tar.bz2 - - Change to the directory: - # cd r8127-11.aaa.bb - - If you are running the target kernel, then you should be able to do : - - # ./autorun.sh (as root or with sudo) - - You can check whether the driver is loaded by using following commands. - - # lsmod | grep r8127 - # ifconfig -a - - If there is a device name, ethX, shown on the monitor, the linux - driver is loaded. Then, you can use the following command to activate - the ethX. - - # ifconfig ethX up - - ,where X=0,1,2,... - - - 1. Set manually - a. Set the IP address of your machine. - - # ifconfig ethX "the IP address of your machine" - - b. Set the IP address of DNS. - - Insert the following configuration in /etc/resolv.conf. - - nameserver "the IP address of DNS" - - c. Set the IP address of gateway. - - # route add default gw "the IP address of gateway" - - 2. Set by doing configurations in /etc/sysconfig/network-scripts - /ifcfg-ethX for Redhat and Fedora, or /etc/sysconfig/network - /ifcfg-ethX for SuSE. There are two examples to set network - configurations. - - a. Fixed IP address: - DEVICE=eth0 - BOOTPROTO=static - ONBOOT=yes - TYPE=ethernet - NETMASK=255.255.255.0 - IPADDR=192.168.1.1 - GATEWAY=192.168.1.254 - BROADCAST=192.168.1.255 - - b. DHCP: - DEVICE=eth0 - BOOTPROTO=dhcp - ONBOOT=yes - - - There are two ways to modify the MAC address of the NIC. - 1. Use ifconfig: - - # ifconfig ethX hw ether YY:YY:YY:YY:YY:YY - - ,where X is the device number assigned by Linux kernel, and - YY:YY:YY:YY:YY:YY is the MAC address assigned by the user. - - 2. Use ip: - - # ip link set ethX address YY:YY:YY:YY:YY:YY - - ,where X is the device number assigned by Linux kernel, and - YY:YY:YY:YY:YY:YY is the MAC address assigned by the user. - - - - 1. Force the link status when insert the driver. - - If the user is in the path ~/r8127, the link status can be forced - to one of the 5 modes as following command. - - # insmod ./src/r8127.ko speed=SPEED_MODE duplex=DUPLEX_MODE autoneg=NWAY_OPTION - - ,where - SPEED_MODE = 1000 for 1000Mbps - = 100 for 100Mbps - = 10 for 10Mbps - DUPLEX_MODE = 0 for half-duplex - = 1 for full-duplex - NWAY_OPTION = 0 for auto-negotiation off (true force) - = 1 for auto-negotiation on (nway force) - For example: - - # insmod ./src/r8127.ko speed=100 duplex=0 autoneg=1 - - will force PHY to operate in 100Mpbs Half-duplex(nway force). - - 2. Force the link status by using ethtool. - a. Insert the driver first. - b. Make sure that ethtool exists in /sbin. - c. Force the link status as the following command. - - 2.5G before kernel v4.10 - # ethtool -s eth0 autoneg on advertise 0x802f - - 2.5G for kernel v4.10 and later - # ethtool -s eth0 autoneg on advertise 0x80000000002f - - 5G for kernel v4.10 and later (Couldn't be supported before kernel v4.10) - # ethtool -s eth0 autoneg on advertise 0x180000000002f - - # ethtool -s eth0 autoneg on advertise 0x1000 (10G) - # ethtool -s eth0 autoneg on advertise 0x002f (1G) - # ethtool -s eth0 autoneg on advertise 0x000f (100M full) - # ethtool -s eth0 autoneg on advertise 0x0003 (10M full) - - - Transmitting Jumbo Frames, whose packet size is bigger than 1500 bytes, please change mtu by the following command. - - # ifconfig ethX mtu MTU - - , where X=0,1,2,..., and MTU is configured by user. - - RTL8127 supports Jumbo Frame size up to 9 kBytes. - - - Get/Set device EEE status - - Get EEE device status - # ethtool --show-eee enp1s0 - - Set EEE device status - # ethtool --set-eee enp1s0 eee on tx-lpi on tx-timer 1546 advertise 0x0008 (100M full) - # ethtool --set-eee enp1s0 eee on tx-lpi on tx-timer 1546 advertise 0x0020 (1G) - # ethtool --set-eee enp1s0 eee on tx-lpi on tx-timer 1546 advertise 0x8000 (2.5G) diff --git a/drivers/net/ethernet/realtek/r8127/autorun.sh b/drivers/net/ethernet/realtek/r8127/autorun.sh deleted file mode 100755 index fd87bced11583..0000000000000 --- a/drivers/net/ethernet/realtek/r8127/autorun.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0-only - -# invoke insmod with all arguments we got -# and use a pathname, as insmod doesn't look in . by default - -TARGET_PATH=$(find /lib/modules/$(uname -r)/kernel/drivers/net/ethernet -name realtek -type d) -if [ "$TARGET_PATH" = "" ]; then - TARGET_PATH=$(find /lib/modules/$(uname -r)/kernel/drivers/net -name realtek -type d) -fi -if [ "$TARGET_PATH" = "" ]; then - TARGET_PATH=/lib/modules/$(uname -r)/kernel/drivers/net -fi -echo -echo "Check old driver and unload it." -check=`lsmod | grep r8169` -if [ "$check" != "" ]; then - echo "rmmod r8169" - /sbin/rmmod r8169 -fi - -check=`lsmod | grep r8127` -if [ "$check" != "" ]; then - echo "rmmod r8127" - /sbin/rmmod r8127 -fi - -echo "Build the module and install" -echo "-------------------------------" >> log.txt -date 1>>log.txt -make $@ all 1>>log.txt || exit 1 -module=`ls src/*.ko` -module=${module#src/} -module=${module%.ko} - -if [ "$module" = "" ]; then - echo "No driver exists!!!" - exit 1 -elif [ "$module" != "r8169" ]; then - if test -e $TARGET_PATH/r8169.ko ; then - echo "Backup r8169.ko" - if test -e $TARGET_PATH/r8169.bak ; then - i=0 - while test -e $TARGET_PATH/r8169.bak$i - do - i=$(($i+1)) - done - echo "rename r8169.ko to r8169.bak$i" - mv $TARGET_PATH/r8169.ko $TARGET_PATH/r8169.bak$i - else - echo "rename r8169.ko to r8169.bak" - mv $TARGET_PATH/r8169.ko $TARGET_PATH/r8169.bak - fi - fi - if test -e $TARGET_PATH/r8169.ko.zst ; then - echo "Backup r8169.ko.zst" - if test -e $TARGET_PATH/r8169.zst.bak ; then - i=0 - while test -e $TARGET_PATH/r8169.zst.bak$i - do - i=$(($i+1)) - done - echo "rename r8169.ko.zst to r8169.zst.bak$i" - mv $TARGET_PATH/r8169.ko.zst $TARGET_PATH/r8169.zst.bak$i - else - echo "rename r8169.ko.zst to r8169.zst.bak" - mv $TARGET_PATH/r8169.ko.zst $TARGET_PATH/r8169.zst.bak - fi - fi -fi - -echo "DEPMOD $(uname -r)" -depmod `uname -r` -echo "load module $module" -modprobe $module - -is_update_initramfs=n -distrib_list="ubuntu debian" - -if [ -r /etc/debian_version ]; then - is_update_initramfs=y -elif [ -r /etc/lsb-release ]; then - for distrib in $distrib_list - do - /bin/grep -i "$distrib" /etc/lsb-release 2>&1 /dev/null && \ - is_update_initramfs=y && break - done -fi - -if [ "$is_update_initramfs" = "y" ]; then - if which update-initramfs >/dev/null ; then - echo "Updating initramfs. Please wait." - update-initramfs -u -k $(uname -r) - else - echo "update-initramfs: command not found" - exit 1 - fi -fi - -echo "Completed." -exit 0 diff --git a/drivers/net/ethernet/realtek/r8127/src/Makefile_linux24x b/drivers/net/ethernet/realtek/r8127/src/Makefile_linux24x deleted file mode 100755 index 7cb3d91a85a64..0000000000000 --- a/drivers/net/ethernet/realtek/r8127/src/Makefile_linux24x +++ /dev/null @@ -1,75 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -################################################################################ -# -# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet -# controllers with PCI-Express interface. -# -# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; either version 2 of the License, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, see . -# -# Author: -# Realtek NIC software team -# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan -# -################################################################################ - -################################################################################ -# This product is covered by one or more of the following patents: -# US6,570,884, US6,115,776, and US6,327,625. -################################################################################ - -CC := gcc -LD := ld -ARCH := $(shell uname -m | sed 's/i.86/i386/') -KSRC := /lib/modules/$(shell uname -r)/build -CONFIG_FILE := $(KSRC)/include/linux/autoconf.h -KMISC := /lib/modules/$(shell uname -r)/kernel/drivers/net/ - - -ifeq ($(ARCH),x86_64) - MODCFLAGS += -mcmodel=kernel -mno-red-zone -endif - -#standard flags for module builds -MODCFLAGS += -DLINUX -D__KERNEL__ -DMODULE -O2 -pipe -Wall -MODCFLAGS += -I$(KSRC)/include -I. -MODCFLAGS += -DMODVERSIONS -DEXPORT_SYMTAB -include $(KSRC)/include/linux/modversions.h -SOURCE := r8127_n.c rtl_eeprom.c rtltool.c -OBJS := $(SOURCE:.c=.o) - - -SMP := $(shell $(CC) $(MODCFLAGS) -E -dM $(CONFIG_FILE) | \ - grep CONFIG_SMP | awk '{print $$3}') - -ifneq ($(SMP),1) - SMP := 0 -endif - -ifeq ($(SMP),1) - MODCFLAGS += -D__SMP__ -endif - -modules: $(OBJS) - $(LD) -r $^ -o r8127.o - strip --strip-debug r8127.o - -%.o: %.c - $(CC) $(MODCFLAGS) -c $< -o $@ - -clean: - rm *.o -f - -install: - install -m 744 -c r8127.o $(KMISC) From 1686180809e4e84fc9de6c6b0cfe0b6084262762 Mon Sep 17 00:00:00 2001 From: Abhishek Sahu Date: Mon, 28 Apr 2025 15:45:44 +0000 Subject: [PATCH 038/311] NVIDIA: SAUCE: r8127: Moved files from r8127/src to r8127 folder BugLink: https://bugs.launchpad.net/bugs/2109730 This commit moved all files from src folder to parent folder itself. Signed-off-by: Abhishek Sahu Acked-by: Matt Ochs Acked-by: Carol L Soto Acked-by: Ian May Acked-by: Jacob Martin Acked-by: Noah Wager Signed-off-by: Ian May (cherry picked from commit a5fe39b0572298d86a23047fa8d9e73ae012c3dc noble:linux-nvidia-6.11) Signed-off-by: Jacob Martin (cherry picked from commit 1802cd38abe2 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit f83397fd097642788c53fc208b4226f987312351 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/net/ethernet/realtek/r8127/{src => }/Makefile | 0 drivers/net/ethernet/realtek/r8127/{src => }/r8127.h | 0 drivers/net/ethernet/realtek/r8127/{src => }/r8127_dash.h | 0 drivers/net/ethernet/realtek/r8127/{src => }/r8127_firmware.c | 0 drivers/net/ethernet/realtek/r8127/{src => }/r8127_firmware.h | 0 drivers/net/ethernet/realtek/r8127/{src => }/r8127_n.c | 0 drivers/net/ethernet/realtek/r8127/{src => }/r8127_ptp.c | 0 drivers/net/ethernet/realtek/r8127/{src => }/r8127_ptp.h | 0 drivers/net/ethernet/realtek/r8127/{src => }/r8127_realwow.h | 0 drivers/net/ethernet/realtek/r8127/{src => }/r8127_rss.c | 0 drivers/net/ethernet/realtek/r8127/{src => }/r8127_rss.h | 0 drivers/net/ethernet/realtek/r8127/{src => }/rtl_eeprom.c | 0 drivers/net/ethernet/realtek/r8127/{src => }/rtl_eeprom.h | 0 drivers/net/ethernet/realtek/r8127/{src => }/rtltool.c | 0 drivers/net/ethernet/realtek/r8127/{src => }/rtltool.h | 0 15 files changed, 0 insertions(+), 0 deletions(-) rename drivers/net/ethernet/realtek/r8127/{src => }/Makefile (100%) rename drivers/net/ethernet/realtek/r8127/{src => }/r8127.h (100%) rename drivers/net/ethernet/realtek/r8127/{src => }/r8127_dash.h (100%) rename drivers/net/ethernet/realtek/r8127/{src => }/r8127_firmware.c (100%) rename drivers/net/ethernet/realtek/r8127/{src => }/r8127_firmware.h (100%) rename drivers/net/ethernet/realtek/r8127/{src => }/r8127_n.c (100%) rename drivers/net/ethernet/realtek/r8127/{src => }/r8127_ptp.c (100%) rename drivers/net/ethernet/realtek/r8127/{src => }/r8127_ptp.h (100%) rename drivers/net/ethernet/realtek/r8127/{src => }/r8127_realwow.h (100%) rename drivers/net/ethernet/realtek/r8127/{src => }/r8127_rss.c (100%) rename drivers/net/ethernet/realtek/r8127/{src => }/r8127_rss.h (100%) rename drivers/net/ethernet/realtek/r8127/{src => }/rtl_eeprom.c (100%) rename drivers/net/ethernet/realtek/r8127/{src => }/rtl_eeprom.h (100%) rename drivers/net/ethernet/realtek/r8127/{src => }/rtltool.c (100%) rename drivers/net/ethernet/realtek/r8127/{src => }/rtltool.h (100%) diff --git a/drivers/net/ethernet/realtek/r8127/src/Makefile b/drivers/net/ethernet/realtek/r8127/Makefile similarity index 100% rename from drivers/net/ethernet/realtek/r8127/src/Makefile rename to drivers/net/ethernet/realtek/r8127/Makefile diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127.h b/drivers/net/ethernet/realtek/r8127/r8127.h similarity index 100% rename from drivers/net/ethernet/realtek/r8127/src/r8127.h rename to drivers/net/ethernet/realtek/r8127/r8127.h diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127_dash.h b/drivers/net/ethernet/realtek/r8127/r8127_dash.h similarity index 100% rename from drivers/net/ethernet/realtek/r8127/src/r8127_dash.h rename to drivers/net/ethernet/realtek/r8127/r8127_dash.h diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127_firmware.c b/drivers/net/ethernet/realtek/r8127/r8127_firmware.c similarity index 100% rename from drivers/net/ethernet/realtek/r8127/src/r8127_firmware.c rename to drivers/net/ethernet/realtek/r8127/r8127_firmware.c diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127_firmware.h b/drivers/net/ethernet/realtek/r8127/r8127_firmware.h similarity index 100% rename from drivers/net/ethernet/realtek/r8127/src/r8127_firmware.h rename to drivers/net/ethernet/realtek/r8127/r8127_firmware.h diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127_n.c b/drivers/net/ethernet/realtek/r8127/r8127_n.c similarity index 100% rename from drivers/net/ethernet/realtek/r8127/src/r8127_n.c rename to drivers/net/ethernet/realtek/r8127/r8127_n.c diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127_ptp.c b/drivers/net/ethernet/realtek/r8127/r8127_ptp.c similarity index 100% rename from drivers/net/ethernet/realtek/r8127/src/r8127_ptp.c rename to drivers/net/ethernet/realtek/r8127/r8127_ptp.c diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127_ptp.h b/drivers/net/ethernet/realtek/r8127/r8127_ptp.h similarity index 100% rename from drivers/net/ethernet/realtek/r8127/src/r8127_ptp.h rename to drivers/net/ethernet/realtek/r8127/r8127_ptp.h diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127_realwow.h b/drivers/net/ethernet/realtek/r8127/r8127_realwow.h similarity index 100% rename from drivers/net/ethernet/realtek/r8127/src/r8127_realwow.h rename to drivers/net/ethernet/realtek/r8127/r8127_realwow.h diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127_rss.c b/drivers/net/ethernet/realtek/r8127/r8127_rss.c similarity index 100% rename from drivers/net/ethernet/realtek/r8127/src/r8127_rss.c rename to drivers/net/ethernet/realtek/r8127/r8127_rss.c diff --git a/drivers/net/ethernet/realtek/r8127/src/r8127_rss.h b/drivers/net/ethernet/realtek/r8127/r8127_rss.h similarity index 100% rename from drivers/net/ethernet/realtek/r8127/src/r8127_rss.h rename to drivers/net/ethernet/realtek/r8127/r8127_rss.h diff --git a/drivers/net/ethernet/realtek/r8127/src/rtl_eeprom.c b/drivers/net/ethernet/realtek/r8127/rtl_eeprom.c similarity index 100% rename from drivers/net/ethernet/realtek/r8127/src/rtl_eeprom.c rename to drivers/net/ethernet/realtek/r8127/rtl_eeprom.c diff --git a/drivers/net/ethernet/realtek/r8127/src/rtl_eeprom.h b/drivers/net/ethernet/realtek/r8127/rtl_eeprom.h similarity index 100% rename from drivers/net/ethernet/realtek/r8127/src/rtl_eeprom.h rename to drivers/net/ethernet/realtek/r8127/rtl_eeprom.h diff --git a/drivers/net/ethernet/realtek/r8127/src/rtltool.c b/drivers/net/ethernet/realtek/r8127/rtltool.c similarity index 100% rename from drivers/net/ethernet/realtek/r8127/src/rtltool.c rename to drivers/net/ethernet/realtek/r8127/rtltool.c diff --git a/drivers/net/ethernet/realtek/r8127/src/rtltool.h b/drivers/net/ethernet/realtek/r8127/rtltool.h similarity index 100% rename from drivers/net/ethernet/realtek/r8127/src/rtltool.h rename to drivers/net/ethernet/realtek/r8127/rtltool.h From e3db3df2714e7763912b2fc3735d2b40555b4d9e Mon Sep 17 00:00:00 2001 From: tbergstrom Date: Tue, 22 Apr 2025 13:26:49 -0700 Subject: [PATCH 039/311] NVIDIA: SAUCE: Add r8127 in kernel build BugLink: https://bugs.launchpad.net/bugs/2109730 In the original code, r8127 driver was build as out of tree module. This commit adds Kconfig and updates Makefile for building it with kernel build. r8127 driver internally uses different config flags and these are set through EXTRA_CFLAGS. These config flags are now set in the Makefile with ccflags-y. All the flags, that were getting enabled by default in the original code, have been enabled in ccflags-y. This commit is not enabling any extra flags. Some of the files compilation are dependent upon a particular flag. Now, only default flags are set, so these files will become unused, This commit has removed these files. Signed-off-by: Terje Bergstrom Signed-off-by: Abhishek Sahu Acked-by: Matt Ochs Acked-by: Carol L Soto Acked-by: Ian May Acked-by: Jacob Martin Acked-by: Noah Wager Signed-off-by: Ian May (backported from commit 04ea6d025e87e2d9b8f88541236ebdd116a15d95 noble:linux-nvidia-6.11) [jacobmartin: adjust context around RTASE definitions introduced in K6.14] Signed-off-by: Jacob Martin (cherry picked from commit 6217feae8e42 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit d423ea701ce8f15137682f6b33118bebf1290964 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/net/ethernet/realtek/Kconfig | 2 + drivers/net/ethernet/realtek/Makefile | 1 + drivers/net/ethernet/realtek/r8127/Kconfig | 42 + drivers/net/ethernet/realtek/r8127/Makefile | 180 +--- .../ethernet/realtek/r8127/r8127_firmware.c | 264 ----- .../net/ethernet/realtek/r8127/r8127_ptp.c | 944 ------------------ .../net/ethernet/realtek/r8127/r8127_ptp.h | 202 ---- .../net/ethernet/realtek/r8127/r8127_rss.c | 583 ----------- 8 files changed, 48 insertions(+), 2170 deletions(-) create mode 100644 drivers/net/ethernet/realtek/r8127/Kconfig delete mode 100755 drivers/net/ethernet/realtek/r8127/r8127_firmware.c delete mode 100755 drivers/net/ethernet/realtek/r8127/r8127_ptp.c delete mode 100755 drivers/net/ethernet/realtek/r8127/r8127_ptp.h delete mode 100755 drivers/net/ethernet/realtek/r8127/r8127_rss.c diff --git a/drivers/net/ethernet/realtek/Kconfig b/drivers/net/ethernet/realtek/Kconfig index 9b0f4f9631dba..ca58b70f6f317 100644 --- a/drivers/net/ethernet/realtek/Kconfig +++ b/drivers/net/ethernet/realtek/Kconfig @@ -126,4 +126,6 @@ config RTASE To compile this driver as a module, choose M here: the module will be called rtase. This is recommended. +source "drivers/net/ethernet/realtek/r8127/Kconfig" + endif # NET_VENDOR_REALTEK diff --git a/drivers/net/ethernet/realtek/Makefile b/drivers/net/ethernet/realtek/Makefile index 12a9c399f40c6..4e1d21508e260 100644 --- a/drivers/net/ethernet/realtek/Makefile +++ b/drivers/net/ethernet/realtek/Makefile @@ -9,3 +9,4 @@ r8169-y += r8169_main.o r8169_firmware.o r8169_phy_config.o r8169-$(CONFIG_R8169_LEDS) += r8169_leds.o obj-$(CONFIG_R8169) += r8169.o obj-$(CONFIG_RTASE) += rtase/ +obj-$(CONFIG_R8127) += r8127/ diff --git a/drivers/net/ethernet/realtek/r8127/Kconfig b/drivers/net/ethernet/realtek/r8127/Kconfig new file mode 100644 index 0000000000000..e5a8e399390cd --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/Kconfig @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: GPL-2.0-only +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ + +################################################################################ +# This product is covered by one or more of the following patents: +# US6,570,884, US6,115,776, and US6,327,625. +################################################################################ + +config R8127 + tristate "RealTek RTL-8127 PCI 10 Gigabit Ethernet Adapter support" + depends on PCI + help + This is a driver for the 10 Gigabit Ethernet PCI network cards based on + the RTL-8127 chips. If you have one of those, say Y here. + + To compile this driver as a module, choose M here: the module + will be called r8127. This is recommended. + diff --git a/drivers/net/ethernet/realtek/r8127/Makefile b/drivers/net/ethernet/realtek/r8127/Makefile index d270904691bf3..d7cf5f65a9d8e 100755 --- a/drivers/net/ethernet/realtek/r8127/Makefile +++ b/drivers/net/ethernet/realtek/r8127/Makefile @@ -30,180 +30,6 @@ # US6,570,884, US6,115,776, and US6,327,625. ################################################################################ -CONFIG_SOC_LAN = y -ENABLE_REALWOW_SUPPORT = n -ENABLE_DASH_SUPPORT = n -ENABLE_DASH_PRINTER_SUPPORT = n -CONFIG_DOWN_SPEED_100 = n -CONFIG_ASPM = y -ENABLE_S5WOL = y -ENABLE_S5_KEEP_CURR_MAC = n -ENABLE_EEE = y -ENABLE_S0_MAGIC_PACKET = n -ENABLE_TX_NO_CLOSE = y -ENABLE_MULTIPLE_TX_QUEUE = n -ENABLE_PTP_SUPPORT = n -ENABLE_RSS_SUPPORT = n -ENABLE_LIB_SUPPORT = n -ENABLE_USE_FIRMWARE_FILE = n -DISABLE_WOL_SUPPORT = n -DISABLE_MULTI_MSIX_VECTOR = n -ENABLE_DOUBLE_VLAN = n -ENABLE_PAGE_REUSE = n -ENABLE_RX_PACKET_FRAGMENT = n -ENABLE_GIGA_LITE = y - -ifneq ($(KERNELRELEASE),) - obj-m := r8127.o - r8127-objs := r8127_n.o rtl_eeprom.o rtltool.o - ifeq ($(CONFIG_SOC_LAN), y) - EXTRA_CFLAGS += -DCONFIG_SOC_LAN - endif - ifeq ($(ENABLE_REALWOW_SUPPORT), y) - r8127-objs += r8127_realwow.o - EXTRA_CFLAGS += -DENABLE_REALWOW_SUPPORT - endif - ifeq ($(ENABLE_DASH_SUPPORT), y) - r8127-objs += r8127_dash.o - EXTRA_CFLAGS += -DENABLE_DASH_SUPPORT - endif - ifeq ($(ENABLE_DASH_PRINTER_SUPPORT), y) - r8127-objs += r8127_dash.o - EXTRA_CFLAGS += -DENABLE_DASH_SUPPORT -DENABLE_DASH_PRINTER_SUPPORT - endif - EXTRA_CFLAGS += -DCONFIG_R8127_NAPI - EXTRA_CFLAGS += -DCONFIG_R8127_VLAN - ifeq ($(CONFIG_DOWN_SPEED_100), y) - EXTRA_CFLAGS += -DCONFIG_DOWN_SPEED_100 - endif - ifeq ($(CONFIG_ASPM), y) - EXTRA_CFLAGS += -DCONFIG_ASPM - endif - ifeq ($(ENABLE_S5WOL), y) - EXTRA_CFLAGS += -DENABLE_S5WOL - endif - ifeq ($(ENABLE_S5_KEEP_CURR_MAC), y) - EXTRA_CFLAGS += -DENABLE_S5_KEEP_CURR_MAC - endif - ifeq ($(ENABLE_EEE), y) - EXTRA_CFLAGS += -DENABLE_EEE - endif - ifeq ($(ENABLE_S0_MAGIC_PACKET), y) - EXTRA_CFLAGS += -DENABLE_S0_MAGIC_PACKET - endif - ifeq ($(ENABLE_TX_NO_CLOSE), y) - EXTRA_CFLAGS += -DENABLE_TX_NO_CLOSE - endif - ifeq ($(ENABLE_MULTIPLE_TX_QUEUE), y) - EXTRA_CFLAGS += -DENABLE_MULTIPLE_TX_QUEUE - endif - ifeq ($(ENABLE_PTP_SUPPORT), y) - r8127-objs += r8127_ptp.o - EXTRA_CFLAGS += -DENABLE_PTP_SUPPORT - endif - ifeq ($(ENABLE_RSS_SUPPORT), y) - r8127-objs += r8127_rss.o - EXTRA_CFLAGS += -DENABLE_RSS_SUPPORT - endif - ifeq ($(ENABLE_LIB_SUPPORT), y) - r8127-objs += r8127_lib.o - EXTRA_CFLAGS += -DENABLE_LIB_SUPPORT - endif - ifeq ($(ENABLE_USE_FIRMWARE_FILE), y) - r8127-objs += r8127_firmware.o - EXTRA_CFLAGS += -DENABLE_USE_FIRMWARE_FILE - endif - ifeq ($(DISABLE_WOL_SUPPORT), y) - EXTRA_CFLAGS += -DDISABLE_WOL_SUPPORT - endif - ifeq ($(DISABLE_MULTI_MSIX_VECTOR), y) - EXTRA_CFLAGS += -DDISABLE_MULTI_MSIX_VECTOR - endif - ifeq ($(ENABLE_DOUBLE_VLAN), y) - EXTRA_CFLAGS += -DENABLE_DOUBLE_VLAN - endif - ifeq ($(ENABLE_PAGE_REUSE), y) - EXTRA_CFLAGS += -DENABLE_PAGE_REUSE - endif - ifeq ($(ENABLE_RX_PACKET_FRAGMENT), y) - EXTRA_CFLAGS += -DENABLE_RX_PACKET_FRAGMENT - endif - ifeq ($(ENABLE_GIGA_LITE), y) - EXTRA_CFLAGS += -DENABLE_GIGA_LITE - endif -else - BASEDIR := /lib/modules/$(shell uname -r) - KERNELDIR ?= $(BASEDIR)/build - PWD :=$(shell pwd) - DRIVERDIR := $(shell find $(BASEDIR)/kernel/drivers/net/ethernet -name realtek -type d) - ifeq ($(DRIVERDIR),) - DRIVERDIR := $(shell find $(BASEDIR)/kernel/drivers/net -name realtek -type d) - endif - ifeq ($(DRIVERDIR),) - DRIVERDIR := $(BASEDIR)/kernel/drivers/net - endif - RTKDIR := $(subst $(BASEDIR)/,,$(DRIVERDIR)) - - KERNEL_GCC_VERSION := $(shell cat /proc/version | sed -n 's/.*gcc version \([[:digit:]]\.[[:digit:]]\.[[:digit:]]\).*/\1/p') - CCVERSION = $(shell $(CC) -dumpversion) - - KVER = $(shell uname -r) - KMAJ = $(shell echo $(KVER) | \ - sed -e 's/^\([0-9][0-9]*\)\.[0-9][0-9]*\.[0-9][0-9]*.*/\1/') - KMIN = $(shell echo $(KVER) | \ - sed -e 's/^[0-9][0-9]*\.\([0-9][0-9]*\)\.[0-9][0-9]*.*/\1/') - KREV = $(shell echo $(KVER) | \ - sed -e 's/^[0-9][0-9]*\.[0-9][0-9]*\.\([0-9][0-9]*\).*/\1/') - - kver_ge = $(shell \ - echo test | awk '{if($(KMAJ) < $(1)) {print 0} else { \ - if($(KMAJ) > $(1)) {print 1} else { \ - if($(KMIN) < $(2)) {print 0} else { \ - if($(KMIN) > $(2)) {print 1} else { \ - if($(KREV) < $(3)) {print 0} else { print 1 } \ - }}}}}' \ - ) - -.PHONY: all -all: print_vars clean modules install - -print_vars: - @echo - @echo "CC: " $(CC) - @echo "CCVERSION: " $(CCVERSION) - @echo "KERNEL_GCC_VERSION: " $(KERNEL_GCC_VERSION) - @echo "KVER: " $(KVER) - @echo "KMAJ: " $(KMAJ) - @echo "KMIN: " $(KMIN) - @echo "KREV: " $(KREV) - @echo "BASEDIR: " $(BASEDIR) - @echo "DRIVERDIR: " $(DRIVERDIR) - @echo "PWD: " $(PWD) - @echo "RTKDIR: " $(RTKDIR) - @echo - -.PHONY:modules -modules: -#ifeq ($(call kver_ge,5,0,0),1) - $(MAKE) -C $(KERNELDIR) M=$(PWD) modules -#else -# $(MAKE) -C $(KERNELDIR) SUBDIRS=$(PWD) modules -#endif - -.PHONY:clean -clean: -#ifeq ($(call kver_ge,5,0,0),1) - $(MAKE) -C $(KERNELDIR) M=$(PWD) clean -#else -# $(MAKE) -C $(KERNELDIR) SUBDIRS=$(PWD) clean -#endif - -.PHONY:install -install: -#ifeq ($(call kver_ge,5,0,0),1) - $(MAKE) -C $(KERNELDIR) M=$(PWD) INSTALL_MOD_DIR=$(RTKDIR) modules_install -#else -# $(MAKE) -C $(KERNELDIR) SUBDIRS=$(PWD) INSTALL_MOD_DIR=$(RTKDIR) modules_install -#endif - -endif +ccflags-y += -DCONFIG_SOC_LAN -DCONFIG_R8127_NAPI -DCONFIG_R8127_VLAN -DCONFIG_ASPM -DENABLE_S5WOL -DENABLE_EEE -DENABLE_TX_NO_CLOSE -DENABLE_GIGA_LITE +obj-$(CONFIG_R8127) += r8127.o +r8127-y := r8127_n.o rtl_eeprom.o rtltool.o diff --git a/drivers/net/ethernet/realtek/r8127/r8127_firmware.c b/drivers/net/ethernet/realtek/r8127/r8127_firmware.c deleted file mode 100755 index 7ab59f641e77a..0000000000000 --- a/drivers/net/ethernet/realtek/r8127/r8127_firmware.c +++ /dev/null @@ -1,264 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* -################################################################################ -# -# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet -# controllers with PCI-Express interface. -# -# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; either version 2 of the License, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, see . -# -# Author: -# Realtek NIC software team -# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan -# -################################################################################ -*/ - -/************************************************************************************ - * This product is covered by one or more of the following patents: - * US6,570,884, US6,115,776, and US6,327,625. - ***********************************************************************************/ - -#include -#include -#include - -#include "r8127_firmware.h" - -enum rtl_fw_opcode { - PHY_READ = 0x0, - PHY_DATA_OR = 0x1, - PHY_DATA_AND = 0x2, - PHY_BJMPN = 0x3, - PHY_MDIO_CHG = 0x4, - PHY_CLEAR_READCOUNT = 0x7, - PHY_WRITE = 0x8, - PHY_READCOUNT_EQ_SKIP = 0x9, - PHY_COMP_EQ_SKIPN = 0xa, - PHY_COMP_NEQ_SKIPN = 0xb, - PHY_WRITE_PREVIOUS = 0xc, - PHY_SKIPN = 0xd, - PHY_DELAY_MS = 0xe, -}; - -struct fw_info { - u32 magic; - char version[RTL8127_VER_SIZE]; - __le32 fw_start; - __le32 fw_len; - u8 chksum; -} __packed; - -#if LINUX_VERSION_CODE < KERNEL_VERSION(4,16,0) -#define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER)) -#endif -#define FW_OPCODE_SIZE sizeof_field(struct rtl8127_fw_phy_action, code[0]) - -static bool rtl8127_fw_format_ok(struct rtl8127_fw *rtl_fw) -{ - const struct firmware *fw = rtl_fw->fw; - struct fw_info *fw_info = (struct fw_info *)fw->data; - struct rtl8127_fw_phy_action *pa = &rtl_fw->phy_action; - - if (fw->size < FW_OPCODE_SIZE) - return false; - - if (!fw_info->magic) { - size_t i, size, start; - u8 checksum = 0; - - if (fw->size < sizeof(*fw_info)) - return false; - - for (i = 0; i < fw->size; i++) - checksum += fw->data[i]; - if (checksum != 0) - return false; - - start = le32_to_cpu(fw_info->fw_start); - if (start > fw->size) - return false; - - size = le32_to_cpu(fw_info->fw_len); - if (size > (fw->size - start) / FW_OPCODE_SIZE) - return false; - - strscpy(rtl_fw->version, fw_info->version, RTL8127_VER_SIZE); - - pa->code = (__le32 *)(fw->data + start); - pa->size = size; - } else { - if (fw->size % FW_OPCODE_SIZE) - return false; - - strscpy(rtl_fw->version, rtl_fw->fw_name, RTL8127_VER_SIZE); - - pa->code = (__le32 *)fw->data; - pa->size = fw->size / FW_OPCODE_SIZE; - } - - return true; -} - -static bool rtl8127_fw_data_ok(struct rtl8127_fw *rtl_fw) -{ - struct rtl8127_fw_phy_action *pa = &rtl_fw->phy_action; - size_t index; - - for (index = 0; index < pa->size; index++) { - u32 action = le32_to_cpu(pa->code[index]); - u32 val = action & 0x0000ffff; - u32 regno = (action & 0x0fff0000) >> 16; - - switch (action >> 28) { - case PHY_READ: - case PHY_DATA_OR: - case PHY_DATA_AND: - case PHY_CLEAR_READCOUNT: - case PHY_WRITE: - case PHY_WRITE_PREVIOUS: - case PHY_DELAY_MS: - break; - - case PHY_MDIO_CHG: - if (val > 1) - goto out; - break; - - case PHY_BJMPN: - if (regno > index) - goto out; - break; - case PHY_READCOUNT_EQ_SKIP: - if (index + 2 >= pa->size) - goto out; - break; - case PHY_COMP_EQ_SKIPN: - case PHY_COMP_NEQ_SKIPN: - case PHY_SKIPN: - if (index + 1 + regno >= pa->size) - goto out; - break; - - default: - dev_err(rtl_fw->dev, "Invalid action 0x%08x\n", action); - return false; - } - } - - return true; -out: - dev_err(rtl_fw->dev, "Out of range of firmware\n"); - return false; -} - -void rtl8127_fw_write_firmware(struct rtl8127_private *tp, struct rtl8127_fw *rtl_fw) -{ - struct rtl8127_fw_phy_action *pa = &rtl_fw->phy_action; - rtl8127_fw_write_t fw_write = rtl_fw->phy_write; - rtl8127_fw_read_t fw_read = rtl_fw->phy_read; - int predata = 0, count = 0; - size_t index; - - for (index = 0; index < pa->size; index++) { - u32 action = le32_to_cpu(pa->code[index]); - u32 data = action & 0x0000ffff; - u32 regno = (action & 0x0fff0000) >> 16; - enum rtl_fw_opcode opcode = action >> 28; - - if (!action) - break; - - switch (opcode) { - case PHY_READ: - predata = fw_read(tp, regno); - count++; - break; - case PHY_DATA_OR: - predata |= data; - break; - case PHY_DATA_AND: - predata &= data; - break; - case PHY_BJMPN: - index -= (regno + 1); - break; - case PHY_MDIO_CHG: - if (data) { - fw_write = rtl_fw->mac_mcu_write; - fw_read = rtl_fw->mac_mcu_read; - } else { - fw_write = rtl_fw->phy_write; - fw_read = rtl_fw->phy_read; - } - - break; - case PHY_CLEAR_READCOUNT: - count = 0; - break; - case PHY_WRITE: - fw_write(tp, regno, data); - break; - case PHY_READCOUNT_EQ_SKIP: - if (count == data) - index++; - break; - case PHY_COMP_EQ_SKIPN: - if (predata == data) - index += regno; - break; - case PHY_COMP_NEQ_SKIPN: - if (predata != data) - index += regno; - break; - case PHY_WRITE_PREVIOUS: - fw_write(tp, regno, predata); - break; - case PHY_SKIPN: - index += regno; - break; - case PHY_DELAY_MS: - mdelay(data); - break; - } - } -} - -void rtl8127_fw_release_firmware(struct rtl8127_fw *rtl_fw) -{ - release_firmware(rtl_fw->fw); -} - -int rtl8127_fw_request_firmware(struct rtl8127_fw *rtl_fw) -{ - int rc; - - rc = request_firmware(&rtl_fw->fw, rtl_fw->fw_name, rtl_fw->dev); - if (rc < 0) - goto out; - - if (!rtl8127_fw_format_ok(rtl_fw) || !rtl8127_fw_data_ok(rtl_fw)) { - release_firmware(rtl_fw->fw); - rc = -EINVAL; - goto out; - } - - return 0; -out: - dev_err(rtl_fw->dev, "Unable to load firmware %s (%d)\n", - rtl_fw->fw_name, rc); - return rc; -} diff --git a/drivers/net/ethernet/realtek/r8127/r8127_ptp.c b/drivers/net/ethernet/realtek/r8127/r8127_ptp.c deleted file mode 100755 index f3fd421625c0c..0000000000000 --- a/drivers/net/ethernet/realtek/r8127/r8127_ptp.c +++ /dev/null @@ -1,944 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* -################################################################################ -# -# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet -# controllers with PCI-Express interface. -# -# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; either version 2 of the License, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, see . -# -# Author: -# Realtek NIC software team -# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan -# -################################################################################ -*/ - -/************************************************************************************ - * This product is covered by one or more of the following patents: - * US6,570,884, US6,115,776, and US6,327,625. - ***********************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "r8127.h" -#include "r8127_ptp.h" - -static void rtl8127_wait_clkadj_ready(struct rtl8127_private *tp) -{ - int i; - - for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) - if (!(rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CLK_CFG_8126) & CLKADJ_MODE_SET)) - break; -} - -static void rtl8127_set_clkadj_mode(struct rtl8127_private *tp, u16 cmd) -{ - rtl8127_clear_and_set_eth_phy_ocp_bit(tp, - PTP_CLK_CFG_8126, - BIT_3 | BIT_2 | BIT_1, - CLKADJ_MODE_SET | cmd); - - rtl8127_wait_clkadj_ready(tp); -} - -static int _rtl8127_phc_gettime(struct rtl8127_private *tp, struct timespec64 *ts64) -{ - unsigned long flags; - - spin_lock_irqsave(&tp->phy_lock, flags); - - //Direct Read - rtl8127_set_clkadj_mode(tp, DIRECT_READ); - - /* nanoseconds */ - //Ns[29:16] E414[13:0] - ts64->tv_nsec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_NS_HI_8126) & 0x3fff; - ts64->tv_nsec <<= 16; - //Ns[15:0] E412[15:0] - ts64->tv_nsec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_NS_LO_8126); - - - /* seconds */ - //S[47:32] E41A[15:0] - ts64->tv_sec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_S_HI_8126); - ts64->tv_sec <<= 16; - //S[31:16] E418[15:0] - ts64->tv_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_S_MI_8126); - ts64->tv_sec <<= 16; - //S[15:0] E416[15:0] - ts64->tv_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_S_LO_8126); - - spin_unlock_irqrestore(&tp->phy_lock, flags); - - return 0; -} - -static int _rtl8127_phc_settime(struct rtl8127_private *tp, const struct timespec64 *ts64) -{ - unsigned long flags; - - spin_lock_irqsave(&tp->phy_lock, flags); - - /* nanoseconds */ - //Ns[15:0] E412[15:0] - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_NS_LO_8126, ts64->tv_nsec); - //Ns[29:16] E414[13:0] - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_NS_HI_8126, (ts64->tv_nsec & 0x3fff0000) >> 16); - - /* seconds */ - //S[15:0] E416[15:0] - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_S_LO_8126, ts64->tv_sec); - //S[31:16] E418[15:0] - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_S_MI_8126, (ts64->tv_sec >> 16)); - //S[47:32] E41A[15:0] - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_S_HI_8126, (ts64->tv_sec >> 32)); - - //Direct Write - rtl8127_set_clkadj_mode(tp, DIRECT_WRITE); - - spin_unlock_irqrestore(&tp->phy_lock, flags); - - return 0; -} - -static int _rtl8127_phc_adjtime(struct rtl8127_private *tp, s64 delta) -{ - unsigned long flags; - struct timespec64 d; - bool negative; - u64 tohw; - u32 nsec; - u64 sec; - - if (delta < 0) { - negative = true; - tohw = -delta; - } else { - negative = false; - tohw = delta; - } - - d = ns_to_timespec64(tohw); - - nsec = d.tv_nsec; - sec = d.tv_sec; - - nsec &= 0x3fffffff; - sec &= 0x0000ffffffffffff; - - spin_lock_irqsave(&tp->phy_lock, flags); - - /* nanoseconds */ - //Ns[15:0] E412[15:0] - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_NS_LO_8126, nsec); - //Ns[29:16] E414[13:0] - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_NS_HI_8126, (nsec >> 16)); - - /* seconds */ - //S[15:0] E416[15:0] - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_S_LO_8126, sec); - //S[31:16] E418[15:0] - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_S_MI_8126, (sec >> 16)); - //S[47:32] E41A[15:0] - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_S_HI_8126, (sec >> 32)); - - if (negative) - rtl8127_set_clkadj_mode(tp, DECREMENT_STEP); - else - rtl8127_set_clkadj_mode(tp, INCREMENT_STEP); - - spin_unlock_irqrestore(&tp->phy_lock, flags); - - return 0; -} - -static int rtl8127_phc_adjtime(struct ptp_clock_info *ptp, s64 delta) -{ - struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info); - int ret; - - //netif_info(tp, drv, tp->dev, "phc adjust time\n"); - - ret = _rtl8127_phc_adjtime(tp, delta); - - return ret; -} - -/* - * delta = delta * 10^6 ppm = delta * 10^9 ppb (in this equation ppm and ppb are not variable) - * - * in adjfreq ppb is a variable - * ppb = delta * 10^9 - * delta = ppb / 10^9 - * rate_value = |delta| * 2^32 = |ppb| / 10^9 * 2^32 = (|ppb| << 32) / 10^9 - */ -static int _rtl8127_phc_adjfreq(struct ptp_clock_info *ptp, s32 ppb) -{ - struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info); - unsigned long flags; - u32 rate_value; - - if (ppb < 0) { - rate_value = ((u64)-ppb << 32) / 1000000000; - rate_value = ~rate_value + 1; - } else - rate_value = ((u64)ppb << 32) / 1000000000; - - spin_lock_irqsave(&tp->phy_lock, flags); - - /* nanoseconds */ - //Ns[15:0] E412[15:0] - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_NS_LO_8126, rate_value); - //Ns[22:16] E414[13:0] - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_NS_HI_8126, (rate_value & 0x003f0000) >> 16); - - rtl8127_set_clkadj_mode(tp, RATE_WRITE); - - spin_unlock_irqrestore(&tp->phy_lock, flags); - - return 0; -} - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,2,0) -static int rtl8127_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) -{ - s32 ppb = scaled_ppm_to_ppb(scaled_ppm); - - if (ppb > ptp->max_adj || ppb < -ptp->max_adj) - return -EINVAL; - - _rtl8127_phc_adjfreq(ptp, ppb); - - return 0; -} - -#else -static int rtl8127_phc_adjfreq(struct ptp_clock_info *ptp, s32 delta) -{ - //struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info); - - //netif_info(tp, drv, tp->dev, "phc adjust freq\n"); - - if (delta > ptp->max_adj || delta < -ptp->max_adj) - return -EINVAL; - - _rtl8127_phc_adjfreq(ptp, delta); - - return 0; -} -#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6,2,0) */ - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0) -static int rtl8127_phc_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts64, - struct ptp_system_timestamp *sts) -{ - struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info); - int ret; - - //netif_info(tp, drv, tp->dev, "phc get ts\n"); - - ptp_read_system_prets(sts); - ret = _rtl8127_phc_gettime(tp, ts64); - ptp_read_system_postts(sts); - - return ret; -} -#else -static int rtl8127_phc_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts64) -{ - struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info); - int ret; - - //netif_info(tp, drv, tp->dev, "phc get ts\n"); - - ret = _rtl8127_phc_gettime(tp, ts64); - - return ret; -} -#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0) */ - -static int rtl8127_phc_settime(struct ptp_clock_info *ptp, - const struct timespec64 *ts64) -{ - struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info); - int ret; - - //netif_info(tp, drv, tp->dev, "phc set ts\n"); - - ret = _rtl8127_phc_settime(tp, ts64); - - return ret; -} - -static void _rtl8127_phc_enable(struct ptp_clock_info *ptp, - struct ptp_clock_request *rq, int on) -{ - struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info); - unsigned long flags; - u16 phy_ocp_data; - - if (on) { - tp->pps_enable = 1; - rtl8127_clear_mac_ocp_bit(tp, 0xDC00, BIT_6); - rtl8127_clear_mac_ocp_bit(tp, 0xDC20, BIT_1); - - spin_lock_irqsave(&tp->phy_lock, flags); - - /* Set periodic pulse 1pps */ - /* E432[8:0] = 0x017d */ - phy_ocp_data = rtl8127_mdio_direct_read_phy_ocp(tp, 0xE432); - phy_ocp_data &= 0xFE00; - phy_ocp_data |= 0x017d; - rtl8127_mdio_direct_write_phy_ocp(tp, 0xE432, phy_ocp_data); - - rtl8127_mdio_direct_write_phy_ocp(tp, 0xE434, 0x7840); - - /* E436[8:0] = 0xbe */ - phy_ocp_data = rtl8127_mdio_direct_read_phy_ocp(tp, 0xE436); - phy_ocp_data &= 0xFE00; - phy_ocp_data |= 0xbe; - rtl8127_mdio_direct_write_phy_ocp(tp, 0xE436, phy_ocp_data); - - rtl8127_mdio_direct_write_phy_ocp(tp, 0xE438, 0xbc20); - - spin_unlock_irqrestore(&tp->phy_lock, flags); - - /* start hrtimer */ - hrtimer_start(&tp->pps_timer, 1000000000, HRTIMER_MODE_REL); - } else - tp->pps_enable = 0; -} - -static int rtl8127_phc_enable(struct ptp_clock_info *ptp, - struct ptp_clock_request *rq, int on) -{ - switch (rq->type) { - case PTP_CLK_REQ_PPS: - _rtl8127_phc_enable(ptp, rq, on); - return 0; - default: - return -EOPNOTSUPP; - } -} - -static void rtl8127_ptp_enable_config(struct rtl8127_private *tp) -{ - if (tp->syncE_en) - rtl8127_set_eth_phy_ocp_bit(tp, PTP_SYNCE_CTL, BIT_0); - else - rtl8127_clear_eth_phy_ocp_bit(tp, PTP_SYNCE_CTL, BIT_0); - - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CTL, PTP_CTL_TYPE_3 | BIT_12); - - rtl8127_set_eth_phy_ocp_bit(tp, 0xA640, BIT_15); -} - -int rtl8127_get_ts_info(struct net_device *netdev, - struct ethtool_ts_info *info) -{ - struct rtl8127_private *tp = netdev_priv(netdev); - - /* we always support timestamping disabled */ - info->rx_filters = BIT(HWTSTAMP_FILTER_NONE); - - if (tp->HwSuppPtpVer == 0) - return ethtool_op_get_ts_info(netdev, info); - - info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | - SOF_TIMESTAMPING_RX_SOFTWARE | - SOF_TIMESTAMPING_SOFTWARE | - SOF_TIMESTAMPING_TX_HARDWARE | - SOF_TIMESTAMPING_RX_HARDWARE | - SOF_TIMESTAMPING_RAW_HARDWARE; - - if (tp->ptp_clock) - info->phc_index = ptp_clock_index(tp->ptp_clock); - else - info->phc_index = -1; - - info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON); - - info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) | - BIT(HWTSTAMP_FILTER_PTP_V2_EVENT) | - BIT(HWTSTAMP_FILTER_PTP_V2_L4_EVENT) | - BIT(HWTSTAMP_FILTER_PTP_V2_SYNC) | - BIT(HWTSTAMP_FILTER_PTP_V2_L4_SYNC) | - BIT(HWTSTAMP_FILTER_PTP_V2_DELAY_REQ) | - BIT(HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ); - - return 0; -} - -static const struct ptp_clock_info rtl_ptp_clock_info = { - .owner = THIS_MODULE, - .n_alarm = 0, - .n_ext_ts = 0, - .n_per_out = 0, - .n_pins = 0, - .pps = 1, -#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,2,0) - .adjfine = rtl8127_ptp_adjfine, -#else - .adjfreq = rtl8127_phc_adjfreq, -#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6,2,0) */ - .adjtime = rtl8127_phc_adjtime, -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0) - .gettimex64 = rtl8127_phc_gettime, -#else - .gettime64 = rtl8127_phc_gettime, -#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0) */ - - .settime64 = rtl8127_phc_settime, - .enable = rtl8127_phc_enable, -}; - -static u16 rtl8127_ptp_get_tx_msgtype(struct rtl8127_private *tp) -{ - u16 tx_ts_ready = 0; - int i; - - for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { - tx_ts_ready = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_STA) & 0xF000; - if (tx_ts_ready) - break; - } - - switch (tx_ts_ready) { - case TX_TS_PDLYRSP_RDY: - return PTP_MSGTYPE_PDELAY_RESP; - case TX_TS_PDLYREQ_RDY: - return PTP_MSGTYPE_PDELAY_REQ; - case TX_TS_DLYREQ_RDY: - return PTP_MSGTYPE_DELAY_REQ; - case TX_TS_SYNC_RDY: - default: - return PTP_MSGTYPE_SYNC; - } -} - -/* -static u16 rtl8127_ptp_get_rx_msgtype(struct rtl8127_private *tp) -{ - u16 rx_ts_ready = 0; - int i; - - for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { - rx_ts_ready = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_STA) & 0x0F00; - if (rx_ts_ready) - break; - } - - switch (rx_ts_ready) { - case RX_TS_PDLYRSP_RDY: - return PTP_MSGTYPE_PDELAY_RESP; - case RX_TS_PDLYREQ_RDY: - return PTP_MSGTYPE_PDELAY_REQ; - case RX_TS_DLYREQ_RDY: - return PTP_MSGTYPE_DELAY_REQ; - case RX_TS_SYNC_RDY: - default: - return PTP_MSGTYPE_SYNC; - } -} -*/ - -static void rtl8127_wait_trx_ts_ready(struct rtl8127_private *tp) -{ - int i; - - for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) - if (!(rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_STA) & TRX_TS_RD)) - break; -} - -static void rtl8127_set_trx_ts_cmd(struct rtl8127_private *tp, u16 cmd) -{ - rtl8127_clear_and_set_eth_phy_ocp_bit(tp, - PTP_TRX_TS_STA, - TRXTS_SEL | BIT_3 | BIT_2, - TRX_TS_RD | cmd); - - rtl8127_wait_trx_ts_ready(tp); -} - -static void rtl8127_ptp_egresstime(struct rtl8127_private *tp, struct timespec64 *ts64) -{ - u16 msgtype; - - msgtype = rtl8127_ptp_get_tx_msgtype(tp); - - msgtype <<= 2; - - rtl8127_set_trx_ts_cmd(tp, (msgtype | BIT_4)); - - /* nanoseconds */ - //Ns[29:16] E448[13:0] - ts64->tv_nsec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_NS_HI) & 0x3fff; - ts64->tv_nsec <<= 16; - //Ns[15:0] E446[15:0] - ts64->tv_nsec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_NS_LO); - - /* seconds */ - //S[47:32] E44E[15:0] - ts64->tv_sec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_S_HI); - ts64->tv_sec <<= 16; - //S[31:16] E44C[15:0] - ts64->tv_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_S_MI); - ts64->tv_sec <<= 16; - //S[15:0] E44A[15:0] - ts64->tv_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_S_LO); -} - -static void rtl8127_ptp_ingresstime(struct rtl8127_private *tp, struct timespec64 *ts64, u8 type) -{ - u16 msgtype; - - switch (type) { - case PTP_MSGTYPE_PDELAY_RESP: - case PTP_MSGTYPE_PDELAY_REQ: - case PTP_MSGTYPE_DELAY_REQ: - case PTP_MSGTYPE_SYNC: - msgtype = type << 2; - break; - default: - return; - } - - rtl8127_set_trx_ts_cmd(tp, (TRXTS_SEL | msgtype | BIT_4)); - - /* nanoseconds */ - //Ns[29:16] E448[13:0] - ts64->tv_nsec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_NS_HI) & 0x3fff; - ts64->tv_nsec <<= 16; - //Ns[15:0] E446[15:0] - ts64->tv_nsec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_NS_LO); - - /* seconds */ - //S[47:32] E44E[15:0] - ts64->tv_sec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_S_HI); - ts64->tv_sec <<= 16; - //S[31:16] E44C[15:0] - ts64->tv_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_S_MI); - ts64->tv_sec <<= 16; - //S[15:0] E44A[15:0] - ts64->tv_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_S_LO); -} - -static void rtl8127_ptp_tx_hwtstamp(struct rtl8127_private *tp) -{ - struct sk_buff *skb = tp->ptp_tx_skb; - struct skb_shared_hwtstamps shhwtstamps = { 0 }; - struct timespec64 ts64; - - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_INSR, TX_TX_INTR); - - rtl8127_ptp_egresstime(tp, &ts64); - - /* Upper 32 bits contain s, lower 32 bits contain ns. */ - shhwtstamps.hwtstamp = ktime_set(ts64.tv_sec, - ts64.tv_nsec); - - /* Clear the lock early before calling skb_tstamp_tx so that - * applications are not woken up before the lock bit is clear. We use - * a copy of the skb pointer to ensure other threads can't change it - * while we're notifying the stack. - */ - tp->ptp_tx_skb = NULL; - clear_bit_unlock(__RTL8127_PTP_TX_IN_PROGRESS, &tp->state); - - /* Notify the stack and free the skb after we've unlocked */ - skb_tstamp_tx(skb, &shhwtstamps); - dev_kfree_skb_any(skb); -} - -#define RTL8127_PTP_TX_TIMEOUT (HZ * 15) -static void rtl8127_ptp_tx_work(struct work_struct *work) -{ - struct rtl8127_private *tp = container_of(work, struct rtl8127_private, - ptp_tx_work); - unsigned long flags; - - if (!tp->ptp_tx_skb) - return; - - if (time_is_before_jiffies(tp->ptp_tx_start + - RTL8127_PTP_TX_TIMEOUT)) { - dev_kfree_skb_any(tp->ptp_tx_skb); - tp->ptp_tx_skb = NULL; - clear_bit_unlock(__RTL8127_PTP_TX_IN_PROGRESS, &tp->state); - tp->tx_hwtstamp_timeouts++; - /* Clear the tx valid bit in TSYNCTXCTL register to enable - * interrupt - */ - spin_lock_irqsave(&tp->phy_lock, flags); - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_INSR, TX_TX_INTR); - spin_unlock_irqrestore(&tp->phy_lock, flags); - return; - } - - spin_lock_irqsave(&tp->phy_lock, flags); - if (rtl8127_mdio_direct_read_phy_ocp(tp, PTP_INSR) & TX_TX_INTR) { - rtl8127_ptp_tx_hwtstamp(tp); - spin_unlock_irqrestore(&tp->phy_lock, flags); - } else { - spin_unlock_irqrestore(&tp->phy_lock, flags); - /* reschedule to check later */ - schedule_work(&tp->ptp_tx_work); - } -} - -static int rtl8127_hwtstamp_enable(struct rtl8127_private *tp, bool enable) -{ - unsigned long flags; - - spin_lock_irqsave(&tp->phy_lock, flags); - - if (enable) { - //trx timestamp interrupt enable - rtl8127_set_eth_phy_ocp_bit(tp, PTP_INER, BIT_2 | BIT_3); - - //set isr clear mode - rtl8127_set_eth_phy_ocp_bit(tp, PTP_GEN_CFG, BIT_0); - - //clear ptp isr - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_INSR, 0xFFFF); - - //enable ptp - rtl8127_ptp_enable_config(tp); - - //rtl8127_set_local_time(tp); - } else { - /* trx timestamp interrupt disable */ - rtl8127_clear_eth_phy_ocp_bit(tp, PTP_INER, BIT_2 | BIT_3); - - /* disable ptp */ - rtl8127_clear_eth_phy_ocp_bit(tp, PTP_SYNCE_CTL, BIT_0); - rtl8127_clear_eth_phy_ocp_bit(tp, PTP_CTL, BIT_0); - rtl8127_set_eth_phy_ocp_bit(tp, 0xA640, BIT_15); - } - - spin_unlock_irqrestore(&tp->phy_lock, flags); - - return 0; -} - -void rtl8127_set_local_time(struct rtl8127_private *tp) -{ - struct timespec64 ts64; - //set system time - ktime_get_real_ts64(&ts64); - _rtl8127_phc_settime(tp, &ts64); -} - -static long rtl8127_ptp_create_clock(struct rtl8127_private *tp) -{ - struct net_device *netdev = tp->dev; - long err; - - if (!IS_ERR_OR_NULL(tp->ptp_clock)) - return 0; - - if (tp->HwSuppPtpVer == 0) { - tp->ptp_clock = NULL; - return -EOPNOTSUPP; - } - - tp->ptp_clock_info = rtl_ptp_clock_info; - tp->ptp_clock_info.max_adj = 488281;//0x1FFFFF * 10^9 / 2^32 - - snprintf(tp->ptp_clock_info.name, sizeof(tp->ptp_clock_info.name), - "%pm", tp->dev->dev_addr); - tp->ptp_clock = ptp_clock_register(&tp->ptp_clock_info, &tp->pci_dev->dev); - if (IS_ERR(tp->ptp_clock)) { - err = PTR_ERR(tp->ptp_clock); - tp->ptp_clock = NULL; - netif_err(tp, drv, tp->dev, "ptp_clock_register failed\n"); - return err; - } else - netif_info(tp, drv, tp->dev, "registered PHC device on %s\n", netdev->name); - - return 0; -} - -static enum hrtimer_restart -rtl8127_hrtimer_for_pps(struct hrtimer *timer) { - struct rtl8127_private *tp = container_of(timer, struct rtl8127_private, pps_timer); - u16 tai_cfg = BIT_8 | BIT_3 | BIT_1 | BIT_0; - s64 pps_sec; - - if (tp->pps_enable) - { - unsigned long flags; - - spin_lock_irqsave(&tp->phy_lock, flags); - - //Direct Read - rtl8127_set_clkadj_mode(tp, DIRECT_READ); - - pps_sec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_S_HI_8126); - pps_sec <<= 16; - pps_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_S_MI_8126); - pps_sec <<= 16; - pps_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_S_LO_8126); - pps_sec++; - - //E42A[15:0] - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_TAI_TS_S_LO, pps_sec & 0xffff); - //E42C[31:16] - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_TAI_TS_S_HI, (pps_sec & 0xffff0000) >> 16); - //Periodic Tai start - rtl8127_mdio_direct_write_phy_ocp(tp, PTP_TAI_CFG, tai_cfg); - - spin_unlock_irqrestore(&tp->phy_lock, flags); - - hrtimer_forward_now(&tp->pps_timer, 1000000000); //rekick - return HRTIMER_RESTART; - } else - return HRTIMER_NORESTART; -} - -void rtl8127_ptp_reset(struct rtl8127_private *tp) -{ - if (!tp->ptp_clock) - return; - - netif_info(tp, drv, tp->dev, "reset PHC clock\n"); - - rtl8127_hwtstamp_enable(tp, false); -} - -void rtl8127_ptp_init(struct rtl8127_private *tp) -{ - /* obtain a PTP device, or re-use an existing device */ - if (rtl8127_ptp_create_clock(tp)) - return; - - /* we have a clock so we can initialize work now */ - INIT_WORK(&tp->ptp_tx_work, rtl8127_ptp_tx_work); - - /* init a hrtimer for pps */ - tp->pps_enable = 0; - hrtimer_init(&tp->pps_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - tp->pps_timer.function = rtl8127_hrtimer_for_pps; - - /* reset the PTP related hardware bits */ - rtl8127_ptp_reset(tp); - - return; -} - -void rtl8127_ptp_suspend(struct rtl8127_private *tp) -{ - if (!tp->ptp_clock) - return; - - netif_info(tp, drv, tp->dev, "suspend PHC clock\n"); - - rtl8127_hwtstamp_enable(tp, false); - - /* ensure that we cancel any pending PTP Tx work item in progress */ - cancel_work_sync(&tp->ptp_tx_work); - - hrtimer_cancel(&tp->pps_timer); -} - -void rtl8127_ptp_stop(struct rtl8127_private *tp) -{ - struct net_device *netdev = tp->dev; - - netif_info(tp, drv, tp->dev, "stop PHC clock\n"); - - /* first, suspend PTP activity */ - rtl8127_ptp_suspend(tp); - - /* disable the PTP clock device */ - if (tp->ptp_clock) { - ptp_clock_unregister(tp->ptp_clock); - tp->ptp_clock = NULL; - netif_info(tp, drv, tp->dev, "removed PHC on %s\n", - netdev->name); - } -} - -static int rtl8127_set_tstamp(struct net_device *netdev, struct ifreq *ifr) -{ - struct rtl8127_private *tp = netdev_priv(netdev); - struct hwtstamp_config config; - bool hwtstamp = 0; - - //netif_info(tp, drv, tp->dev, "ptp set ts\n"); - - if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) - return -EFAULT; - - if (config.flags) - return -EINVAL; - - switch (config.tx_type) { - case HWTSTAMP_TX_ON: - hwtstamp = 1; - break; - case HWTSTAMP_TX_OFF: - break; - case HWTSTAMP_TX_ONESTEP_SYNC: - default: - return -ERANGE; - } - - switch (config.rx_filter) { - case HWTSTAMP_FILTER_PTP_V2_EVENT: - case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: - case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: - case HWTSTAMP_FILTER_PTP_V2_SYNC: - case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: - case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: - case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: - case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: - case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: - config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; - hwtstamp = 1; - tp->flags |= RTL_FLAG_RX_HWTSTAMP_ENABLED; - break; - case HWTSTAMP_FILTER_NONE: - tp->flags &= ~RTL_FLAG_RX_HWTSTAMP_ENABLED; - break; - default: - tp->flags &= ~RTL_FLAG_RX_HWTSTAMP_ENABLED; - return -ERANGE; - } - - if (tp->hwtstamp_config.tx_type != config.tx_type || - tp->hwtstamp_config.rx_filter != config.rx_filter) { - tp->hwtstamp_config = config; - - rtl8127_hwtstamp_enable(tp, hwtstamp); - } - - return copy_to_user(ifr->ifr_data, &config, - sizeof(config)) ? -EFAULT : 0; -} - -static int rtl8127_get_tstamp(struct net_device *netdev, struct ifreq *ifr) -{ - struct rtl8127_private *tp = netdev_priv(netdev); - - //netif_info(tp, drv, tp->dev, "ptp get ts\n"); - - return copy_to_user(ifr->ifr_data, &tp->hwtstamp_config, - sizeof(tp->hwtstamp_config)) ? -EFAULT : 0; -} - -int rtl8127_ptp_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) -{ - int ret; - - //netif_info(tp, drv, tp->dev, "ptp ioctl\n"); - - switch (cmd) { -#ifdef ENABLE_PTP_SUPPORT - case SIOCSHWTSTAMP: - ret = rtl8127_set_tstamp(netdev, ifr); - break; - case SIOCGHWTSTAMP: - ret = rtl8127_get_tstamp(netdev, ifr); - break; -#endif - default: - ret = -EOPNOTSUPP; - break; - } - - return ret; -} - -static void rtl8127_rx_ptp_pktstamp(struct rtl8127_private *tp, struct sk_buff *skb, u8 type) -{ - struct timespec64 ts64; - unsigned long flags; - - spin_lock_irqsave(&tp->phy_lock, flags); - - rtl8127_ptp_ingresstime(tp, &ts64, type); - - spin_unlock_irqrestore(&tp->phy_lock, flags); - - skb_hwtstamps(skb)->hwtstamp = ktime_set(ts64.tv_sec, ts64.tv_nsec); - - return; -} - -void rtl8127_rx_ptp_timestamp(struct rtl8127_private *tp, struct sk_buff *skb) -{ - unsigned int ptp_class; - struct ptp_header *hdr; - u8 msgtype; - - ptp_class = ptp_classify_raw(skb); - if (ptp_class == PTP_CLASS_NONE) - return; - - skb_reset_mac_header(skb); - hdr = ptp_parse_header(skb, ptp_class); - if (unlikely(!hdr)) - return; - - msgtype = ptp_get_msgtype(hdr, ptp_class); - rtl8127_rx_ptp_pktstamp(tp, skb, msgtype); - - return; -} - -#if LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0) -struct ptp_header *ptp_parse_header(struct sk_buff *skb, unsigned int type) -{ - u8 *ptr = skb_mac_header(skb); - - if (type & PTP_CLASS_VLAN) - //ptr += VLAN_HLEN; - ptr += 4; - - switch (type & PTP_CLASS_PMASK) { - case PTP_CLASS_IPV4: - ptr += IPV4_HLEN(ptr) + UDP_HLEN; - break; - case PTP_CLASS_IPV6: - ptr += IP6_HLEN + UDP_HLEN; - break; - case PTP_CLASS_L2: - break; - default: - return NULL; - } - - ptr += ETH_HLEN; - - /* Ensure that the entire header is present in this packet. */ - if (ptr + sizeof(struct ptp_header) > skb->data + skb->len) - return NULL; - - return (struct ptp_header *)ptr; -} -#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0) */ diff --git a/drivers/net/ethernet/realtek/r8127/r8127_ptp.h b/drivers/net/ethernet/realtek/r8127/r8127_ptp.h deleted file mode 100755 index e96afafd70a85..0000000000000 --- a/drivers/net/ethernet/realtek/r8127/r8127_ptp.h +++ /dev/null @@ -1,202 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* -################################################################################ -# -# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet -# controllers with PCI-Express interface. -# -# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; either version 2 of the License, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, see . -# -# Author: -# Realtek NIC software team -# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan -# -################################################################################ -*/ - -/************************************************************************************ - * This product is covered by one or more of the following patents: - * US6,570,884, US6,115,776, and US6,327,625. - ***********************************************************************************/ - -#ifndef _LINUX_R8127_PTP_H -#define _LINUX_R8127_PTP_H - -#include -#include -#include -#include -#include - -#ifndef PTP_CLASS_NONE -#define PTP_CLASS_NONE 0x00 -#endif - -#ifndef PTP_MSGTYPE_SYNC -#define PTP_MSGTYPE_SYNC 0x0 -#endif -#ifndef PTP_MSGTYPE_DELAY_REQ -#define PTP_MSGTYPE_DELAY_REQ 0x1 -#endif -#ifndef PTP_MSGTYPE_PDELAY_REQ -#define PTP_MSGTYPE_PDELAY_REQ 0x2 -#endif -#ifndef PTP_MSGTYPE_PDELAY_RESP -#define PTP_MSGTYPE_PDELAY_RESP 0x3 -#endif - -struct rtl8127_ptp_info { - s64 time_sec; - u32 time_ns; - u16 ts_info; -}; - -#ifndef _STRUCT_TIMESPEC -#define _STRUCT_TIMESPEC -struct timespec { - __kernel_old_time_t tv_sec; /* seconds */ - long tv_nsec; /* nanoseconds */ -}; -#endif - -enum PTP_CMD_TYPE { - PTP_CMD_SET_LOCAL_TIME = 0, - PTP_CMD_DRIFT_LOCAL_TIME, - PTP_CMD_LATCHED_LOCAL_TIME, -}; - -enum PTP_CLKADJ_MOD_TYPE { - NO_FUNCTION = 0, - CLKADJ_MODE_SET = 1, - RESERVED = 2, - DIRECT_READ = 4, - DIRECT_WRITE = 6, - INCREMENT_STEP = 8, - DECREMENT_STEP = 10, - RATE_READ = 12, - RATE_WRITE = 14, -}; - -enum PTP_INSR_TYPE { - EVENT_CAP_INTR = (1 << 0), - TRIG_GEN_INTR = (1 << 1), - RX_TS_INTR = (1 << 2), - TX_TX_INTR = (1 << 3), -}; - -enum PTP_TRX_TS_STA_REG { - TRX_TS_RD = (1 << 0), - TRXTS_SEL = (1 << 1), - RX_TS_PDLYRSP_RDY = (1 << 8), - RX_TS_PDLYREQ_RDY = (1 << 9), - RX_TS_DLYREQ_RDY = (1 << 10), - RX_TS_SYNC_RDY = (1 << 11), - TX_TS_PDLYRSP_RDY = (1 << 12), - TX_TS_PDLYREQ_RDY = (1 << 13), - TX_TS_DLYREQ_RDY = (1 << 14), - TX_TS_SYNC_RDY = (1 << 15), -}; - -#define PTP_CTL_TYPE_0 (0xF3F) -#define PTP_CTL_TYPE_1 (0x2FF) -#define PTP_CTL_TYPE_2 (0x0FF) -#define PTP_CTL_TYPE_3 (0x03F) - -#if LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0) -struct clock_identity { - u8 id[8]; -} __packed; - -struct port_identity { - struct clock_identity clock_identity; - __be16 port_number; -} __packed; - -struct ptp_header { - u8 tsmt; /* transportSpecific | messageType */ - u8 ver; /* reserved | versionPTP */ - __be16 message_length; - u8 domain_number; - u8 reserved1; - u8 flag_field[2]; - __be64 correction; - __be32 reserved2; - struct port_identity source_port_identity; - __be16 sequence_id; - u8 control; - u8 log_message_interval; -} __packed; - -/** - * ptp_parse_header - Get pointer to the PTP v2 header - * @skb: packet buffer - * @type: type of the packet (see ptp_classify_raw()) - * - * This function takes care of the VLAN, UDP, IPv4 and IPv6 headers. The length - * is checked. - * - * Note, internally skb_mac_header() is used. Make sure that the @skb is - * initialized accordingly. - * - * Return: Pointer to the ptp v2 header or NULL if not found - */ -struct ptp_header *ptp_parse_header(struct sk_buff *skb, unsigned int type); - -/** - * ptp_get_msgtype - Extract ptp message type from given header - * @hdr: ptp header - * @type: type of the packet (see ptp_classify_raw()) - * - * This function returns the message type for a given ptp header. It takes care - * of the different ptp header versions (v1 or v2). - * - * Return: The message type - */ -static inline u8 ptp_get_msgtype(const struct ptp_header *hdr, - unsigned int type) -{ - u8 msgtype; - - if (unlikely(type & PTP_CLASS_V1)) { - /* msg type is located at the control field for ptp v1 */ - msgtype = hdr->control; - } else { - msgtype = hdr->tsmt & 0x0f; - } - - return msgtype; -} - -#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0) */ - -struct rtl8127_private; -struct RxDescV3; - -int rtl8127_get_ts_info(struct net_device *netdev, - struct ethtool_ts_info *info); - -void rtl8127_ptp_reset(struct rtl8127_private *tp); -void rtl8127_ptp_init(struct rtl8127_private *tp); -void rtl8127_ptp_suspend(struct rtl8127_private *tp); -void rtl8127_ptp_stop(struct rtl8127_private *tp); - -int rtl8127_ptp_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd); - -void rtl8127_rx_ptp_timestamp(struct rtl8127_private *tp, struct sk_buff *skb); - -void rtl8127_set_local_time(struct rtl8127_private *tp); - -#endif /* _LINUX_R8127_PTP_H */ diff --git a/drivers/net/ethernet/realtek/r8127/r8127_rss.c b/drivers/net/ethernet/realtek/r8127/r8127_rss.c deleted file mode 100755 index e364621910052..0000000000000 --- a/drivers/net/ethernet/realtek/r8127/r8127_rss.c +++ /dev/null @@ -1,583 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* -################################################################################ -# -# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet -# controllers with PCI-Express interface. -# -# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; either version 2 of the License, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, see . -# -# Author: -# Realtek NIC software team -# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan -# -################################################################################ -*/ - -/************************************************************************************ - * This product is covered by one or more of the following patents: - * US6,570,884, US6,115,776, and US6,327,625. - ***********************************************************************************/ - -#include -#include "r8127.h" - -enum rtl8127_rss_register_content { - /* RSS */ - RSS_CTRL_TCP_IPV4_SUPP = (1 << 0), - RSS_CTRL_IPV4_SUPP = (1 << 1), - RSS_CTRL_TCP_IPV6_SUPP = (1 << 2), - RSS_CTRL_IPV6_SUPP = (1 << 3), - RSS_CTRL_IPV6_EXT_SUPP = (1 << 4), - RSS_CTRL_TCP_IPV6_EXT_SUPP = (1 << 5), - RSS_HALF_SUPP = (1 << 7), - RSS_CTRL_UDP_IPV4_SUPP = (1 << 11), - RSS_CTRL_UDP_IPV6_SUPP = (1 << 12), - RSS_CTRL_UDP_IPV6_EXT_SUPP = (1 << 13), - RSS_QUAD_CPU_EN = (1 << 16), - RSS_HQ_Q_SUP_R = (1 << 31), -}; - -static int rtl8127_get_rss_hash_opts(struct rtl8127_private *tp, - struct ethtool_rxnfc *cmd) -{ - cmd->data = 0; - - /* Report default options for RSS */ - switch (cmd->flow_type) { - case TCP_V4_FLOW: - cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3; - fallthrough; - case UDP_V4_FLOW: - if (tp->rss_flags & RTL_8125_RSS_FLAG_HASH_UDP_IPV4) - cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3; - fallthrough; - case IPV4_FLOW: - cmd->data |= RXH_IP_SRC | RXH_IP_DST; - break; - case TCP_V6_FLOW: - cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3; - fallthrough; - case UDP_V6_FLOW: - if (tp->rss_flags & RTL_8125_RSS_FLAG_HASH_UDP_IPV6) - cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3; - fallthrough; - case IPV6_FLOW: - cmd->data |= RXH_IP_SRC | RXH_IP_DST; - break; - default: - return -EINVAL; - } - - return 0; -} - -int rtl8127_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd, - u32 *rule_locs) -{ - struct rtl8127_private *tp = netdev_priv(dev); - int ret = -EOPNOTSUPP; - - if (!(dev->features & NETIF_F_RXHASH)) - return ret; - - switch (cmd->cmd) { - case ETHTOOL_GRXRINGS: - cmd->data = rtl8127_tot_rx_rings(tp); - ret = 0; - break; - case ETHTOOL_GRXFH: - ret = rtl8127_get_rss_hash_opts(tp, cmd); - break; - default: - break; - } - - return ret; -} - -u32 rtl8127_rss_indir_tbl_entries(struct rtl8127_private *tp) -{ - return tp->HwSuppIndirTblEntries; -} - -#define RSS_MASK_BITS_OFFSET (8) -#define RSS_CPU_NUM_OFFSET (16) -#define RTL8127_UDP_RSS_FLAGS (RTL_8125_RSS_FLAG_HASH_UDP_IPV4 | \ - RTL_8125_RSS_FLAG_HASH_UDP_IPV6) -static int _rtl8127_set_rss_hash_opt(struct rtl8127_private *tp) -{ - u32 rss_flags = tp->rss_flags; - u32 hash_mask_len; - u32 rss_ctrl; - - rss_ctrl = ilog2(rtl8127_tot_rx_rings(tp)); - rss_ctrl &= (BIT_0 | BIT_1 | BIT_2); - rss_ctrl <<= RSS_CPU_NUM_OFFSET; - - /* Perform hash on these packet types */ - rss_ctrl |= RSS_CTRL_TCP_IPV4_SUPP - | RSS_CTRL_IPV4_SUPP - | RSS_CTRL_IPV6_SUPP - | RSS_CTRL_IPV6_EXT_SUPP - | RSS_CTRL_TCP_IPV6_SUPP - | RSS_CTRL_TCP_IPV6_EXT_SUPP; - - if (rss_flags & RTL_8125_RSS_FLAG_HASH_UDP_IPV4) - rss_ctrl |= RSS_CTRL_UDP_IPV4_SUPP; - - if (rss_flags & RTL_8125_RSS_FLAG_HASH_UDP_IPV6) - rss_ctrl |= RSS_CTRL_UDP_IPV6_SUPP | - RSS_CTRL_UDP_IPV6_EXT_SUPP; - - hash_mask_len = ilog2(rtl8127_rss_indir_tbl_entries(tp)); - hash_mask_len &= (BIT_0 | BIT_1 | BIT_2); - rss_ctrl |= hash_mask_len << RSS_MASK_BITS_OFFSET; - - RTL_W32(tp, RSS_CTRL_8125, rss_ctrl); - - return 0; -} - -static int rtl8127_set_rss_hash_opt(struct rtl8127_private *tp, - struct ethtool_rxnfc *nfc) -{ - u32 rss_flags = tp->rss_flags; - - /* - * RSS does not support anything other than hashing - * to queues on src and dst IPs and ports - */ - if (nfc->data & ~(RXH_IP_SRC | RXH_IP_DST | - RXH_L4_B_0_1 | RXH_L4_B_2_3)) - return -EINVAL; - - switch (nfc->flow_type) { - case TCP_V4_FLOW: - case TCP_V6_FLOW: - if (!(nfc->data & RXH_IP_SRC) || - !(nfc->data & RXH_IP_DST) || - !(nfc->data & RXH_L4_B_0_1) || - !(nfc->data & RXH_L4_B_2_3)) - return -EINVAL; - break; - case UDP_V4_FLOW: - if (!(nfc->data & RXH_IP_SRC) || - !(nfc->data & RXH_IP_DST)) - return -EINVAL; - switch (nfc->data & (RXH_L4_B_0_1 | RXH_L4_B_2_3)) { - case 0: - rss_flags &= ~RTL_8125_RSS_FLAG_HASH_UDP_IPV4; - break; - case (RXH_L4_B_0_1 | RXH_L4_B_2_3): - rss_flags |= RTL_8125_RSS_FLAG_HASH_UDP_IPV4; - break; - default: - return -EINVAL; - } - break; - case UDP_V6_FLOW: - if (!(nfc->data & RXH_IP_SRC) || - !(nfc->data & RXH_IP_DST)) - return -EINVAL; - switch (nfc->data & (RXH_L4_B_0_1 | RXH_L4_B_2_3)) { - case 0: - rss_flags &= ~RTL_8125_RSS_FLAG_HASH_UDP_IPV6; - break; - case (RXH_L4_B_0_1 | RXH_L4_B_2_3): - rss_flags |= RTL_8125_RSS_FLAG_HASH_UDP_IPV6; - break; - default: - return -EINVAL; - } - break; - case SCTP_V4_FLOW: - case AH_ESP_V4_FLOW: - case AH_V4_FLOW: - case ESP_V4_FLOW: - case SCTP_V6_FLOW: - case AH_ESP_V6_FLOW: - case AH_V6_FLOW: - case ESP_V6_FLOW: - case IP_USER_FLOW: - case ETHER_FLOW: - /* RSS is not supported for these protocols */ - if (nfc->data) { - netif_err(tp, drv, tp->dev, "Command parameters not supported\n"); - return -EINVAL; - } - return 0; - break; - default: - return -EINVAL; - } - - /* if we changed something we need to update flags */ - if (rss_flags != tp->rss_flags) { - u32 rss_ctrl = RTL_R32(tp, RSS_CTRL_8125); - - if ((rss_flags & RTL8127_UDP_RSS_FLAGS) && - !(tp->rss_flags & RTL8127_UDP_RSS_FLAGS)) - netdev_warn(tp->dev, - "enabling UDP RSS: fragmented packets may " - "arrive out of order to the stack above\n"); - - tp->rss_flags = rss_flags; - - /* Perform hash on these packet types */ - rss_ctrl |= RSS_CTRL_TCP_IPV4_SUPP - | RSS_CTRL_IPV4_SUPP - | RSS_CTRL_IPV6_SUPP - | RSS_CTRL_IPV6_EXT_SUPP - | RSS_CTRL_TCP_IPV6_SUPP - | RSS_CTRL_TCP_IPV6_EXT_SUPP; - - rss_ctrl &= ~(RSS_CTRL_UDP_IPV4_SUPP | - RSS_CTRL_UDP_IPV6_SUPP | - RSS_CTRL_UDP_IPV6_EXT_SUPP); - - if (rss_flags & RTL_8125_RSS_FLAG_HASH_UDP_IPV4) - rss_ctrl |= RSS_CTRL_UDP_IPV4_SUPP; - - if (rss_flags & RTL_8125_RSS_FLAG_HASH_UDP_IPV6) - rss_ctrl |= RSS_CTRL_UDP_IPV6_SUPP | - RSS_CTRL_UDP_IPV6_EXT_SUPP; - - RTL_W32(tp, RSS_CTRL_8125, rss_ctrl); - } - - return 0; -} - -int rtl8127_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd) -{ - struct rtl8127_private *tp = netdev_priv(dev); - int ret = -EOPNOTSUPP; - - if (!(dev->features & NETIF_F_RXHASH)) - return ret; - - switch (cmd->cmd) { - case ETHTOOL_SRXFH: - ret = rtl8127_set_rss_hash_opt(tp, cmd); - break; - default: - break; - } - - return ret; -} - -static u32 _rtl8127_get_rxfh_key_size(struct rtl8127_private *tp) -{ - return sizeof(tp->rss_key); -} - -u32 rtl8127_get_rxfh_key_size(struct net_device *dev) -{ - struct rtl8127_private *tp = netdev_priv(dev); - - if (!(dev->features & NETIF_F_RXHASH)) - return 0; - - return _rtl8127_get_rxfh_key_size(tp); -} - -u32 rtl8127_rss_indir_size(struct net_device *dev) -{ - struct rtl8127_private *tp = netdev_priv(dev); - - if (!(dev->features & NETIF_F_RXHASH)) - return 0; - - return rtl8127_rss_indir_tbl_entries(tp); -} - -static void rtl8127_get_reta(struct rtl8127_private *tp, u32 *indir) -{ - int i, reta_size = rtl8127_rss_indir_tbl_entries(tp); - - for (i = 0; i < reta_size; i++) - indir[i] = tp->rss_indir_tbl[i]; -} - -static u32 rtl8127_rss_key_reg(struct rtl8127_private *tp) -{ - return RSS_KEY_8125; -} - -static u32 rtl8127_rss_indir_tbl_reg(struct rtl8127_private *tp) -{ - return RSS_INDIRECTION_TBL_8125_V2; -} - -static void rtl8127_store_reta(struct rtl8127_private *tp) -{ - u16 indir_tbl_reg = rtl8127_rss_indir_tbl_reg(tp); - u32 i, reta_entries = rtl8127_rss_indir_tbl_entries(tp); - u32 reta = 0; - u8 *indir_tbl = tp->rss_indir_tbl; - - /* Write redirection table to HW */ - for (i = 0; i < reta_entries; i++) { - reta |= indir_tbl[i] << (i & 0x3) * 8; - if ((i & 3) == 3) { - RTL_W32(tp, indir_tbl_reg, reta); - - indir_tbl_reg += 4; - reta = 0; - } - } -} - -static void rtl8127_store_rss_key(struct rtl8127_private *tp) -{ - const u16 rss_key_reg = rtl8127_rss_key_reg(tp); - u32 i, rss_key_size = _rtl8127_get_rxfh_key_size(tp); - u32 *rss_key = (u32*)tp->rss_key; - - /* Write redirection table to HW */ - for (i = 0; i < rss_key_size; i+=4) - RTL_W32(tp, rss_key_reg + i, *rss_key++); -} - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,8,0) -int rtl8127_get_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh) -{ - struct rtl8127_private *tp = netdev_priv(dev); - - if (!(dev->features & NETIF_F_RXHASH)) - return -EOPNOTSUPP; - - rxfh->hfunc = ETH_RSS_HASH_TOP; - - if (rxfh->indir) - rtl8127_get_reta(tp, rxfh->indir); - - if (rxfh->key) - memcpy(rxfh->key, tp->rss_key, RTL8127_RSS_KEY_SIZE); - - return 0; -} - -int rtl8127_set_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh, - struct netlink_ext_ack *extack) -{ - struct rtl8127_private *tp = netdev_priv(dev); - int i; - u32 reta_entries = rtl8127_rss_indir_tbl_entries(tp); - - /* We require at least one supported parameter to be changed and no - * change in any of the unsupported parameters - */ - if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && rxfh->hfunc != ETH_RSS_HASH_TOP) - return -EOPNOTSUPP; - - /* Fill out the redirection table */ - if (rxfh->indir) { - int max_queues = tp->num_rx_rings; - - /* Verify user input. */ - for (i = 0; i < reta_entries; i++) - if (rxfh->indir[i] >= max_queues) - return -EINVAL; - - for (i = 0; i < reta_entries; i++) - tp->rss_indir_tbl[i] = rxfh->indir[i]; - } - - /* Fill out the rss hash key */ - if (rxfh->key) - memcpy(tp->rss_key, rxfh->key, RTL8127_RSS_KEY_SIZE); - - rtl8127_store_reta(tp); - - rtl8127_store_rss_key(tp); - - return 0; -} -#else -int rtl8127_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, - u8 *hfunc) -{ - struct rtl8127_private *tp = netdev_priv(dev); - - if (!(dev->features & NETIF_F_RXHASH)) - return -EOPNOTSUPP; - - if (hfunc) - *hfunc = ETH_RSS_HASH_TOP; - - if (indir) - rtl8127_get_reta(tp, indir); - - if (key) - memcpy(key, tp->rss_key, RTL8127_RSS_KEY_SIZE); - - return 0; -} - -int rtl8127_set_rxfh(struct net_device *dev, const u32 *indir, - const u8 *key, const u8 hfunc) -{ - struct rtl8127_private *tp = netdev_priv(dev); - int i; - u32 reta_entries = rtl8127_rss_indir_tbl_entries(tp); - - /* We require at least one supported parameter to be changed and no - * change in any of the unsupported parameters - */ - if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) - return -EOPNOTSUPP; - - /* Fill out the redirection table */ - if (indir) { - int max_queues = tp->num_rx_rings; - - /* Verify user input. */ - for (i = 0; i < reta_entries; i++) - if (indir[i] >= max_queues) - return -EINVAL; - - for (i = 0; i < reta_entries; i++) - tp->rss_indir_tbl[i] = indir[i]; - } - - /* Fill out the rss hash key */ - if (key) - memcpy(tp->rss_key, key, RTL8127_RSS_KEY_SIZE); - - rtl8127_store_reta(tp); - - rtl8127_store_rss_key(tp); - - return 0; -} -#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6,8,0) */ - -static u32 rtl8127_get_rx_desc_hash(struct rtl8127_private *tp, - struct RxDesc *desc) -{ - switch (tp->InitRxDescType) { - case RX_DESC_RING_TYPE_3: - return le32_to_cpu(((struct RxDescV3 *)desc)->RxDescNormalDDWord2.RSSResult); - case RX_DESC_RING_TYPE_4: - return le32_to_cpu(((struct RxDescV4 *)desc)->RxDescNormalDDWord1.RSSResult); - default: - return 0; - } -} - -#define RXS_8125B_RSS_UDP BIT(9) -#define RXS_8125_RSS_IPV4 BIT(10) -#define RXS_8125_RSS_IPV6 BIT(12) -#define RXS_8125_RSS_TCP BIT(13) -#define RTL8127_RXS_RSS_L3_TYPE_MASK (RXS_8125_RSS_IPV4 | RXS_8125_RSS_IPV6) -#define RTL8127_RXS_RSS_L4_TYPE_MASK (RXS_8125_RSS_TCP | RXS_8125B_RSS_UDP) - -#define RXS_8125B_RSS_UDP_V4 BIT(27) -#define RXS_8125_RSS_IPV4_V4 BIT(28) -#define RXS_8125_RSS_IPV6_V4 BIT(29) -#define RXS_8125_RSS_TCP_V4 BIT(30) -#define RTL8127_RXS_RSS_L3_TYPE_MASK_V4 (RXS_8125_RSS_IPV4_V4 | RXS_8125_RSS_IPV6_V4) -#define RTL8127_RXS_RSS_L4_TYPE_MASK_V4 (RXS_8125_RSS_TCP_V4 | RXS_8125B_RSS_UDP_V4) -static void rtl8127_rx_hash_v3(struct rtl8127_private *tp, - struct RxDescV3 *descv3, - struct sk_buff *skb) -{ - u16 rss_header_info; - - if (!(tp->dev->features & NETIF_F_RXHASH)) - return; - - rss_header_info = le16_to_cpu(descv3->RxDescNormalDDWord2.HeaderInfo); - - if (!(rss_header_info & RTL8127_RXS_RSS_L3_TYPE_MASK)) - return; - - skb_set_hash(skb, rtl8127_get_rx_desc_hash(tp, (struct RxDesc *)descv3), - (RTL8127_RXS_RSS_L4_TYPE_MASK & rss_header_info) ? - PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); -} - -static void rtl8127_rx_hash_v4(struct rtl8127_private *tp, - struct RxDescV4 *descv4, - struct sk_buff *skb) -{ - u32 rss_header_info; - - if (!(tp->dev->features & NETIF_F_RXHASH)) - return; - - rss_header_info = le32_to_cpu(descv4->RxDescNormalDDWord1.RSSInfo); - - if (!(rss_header_info & RTL8127_RXS_RSS_L3_TYPE_MASK_V4)) - return; - - skb_set_hash(skb, rtl8127_get_rx_desc_hash(tp, (struct RxDesc *)descv4), - (RTL8127_RXS_RSS_L4_TYPE_MASK_V4 & rss_header_info) ? - PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); -} - -void rtl8127_rx_hash(struct rtl8127_private *tp, - struct RxDesc *desc, - struct sk_buff *skb) -{ - switch (tp->InitRxDescType) { - case RX_DESC_RING_TYPE_3: - rtl8127_rx_hash_v3(tp, (struct RxDescV3 *)desc, skb); - break; - case RX_DESC_RING_TYPE_4: - rtl8127_rx_hash_v4(tp, (struct RxDescV4 *)desc, skb); - break; - default: - return; - } -} - -void rtl8127_disable_rss(struct rtl8127_private *tp) -{ - RTL_W32(tp, RSS_CTRL_8125, 0x00); -} - -void _rtl8127_config_rss(struct rtl8127_private *tp) -{ - _rtl8127_set_rss_hash_opt(tp); - - rtl8127_store_reta(tp); - - rtl8127_store_rss_key(tp); -} - -void rtl8127_config_rss(struct rtl8127_private *tp) -{ - if (!tp->EnableRss) { - rtl8127_disable_rss(tp); - return; - } - - _rtl8127_config_rss(tp); -} - -void rtl8127_init_rss(struct rtl8127_private *tp) -{ - int i; - - for (i = 0; i < rtl8127_rss_indir_tbl_entries(tp); i++) - tp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, tp->num_rx_rings); - - netdev_rss_key_fill(tp->rss_key, RTL8127_RSS_KEY_SIZE); -} From 1a6140b52b97c167ceba90388bc12d2cbd6a91d2 Mon Sep 17 00:00:00 2001 From: Abhishek Sahu Date: Thu, 24 Apr 2025 10:10:11 +0000 Subject: [PATCH 040/311] UBUNTU: [Config] nvidia-6.11: Update annotations to enable realtek R8127 module BugLink: https://bugs.launchpad.net/bugs/2109730 Signed-off-by: Abhishek Sahu Acked-by: Matt Ochs Acked-by: Carol L Soto Acked-by: Ian May Acked-by: Jacob Martin Acked-by: Noah Wager Signed-off-by: Ian May (cherry picked from commit 59db3944a96c71fbe6c1659faae36a73e07b0d16 noble:linux-nvidia-6.11) Signed-off-by: Jacob Martin (cherry picked from commit aaa549042b742a74100ad1fcb28e003d965cd479) (cherry picked from commit aaa549042b74 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 1edd05ed496e76de824cb4600b8d0e59bb633874 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- debian.nvidia/config/annotations | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index 7bfa5bcee00ba..c38faf08e2a3d 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -156,6 +156,9 @@ CONFIG_NR_CPUS note<'LP: #1864198'> CONFIG_PID_IN_CONTEXTIDR policy<{'arm64': 'y'}> CONFIG_PID_IN_CONTEXTIDR note<'Required for Grace enablement'> +CONFIG_R8127 policy<{'amd64': 'n', 'arm64': 'm'}> +CONFIG_R8127 note<'LP: #2109730'> + CONFIG_SAMPLE_CORESIGHT_SYSCFG policy<{'arm64': 'n'}> CONFIG_SAMPLE_CORESIGHT_SYSCFG note<'Required for Grace enablement'> From e8e9fb2331448649e099084d7216297c86d9b24b Mon Sep 17 00:00:00 2001 From: Abhishek Sahu Date: Thu, 22 May 2025 04:53:00 +0000 Subject: [PATCH 041/311] UBUNTU: [Config] nvidia-6.14: Update annotations to enable TPM over FFA BugLink: https://bugs.launchpad.net/bugs/2111511 - crb_acpi_add() checks for start method - If start method is ACPI_TPM2_CRB_WITH_ARM_FFA, then it invokes tpm_crb_ffa_init(). - The tpm_crb_ffa_init() uses IS_REACHABLE() #if IS_REACHABLE(CONFIG_TCG_ARM_CRB_FFA) int tpm_crb_ffa_init(void); #else static inline int tpm_crb_ffa_init(void) { return 0; } #endif So, either tpm_crb (configured with CONFIG_TCG_CRB) should be module or we need to make tpm_crb_ffa (CONFIG_TCG_ARM_CRB_FFA) built-in. - CONFIG_TCG_CRB is selected by other configs so making it module won't be feasible. We can enable CONFIG_TCG_ARM_CRB_FFA to make tpm_crb_ffa built-in. - This also requires to select CONFIG_ARM_FFA_TRANSPORT=y Signed-off-by: Abhishek Sahu Acked-by: Carol L Soto Acked-by: Matthew R. Ochs Acked-by: Jacob Martin Acked-by: Noah Wager Signed-off-by: Brad Figg (cherry picked from commit 60809f8e7ee9efa255455b3393d96e4d6a2c7306) (cherry picked from commit 60809f8e7ee9 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit b054f0bb84d60776885e1333833c80512808346a noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- debian.nvidia/config/annotations | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index c38faf08e2a3d..20ede527308a9 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -33,6 +33,9 @@ CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE note<'Required for Grace enable CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE policy<{'arm64': 'y'}> CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE note<'Required for Grace enablement'> +CONFIG_ARM_FFA_TRANSPORT policy<{'arm64': 'y'}> +CONFIG_ARM_FFA_TRANSPORT note<'LP: #2111511'> + CONFIG_ARM_SMMU_V3_IOMMUFD policy<{'arm64': 'y'}> CONFIG_ARM_SMMU_V3_IOMMUFD note<'LP: #2095028'> @@ -168,6 +171,9 @@ CONFIG_SENSORS_AAEON note<'Disable all Ubuntu ODM dri CONFIG_SPI_TEGRA210_QUAD policy<{'arm64': 'y'}> CONFIG_SPI_TEGRA210_QUAD note<'Ensures the TPM is available before the IMA driver initializes'> +CONFIG_TCG_ARM_CRB_FFA policy<{'arm64': 'y'}> +CONFIG_TCG_ARM_CRB_FFA note<'LP: #2111511'> + CONFIG_TCG_TIS_SPI policy<{'amd64': 'm', 'arm64': 'y'}> CONFIG_TCG_TIS_SPI note<'Ensures the TPM is available before the IMA driver initializes'> From 63c841f1c6481539e44e91450cdee4cda034b9f0 Mon Sep 17 00:00:00 2001 From: Abhishek Sahu Date: Wed, 7 May 2025 06:19:42 +0000 Subject: [PATCH 042/311] NVIDIA: SAUCE: Add support for custom ARM FFH offset handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BugLink: https://bugs.launchpad.net/bugs/2114230 The FFH (Functional Fixed Hardware) operation region is maintained by ARM in https://developer.arm.com/documentation/den0048/latest/ OperationRegion (RegionName, RegionSpace, Offset, Length) For ARM FFH, Offset is used to identify the functionality offered by this FFH address space. It must be set to one of the following values: - 0x0 to indicate usage of 32-bit calling convention - 0x1 to indicate usage of 64-bit calling convention. - All other values are reserved. For GB10 and other similar SOC’s, to communicate with embedded controller, a new specification is being defined. It is currently in draft stage and maintained in https://github.com/OpenDevicePartnership/documentation/blob/main/bookshelf/Shelf%204%20Specifications/EC%20Interface/src/README.md https://github.com/OpenDevicePartnership/documentation/blob/main/bookshelf/Shelf%204%20Specifications/EC%20Interface/src/secure-ec-services-overview.md Offset 4 section: https://github.com/OpenDevicePartnership/documentation/blob/main/bookshelf/Shelf%204%20Specifications/EC%20Interface/src/secure-ec-services-overview.md#operation-region-definition This specification internally uses offset 0x4 which is not defined in published ARM specification. So, when ACPI request comes with offset 0x4, then it will fail due to missing support. This commit adds support for custom offset handler. A new EC interface driver will be added in subsequent patches which will registers it callback function. When FFH operation region will be executed with offsets other than 0x0 and 0x1, then it will be forwarded to custom handler. Signed-off-by: Abhishek Sahu Acked-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 89b7d0384a9c noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit df76ec3fc884563f2d73013be08371e870a246d9 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/acpi/arm64/ffh.c | 32 ++++++++++++++++++++++++++++++++ include/linux/acpi.h | 16 ++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/drivers/acpi/arm64/ffh.c b/drivers/acpi/arm64/ffh.c index 04380bab193df..8bce1070d3716 100644 --- a/drivers/acpi/arm64/ffh.c +++ b/drivers/acpi/arm64/ffh.c @@ -19,6 +19,9 @@ struct acpi_ffh_data { struct arm_smccc_1_2_regs *res); }; +static int (*ffh_custom_handler)(struct acpi_ffh_info *info, + acpi_integer *value, void *region_context); + int acpi_ffh_address_space_arch_setup(void *handler_ctxt, void **region_ctxt) { enum arm_smccc_conduit conduit; @@ -99,9 +102,38 @@ int acpi_ffh_address_space_arch_handler(acpi_integer *value, void *region_contex ffh_ctxt->invoke_ffh64_fn(r, r); memcpy(value, r, ffh_ctxt->info.length); } + } else if (ffh_custom_handler) { + int err = ffh_custom_handler(&ffh_ctxt->info, value, + region_context); + if (err) { + pr_err("ARM FFH custom offset handler returned error=%d\n", + err); + ret = AE_ERROR; + } } else { ret = AE_ERROR; } return ret; } + +int acpi_arm64_ffh_update_custom_offset_handler( + int (*handler)(struct acpi_ffh_info *info, acpi_integer *value, + void *region_context)) +{ + if (!handler) { + pr_debug("ARM FFH custom offset handler unregistered\n"); + ffh_custom_handler = NULL; + return 0; + } + + if (ffh_custom_handler) + pr_debug("ARM FFH custom offset handler updated\n"); + else + pr_debug("ARM FFH custom offset handler registered\n"); + + ffh_custom_handler = handler; + + return 0; +} +EXPORT_SYMBOL_GPL(acpi_arm64_ffh_update_custom_offset_handler); diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 4d2f0bed7a06d..ee3aa5604512a 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1596,10 +1596,26 @@ extern int acpi_ffh_address_space_arch_setup(void *handler_ctxt, void **region_ctxt); extern int acpi_ffh_address_space_arch_handler(acpi_integer *value, void *region_context); +int acpi_ffh_address_space_arch_update_custom_offset_handler( + int (*handler)(struct acpi_ffh_info *info, acpi_integer *value, + void *region_context)); #else static inline void acpi_init_ffh(void) { } #endif +#if defined(CONFIG_ACPI_FFH) && defined(CONFIG_ARM64) +int acpi_arm64_ffh_update_custom_offset_handler( + int (*handler)(struct acpi_ffh_info *info, acpi_integer *value, + void *region_context)); +#else +static inline int acpi_arm64_ffh_update_custom_offset_handler( + int (*handler)(struct acpi_ffh_info *info, acpi_integer *value, + void *region_context)) +{ + return -EOPNOTSUPP; +} +#endif + #ifdef CONFIG_ACPI extern void acpi_device_notify(struct device *dev); extern void acpi_device_notify_remove(struct device *dev); From 7e5c025d98722aaa3d4c3fa24a7c321e08c45cb3 Mon Sep 17 00:00:00 2001 From: Abhishek Sahu Date: Wed, 7 May 2025 07:32:22 +0000 Subject: [PATCH 043/311] NVIDIA: SAUCE: Add nvidia ffa driver for EC communication BugLink: https://bugs.launchpad.net/bugs/2114230 Please refer https://github.com/OpenDevicePartnership/documentation/blob/main/bookshelf/Shelf%204%20Specifications/EC%20Interface/src/secure-ec-services-overview.md for details regarding FFA device details for secure EC services communication. The HID 'MSFT000C' is reserved for FFA devices. This HID is documented in https://github.com/OpenDevicePartnership/documentation/blob/main/bookshelf/Shelf%204%20Specifications/EC%20Interface/src/secure-ec-services-overview.md#hid-definition This commit adds a platform driver which binds with FFA device. In its probe routine, it executes the AVAL method to check if FFA can be used for secure EC services communication. Signed-off-by: Abhishek Sahu Acked-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 555e41e166a4 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit bdd6ed09666656b94b7d75c56f3c7d9254e6c53f noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/platform/arm64/Kconfig | 13 ++++ drivers/platform/arm64/Makefile | 1 + drivers/platform/arm64/nvidia-ffa-ec.c | 92 ++++++++++++++++++++++++++ 3 files changed, 106 insertions(+) create mode 100644 drivers/platform/arm64/nvidia-ffa-ec.c diff --git a/drivers/platform/arm64/Kconfig b/drivers/platform/arm64/Kconfig index c1ca1d78eeb86..80cefd5772cec 100644 --- a/drivers/platform/arm64/Kconfig +++ b/drivers/platform/arm64/Kconfig @@ -102,4 +102,17 @@ config EC_LENOVO_YOGA_SLIM7X mute button, and reporting device suspend to the EC so it can take appropriate actions. +config NVIDIA_FFA_EC + tristate "NVIDIA FFA EC services driver" + depends on ARM_FFA_TRANSPORT || COMPILE_TEST + depends on ACPI + depends on ACPI_FFH + help + Enable NVIDIA FFA EC services. + For GB10 and other similar SOC’s, to communicate with embedded controller, a new + specification is being defined. It is currently in draft stage and maintained in + https://github.com/OpenDevicePartnership/documentation/blob/main/bookshelf/Shelf%204%20Specifications/EC%20Interface/src/secure-ec-services-overview.md + + Say M or Y here to include this support. + endif # ARM64_PLATFORM_DEVICES diff --git a/drivers/platform/arm64/Makefile b/drivers/platform/arm64/Makefile index c135a895a3ea5..c693a0501631b 100644 --- a/drivers/platform/arm64/Makefile +++ b/drivers/platform/arm64/Makefile @@ -10,3 +10,4 @@ obj-$(CONFIG_EC_HUAWEI_GAOKUN) += huawei-gaokun-ec.o obj-$(CONFIG_EC_LENOVO_YOGA_C630) += lenovo-yoga-c630.o obj-$(CONFIG_EC_LENOVO_THINKPAD_T14S) += lenovo-thinkpad-t14s.o obj-$(CONFIG_EC_LENOVO_YOGA_SLIM7X) += lenovo-yoga-slim7x.o +obj-$(CONFIG_NVIDIA_FFA_EC) += nvidia-ffa-ec.o diff --git a/drivers/platform/arm64/nvidia-ffa-ec.c b/drivers/platform/arm64/nvidia-ffa-ec.c new file mode 100644 index 0000000000000..22dd49a1355cc --- /dev/null +++ b/drivers/platform/arm64/nvidia-ffa-ec.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved + */ + +#include +#include +#include +#include + +#define DRV_NAME "nvidia-ffa-ec" + +/* platform device for FFA ACPI device (HID MSFT000C) */ +static struct platform_device *ffa_pdev; + +static const struct acpi_device_id nvidia_ffa_device_ids[] = { + /* + * Please refer + * https://github.com/OpenDevicePartnership/documentation/blob/main/bookshelf/Shelf%204%20Specifications/EC%20Interface/src/secure-ec-services-overview.md#hid-definition + * where MSFT000C is documented. + * + * The _HID 'MSFT000C' is reserved for FFA device which uses + * FFA interface for secure EC communication. + */ + {"MSFT000C", 0}, + {"", 0}, +}; + +MODULE_DEVICE_TABLE(acpi, nvidia_ffa_device_ids); + +static int nvidia_ffa_probe(struct platform_device *pdev) +{ + struct acpi_device *adev = ACPI_COMPANION(&pdev->dev); + acpi_status status; + unsigned long long data = 0; + + if (ffa_pdev) { + dev_err(&pdev->dev, "FFA device already registered\n"); + return -EINVAL; + } + + if (!adev) { + dev_err(&pdev->dev, "No ACPI companion found\n"); + return -ENODEV; + } + + status = acpi_evaluate_integer(adev->handle, "AVAL", NULL, &data); + if (ACPI_FAILURE(status)) { + dev_err(&pdev->dev, "Failed to execute AVAL method\n"); + return -ENODEV; + } + + if (data != 1) { + dev_err(&pdev->dev, "FFA not available\n"); + return -ENODEV; + } + + ffa_pdev = pdev; + + return 0; +} + +static void nvidia_ffa_remove(struct platform_device *pdev) +{ + ffa_pdev = NULL; +} + +static struct platform_driver nvidia_ffa_driver = { + .probe = nvidia_ffa_probe, + .remove = nvidia_ffa_remove, + .driver = { + .name = "nvidia-ffa", + .acpi_match_table = nvidia_ffa_device_ids, + }, +}; + +static int __init nvidia_ffa_init(void) +{ + return platform_driver_register(&nvidia_ffa_driver); +} +module_init(nvidia_ffa_init); + +static void __exit nvidia_ffa_exit(void) +{ + platform_driver_unregister(&nvidia_ffa_driver); +} +module_exit(nvidia_ffa_exit); + +MODULE_SOFTDEP("pre: arm-ffa"); +MODULE_AUTHOR("NVIDIA CORPORATION"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("NVIDIA FFA EC services driver"); From 4d9f8f9e362907700b7697e5729cea2c44544932 Mon Sep 17 00:00:00 2001 From: Abhishek Sahu Date: Wed, 7 May 2025 08:12:27 +0000 Subject: [PATCH 044/311] NVIDIA: SAUCE: Add ffa driver for each secure EC service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BugLink: https://bugs.launchpad.net/bugs/2114230 Please refer https://github.com/OpenDevicePartnership/documentation/blob/main/bookshelf/Shelf%204%20Specifications/EC%20Interface/src/secure-ec-services-overview.md for details regarding FFA device details for secure EC services communication. Each secure EC service is identified by separate UUID. When generic FFA module loads (ffa_module), then it gets the list of partitions. Each EC service is a FFA partition and ffa_module creates a device for each partition. These devices will be added in arm_ffa bus type. The device will be named as arm-ffa-. For binding with these devices, a driver needs to be registered in arm_ffa bus type. This driver uses structure ‘struct ffa_driver’ where it uses UUID as ID table. The binding of the driver to device happens on basis of UUID. The secure EC services FFA driver is dependent upon main FFA device to be created (which uses ACPI ID MSFT000C), so ffa_driver_register()/ffa_driver_unregister() is invoked from nvidia_ffa_probe()/nvidia_ffa_remove(). Signed-off-by: Abhishek Sahu Acked-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 9613a5c07163 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 5ede0e8d753f77d7a792351102f800dda882032f noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/platform/arm64/nvidia-ffa-ec.c | 109 +++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/drivers/platform/arm64/nvidia-ffa-ec.c b/drivers/platform/arm64/nvidia-ffa-ec.c index 22dd49a1355cc..537e7a51bd133 100644 --- a/drivers/platform/arm64/nvidia-ffa-ec.c +++ b/drivers/platform/arm64/nvidia-ffa-ec.c @@ -7,12 +7,111 @@ #include #include #include +#include +#include #define DRV_NAME "nvidia-ffa-ec" /* platform device for FFA ACPI device (HID MSFT000C) */ static struct platform_device *ffa_pdev; +static const uuid_t nvidia_ec_managment_service_uuid = + UUID_INIT(0x330c1273, 0xfde5, 0x4757, 0x98, 0x19, 0x5b, 0x65, 0x39, 0x03, 0x75, 0x02); + +static const uuid_t nvidia_ec_power_service_uuid = + UUID_INIT(0x7157addf, 0x2fbe, 0x4c63, 0xae, 0x95, 0xef, 0xac, 0x16, 0xe3, 0xb0, 0x1c); + +static const uuid_t nvidia_ec_battery_service_uuid = + UUID_INIT(0x25cb5207, 0xac36, 0x427d, 0xaa, 0xef, 0x3a, 0xa7, 0x88, 0x77, 0xd2, 0x7e); + +static const uuid_t nvidia_ec_thermal_service_uuid = + UUID_INIT(0x31f56da7, 0x593c, 0x4d72, 0xa4, 0xb3, 0x8f, 0xc7, 0x17, 0x1a, 0xc0, 0x73); + +static const uuid_t nvidia_ec_fan_service_uuid = + UUID_INIT(0x7697530c, 0xd079, 0x4ec1, 0xa4, 0xc4, 0xcf, 0x0d, 0x2b, 0xdc, 0x93, 0xfa); + +static const uuid_t nvidia_ec_ucsi_service_uuid = + UUID_INIT(0x65467f50, 0x827f, 0x4e4f, 0x87, 0x70, 0xdb, 0xf4, 0xc3, 0xf7, 0x7f, 0x45); + +static const uuid_t nvidia_ec_input_service_uuid = + UUID_INIT(0xe3168a99, 0x4a57, 0x4a2b, 0x8c, 0x5e, 0x11, 0xbc, 0xfe, 0xc7, 0x34, 0x06); + +static const uuid_t nvidia_ec_time_alarm_service_uuid = + UUID_INIT(0x23ea63ed, 0xb593, 0x46ea, 0xb0, 0x27, 0x89, 0x24, 0xdf, 0x88, 0xe9, 0x2f); + +/* EC service FFA device structure */ +struct nvidia_ec_ffa_device { + struct ffa_device *ffa_dev; + struct list_head list; +}; + +/* List to contain all EC services FFA device */ +static LIST_HEAD(nvidia_ec_ffa_dev_head); + +/* Lock to serialize EC services FFA device list access */ +static DEFINE_MUTEX(nvidia_ffa_lock); + +static int nvidia_ffa_ec_service_probe(struct ffa_device *ffa_dev) +{ + struct nvidia_ec_ffa_device *nvidia_ec_ffa_dev; + + if (!ffa_pdev) { + dev_err(&ffa_dev->dev, "nvidia ffa device not available\n"); + return -ENODEV; + } + + nvidia_ec_ffa_dev = devm_kmalloc(&ffa_dev->dev, + sizeof(*nvidia_ec_ffa_dev), + GFP_KERNEL); + if (!nvidia_ec_ffa_dev) { + dev_err(&ffa_dev->dev, "Failed to allocate memory\n"); + return -ENOMEM; + } + + nvidia_ec_ffa_dev->ffa_dev = ffa_dev; + INIT_LIST_HEAD(&nvidia_ec_ffa_dev->list); + + mutex_lock(&nvidia_ffa_lock); + list_add(&nvidia_ec_ffa_dev->list, &nvidia_ec_ffa_dev_head); + mutex_unlock(&nvidia_ffa_lock); + + return 0; +} + +static void nvidia_ffa_ec_service_remove(struct ffa_device *ffa_dev) +{ + struct nvidia_ec_ffa_device *cur, *tmp; + + mutex_lock(&nvidia_ffa_lock); + list_for_each_entry_safe(cur, tmp, &nvidia_ec_ffa_dev_head, list) { + if (cur->ffa_dev == ffa_dev) { + list_del(&cur->list); + devm_kfree(&ffa_dev->dev, cur); + break; + } + } + mutex_unlock(&nvidia_ffa_lock); +} + +static const struct ffa_device_id nvidia_ffa_ec_service_ids[] = { + { nvidia_ec_managment_service_uuid }, + { nvidia_ec_power_service_uuid }, + { nvidia_ec_battery_service_uuid }, + { nvidia_ec_thermal_service_uuid }, + { nvidia_ec_fan_service_uuid }, + { nvidia_ec_ucsi_service_uuid }, + { nvidia_ec_input_service_uuid }, + { nvidia_ec_time_alarm_service_uuid }, + {} +}; + +static struct ffa_driver nvidia_ffa_ec_service_driver = { + .name = DRV_NAME, + .probe = nvidia_ffa_ec_service_probe, + .remove = nvidia_ffa_ec_service_remove, + .id_table = nvidia_ffa_ec_service_ids, +}; + static const struct acpi_device_id nvidia_ffa_device_ids[] = { /* * Please refer @@ -33,6 +132,7 @@ static int nvidia_ffa_probe(struct platform_device *pdev) struct acpi_device *adev = ACPI_COMPANION(&pdev->dev); acpi_status status; unsigned long long data = 0; + int ret; if (ffa_pdev) { dev_err(&pdev->dev, "FFA device already registered\n"); @@ -57,11 +157,20 @@ static int nvidia_ffa_probe(struct platform_device *pdev) ffa_pdev = pdev; + ret = ffa_driver_register(&nvidia_ffa_ec_service_driver, THIS_MODULE, DRV_NAME); + if (ret) { + dev_err(&pdev->dev, + "Failed to register ec service driver error=%d\n", ret); + ffa_pdev = NULL; + return ret; + } + return 0; } static void nvidia_ffa_remove(struct platform_device *pdev) { + ffa_driver_unregister(&nvidia_ffa_ec_service_driver); ffa_pdev = NULL; } From e4a36cf3a42272b96eed204edded11dcdcb6c5d5 Mon Sep 17 00:00:00 2001 From: Abhishek Sahu Date: Wed, 7 May 2025 09:06:47 +0000 Subject: [PATCH 045/311] NVIDIA: SAUCE: Add support for EC secure service communication BugLink: https://bugs.launchpad.net/bugs/2114230 Please refer https://github.com/OpenDevicePartnership/documentation/blob/main/bookshelf/Shelf%204%20Specifications/EC%20Interface/src/secure-ec-services-overview.md for details regarding FFA device details for secure EC services communication. When ACPI interpreter runs code with FFH operation region offset 4, then this data is meant for EC secure services. The FFH buffer has data in FFA_REQ_PACKET format. In this packet, it has UUID for EC service and then the service specific raw data. This commit adds a custom FFH offset handler. When request comes with custom offset then it will be handled by nvdia FFA EC driver. Inside the custom ffh callback, it extracts the UUID and gets the ffa_device for it. Then it fills raw data in ffa_send_direct_data2 and invoke sync_send_receive2() routine for that ffa_device. Once it gets the response back, then it fill data in FFA_RESP_PACKET format and ACPI interpreter passes that data to upper layer. NOTE: In the above document, the FFA_REQ_PACKET and FFA_RESP_PACKET uses different format. But in latest firmware code, the ACPI implementation is done using same format for both request and response (follows the FFA_REQ_PACKET format). The status bit will be updated in the response (0 for success and 1 for failure). This mixed endian is documented in https://cdrdv2-public.intel.com/772722/asl-tutorial-v20190625.pdf In addition to Concatenate, there are several useful macros that generate buffers from strings. For example, the ToUUID macro takes a string of the form aabbccdd-eeff-gghh-iijj-kkllmmnnoopp where aa through pp represent one byte values encoded with hexadecimal characters. This string gets converted to a 16-byte buffer that looks like the following: Buffer() { dd, cc, bb, aa, ff, ee, hh, gg, ii, jj, kk, ll, mm, nn, oo, pp } This mixture of little endian and big-endian encoding UUID is called a mixed-endian format. The use of strings and the ToUUID macro is a convenient way to avoid having to manually encode the mixed-endian format. There are many other macros that provide similar conveniences, such as EISAID. In kernel, it is represented with guid_t. Inside nvidia_ffh_handler(), we need to covert buffer of 16 bytes from FFA UUID to AML UUID format. nvidia_get_uuid_from_aml_buf() converts the AML UUID buffer into FFA UUID format. Signed-off-by: Abhishek Sahu Acked-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 40ca7bcc7774 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 613505b042bcdd9ac64a33690d9b175f596b0828 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/platform/arm64/nvidia-ffa-ec.c | 115 +++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/drivers/platform/arm64/nvidia-ffa-ec.c b/drivers/platform/arm64/nvidia-ffa-ec.c index 537e7a51bd133..4181addad8108 100644 --- a/drivers/platform/arm64/nvidia-ffa-ec.c +++ b/drivers/platform/arm64/nvidia-ffa-ec.c @@ -51,6 +51,113 @@ static LIST_HEAD(nvidia_ec_ffa_dev_head); /* Lock to serialize EC services FFA device list access */ static DEFINE_MUTEX(nvidia_ffa_lock); +/* EC secure services FFA packet structure sent via ACPI */ +struct nvidia_ec_ffa_packet { + u8 status; + u8 length; + u8 uuid[UUID_SIZE]; + u8 rawdata[]; +} __packed; + +/* + * ACPI ASL code uses ToUUID() macro which encodes it in mixed-endian format. + * Convert the AML UUID buffer into FFA UUID format. + */ +static uuid_t nvidia_get_uuid_from_aml_buf(const u8 *buf) +{ + return (uuid_t) {{ buf[3], buf[2], buf[1], buf[0], + buf[5], buf[4], buf[7], buf[6], + buf[8], buf[9], buf[10], buf[11], + buf[12], buf[13], buf[14], buf[15] }}; +} + +/* + * Handler function for FFH operation region offset 4. + * When ACPI interpreter runs code with FFH operation region offset 4, + * then this data is meant for EC secure services. The FFH buffer has + * data in 'struct nvidia_ec_ffa_packet' format. In this packet, it has UUID + * for EC secure service and then the service specific raw data. + * + * 1. Extract the UUID from this packet and get ffa_device for it. + * 2. Fill raw data in 'struct ffa_send_direct_data2' and + * invoke sync_send_receive2() routine for the ffa_device. + * 3. From response, fill the data in 'struct ffa_send_direct_data2' + * and return. + */ +static int nvidia_ffh_handler(struct acpi_ffh_info *info, acpi_integer *value, void *region_context) +{ + struct ffa_send_direct_data2 ffa_data = { 0 }; + struct nvidia_ec_ffa_packet *ffa_packet = (struct nvidia_ec_ffa_packet *)value; + struct nvidia_ec_ffa_device *cur, *ec_dev = NULL; + int ret; + uuid_t uuid; + + /* Only offset 4 is supported */ + if (info->offset != 4) + return -EOPNOTSUPP; + + /* Length should not be less than header length */ + if (info->length < offsetof(struct nvidia_ec_ffa_packet, rawdata)) + return -EINVAL; + + /* Length should not be less than actual packet length */ + if (info->length < + ffa_packet->length + offsetof(struct nvidia_ec_ffa_packet, rawdata)) { + ffa_packet->status = 1; + return -EINVAL; + } + + /* Packet length should not greater than FFA supported data length */ + if (ffa_packet->length > sizeof(ffa_data.data)) { + ffa_packet->status = 1; + return -EINVAL; + } + + /* Convert AML UUID to FFA UUID */ + uuid = nvidia_get_uuid_from_aml_buf((u8 *)ffa_packet->uuid); + + mutex_lock(&nvidia_ffa_lock); + /* Get nvidia_ec_ffa_device for the current UUID */ + list_for_each_entry(cur, &nvidia_ec_ffa_dev_head, list) { + if (uuid_equal(&uuid, &cur->ffa_dev->uuid)) { + ec_dev = cur; + break; + } + } + mutex_unlock(&nvidia_ffa_lock); + + if (!ec_dev) { + ffa_packet->status = 1; + return -EINVAL; + } + + /* Copy the ACPI FFH packet data into FFA data */ + memcpy(ffa_data.data, ffa_packet->rawdata, ffa_packet->length); + + if (!ec_dev->ffa_dev->ops || + !ec_dev->ffa_dev->ops->msg_ops || + !ec_dev->ffa_dev->ops->msg_ops->sync_send_receive2) { + return -EINVAL; + } + + ret = ec_dev->ffa_dev->ops->msg_ops->sync_send_receive2(ec_dev->ffa_dev, + &ffa_data); + if (ret) { + dev_err(&ec_dev->ffa_dev->dev, + "Failed to send FFA messages error=%d\n", ret); + ffa_packet->status = 1; + return ret; + } + + /* Set the status as success */ + ffa_packet->status = 0; + + /* Copy the ACPI FFA data back into ACPI FFH packet */ + memcpy(ffa_packet->rawdata, ffa_data.data, ffa_packet->length); + + return 0; +} + static int nvidia_ffa_ec_service_probe(struct ffa_device *ffa_dev) { struct nvidia_ec_ffa_device *nvidia_ec_ffa_dev; @@ -155,6 +262,13 @@ static int nvidia_ffa_probe(struct platform_device *pdev) return -ENODEV; } + ret = acpi_arm64_ffh_update_custom_offset_handler(nvidia_ffh_handler); + if (ret) { + dev_err(&pdev->dev, + "Failed to register custom offset handler error=%d\n", ret); + return ret; + } + ffa_pdev = pdev; ret = ffa_driver_register(&nvidia_ffa_ec_service_driver, THIS_MODULE, DRV_NAME); @@ -172,6 +286,7 @@ static void nvidia_ffa_remove(struct platform_device *pdev) { ffa_driver_unregister(&nvidia_ffa_ec_service_driver); ffa_pdev = NULL; + acpi_arm64_ffh_update_custom_offset_handler(NULL); } static struct platform_driver nvidia_ffa_driver = { From fe037c325be5384dc025710c0d3650eb800c3b39 Mon Sep 17 00:00:00 2001 From: Abhishek Sahu Date: Wed, 7 May 2025 09:22:14 +0000 Subject: [PATCH 046/311] NVIDIA: SAUCE: Rescan acpi devices that uses secure EC communication BugLink: https://bugs.launchpad.net/bugs/2114230 - During boot time, ACPI probe happens first. It calls _STA method for each added device. - Inside _STA method for device managed by EC, it uses FFH offset 4. - The request will fail since there is no custom handler registered for offset 0x4 and device will be disabled. - If rescan happens on acpi bus, then device _STA method will be called again. This commit adds support to get acpi id from UUID and invokes acpi_bus_scan(). NOTE: nvidia_get_acpi_id_from_uuid() returns ACPI ID only for few services. We don't have a corresponding driver available for all the services in the current code. For few services only, its node uses generic ACPI ID and has driver available. For rest of the service, the driver is not yet available, or the published spec is not updated with full ACPI sample code. Once we have driver available for that, then we can add those ACPI IDs in this list. Signed-off-by: Abhishek Sahu Acked-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 971a25e19691 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit e4ec4146cc8afe52290d15148ecbff121d06a140 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/platform/arm64/nvidia-ffa-ec.c | 42 ++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/drivers/platform/arm64/nvidia-ffa-ec.c b/drivers/platform/arm64/nvidia-ffa-ec.c index 4181addad8108..b47a99a5d45c3 100644 --- a/drivers/platform/arm64/nvidia-ffa-ec.c +++ b/drivers/platform/arm64/nvidia-ffa-ec.c @@ -71,6 +71,35 @@ static uuid_t nvidia_get_uuid_from_aml_buf(const u8 *buf) buf[12], buf[13], buf[14], buf[15] }}; } +static int nvidia_ffa_rescan_acpi_device(struct device *dev, void *data) +{ + struct acpi_device *adev = to_acpi_device(dev); + + if (acpi_dev_hid_uid_match(adev, data, NULL)) { + acpi_bus_scan(adev->handle); + return 1; + } + + return 0; +} + +static const char *nvidia_get_acpi_id_from_uuid(uuid_t *uuid) +{ + if (uuid_equal(uuid, &nvidia_ec_battery_service_uuid)) + return "PNP0C0A"; + + if (uuid_equal(uuid, &nvidia_ec_time_alarm_service_uuid)) + return "ACPI000E"; + + if (uuid_equal(uuid, &nvidia_ec_fan_service_uuid)) + return "PNP0C0B"; + + if (uuid_equal(uuid, &nvidia_ec_ucsi_service_uuid)) + return "PNP0CA0"; + + return NULL; +} + /* * Handler function for FFH operation region offset 4. * When ACPI interpreter runs code with FFH operation region offset 4, @@ -161,6 +190,7 @@ static int nvidia_ffh_handler(struct acpi_ffh_info *info, acpi_integer *value, v static int nvidia_ffa_ec_service_probe(struct ffa_device *ffa_dev) { struct nvidia_ec_ffa_device *nvidia_ec_ffa_dev; + const char *acpi_id = NULL; if (!ffa_pdev) { dev_err(&ffa_dev->dev, "nvidia ffa device not available\n"); @@ -182,6 +212,18 @@ static int nvidia_ffa_ec_service_probe(struct ffa_device *ffa_dev) list_add(&nvidia_ec_ffa_dev->list, &nvidia_ec_ffa_dev_head); mutex_unlock(&nvidia_ffa_lock); + /* + * When acpi subsystem probe all ACPI devices, then it execute _STA + * method for each device. The _STA method fails at that time since + * custom FFA driver won't be ready. Get ACPI ID from UUID and + * rescan the device again. + */ + acpi_id = nvidia_get_acpi_id_from_uuid(&ffa_dev->uuid); + if (acpi_id) { + acpi_bus_for_each_dev(nvidia_ffa_rescan_acpi_device, + (void *)acpi_id); + } + return 0; } From ab5ca15f5f907fc86a1e31972d7b2d3bd0cf8223 Mon Sep 17 00:00:00 2001 From: Shanker Donthineni Date: Mon, 12 Aug 2024 22:39:25 -0500 Subject: [PATCH 047/311] NVIDIA: SAUCE: irqchip/gic-v3: Allow unused SGIs for drivers/modules BugLink: https://bugs.launchpad.net/bugs/2114230 The commit 897e9e60c016 ("firmware: arm_ffa: Initial support for scheduler receiver interrupt") adds support for SGI interrupts in the FFA driver. However, the validation for SGIs in the GICv3 is too strict, causing the driver probe to fail. This patch relaxes the SGI validation check, allowing callers to use SGIs if the requested SGI number is greater than or equal to MAX_IPI, which fixes the TFA driver probe failure. This issue is observed on NVIDIA server platform with FFA-v1.1. PTP clock support registered EDAC MC: Ver: 3.0.0 ARM FF-A: Driver version 1.1 ARM FF-A: Firmware version 1.1 found GICv3: [Firmware Bug]: Illegal GSI8 translation request ARM FF-A: Failed to create IRQ mapping! ARM FF-A: Notification setup failed -61, not enabled ARM FF-A: Failed to register driver sched callback -95 scmi_core: SCMI protocol bus registered This patch was sent in arm mailing list for upstream but it got rejected. https://patchwork.kernel.org/project/linux-arm-kernel/patch/20240813033925.925947-1-sdonthineni@nvidia.com/ The proper fix requires some kind of mechanism by which a SGI can be requested by module but that needs discussion with arm and it will take time. This patch will break only if MAX_IPI value gets changed. This patch adds a BUILD_BUG_ON() to catch that situation. Once proper solution is concluded then this patch will be reverted. Signed-off-by: Shanker Donthineni Signed-off-by: Abhishek Sahu Acked-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (backported from commit fd136cf979db) [maskedarray: removed enum ipi_msg_type definition as it appears in upstream commit "irqchip/gic-v5: Add GICv5 LPI/IPI support"] Signed-off-by: Abdur Rahman (cherry picked from commit df84d5ddd125e925e912c8af1c76a71d3beb1de6 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/irqchip/irq-gic-v3.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 20f13b686ab22..3dd47cb01bb16 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -1634,7 +1634,13 @@ static int gic_irq_domain_translate(struct irq_domain *d, if(fwspec->param_count != 2) return -EINVAL; - if (fwspec->param[0] < 16) { + /* + * Below check was added on assumption that MAX_IPI + * value will not be greater than 8. + */ + BUILD_BUG_ON(MAX_IPI > 8); + + if (fwspec->param[0] < MAX_IPI) { pr_err(FW_BUG "Illegal GSI%d translation request\n", fwspec->param[0]); return -EINVAL; From ac1ad20e63732ef6388e7c3e5206ff7dd81309d3 Mon Sep 17 00:00:00 2001 From: Abhishek Sahu Date: Thu, 8 May 2025 20:26:27 +0000 Subject: [PATCH 048/311] NVIDIA: SAUCE: Add support for notifications from secure EC services BugLink: https://bugs.launchpad.net/bugs/2114230 Please refer https://github.com/OpenDevicePartnership/documentation/blob/main/bookshelf/Shelf%204%20Specifications/EC%20Interface/src/secure-ec-services-overview.md for details regarding FFA device details for secure EC services communication. 1. We need to get virtual IDs which a EC service supports. In the FFA node, the _DSD object contains this information. If we look the sample from above document, Name(_DSD, Package() { ToUUID("daffd814-6eba-4d8c-8a91-bc9bbf4aa301"), //Device Prop UUID Package() { Package(2) { "arm-arml0002-ffa-ntf-bind", Package() { 1, // Revision 2, // Count of following packages Package () { ToUUID("330c1273-fde5-4757-9819-5b6539037502"), // Service1 UUID Package () { 0x01, //Cookie1 (UINT32) 0x07, //Cookie2 } }, Package () { ToUUID("b510b3a3-59f6-4054-ba7a-ff2eb1eac765"), // Service2 UUID Package () { 0x01, //Cookie1 0x03, //Cookie2 } } } } } }) // _DSD() Then it uses a nexted package structure. nvidia_ffa_fill_notification_map() added in this commit parses the _DSD object and fill the notification id map for that service. 2. Once the virtual ID is get then it needs to map to physical ID by invoking function 1 in the notify service. 3. The UUID for notification service is B510B3A3-59F6-4054-BA7A-FF2EB1EAC765. An FFA device will be created for this notification service by ffa_module. This notify service needs to be probed first. To make that happen, a separate ffa_driver instance is created and it is getting registered first. 4. We can do 1:1 mapping between virtual ID and hardware ID. 5. We need to invoke notify_request() with hardware notification ID. It registers callback function for notification. 6. Once notification comes then we need to evaluate _DSM method with virtual ID (which will be mapped same as hardware ID). 7. The function 2 in the notify service should destroy the mapping. But it is nither implemented in the firmware not its documentation is available. A TODO comment is added in nvidia_ffa_notification_destroy(). Also, if we unload and reload the modules, the existing mapping still exists. In nvidia_ffa_notification_setup(), ignore the error for this case. When firmware is updated, then the error will be returned. 8. The notification service FFA device is needed by each EC secure services FFA device to get virtual notification list. Now following device dependency chain is created. FFA device <- notification service FFA device <- EC secure services FFA device To satisfy this, call driver registration in its dependent driver probe routine. Similarly, do the driver registration in its dependent driver removed routine. Signed-off-by: Abhishek Sahu Acked-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 1287a1d24fd0 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 605dde1a0254e9eebe7ab4142891430a71f4da39 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/platform/arm64/nvidia-ffa-ec.c | 473 ++++++++++++++++++++++++- 1 file changed, 468 insertions(+), 5 deletions(-) diff --git a/drivers/platform/arm64/nvidia-ffa-ec.c b/drivers/platform/arm64/nvidia-ffa-ec.c index b47a99a5d45c3..78068f1237b57 100644 --- a/drivers/platform/arm64/nvidia-ffa-ec.c +++ b/drivers/platform/arm64/nvidia-ffa-ec.c @@ -15,6 +15,12 @@ /* platform device for FFA ACPI device (HID MSFT000C) */ static struct platform_device *ffa_pdev; +/* FFA device for EC notification service */ +static struct ffa_device *notify_ffa_dev; + +static const uuid_t nvidia_ec_notify_service_uuid = + UUID_INIT(0xb510b3a3, 0x59f6, 0x4054, 0xba, 0x7a, 0xff, 0x2e, 0xb1, 0xea, 0xc7, 0x65); + static const uuid_t nvidia_ec_managment_service_uuid = UUID_INIT(0x330c1273, 0xfde5, 0x4757, 0x98, 0x19, 0x5b, 0x65, 0x39, 0x03, 0x75, 0x02); @@ -39,9 +45,19 @@ static const uuid_t nvidia_ec_input_service_uuid = static const uuid_t nvidia_ec_time_alarm_service_uuid = UUID_INIT(0x23ea63ed, 0xb593, 0x46ea, 0xb0, 0x27, 0x89, 0x24, 0xdf, 0x88, 0xe9, 0x2f); +static const guid_t nvidia_notify_bind_guid = + GUID_INIT(0xdaffd814, 0x6eba, 0x4d8c, 0x8a, 0x91, 0xbc, 0x9b, 0xbf, 0x4a, 0xa3, 0x01); + +static const guid_t nvidia_notify_dsm_guid = + GUID_INIT(0x7681541e, 0x8827, 0x4239, 0x8d, 0x9d, 0x36, 0xbe, 0x7f, 0xe1, 0x25, 0x42); + +#define NVIDIA_FFA_MAX_NOTIFICATIONS 64 + /* EC service FFA device structure */ struct nvidia_ec_ffa_device { struct ffa_device *ffa_dev; + u8 notification_count; + u8 notification_id[NVIDIA_FFA_MAX_NOTIFICATIONS]; struct list_head list; }; @@ -71,6 +87,27 @@ static uuid_t nvidia_get_uuid_from_aml_buf(const u8 *buf) buf[12], buf[13], buf[14], buf[15] }}; } +/* + * ACPI ASL code uses ToUUID() macro which encodes it in mixed-endian format. + * Convert UUID buffer to AML UUID. + */ +static void nvidia_uuid_to_aml_uuid_buf(const uuid_t *uuid, u8 *buf) +{ + const u8 *src = (u8 *)uuid; + + buf[0] = src[3]; + buf[1] = src[2]; + buf[2] = src[1]; + buf[3] = src[0]; + + buf[4] = src[5]; + buf[5] = src[4]; + buf[6] = src[7]; + buf[7] = src[6]; + + memcpy(buf + 8, src + 8, 8); +} + static int nvidia_ffa_rescan_acpi_device(struct device *dev, void *data) { struct acpi_device *adev = to_acpi_device(dev); @@ -100,6 +137,380 @@ static const char *nvidia_get_acpi_id_from_uuid(uuid_t *uuid) return NULL; } +/* + * Fill the virtual notification IDs array supported by the current FFA device. + * ACPI _DSD object contains notification mapping. It uses nexted package + * acpi object. + * + * From the example given in + * https://github.com/OpenDevicePartnership/documentation/blob/main/bookshelf/Shelf%204%20Specifications/EC%20Interface/src/secure-ec-services-overview.md#register-notification + * + * pkg1 Name(_DSD, Package() { + * pkg1_guid ToUUID("daffd814-6eba-4d8c-8a91-bc9bbf4aa301"), // Device Prop UUID + * pkg2 Package() { + * pkg3 Package(2) { + * pkg3_prop "arm-arml0002-ffa-ntf-bind", + * pkg4 Package() { + * pkg4_rev 1, // Revision + * pkg4_count 1, // Count of following packages + * pkg5 Package () { + * pkg5_uuid ToUUID("330c1273-fde5-4757-9819-5b6539037502"), // Service1 UUID + * pkg6 Package () { + * pkg6_notify_id[] 0x01, // Cookie1 (UINT32) + * 0x07, // Cookie2 + * } + * }, + * } + * } + * } + * }) // _DSD() + * + * The variable names in this function are according to above. + */ +static int nvidia_ffa_fill_notification_map(struct nvidia_ec_ffa_device *ec_ffa_dev) +{ + struct acpi_device *adev = ACPI_COMPANION(&ffa_pdev->dev); + struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL }; + union acpi_object *pkg1, *pkg1_guid; + union acpi_object *pkg2; + union acpi_object *pkg3, *pkg3_prop; + union acpi_object *pkg4, *pkg4_rev, *pkg4_count; + acpi_status status; + int i; + + status = acpi_evaluate_object_typed(adev->handle, "_DSD", NULL, + &output, ACPI_TYPE_PACKAGE); + if (ACPI_FAILURE(status)) { + dev_err(&ffa_pdev->dev, "ACPI _DSD object not found\n"); + return -ENODEV; + } + + pkg1 = output.pointer; + + /* + * _DSD returns a Package() with one or more pairs of elements. + * The first element of each pair is a Universal Unique Identifier (UUID). + * The second element of each pair is another Package() Data Structure. + * + * The _DSD for FFA device will have only one pair of elements so + * pkg1 elements count should be 2. + */ + if (pkg1->package.count != 2) { + kfree(output.pointer); + return -EINVAL; + } + + pkg1_guid = &pkg1->package.elements[0]; + pkg2 = &pkg1->package.elements[1]; + if (pkg1_guid->type != ACPI_TYPE_BUFFER || + pkg1_guid->buffer.length != UUID_SIZE || + pkg2->type != ACPI_TYPE_PACKAGE) { + kfree(output.pointer); + return -EINVAL; + } + + /* Check if GUID macthes with notify device prop GUID */ + if (!guid_equal((guid_t *)pkg1_guid->buffer.pointer, + &nvidia_notify_bind_guid)) { + kfree(output.pointer); + return -EINVAL; + } + + /* pkg3 should conatin 1 element with package type */ + if (pkg2->package.count != 1) { + kfree(output.pointer); + return -EINVAL; + } + + pkg3 = &pkg2->package.elements[0]; + if (pkg3->type != ACPI_TYPE_PACKAGE) { + kfree(output.pointer); + return -EINVAL; + } + + pkg3_prop = &pkg3->package.elements[0]; + if (pkg3_prop->type != ACPI_TYPE_STRING || + strncmp(pkg3_prop->string.pointer, + "arm-arml0002-ffa-ntf-bind", + pkg3_prop->string.length)) { + kfree(output.pointer); + return -EINVAL; + } + + pkg4 = &pkg3->package.elements[1]; + /* + * pkg4 should have minimum 3 elements (revision, count and minimum + * one notification map package) + */ + if (pkg4->type != ACPI_TYPE_PACKAGE || + pkg4->package.count < 3) { + kfree(output.pointer); + return -EINVAL; + } + + pkg4_rev = &pkg4->package.elements[0]; + pkg4_count = &pkg4->package.elements[1]; + + /* Check if revision is 1 */ + if (pkg4_rev->type != ACPI_TYPE_INTEGER || + pkg4_rev->integer.value != 1) { + kfree(output.pointer); + return -EINVAL; + } + + /* + * The pkg4_count represents the count of following packages. + * pkg4_count + 1 (for revision) + 1 (for pkg4_count itself) should + * match total number of elements in pkg4. + */ + if (pkg4_count->type != ACPI_TYPE_INTEGER || + (pkg4_count->integer.value + 2) != pkg4->package.count) { + kfree(output.pointer); + return -EINVAL; + } + + /* + * Traverse the array of notification map packages. + * Each notification map package contains 2 elements, UUID + * and notification ID array package. Check if there is a notification + * map for the FFA device by comparing UUID and update the + * notification_id[] and notification_count. + */ + for (i = 2; i < pkg4->package.count; i++) { + union acpi_object *pkg5_uuid, *pkg5 = &pkg4->package.elements[2]; + union acpi_object *pkg6; + uuid_t uuid; + int j; + + if (pkg5->type != ACPI_TYPE_PACKAGE && + pkg5->package.count != 2) { + kfree(output.pointer); + return -EINVAL; + } + + pkg5_uuid = &pkg5->package.elements[0]; + pkg6 = &pkg5->package.elements[1]; + if (pkg5_uuid->type != ACPI_TYPE_BUFFER || + pkg5_uuid->buffer.length != UUID_SIZE || + pkg6->type != ACPI_TYPE_PACKAGE) { + kfree(output.pointer); + return -EINVAL; + } + + uuid = nvidia_get_uuid_from_aml_buf(pkg5_uuid->buffer.pointer); + if (!uuid_equal(&uuid, &ec_ffa_dev->ffa_dev->uuid)) + continue; + + for (j = 0; j < pkg6->package.count; j++) { + union acpi_object *pkg6_notify_id = &pkg6->package.elements[j]; + + if (pkg6_notify_id->type != ACPI_TYPE_INTEGER) { + kfree(output.pointer); + return -EINVAL; + } + + ec_ffa_dev->notification_id[j] = pkg6_notify_id->integer.value; + } + + ec_ffa_dev->notification_count = pkg6->package.count; + kfree(output.pointer); + return 0; + } + + kfree(output.pointer); + return 0; +} + +/* + * Notification EC service callback. + * Get the ffa device from callback data and invoke notification _DSM with + * notify_id. + * + * The details regarding _DSM is documented in + * https://github.com/OpenDevicePartnership/documentation/tree/main/bookshelf/Shelf%204%20Specifications#notification-events + */ +static void nvidia_ffa_ec_service_notif_callback(int notify_id, void *cb_data) +{ + struct acpi_device *adev = ACPI_COMPANION(&ffa_pdev->dev); + struct ffa_device *ffa_dev = (struct ffa_device *)cb_data; + union acpi_object args[2], input_pkg; + union acpi_object *output; + u8 uuid[UUID_SIZE]; + + nvidia_uuid_to_aml_uuid_buf(&ffa_dev->uuid, uuid); + + args[0].type = ACPI_TYPE_BUFFER; + args[0].buffer.length = sizeof(uuid); + args[0].buffer.pointer = uuid; + + args[1].type = ACPI_TYPE_INTEGER; + args[1].integer.value = notify_id; + + input_pkg.type = ACPI_TYPE_PACKAGE; + input_pkg.package.count = 2; + input_pkg.package.elements = args; + + output = acpi_evaluate_dsm(adev->handle, &nvidia_notify_dsm_guid, + 1, 1, &input_pkg); + if (!output) + dev_err(&ffa_pdev->dev, "Failed to execute notify\n"); + else + ACPI_FREE(output); +} + +/* + * Create notification setup for the notification_id. + * + * The details regarding notification setup is documented in + * https://github.com/OpenDevicePartnership/documentation/tree/main/bookshelf/Shelf%204%20Specifications#register-notification + * + * This function setup 1:1 mapping between hardware notification ID and + * virtual notification ID. + */ +static int nvidia_ffa_notification_setup(struct nvidia_ec_ffa_device *ec_ffa_dev, + u8 notification_id) +{ + struct ffa_send_direct_data2 ffa_data = { 0 }; + u8 *uuid = (u8 *)&ec_ffa_dev->ffa_dev->uuid; + int ret; + + /* X4 register, function 1 */ + ffa_data.data[0] = 1; + + BUILD_BUG_ON(UUID_SIZE != 16); + BUILD_BUG_ON(sizeof(ffa_data.data[1]) < 8); + + /* X5 and X6 registers contain UUID */ + memcpy(&ffa_data.data[1], uuid, 8); + memcpy(&ffa_data.data[2], uuid + 8, 8); + + /* X7 register, the number of notification mappings */ + ffa_data.data[3] = 1; + + /* X7 register, notification ID and notification bitmap bit number */ + ffa_data.data[4] = ((u64)notification_id << 32) | notification_id; + + if (!notify_ffa_dev->ops || + !notify_ffa_dev->ops->msg_ops || + !notify_ffa_dev->ops->msg_ops->sync_send_receive2) { + return -EINVAL; + } + + ret = notify_ffa_dev->ops->msg_ops->sync_send_receive2(notify_ffa_dev, + &ffa_data); + if (ret) { + dev_err(&ec_ffa_dev->ffa_dev->dev, + "Failed to send NOTIFY_SETUP id=%d error=%d\n", + notification_id, ret); + return ret; + } + + if (ffa_data.data[0]) { + dev_err(&ec_ffa_dev->ffa_dev->dev, + "NOTIFY_SETUP returned failure id=%d error=%ld\n", + notification_id, ffa_data.data[0]); + + /* + * TODO: destroy operation is not yet implemented in the firmware + * So, if driver is reloaded, then the previous notification + * still exists and failure will be returned. Once destroy + * is implemented in firmware, update code here to return error + */ + } + + return 0; +} + +/* Destroy notification setup for the notification_id */ +static void nvidia_ffa_notification_destroy(struct nvidia_ec_ffa_device *ec_ffa_dev, + u8 notification_id) +{ + /* + * TODO: destroy operation is not yet implemented in the firmware. + * Once implemented in firmware, update code here. + */ +} + +/* + * Create notifications for the FFA device. + * + * 1. Get notification map array for FFA device. + * 2. For each notification, setup notification with notify service and + * then invoke notify_request method to enable notification for FFA device. + */ +static int nvidia_ffa_create_notifications(struct nvidia_ec_ffa_device *ec_ffa_dev) +{ + int i, ret = 0; + + if (!ec_ffa_dev->ffa_dev->ops || + !ec_ffa_dev->ffa_dev->ops->notifier_ops || + !ec_ffa_dev->ffa_dev->ops->notifier_ops->notify_request || + !ec_ffa_dev->ffa_dev->ops->notifier_ops->notify_relinquish) { + return -EOPNOTSUPP; + } + + ret = nvidia_ffa_fill_notification_map(ec_ffa_dev); + if (ret) { + dev_err(&ffa_pdev->dev, "Error in filling notification map error=%d\n", ret); + return ret; + } + + for (i = 0; i < ec_ffa_dev->notification_count; i++) { + ret = nvidia_ffa_notification_setup(ec_ffa_dev, + ec_ffa_dev->notification_id[i]); + if (ret) { + dev_err(&ec_ffa_dev->ffa_dev->dev, + "Failed to setup notification id=%d error=%d\n", + ec_ffa_dev->notification_id[i], ret); + break; + } + + ret = ec_ffa_dev->ffa_dev->ops->notifier_ops->notify_request( + ec_ffa_dev->ffa_dev, false, + nvidia_ffa_ec_service_notif_callback, + ec_ffa_dev->ffa_dev, ec_ffa_dev->notification_id[i]); + if (ret) { + nvidia_ffa_notification_destroy(ec_ffa_dev, + ec_ffa_dev->notification_id[i]); + dev_err(&ec_ffa_dev->ffa_dev->dev, + "Failed to request notification id=%d error=%d\n", + ec_ffa_dev->notification_id[i], ret); + break; + } + } + + /* Remove already setup notification in case of error */ + if (ret) { + int j; + + for (j = 0; j < i; j++) { + ec_ffa_dev->ffa_dev->ops->notifier_ops->notify_relinquish( + ec_ffa_dev->ffa_dev, + ec_ffa_dev->notification_id[j]); + nvidia_ffa_notification_destroy(ec_ffa_dev, + ec_ffa_dev->notification_id[j]); + } + + ec_ffa_dev->notification_count = 0; + } + + return ret; +} + +/* Remove notifications for the FFA device. */ +static void nvidia_ffa_remove_notifications(struct nvidia_ec_ffa_device *ec_ffa_dev) +{ + int i; + + for (i = 0; i < ec_ffa_dev->notification_count; i++) { + ec_ffa_dev->ffa_dev->ops->notifier_ops->notify_relinquish( + ec_ffa_dev->ffa_dev, + ec_ffa_dev->notification_id[i]); + nvidia_ffa_notification_destroy(ec_ffa_dev, + ec_ffa_dev->notification_id[i]); + } +} + /* * Handler function for FFH operation region offset 4. * When ACPI interpreter runs code with FFH operation region offset 4, @@ -191,9 +602,10 @@ static int nvidia_ffa_ec_service_probe(struct ffa_device *ffa_dev) { struct nvidia_ec_ffa_device *nvidia_ec_ffa_dev; const char *acpi_id = NULL; + int ret; - if (!ffa_pdev) { - dev_err(&ffa_dev->dev, "nvidia ffa device not available\n"); + if (!ffa_pdev || !notify_ffa_dev) { + dev_err(&ffa_dev->dev, "nvidia ffa or notify device not available\n"); return -ENODEV; } @@ -208,6 +620,15 @@ static int nvidia_ffa_ec_service_probe(struct ffa_device *ffa_dev) nvidia_ec_ffa_dev->ffa_dev = ffa_dev; INIT_LIST_HEAD(&nvidia_ec_ffa_dev->list); + ret = nvidia_ffa_create_notifications(nvidia_ec_ffa_dev); + if (ret) { + dev_info(&ffa_dev->dev, + "Failed to create ffa notifications error=%d\n", + ret); + devm_kfree(&ffa_dev->dev, nvidia_ec_ffa_dev); + return ret; + } + mutex_lock(&nvidia_ffa_lock); list_add(&nvidia_ec_ffa_dev->list, &nvidia_ec_ffa_dev_head); mutex_unlock(&nvidia_ffa_lock); @@ -235,6 +656,7 @@ static void nvidia_ffa_ec_service_remove(struct ffa_device *ffa_dev) list_for_each_entry_safe(cur, tmp, &nvidia_ec_ffa_dev_head, list) { if (cur->ffa_dev == ffa_dev) { list_del(&cur->list); + nvidia_ffa_remove_notifications(cur); devm_kfree(&ffa_dev->dev, cur); break; } @@ -261,6 +683,46 @@ static struct ffa_driver nvidia_ffa_ec_service_driver = { .id_table = nvidia_ffa_ec_service_ids, }; +static int nvidia_ffa_notify_service_probe(struct ffa_device *ffa_dev) +{ + int ret; + + if (!ffa_pdev) { + dev_err(&ffa_dev->dev, "nvidia ffa device not available\n"); + return -ENODEV; + } + + notify_ffa_dev = ffa_dev; + + ret = ffa_driver_register(&nvidia_ffa_ec_service_driver, THIS_MODULE, DRV_NAME); + if (ret) { + dev_err(&ffa_dev->dev, + "Failed to register ec service driver error=%d\n", ret); + notify_ffa_dev = NULL; + return ret; + } + + return 0; +} + +static void nvidia_ffa_notify_service_remove(struct ffa_device *ffa_dev) +{ + ffa_driver_unregister(&nvidia_ffa_ec_service_driver); + notify_ffa_dev = NULL; +} + +static const struct ffa_device_id nvidia_ffa_notify_service_ids[] = { + { nvidia_ec_notify_service_uuid }, + {} +}; + +static struct ffa_driver nvidia_ffa_notify_service_driver = { + .name = "nvidia-ffa-notify", + .probe = nvidia_ffa_notify_service_probe, + .remove = nvidia_ffa_notify_service_remove, + .id_table = nvidia_ffa_notify_service_ids, +}; + static const struct acpi_device_id nvidia_ffa_device_ids[] = { /* * Please refer @@ -313,10 +775,11 @@ static int nvidia_ffa_probe(struct platform_device *pdev) ffa_pdev = pdev; - ret = ffa_driver_register(&nvidia_ffa_ec_service_driver, THIS_MODULE, DRV_NAME); + ret = ffa_driver_register(&nvidia_ffa_notify_service_driver, THIS_MODULE, DRV_NAME); if (ret) { dev_err(&pdev->dev, - "Failed to register ec service driver error=%d\n", ret); + "Failed to register notify service driver error=%d\n", ret); + acpi_arm64_ffh_update_custom_offset_handler(NULL); ffa_pdev = NULL; return ret; } @@ -326,7 +789,7 @@ static int nvidia_ffa_probe(struct platform_device *pdev) static void nvidia_ffa_remove(struct platform_device *pdev) { - ffa_driver_unregister(&nvidia_ffa_ec_service_driver); + ffa_driver_unregister(&nvidia_ffa_notify_service_driver); ffa_pdev = NULL; acpi_arm64_ffh_update_custom_offset_handler(NULL); } From 3d46291ecd20d649d744205ce2f90a19e11f4644 Mon Sep 17 00:00:00 2001 From: Abhishek Sahu Date: Fri, 13 Jun 2025 05:42:10 +0000 Subject: [PATCH 049/311] UBUNTU: [Config] nvidia: Update annotations to enable NVIDIA FFA EC driver BugLink: https://bugs.launchpad.net/bugs/2114230 The NVIDIA FFA and EC secure services driver enables the communication with EC (Embedded Controller). Make this driver built-in to enable EC communication at early boot. Signed-off-by: Abhishek Sahu Acked-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 9ea0251a632d4455a4ae3e9878a928d65f48ad30) (cherry picked from commit 9ea0251a632d noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 31b28eae7597f7e0a38375f933e4fd1af9e246d7 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- debian.nvidia/config/annotations | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index 20ede527308a9..8404149f7f758 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -156,6 +156,9 @@ CONFIG_NOUVEAU_PLATFORM_DRIVER note<'Disable nouveau for NVIDIA CONFIG_NR_CPUS policy<{'amd64': '8192', 'arm64': '512'}> CONFIG_NR_CPUS note<'LP: #1864198'> +CONFIG_NVIDIA_FFA_EC policy<{'arm64': 'y'}> +CONFIG_NVIDIA_FFA_EC note<'LP: #2114230'> + CONFIG_PID_IN_CONTEXTIDR policy<{'arm64': 'y'}> CONFIG_PID_IN_CONTEXTIDR note<'Required for Grace enablement'> From 4de74386b88eb2e99018f569421cd5790edb750b Mon Sep 17 00:00:00 2001 From: Jonas Chen Date: Mon, 21 Apr 2025 17:17:38 +0800 Subject: [PATCH 050/311] NVIDIA: SAUCE: MEDIATEK: pinctrl: mediatek: Add gpio-range record in pinctrl driver BugLink: https://bugs.launchpad.net/bugs/2117784 Kernel GPIO subsystem mapping hardware pin number to a different range of gpio number. Add gpio-range structure to hold the mapped gpio range in pinctrl driver. That enables the kernel to search a range of mapped gpio range against a pinctrl device. Signed-off-by: Jonas Chen Signed-off-by: Yenchia Chen Signed-off-by: Abhishek Sahu Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Acked-by: nvmochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 1049985ca252 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit c113e8d8f903eccbe4fa0c075af1aadd5f4fe014 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.h | 1 + drivers/pinctrl/mediatek/pinctrl-paris.c | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.h b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.h index fa7c0ed493464..df8dce14744f9 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.h +++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.h @@ -302,6 +302,7 @@ struct mtk_pinctrl { spinlock_t lock; /* identify rsel setting by si unit or rsel define in dts node */ bool rsel_si_unit; + struct pinctrl_gpio_range range; }; void mtk_rmw(struct mtk_pinctrl *pctl, u8 i, u32 reg, u32 mask, u32 set); diff --git a/drivers/pinctrl/mediatek/pinctrl-paris.c b/drivers/pinctrl/mediatek/pinctrl-paris.c index 6bf37d8085fae..2cf61cfe809ed 100644 --- a/drivers/pinctrl/mediatek/pinctrl-paris.c +++ b/drivers/pinctrl/mediatek/pinctrl-paris.c @@ -3,7 +3,7 @@ * MediaTek Pinctrl Paris Driver, which implement the vendor per-pin * bindings for MediaTek SoC. * - * Copyright (C) 2018 MediaTek Inc. + * Copyright (C) 2018-2025 MediaTek Inc. * Author: Sean Wang * Zhiyong Tao * Hongzhou.Yang @@ -936,6 +936,15 @@ static int mtk_gpio_set_config(struct gpio_chip *chip, unsigned int offset, return mtk_eint_set_debounce(hw->eint, desc->eint.eint_n, debounce); } +static void mtk_pinctrl_gpio_range_init(struct mtk_pinctrl *hw, struct gpio_chip *chip) +{ + hw->range.name = "mtk_pinctrl_gpio_range"; + hw->range.id = 0; + hw->range.pin_base = 0; + hw->range.base = chip->base; + hw->range.npins = hw->soc->npins; +} + static int mtk_build_gpiochip(struct mtk_pinctrl *hw) { struct gpio_chip *chip = &hw->chip; @@ -959,6 +968,8 @@ static int mtk_build_gpiochip(struct mtk_pinctrl *hw) if (ret < 0) return ret; + mtk_pinctrl_gpio_range_init(hw, chip); + return 0; } @@ -1077,6 +1088,8 @@ int mtk_paris_pinctrl_probe(struct platform_device *pdev) if (err) return dev_err_probe(dev, err, "Failed to add gpio_chip\n"); + pinctrl_add_gpio_range(hw->pctrl, &hw->range); + platform_set_drvdata(pdev, hw); return 0; From 8607439009b1fda8499137d8f9a5c0b8a7b3e20d Mon Sep 17 00:00:00 2001 From: Jonas Chen Date: Tue, 22 Apr 2025 09:18:17 +0800 Subject: [PATCH 051/311] NVIDIA: SAUCE: MEDIATEK: pinctrl: mediatek: Add acpi support BugLink: https://bugs.launchpad.net/bugs/2117784 Add acpi support in the shared part of pinctrl driver. Parsing hardware base addresses and irq naumber to initialize eint accroding to the acpi table data. Signed-off-by: Jonas Chen Signed-off-by: Yenchia Chen Signed-off-by: Abhishek Sahu Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Acked-by: nvmochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (backported from commit cdce65d91ea9 noble:linux-nvidia-6.14) [maskedarray: context adjusted due to commit 86dee87: "pinctrl: mediatek: Fix the invalid conditions"] Signed-off-by: Abdur Rahman (cherry picked from commit 84076e8dd9e065d3ab6ce59aec774a14c7f8ff32 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- .../pinctrl/mediatek/pinctrl-mtk-common-v2.c | 26 +++++++++++++++---- drivers/pinctrl/mediatek/pinctrl-paris.c | 11 +++++--- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c index 4918d38abfc29..fc71f9b267c52 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c +++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (C) 2018 MediaTek Inc. + * Copyright (C) 2018-2025 MediaTek Inc. * * Author: Sean Wang * @@ -369,18 +369,30 @@ int mtk_build_eint(struct mtk_pinctrl *hw, struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; int ret, i, j, count_reg_names; + struct fwnode_handle *fwnode = dev_fwnode(&pdev->dev); + struct resource *res; if (!IS_ENABLED(CONFIG_EINT_MTK)) return 0; - if (!of_property_read_bool(np, "interrupt-controller")) + if (is_of_node(fwnode) && !of_property_read_bool(np, "interrupt-controller")) return -ENODEV; hw->eint = devm_kzalloc(hw->dev, sizeof(*hw->eint), GFP_KERNEL); if (!hw->eint) return -ENOMEM; - count_reg_names = of_property_count_strings(np, "reg-names"); + if (is_of_node(fwnode)) { + count_reg_names = of_property_count_strings(np, "reg-names"); + } else { + count_reg_names = 0; + for (i = 0; i < pdev->num_resources; i++) { + struct resource *r = &pdev->resource[i]; + + if (resource_type(r) == IORESOURCE_MEM) + count_reg_names++; + } + } if (count_reg_names < 0) return -EINVAL; @@ -396,14 +408,18 @@ int mtk_build_eint(struct mtk_pinctrl *hw, struct platform_device *pdev) } for (i = hw->soc->nbase_names, j = 0; i < count_reg_names; i++, j++) { - hw->eint->base[j] = of_iomap(np, i); + res = platform_get_resource(pdev, IORESOURCE_MEM, i); + hw->eint->base[j] = is_of_node(fwnode) ? of_iomap(np, i) : + ioremap(res->start, resource_size(res)); if (IS_ERR(hw->eint->base[j])) { ret = PTR_ERR(hw->eint->base[j]); goto err_free_eint; } } - hw->eint->irq = irq_of_parse_and_map(np, 0); + hw->eint->irq = is_of_node(fwnode) + ? irq_of_parse_and_map(np, 0) + : platform_get_irq(pdev, 0); if (!hw->eint->irq) { ret = -EINVAL; goto err_free_eint; diff --git a/drivers/pinctrl/mediatek/pinctrl-paris.c b/drivers/pinctrl/mediatek/pinctrl-paris.c index 2cf61cfe809ed..f74221acba8cb 100644 --- a/drivers/pinctrl/mediatek/pinctrl-paris.c +++ b/drivers/pinctrl/mediatek/pinctrl-paris.c @@ -1008,6 +1008,7 @@ int mtk_paris_pinctrl_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct pinctrl_pin_desc *pins; struct mtk_pinctrl *hw; + struct fwnode_handle *fwnode = dev_fwnode(&pdev->dev); int err, i; hw = devm_kzalloc(&pdev->dev, sizeof(*hw), GFP_KERNEL); @@ -1032,16 +1033,20 @@ int mtk_paris_pinctrl_probe(struct platform_device *pdev) return -ENOMEM; for (i = 0; i < hw->soc->nbase_names; i++) { - hw->base[i] = devm_platform_ioremap_resource_byname(pdev, - hw->soc->base_names[i]); + hw->base[i] = is_of_node(fwnode) + ? devm_platform_ioremap_resource_byname(pdev, hw->soc->base_names[i]) + : devm_platform_get_and_ioremap_resource(pdev, i, NULL); if (IS_ERR(hw->base[i])) return PTR_ERR(hw->base[i]); } hw->nbase = hw->soc->nbase_names; - hw->rsel_si_unit = of_property_read_bool(hw->dev->of_node, + if (is_of_node(fwnode)) + hw->rsel_si_unit = of_property_read_bool(hw->dev->of_node, "mediatek,rsel-resistance-in-si-unit"); + else + hw->rsel_si_unit = false; spin_lock_init(&hw->lock); From 3f23b1456e73b7064f5f3a59d701355c72eed343 Mon Sep 17 00:00:00 2001 From: Jonas Chen Date: Tue, 22 Apr 2025 09:30:44 +0800 Subject: [PATCH 052/311] NVIDIA: SAUCE: MEDIATEK: pinctrl: mt8901: Add pinctrl driver BugLink: https://bugs.launchpad.net/bugs/2117784 Add mt8901 pinctrl, gpio and eint driver implementation. Signed-off-by: Jonas Chen Signed-off-by: Yenchia Chen Signed-off-by: Abhishek Sahu Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Acked-by: nvmochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off-by: Brad Figg (backported from commit 1fc7a586c54c noble:linux-nvidia-6.14) [maskedarray: context adjusted for missing commit a3fe132: "pinctrl: mediatek: Add pinctrl driver for mt8189"] Signed-off-by: Abdur Rahman (backported from commit 81bfb0635b75667abd430efc1d2ea50813b0fdf5 noble:linux-nvidia-6.17) [jacobmartin: context adjusted for new pinctrl-mt8901 driver from upstream] Signed-off-by: Jacob Martin --- drivers/pinctrl/mediatek/Kconfig | 12 + drivers/pinctrl/mediatek/Makefile | 1 + drivers/pinctrl/mediatek/mtk-eint.c | 4 + drivers/pinctrl/mediatek/mtk-eint.h | 1 + drivers/pinctrl/mediatek/pinctrl-mt8901.c | 1460 +++++++++++ drivers/pinctrl/mediatek/pinctrl-mtk-mt8901.h | 2130 +++++++++++++++++ 6 files changed, 3608 insertions(+) create mode 100644 drivers/pinctrl/mediatek/pinctrl-mt8901.c create mode 100644 drivers/pinctrl/mediatek/pinctrl-mtk-mt8901.h diff --git a/drivers/pinctrl/mediatek/Kconfig b/drivers/pinctrl/mediatek/Kconfig index 4819617d93683..92f4f394b71e7 100644 --- a/drivers/pinctrl/mediatek/Kconfig +++ b/drivers/pinctrl/mediatek/Kconfig @@ -281,6 +281,18 @@ config PINCTRL_MT8189 In MTK platform, we support virtual gpio and use it to map specific eint which doesn't have real gpio pin. +config PINCTRL_MT8901 + bool "MediaTek MT8901 pin control" + depends on ACPI + depends on ARM64 || COMPILE_TEST + default ARM64 && ARCH_MEDIATEK + select PINCTRL_MTK_PARIS + help + Say yes here to support pin controller and gpio driver + on MediaTek MT8901 SoC. + In MTK platform, we support virtual gpio and use it to + map specific eint which doesn't have real gpio pin. + config PINCTRL_MT8192 bool "MediaTek MT8192 pin control" depends on OF diff --git a/drivers/pinctrl/mediatek/Makefile b/drivers/pinctrl/mediatek/Makefile index ae765bd999657..57c69b1e5c2d4 100644 --- a/drivers/pinctrl/mediatek/Makefile +++ b/drivers/pinctrl/mediatek/Makefile @@ -43,3 +43,4 @@ obj-$(CONFIG_PINCTRL_MT8196) += pinctrl-mt8196.o obj-$(CONFIG_PINCTRL_MT8365) += pinctrl-mt8365.o obj-$(CONFIG_PINCTRL_MT8516) += pinctrl-mt8516.o obj-$(CONFIG_PINCTRL_MT6397) += pinctrl-mt6397.o +obj-$(CONFIG_PINCTRL_MT8901) += pinctrl-mt8901.o diff --git a/drivers/pinctrl/mediatek/mtk-eint.c b/drivers/pinctrl/mediatek/mtk-eint.c index 2a3c04eedc5f3..3e6b121cf593a 100644 --- a/drivers/pinctrl/mediatek/mtk-eint.c +++ b/drivers/pinctrl/mediatek/mtk-eint.c @@ -71,6 +71,10 @@ const unsigned int debounce_time_mt6878[] = { }; EXPORT_SYMBOL_GPL(debounce_time_mt6878); +const unsigned int debounce_time_mt8901[] = { + 156, 313, 625, 1250, 20000, 40000, 80000, 160000, 320000, 640000, 0}; +EXPORT_SYMBOL_GPL(debounce_time_mt8901); + static void __iomem *mtk_eint_get_offset(struct mtk_eint *eint, unsigned int eint_num, unsigned int offset) diff --git a/drivers/pinctrl/mediatek/mtk-eint.h b/drivers/pinctrl/mediatek/mtk-eint.h index 3cdd6f6310cd0..1b185f660affa 100644 --- a/drivers/pinctrl/mediatek/mtk-eint.h +++ b/drivers/pinctrl/mediatek/mtk-eint.h @@ -53,6 +53,7 @@ extern const unsigned int debounce_time_mt2701[]; extern const unsigned int debounce_time_mt6765[]; extern const unsigned int debounce_time_mt6795[]; extern const unsigned int debounce_time_mt6878[]; +extern const unsigned int debounce_time_mt8901[]; struct mtk_eint; diff --git a/drivers/pinctrl/mediatek/pinctrl-mt8901.c b/drivers/pinctrl/mediatek/pinctrl-mt8901.c new file mode 100644 index 0000000000000..623cd0cd58f05 --- /dev/null +++ b/drivers/pinctrl/mediatek/pinctrl-mt8901.c @@ -0,0 +1,1460 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2025 MediaTek Inc. + * + */ + +#include +#include +#include "pinctrl-mtk-mt8901.h" +#include "pinctrl-paris.h" + +#define PIN_FIELD_BASE(s_pin, e_pin, i_base, s_addr, x_addrs, s_bit, x_bits) \ + PIN_FIELD_CALC(s_pin, e_pin, i_base, s_addr, x_addrs, s_bit, x_bits, 32, 0) +#define PINS_FIELD_BASE(s_pin, e_pin, i_base, s_addr, x_addrs, s_bit, x_bits) \ + PIN_FIELD_CALC(s_pin, e_pin, i_base, s_addr, x_addrs, s_bit, x_bits, 32, 1) + +static const struct mtk_pin_field_calc mt8901_pin_mode_range[] = { + PIN_FIELD(0, 181, 0x0300, 0x10, 0, 4), +}; + +static const struct mtk_pin_field_calc mt8901_pin_dir_range[] = { + PIN_FIELD(0, 181, 0x0000, 0x10, 0, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_di_range[] = { + PIN_FIELD(0, 181, 0x0200, 0x10, 0, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_do_range[] = { + PIN_FIELD(0, 181, 0x0100, 0x10, 0, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_smt_range[] = { + PIN_FIELD_BASE(0, 0, 8, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(1, 1, 8, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(2, 2, 8, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(3, 3, 8, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(4, 4, 8, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(5, 5, 8, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(6, 6, 8, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(7, 7, 8, 0x00c0, 0x10, 7, 1), + PIN_FIELD_BASE(8, 8, 2, 0x0120, 0x10, 13, 1), + PIN_FIELD_BASE(9, 9, 2, 0x0120, 0x10, 14, 1), + PIN_FIELD_BASE(10, 10, 2, 0x0120, 0x10, 15, 1), + PIN_FIELD_BASE(11, 11, 2, 0x0120, 0x10, 16, 1), + PIN_FIELD_BASE(12, 12, 1, 0x0140, 0x10, 17, 1), + PIN_FIELD_BASE(13, 13, 1, 0x0140, 0x10, 18, 1), + PIN_FIELD_BASE(14, 14, 1, 0x0140, 0x10, 14, 1), + PIN_FIELD_BASE(15, 15, 1, 0x0140, 0x10, 16, 1), + PIN_FIELD_BASE(16, 16, 1, 0x0140, 0x10, 19, 1), + PIN_FIELD_BASE(17, 17, 1, 0x0140, 0x10, 20, 1), + PIN_FIELD_BASE(18, 18, 1, 0x0140, 0x10, 21, 1), + PIN_FIELD_BASE(19, 19, 1, 0x0140, 0x10, 27, 1), + PIN_FIELD_BASE(20, 20, 1, 0x0140, 0x10, 28, 1), + PIN_FIELD_BASE(21, 21, 1, 0x0140, 0x10, 26, 1), + PIN_FIELD_BASE(22, 22, 1, 0x0140, 0x10, 25, 1), + PIN_FIELD_BASE(23, 23, 1, 0x0140, 0x10, 0, 1), + PIN_FIELD_BASE(24, 24, 1, 0x0140, 0x10, 1, 1), + PIN_FIELD_BASE(25, 25, 1, 0x0140, 0x10, 2, 1), + PIN_FIELD_BASE(26, 26, 1, 0x0140, 0x10, 3, 1), + PIN_FIELD_BASE(27, 27, 1, 0x0140, 0x10, 4, 1), + PIN_FIELD_BASE(28, 28, 1, 0x0140, 0x10, 5, 1), + PIN_FIELD_BASE(29, 29, 1, 0x0140, 0x10, 6, 1), + PIN_FIELD_BASE(30, 30, 1, 0x0140, 0x10, 7, 1), + PIN_FIELD_BASE(31, 31, 1, 0x0140, 0x10, 8, 1), + PIN_FIELD_BASE(32, 32, 9, 0x0100, 0x10, 0, 1), + PIN_FIELD_BASE(33, 33, 1, 0x0140, 0x10, 9, 1), + PIN_FIELD_BASE(34, 34, 1, 0x0140, 0x10, 10, 1), + PIN_FIELD_BASE(35, 35, 1, 0x0140, 0x10, 11, 1), + PIN_FIELD_BASE(36, 36, 9, 0x0100, 0x10, 8, 1), + PIN_FIELD_BASE(37, 37, 2, 0x0120, 0x10, 17, 1), + PIN_FIELD_BASE(38, 38, 2, 0x0120, 0x10, 18, 1), + PIN_FIELD_BASE(39, 39, 1, 0x0140, 0x10, 12, 1), + PIN_FIELD_BASE(40, 40, 2, 0x0120, 0x10, 1, 1), + PIN_FIELD_BASE(41, 41, 2, 0x0120, 0x10, 2, 1), + PIN_FIELD_BASE(42, 42, 2, 0x0120, 0x10, 3, 1), + PIN_FIELD_BASE(43, 43, 2, 0x0120, 0x10, 4, 1), + PIN_FIELD_BASE(44, 44, 2, 0x0120, 0x10, 5, 1), + PIN_FIELD_BASE(45, 45, 2, 0x0120, 0x10, 6, 1), + PIN_FIELD_BASE(46, 46, 1, 0x0140, 0x10, 13, 1), + PIN_FIELD_BASE(47, 47, 1, 0x0140, 0x10, 15, 1), + PIN_FIELD_BASE(48, 48, 2, 0x0120, 0x10, 7, 1), + PIN_FIELD_BASE(49, 49, 2, 0x0120, 0x10, 8, 1), + PIN_FIELD_BASE(50, 50, 2, 0x0120, 0x10, 9, 1), + PIN_FIELD_BASE(51, 51, 2, 0x0120, 0x10, 10, 1), + PIN_FIELD_BASE(52, 52, 2, 0x0120, 0x10, 11, 1), + PIN_FIELD_BASE(53, 53, 2, 0x0120, 0x10, 12, 1), + PIN_FIELD_BASE(54, 54, 5, 0x0120, 0x10, 10, 1), + PIN_FIELD_BASE(55, 55, 5, 0x0120, 0x10, 11, 1), + PIN_FIELD_BASE(56, 56, 1, 0x0140, 0x10, 22, 1), + PIN_FIELD_BASE(57, 57, 1, 0x0140, 0x10, 23, 1), + PIN_FIELD_BASE(58, 58, 1, 0x0140, 0x10, 24, 1), + PIN_FIELD_BASE(59, 59, 2, 0x0120, 0x10, 0, 1), + PIN_FIELD_BASE(60, 60, 9, 0x0100, 0x10, 1, 1), + PIN_FIELD_BASE(61, 61, 9, 0x0100, 0x10, 2, 1), + PIN_FIELD_BASE(62, 62, 9, 0x0100, 0x10, 3, 1), + PIN_FIELD_BASE(63, 63, 9, 0x0100, 0x10, 4, 1), + PIN_FIELD_BASE(64, 64, 9, 0x0100, 0x10, 5, 1), + PIN_FIELD_BASE(65, 65, 9, 0x0100, 0x10, 6, 1), + PIN_FIELD_BASE(66, 66, 5, 0x0120, 0x10, 0, 1), + PIN_FIELD_BASE(67, 67, 5, 0x0120, 0x10, 1, 1), + PIN_FIELD_BASE(68, 68, 9, 0x0100, 0x10, 7, 1), + PIN_FIELD_BASE(69, 69, 7, 0x0110, 0x10, 0, 1), + PIN_FIELD_BASE(70, 70, 7, 0x0110, 0x10, 1, 1), + PIN_FIELD_BASE(71, 71, 7, 0x0110, 0x10, 2, 1), + PIN_FIELD_BASE(72, 72, 7, 0x0110, 0x10, 3, 1), + PIN_FIELD_BASE(73, 73, 7, 0x0110, 0x10, 4, 1), + PIN_FIELD_BASE(74, 74, 7, 0x0110, 0x10, 5, 1), + PIN_FIELD_BASE(75, 75, 7, 0x0110, 0x10, 6, 1), + PIN_FIELD_BASE(76, 76, 7, 0x0110, 0x10, 7, 1), + PIN_FIELD_BASE(77, 77, 7, 0x0110, 0x10, 8, 1), + PIN_FIELD_BASE(78, 78, 7, 0x0110, 0x10, 9, 1), + PIN_FIELD_BASE(79, 79, 7, 0x0110, 0x10, 10, 1), + PIN_FIELD_BASE(80, 80, 7, 0x0110, 0x10, 11, 1), + PIN_FIELD_BASE(81, 81, 7, 0x0110, 0x10, 12, 1), + PIN_FIELD_BASE(82, 82, 7, 0x0110, 0x10, 13, 1), + PIN_FIELD_BASE(83, 83, 7, 0x0110, 0x10, 14, 1), + PIN_FIELD_BASE(84, 84, 7, 0x0110, 0x10, 15, 1), + PIN_FIELD_BASE(85, 85, 7, 0x0110, 0x10, 16, 1), + PIN_FIELD_BASE(86, 86, 7, 0x0110, 0x10, 17, 1), + PIN_FIELD_BASE(87, 87, 7, 0x0110, 0x10, 18, 1), + PIN_FIELD_BASE(88, 88, 7, 0x0110, 0x10, 19, 1), + PIN_FIELD_BASE(89, 89, 7, 0x0110, 0x10, 20, 1), + PIN_FIELD_BASE(90, 90, 7, 0x0110, 0x10, 21, 1), + PIN_FIELD_BASE(91, 91, 7, 0x0110, 0x10, 22, 1), + PIN_FIELD_BASE(92, 92, 3, 0x0130, 0x10, 16, 1), + PIN_FIELD_BASE(93, 93, 3, 0x0130, 0x10, 17, 1), + PIN_FIELD_BASE(94, 94, 3, 0x0130, 0x10, 18, 1), + PIN_FIELD_BASE(95, 95, 3, 0x0130, 0x10, 19, 1), + PIN_FIELD_BASE(96, 96, 3, 0x0130, 0x10, 20, 1), + PIN_FIELD_BASE(97, 97, 3, 0x0130, 0x10, 21, 1), + PIN_FIELD_BASE(98, 98, 3, 0x0130, 0x10, 22, 1), + PIN_FIELD_BASE(99, 99, 3, 0x0130, 0x10, 23, 1), + PIN_FIELD_BASE(100, 100, 3, 0x0130, 0x10, 24, 1), + PIN_FIELD_BASE(101, 101, 3, 0x0130, 0x10, 25, 1), + PIN_FIELD_BASE(102, 102, 3, 0x0130, 0x10, 26, 1), + PIN_FIELD_BASE(103, 103, 3, 0x0130, 0x10, 27, 1), + PIN_FIELD_BASE(104, 104, 3, 0x0130, 0x10, 28, 1), + PIN_FIELD_BASE(105, 105, 3, 0x0130, 0x10, 29, 1), + PIN_FIELD_BASE(106, 106, 8, 0x00c0, 0x10, 8, 1), + PIN_FIELD_BASE(107, 107, 8, 0x00c0, 0x10, 9, 1), + PIN_FIELD_BASE(108, 108, 8, 0x00c0, 0x10, 10, 1), + PIN_FIELD_BASE(109, 109, 8, 0x00c0, 0x10, 11, 1), + PIN_FIELD_BASE(110, 110, 8, 0x00c0, 0x10, 12, 1), + PIN_FIELD_BASE(111, 111, 8, 0x00c0, 0x10, 13, 1), + PIN_FIELD_BASE(112, 112, 5, 0x0120, 0x10, 15, 1), + PIN_FIELD_BASE(113, 113, 5, 0x0120, 0x10, 16, 1), + PIN_FIELD_BASE(114, 114, 5, 0x0120, 0x10, 17, 1), + PIN_FIELD_BASE(115, 115, 5, 0x0120, 0x10, 18, 1), + PIN_FIELD_BASE(116, 116, 5, 0x0120, 0x10, 19, 1), + PIN_FIELD_BASE(117, 117, 4, 0x0110, 0x10, 8, 1), + PIN_FIELD_BASE(118, 118, 4, 0x0110, 0x10, 9, 1), + PIN_FIELD_BASE(119, 119, 4, 0x0110, 0x10, 10, 1), + PIN_FIELD_BASE(120, 120, 4, 0x0110, 0x10, 11, 1), + PIN_FIELD_BASE(121, 121, 4, 0x0110, 0x10, 12, 1), + PIN_FIELD_BASE(122, 122, 4, 0x0110, 0x10, 13, 1), + PIN_FIELD_BASE(123, 123, 4, 0x0110, 0x10, 14, 1), + PIN_FIELD_BASE(124, 124, 4, 0x0110, 0x10, 15, 1), + PIN_FIELD_BASE(125, 125, 4, 0x0110, 0x10, 16, 1), + PIN_FIELD_BASE(126, 126, 5, 0x0120, 0x10, 6, 1), + PIN_FIELD_BASE(127, 127, 5, 0x0120, 0x10, 7, 1), + PIN_FIELD_BASE(128, 128, 5, 0x0120, 0x10, 8, 1), + PIN_FIELD_BASE(129, 129, 5, 0x0120, 0x10, 9, 1), + PIN_FIELD_BASE(130, 130, 4, 0x0110, 0x10, 17, 1), + PIN_FIELD_BASE(131, 131, 4, 0x0110, 0x10, 18, 1), + PIN_FIELD_BASE(132, 132, 4, 0x0110, 0x10, 19, 1), + PIN_FIELD_BASE(133, 133, 4, 0x0110, 0x10, 20, 1), + PIN_FIELD_BASE(134, 134, 3, 0x0130, 0x10, 0, 1), + PIN_FIELD_BASE(135, 135, 3, 0x0130, 0x10, 1, 1), + PIN_FIELD_BASE(136, 136, 3, 0x0130, 0x10, 2, 1), + PIN_FIELD_BASE(137, 137, 3, 0x0130, 0x10, 3, 1), + PIN_FIELD_BASE(138, 138, 3, 0x0130, 0x10, 4, 1), + PIN_FIELD_BASE(139, 139, 3, 0x0130, 0x10, 5, 1), + PIN_FIELD_BASE(140, 140, 3, 0x0130, 0x10, 6, 1), + PIN_FIELD_BASE(141, 141, 3, 0x0130, 0x10, 7, 1), + PIN_FIELD_BASE(142, 142, 3, 0x0130, 0x10, 8, 1), + PIN_FIELD_BASE(143, 143, 3, 0x0130, 0x10, 9, 1), + PIN_FIELD_BASE(144, 144, 3, 0x0130, 0x10, 10, 1), + PIN_FIELD_BASE(145, 145, 3, 0x0130, 0x10, 11, 1), + PIN_FIELD_BASE(146, 146, 3, 0x0130, 0x10, 12, 1), + PIN_FIELD_BASE(147, 147, 3, 0x0130, 0x10, 13, 1), + PIN_FIELD_BASE(148, 148, 3, 0x0130, 0x10, 14, 1), + PIN_FIELD_BASE(149, 149, 3, 0x0130, 0x10, 15, 1), + PIN_FIELD_BASE(150, 150, 4, 0x0110, 0x10, 21, 1), + PIN_FIELD_BASE(151, 151, 4, 0x0110, 0x10, 26, 1), + PIN_FIELD_BASE(152, 152, 4, 0x0110, 0x10, 25, 1), + PIN_FIELD_BASE(153, 153, 4, 0x0110, 0x10, 24, 1), + PIN_FIELD_BASE(154, 154, 4, 0x0110, 0x10, 22, 1), + PIN_FIELD_BASE(155, 155, 4, 0x0110, 0x10, 23, 1), + PIN_FIELD_BASE(156, 156, 4, 0x0110, 0x10, 0, 1), + PIN_FIELD_BASE(157, 157, 4, 0x0110, 0x10, 1, 1), + PIN_FIELD_BASE(158, 158, 4, 0x0110, 0x10, 2, 1), + PIN_FIELD_BASE(159, 159, 4, 0x0110, 0x10, 3, 1), + PIN_FIELD_BASE(160, 160, 4, 0x0110, 0x10, 4, 1), + PIN_FIELD_BASE(161, 161, 4, 0x0110, 0x10, 5, 1), + PIN_FIELD_BASE(162, 162, 4, 0x0110, 0x10, 6, 1), + PIN_FIELD_BASE(163, 163, 4, 0x0110, 0x10, 7, 1), + PIN_FIELD_BASE(164, 164, 5, 0x0120, 0x10, 12, 1), + PIN_FIELD_BASE(165, 165, 5, 0x0120, 0x10, 3, 1), + PIN_FIELD_BASE(166, 166, 5, 0x0120, 0x10, 4, 1), + PIN_FIELD_BASE(167, 167, 5, 0x0120, 0x10, 2, 1), + PIN_FIELD_BASE(168, 168, 5, 0x0120, 0x10, 13, 1), + PIN_FIELD_BASE(169, 169, 5, 0x0120, 0x10, 14, 1), + PIN_FIELD_BASE(170, 170, 5, 0x0120, 0x10, 5, 1), + PIN_FIELD_BASE(171, 171, 6, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(172, 172, 6, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(173, 173, 6, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(174, 174, 6, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(175, 175, 6, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(176, 176, 6, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(177, 177, 6, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(178, 178, 6, 0x00c0, 0x10, 7, 1), + PIN_FIELD_BASE(179, 179, 6, 0x00c0, 0x10, 8, 1), + PIN_FIELD_BASE(180, 180, 6, 0x00c0, 0x10, 9, 1), + PIN_FIELD_BASE(181, 181, 10, 0x0080, 0x10, 0, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_ies_range[] = { + PIN_FIELD_BASE(0, 0, 8, 0x0050, 0x10, 0, 1), + PIN_FIELD_BASE(1, 1, 8, 0x0050, 0x10, 1, 1), + PIN_FIELD_BASE(2, 2, 8, 0x0050, 0x10, 2, 1), + PIN_FIELD_BASE(3, 3, 8, 0x0050, 0x10, 3, 1), + PIN_FIELD_BASE(4, 4, 8, 0x0050, 0x10, 4, 1), + PIN_FIELD_BASE(5, 5, 8, 0x0050, 0x10, 5, 1), + PIN_FIELD_BASE(6, 6, 8, 0x0050, 0x10, 6, 1), + PIN_FIELD_BASE(7, 7, 8, 0x0050, 0x10, 7, 1), + PIN_FIELD_BASE(8, 8, 2, 0x0070, 0x10, 13, 1), + PIN_FIELD_BASE(9, 9, 2, 0x0070, 0x10, 14, 1), + PIN_FIELD_BASE(10, 10, 2, 0x0070, 0x10, 15, 1), + PIN_FIELD_BASE(11, 11, 2, 0x0070, 0x10, 16, 1), + PIN_FIELD_BASE(12, 12, 1, 0x0080, 0x10, 17, 1), + PIN_FIELD_BASE(13, 13, 1, 0x0080, 0x10, 18, 1), + PIN_FIELD_BASE(14, 14, 1, 0x0080, 0x10, 14, 1), + PIN_FIELD_BASE(15, 15, 1, 0x0080, 0x10, 16, 1), + PIN_FIELD_BASE(16, 16, 1, 0x0080, 0x10, 19, 1), + PIN_FIELD_BASE(17, 17, 1, 0x0080, 0x10, 20, 1), + PIN_FIELD_BASE(18, 18, 1, 0x0080, 0x10, 21, 1), + PIN_FIELD_BASE(19, 19, 1, 0x0080, 0x10, 27, 1), + PIN_FIELD_BASE(20, 20, 1, 0x0080, 0x10, 28, 1), + PIN_FIELD_BASE(21, 21, 1, 0x0080, 0x10, 26, 1), + PIN_FIELD_BASE(22, 22, 1, 0x0080, 0x10, 25, 1), + PIN_FIELD_BASE(23, 23, 1, 0x0080, 0x10, 0, 1), + PIN_FIELD_BASE(24, 24, 1, 0x0080, 0x10, 1, 1), + PIN_FIELD_BASE(25, 25, 1, 0x0080, 0x10, 2, 1), + PIN_FIELD_BASE(26, 26, 1, 0x0080, 0x10, 3, 1), + PIN_FIELD_BASE(27, 27, 1, 0x0080, 0x10, 4, 1), + PIN_FIELD_BASE(28, 28, 1, 0x0080, 0x10, 5, 1), + PIN_FIELD_BASE(29, 29, 1, 0x0080, 0x10, 6, 1), + PIN_FIELD_BASE(30, 30, 1, 0x0080, 0x10, 7, 1), + PIN_FIELD_BASE(31, 31, 1, 0x0080, 0x10, 8, 1), + PIN_FIELD_BASE(32, 32, 9, 0x0060, 0x10, 0, 1), + PIN_FIELD_BASE(33, 33, 1, 0x0080, 0x10, 9, 1), + PIN_FIELD_BASE(34, 34, 1, 0x0080, 0x10, 10, 1), + PIN_FIELD_BASE(35, 35, 1, 0x0080, 0x10, 11, 1), + PIN_FIELD_BASE(36, 36, 9, 0x0060, 0x10, 8, 1), + PIN_FIELD_BASE(37, 37, 2, 0x0070, 0x10, 17, 1), + PIN_FIELD_BASE(38, 38, 2, 0x0070, 0x10, 18, 1), + PIN_FIELD_BASE(39, 39, 1, 0x0080, 0x10, 12, 1), + PIN_FIELD_BASE(40, 40, 2, 0x0070, 0x10, 1, 1), + PIN_FIELD_BASE(41, 41, 2, 0x0070, 0x10, 2, 1), + PIN_FIELD_BASE(42, 42, 2, 0x0070, 0x10, 3, 1), + PIN_FIELD_BASE(43, 43, 2, 0x0070, 0x10, 4, 1), + PIN_FIELD_BASE(44, 44, 2, 0x0070, 0x10, 5, 1), + PIN_FIELD_BASE(45, 45, 2, 0x0070, 0x10, 6, 1), + PIN_FIELD_BASE(46, 46, 1, 0x0080, 0x10, 13, 1), + PIN_FIELD_BASE(47, 47, 1, 0x0080, 0x10, 15, 1), + PIN_FIELD_BASE(48, 48, 2, 0x0070, 0x10, 7, 1), + PIN_FIELD_BASE(49, 49, 2, 0x0070, 0x10, 8, 1), + PIN_FIELD_BASE(50, 50, 2, 0x0070, 0x10, 9, 1), + PIN_FIELD_BASE(51, 51, 2, 0x0070, 0x10, 10, 1), + PIN_FIELD_BASE(52, 52, 2, 0x0070, 0x10, 11, 1), + PIN_FIELD_BASE(53, 53, 2, 0x0070, 0x10, 12, 1), + PIN_FIELD_BASE(54, 54, 5, 0x0060, 0x10, 10, 1), + PIN_FIELD_BASE(55, 55, 5, 0x0060, 0x10, 11, 1), + PIN_FIELD_BASE(56, 56, 1, 0x0080, 0x10, 22, 1), + PIN_FIELD_BASE(57, 57, 1, 0x0080, 0x10, 23, 1), + PIN_FIELD_BASE(58, 58, 1, 0x0080, 0x10, 24, 1), + PIN_FIELD_BASE(59, 59, 2, 0x0070, 0x10, 0, 1), + PIN_FIELD_BASE(60, 60, 9, 0x0060, 0x10, 1, 1), + PIN_FIELD_BASE(61, 61, 9, 0x0060, 0x10, 2, 1), + PIN_FIELD_BASE(62, 62, 9, 0x0060, 0x10, 3, 1), + PIN_FIELD_BASE(63, 63, 9, 0x0060, 0x10, 4, 1), + PIN_FIELD_BASE(64, 64, 9, 0x0060, 0x10, 5, 1), + PIN_FIELD_BASE(65, 65, 9, 0x0060, 0x10, 6, 1), + PIN_FIELD_BASE(66, 66, 5, 0x0060, 0x10, 0, 1), + PIN_FIELD_BASE(67, 67, 5, 0x0060, 0x10, 1, 1), + PIN_FIELD_BASE(68, 68, 9, 0x0060, 0x10, 7, 1), + PIN_FIELD_BASE(69, 69, 7, 0x0070, 0x10, 0, 1), + PIN_FIELD_BASE(70, 70, 7, 0x0070, 0x10, 1, 1), + PIN_FIELD_BASE(71, 71, 7, 0x0070, 0x10, 2, 1), + PIN_FIELD_BASE(72, 72, 7, 0x0070, 0x10, 3, 1), + PIN_FIELD_BASE(73, 73, 7, 0x0070, 0x10, 4, 1), + PIN_FIELD_BASE(74, 74, 7, 0x0070, 0x10, 5, 1), + PIN_FIELD_BASE(75, 75, 7, 0x0070, 0x10, 6, 1), + PIN_FIELD_BASE(76, 76, 7, 0x0070, 0x10, 7, 1), + PIN_FIELD_BASE(77, 77, 7, 0x0070, 0x10, 8, 1), + PIN_FIELD_BASE(78, 78, 7, 0x0070, 0x10, 9, 1), + PIN_FIELD_BASE(79, 79, 7, 0x0070, 0x10, 10, 1), + PIN_FIELD_BASE(80, 80, 7, 0x0070, 0x10, 11, 1), + PIN_FIELD_BASE(81, 81, 7, 0x0070, 0x10, 12, 1), + PIN_FIELD_BASE(82, 82, 7, 0x0070, 0x10, 13, 1), + PIN_FIELD_BASE(83, 83, 7, 0x0070, 0x10, 14, 1), + PIN_FIELD_BASE(84, 84, 7, 0x0070, 0x10, 15, 1), + PIN_FIELD_BASE(85, 85, 7, 0x0070, 0x10, 16, 1), + PIN_FIELD_BASE(86, 86, 7, 0x0070, 0x10, 17, 1), + PIN_FIELD_BASE(87, 87, 7, 0x0070, 0x10, 18, 1), + PIN_FIELD_BASE(88, 88, 7, 0x0070, 0x10, 19, 1), + PIN_FIELD_BASE(89, 89, 7, 0x0070, 0x10, 20, 1), + PIN_FIELD_BASE(90, 90, 7, 0x0070, 0x10, 21, 1), + PIN_FIELD_BASE(91, 91, 7, 0x0070, 0x10, 22, 1), + PIN_FIELD_BASE(92, 92, 3, 0x0080, 0x10, 16, 1), + PIN_FIELD_BASE(93, 93, 3, 0x0080, 0x10, 17, 1), + PIN_FIELD_BASE(94, 94, 3, 0x0080, 0x10, 18, 1), + PIN_FIELD_BASE(95, 95, 3, 0x0080, 0x10, 19, 1), + PIN_FIELD_BASE(96, 96, 3, 0x0080, 0x10, 20, 1), + PIN_FIELD_BASE(97, 97, 3, 0x0080, 0x10, 21, 1), + PIN_FIELD_BASE(98, 98, 3, 0x0080, 0x10, 22, 1), + PIN_FIELD_BASE(99, 99, 3, 0x0080, 0x10, 23, 1), + PIN_FIELD_BASE(100, 100, 3, 0x0080, 0x10, 24, 1), + PIN_FIELD_BASE(101, 101, 3, 0x0080, 0x10, 25, 1), + PIN_FIELD_BASE(102, 102, 3, 0x0080, 0x10, 26, 1), + PIN_FIELD_BASE(103, 103, 3, 0x0080, 0x10, 27, 1), + PIN_FIELD_BASE(104, 104, 3, 0x0080, 0x10, 28, 1), + PIN_FIELD_BASE(105, 105, 3, 0x0080, 0x10, 29, 1), + PIN_FIELD_BASE(106, 106, 8, 0x0050, 0x10, 8, 1), + PIN_FIELD_BASE(107, 107, 8, 0x0050, 0x10, 9, 1), + PIN_FIELD_BASE(108, 108, 8, 0x0050, 0x10, 10, 1), + PIN_FIELD_BASE(109, 109, 8, 0x0050, 0x10, 11, 1), + PIN_FIELD_BASE(110, 110, 8, 0x0050, 0x10, 12, 1), + PIN_FIELD_BASE(111, 111, 8, 0x0050, 0x10, 13, 1), + PIN_FIELD_BASE(112, 112, 5, 0x0060, 0x10, 15, 1), + PIN_FIELD_BASE(113, 113, 5, 0x0060, 0x10, 16, 1), + PIN_FIELD_BASE(114, 114, 5, 0x0060, 0x10, 17, 1), + PIN_FIELD_BASE(115, 115, 5, 0x0060, 0x10, 18, 1), + PIN_FIELD_BASE(116, 116, 5, 0x0060, 0x10, 19, 1), + PIN_FIELD_BASE(117, 117, 4, 0x0060, 0x10, 8, 1), + PIN_FIELD_BASE(118, 118, 4, 0x0060, 0x10, 9, 1), + PIN_FIELD_BASE(119, 119, 4, 0x0060, 0x10, 10, 1), + PIN_FIELD_BASE(120, 120, 4, 0x0060, 0x10, 11, 1), + PIN_FIELD_BASE(121, 121, 4, 0x0060, 0x10, 12, 1), + PIN_FIELD_BASE(122, 122, 4, 0x0060, 0x10, 13, 1), + PIN_FIELD_BASE(123, 123, 4, 0x0060, 0x10, 14, 1), + PIN_FIELD_BASE(124, 124, 4, 0x0060, 0x10, 15, 1), + PIN_FIELD_BASE(125, 125, 4, 0x0060, 0x10, 16, 1), + PIN_FIELD_BASE(126, 126, 5, 0x0060, 0x10, 6, 1), + PIN_FIELD_BASE(127, 127, 5, 0x0060, 0x10, 7, 1), + PIN_FIELD_BASE(128, 128, 5, 0x0060, 0x10, 8, 1), + PIN_FIELD_BASE(129, 129, 5, 0x0060, 0x10, 9, 1), + PIN_FIELD_BASE(130, 130, 4, 0x0060, 0x10, 17, 1), + PIN_FIELD_BASE(131, 131, 4, 0x0060, 0x10, 18, 1), + PIN_FIELD_BASE(132, 132, 4, 0x0060, 0x10, 19, 1), + PIN_FIELD_BASE(133, 133, 4, 0x0060, 0x10, 20, 1), + PIN_FIELD_BASE(134, 134, 3, 0x0080, 0x10, 0, 1), + PIN_FIELD_BASE(135, 135, 3, 0x0080, 0x10, 1, 1), + PIN_FIELD_BASE(136, 136, 3, 0x0080, 0x10, 2, 1), + PIN_FIELD_BASE(137, 137, 3, 0x0080, 0x10, 3, 1), + PIN_FIELD_BASE(138, 138, 3, 0x0080, 0x10, 4, 1), + PIN_FIELD_BASE(139, 139, 3, 0x0080, 0x10, 5, 1), + PIN_FIELD_BASE(140, 140, 3, 0x0080, 0x10, 6, 1), + PIN_FIELD_BASE(141, 141, 3, 0x0080, 0x10, 7, 1), + PIN_FIELD_BASE(142, 142, 3, 0x0080, 0x10, 8, 1), + PIN_FIELD_BASE(143, 143, 3, 0x0080, 0x10, 9, 1), + PIN_FIELD_BASE(144, 144, 3, 0x0080, 0x10, 10, 1), + PIN_FIELD_BASE(145, 145, 3, 0x0080, 0x10, 11, 1), + PIN_FIELD_BASE(146, 146, 3, 0x0080, 0x10, 12, 1), + PIN_FIELD_BASE(147, 147, 3, 0x0080, 0x10, 13, 1), + PIN_FIELD_BASE(148, 148, 3, 0x0080, 0x10, 14, 1), + PIN_FIELD_BASE(149, 149, 3, 0x0080, 0x10, 15, 1), + PIN_FIELD_BASE(150, 150, 4, 0x0060, 0x10, 21, 1), + PIN_FIELD_BASE(151, 151, 4, 0x0060, 0x10, 26, 1), + PIN_FIELD_BASE(152, 152, 4, 0x0060, 0x10, 25, 1), + PIN_FIELD_BASE(153, 153, 4, 0x0060, 0x10, 24, 1), + PIN_FIELD_BASE(154, 154, 4, 0x0060, 0x10, 22, 1), + PIN_FIELD_BASE(155, 155, 4, 0x0060, 0x10, 23, 1), + PIN_FIELD_BASE(156, 156, 4, 0x0060, 0x10, 0, 1), + PIN_FIELD_BASE(157, 157, 4, 0x0060, 0x10, 1, 1), + PIN_FIELD_BASE(158, 158, 4, 0x0060, 0x10, 2, 1), + PIN_FIELD_BASE(159, 159, 4, 0x0060, 0x10, 3, 1), + PIN_FIELD_BASE(160, 160, 4, 0x0060, 0x10, 4, 1), + PIN_FIELD_BASE(161, 161, 4, 0x0060, 0x10, 5, 1), + PIN_FIELD_BASE(162, 162, 4, 0x0060, 0x10, 6, 1), + PIN_FIELD_BASE(163, 163, 4, 0x0060, 0x10, 7, 1), + PIN_FIELD_BASE(164, 164, 5, 0x0060, 0x10, 12, 1), + PIN_FIELD_BASE(165, 165, 5, 0x0060, 0x10, 3, 1), + PIN_FIELD_BASE(166, 166, 5, 0x0060, 0x10, 4, 1), + PIN_FIELD_BASE(167, 167, 5, 0x0060, 0x10, 2, 1), + PIN_FIELD_BASE(168, 168, 5, 0x0060, 0x10, 13, 1), + PIN_FIELD_BASE(169, 169, 5, 0x0060, 0x10, 14, 1), + PIN_FIELD_BASE(170, 170, 5, 0x0060, 0x10, 5, 1), + PIN_FIELD_BASE(171, 171, 6, 0x0050, 0x10, 0, 1), + PIN_FIELD_BASE(172, 172, 6, 0x0050, 0x10, 1, 1), + PIN_FIELD_BASE(173, 173, 6, 0x0050, 0x10, 2, 1), + PIN_FIELD_BASE(174, 174, 6, 0x0050, 0x10, 3, 1), + PIN_FIELD_BASE(175, 175, 6, 0x0050, 0x10, 4, 1), + PIN_FIELD_BASE(176, 176, 6, 0x0050, 0x10, 5, 1), + PIN_FIELD_BASE(177, 177, 6, 0x0050, 0x10, 6, 1), + PIN_FIELD_BASE(178, 178, 6, 0x0050, 0x10, 7, 1), + PIN_FIELD_BASE(179, 179, 6, 0x0050, 0x10, 8, 1), + PIN_FIELD_BASE(180, 180, 6, 0x0050, 0x10, 9, 1), + PIN_FIELD_BASE(181, 181, 10, 0x0020, 0x10, 0, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_pupd_range[] = { + PIN_FIELD_BASE(0, 0, 8, 0x0080, 0x10, 0, 1), + PIN_FIELD_BASE(1, 1, 8, 0x0080, 0x10, 1, 1), + PIN_FIELD_BASE(2, 2, 8, 0x0080, 0x10, 2, 1), + PIN_FIELD_BASE(3, 3, 8, 0x0080, 0x10, 3, 1), + PIN_FIELD_BASE(4, 4, 8, 0x0080, 0x10, 4, 1), + PIN_FIELD_BASE(5, 5, 8, 0x0080, 0x10, 5, 1), + PIN_FIELD_BASE(6, 6, 8, 0x0080, 0x10, 6, 1), + PIN_FIELD_BASE(7, 7, 8, 0x0080, 0x10, 7, 1), + PIN_FIELD_BASE(14, 14, 1, 0x00c0, 0x10, 14, 1), + PIN_FIELD_BASE(15, 15, 1, 0x00c0, 0x10, 16, 1), + PIN_FIELD_BASE(16, 16, 1, 0x00c0, 0x10, 17, 1), + PIN_FIELD_BASE(19, 19, 1, 0x00c0, 0x10, 21, 1), + PIN_FIELD_BASE(20, 20, 1, 0x00c0, 0x10, 22, 1), + PIN_FIELD_BASE(21, 21, 1, 0x00c0, 0x10, 20, 1), + PIN_FIELD_BASE(22, 22, 1, 0x00c0, 0x10, 19, 1), + PIN_FIELD_BASE(23, 23, 1, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(24, 24, 1, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(25, 25, 1, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(26, 26, 1, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(27, 27, 1, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(28, 28, 1, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(29, 29, 1, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(30, 30, 1, 0x00c0, 0x10, 7, 1), + PIN_FIELD_BASE(31, 31, 1, 0x00c0, 0x10, 8, 1), + PIN_FIELD_BASE(32, 32, 9, 0x00a0, 0x10, 0, 1), + PIN_FIELD_BASE(33, 33, 1, 0x00c0, 0x10, 9, 1), + PIN_FIELD_BASE(34, 34, 1, 0x00c0, 0x10, 10, 1), + PIN_FIELD_BASE(35, 35, 1, 0x00c0, 0x10, 11, 1), + PIN_FIELD_BASE(36, 36, 9, 0x00a0, 0x10, 6, 1), + PIN_FIELD_BASE(37, 37, 2, 0x00b0, 0x10, 10, 1), + PIN_FIELD_BASE(38, 38, 2, 0x00b0, 0x10, 11, 1), + PIN_FIELD_BASE(39, 39, 1, 0x00c0, 0x10, 12, 1), + PIN_FIELD_BASE(40, 40, 2, 0x00b0, 0x10, 0, 1), + PIN_FIELD_BASE(41, 41, 2, 0x00b0, 0x10, 1, 1), + PIN_FIELD_BASE(42, 42, 2, 0x00b0, 0x10, 2, 1), + PIN_FIELD_BASE(43, 43, 2, 0x00b0, 0x10, 3, 1), + PIN_FIELD_BASE(44, 44, 2, 0x00b0, 0x10, 4, 1), + PIN_FIELD_BASE(45, 45, 2, 0x00b0, 0x10, 5, 1), + PIN_FIELD_BASE(46, 46, 1, 0x00c0, 0x10, 13, 1), + PIN_FIELD_BASE(47, 47, 1, 0x00c0, 0x10, 15, 1), + PIN_FIELD_BASE(48, 48, 2, 0x00b0, 0x10, 6, 1), + PIN_FIELD_BASE(49, 49, 2, 0x00b0, 0x10, 7, 1), + PIN_FIELD_BASE(50, 50, 2, 0x00b0, 0x10, 8, 1), + PIN_FIELD_BASE(51, 51, 2, 0x00b0, 0x10, 9, 1), + PIN_FIELD_BASE(58, 58, 1, 0x00c0, 0x10, 18, 1), + PIN_FIELD_BASE(62, 62, 9, 0x00a0, 0x10, 1, 1), + PIN_FIELD_BASE(63, 63, 9, 0x00a0, 0x10, 2, 1), + PIN_FIELD_BASE(64, 64, 9, 0x00a0, 0x10, 3, 1), + PIN_FIELD_BASE(65, 65, 9, 0x00a0, 0x10, 4, 1), + PIN_FIELD_BASE(68, 68, 9, 0x00a0, 0x10, 5, 1), + PIN_FIELD_BASE(74, 74, 7, 0x00b0, 0x10, 0, 1), + PIN_FIELD_BASE(75, 75, 7, 0x00b0, 0x10, 1, 1), + PIN_FIELD_BASE(76, 76, 7, 0x00b0, 0x10, 2, 1), + PIN_FIELD_BASE(77, 77, 7, 0x00b0, 0x10, 3, 1), + PIN_FIELD_BASE(78, 78, 7, 0x00b0, 0x10, 4, 1), + PIN_FIELD_BASE(79, 79, 7, 0x00b0, 0x10, 5, 1), + PIN_FIELD_BASE(80, 80, 7, 0x00b0, 0x10, 6, 1), + PIN_FIELD_BASE(81, 81, 7, 0x00b0, 0x10, 7, 1), + PIN_FIELD_BASE(82, 82, 7, 0x00b0, 0x10, 8, 1), + PIN_FIELD_BASE(83, 83, 7, 0x00b0, 0x10, 9, 1), + PIN_FIELD_BASE(84, 84, 7, 0x00b0, 0x10, 10, 1), + PIN_FIELD_BASE(85, 85, 7, 0x00b0, 0x10, 11, 1), + PIN_FIELD_BASE(86, 86, 7, 0x00b0, 0x10, 12, 1), + PIN_FIELD_BASE(87, 87, 7, 0x00b0, 0x10, 13, 1), + PIN_FIELD_BASE(90, 90, 7, 0x00b0, 0x10, 14, 1), + PIN_FIELD_BASE(91, 91, 7, 0x00b0, 0x10, 15, 1), + PIN_FIELD_BASE(94, 94, 3, 0x00c0, 0x10, 12, 1), + PIN_FIELD_BASE(95, 95, 3, 0x00c0, 0x10, 13, 1), + PIN_FIELD_BASE(96, 96, 3, 0x00c0, 0x10, 14, 1), + PIN_FIELD_BASE(97, 97, 3, 0x00c0, 0x10, 15, 1), + PIN_FIELD_BASE(98, 98, 3, 0x00c0, 0x10, 16, 1), + PIN_FIELD_BASE(99, 99, 3, 0x00c0, 0x10, 17, 1), + PIN_FIELD_BASE(100, 100, 3, 0x00c0, 0x10, 18, 1), + PIN_FIELD_BASE(101, 101, 3, 0x00c0, 0x10, 19, 1), + PIN_FIELD_BASE(102, 102, 3, 0x00c0, 0x10, 20, 1), + PIN_FIELD_BASE(103, 103, 3, 0x00c0, 0x10, 21, 1), + PIN_FIELD_BASE(104, 104, 3, 0x00c0, 0x10, 22, 1), + PIN_FIELD_BASE(105, 105, 3, 0x00c0, 0x10, 23, 1), + PIN_FIELD_BASE(106, 106, 8, 0x0080, 0x10, 8, 1), + PIN_FIELD_BASE(107, 107, 8, 0x0080, 0x10, 9, 1), + PIN_FIELD_BASE(108, 108, 8, 0x0080, 0x10, 10, 1), + PIN_FIELD_BASE(109, 109, 8, 0x0080, 0x10, 11, 1), + PIN_FIELD_BASE(110, 110, 8, 0x0080, 0x10, 12, 1), + PIN_FIELD_BASE(111, 111, 8, 0x0080, 0x10, 13, 1), + PIN_FIELD_BASE(112, 112, 5, 0x00a0, 0x10, 5, 1), + PIN_FIELD_BASE(113, 113, 5, 0x00a0, 0x10, 6, 1), + PIN_FIELD_BASE(114, 114, 5, 0x00a0, 0x10, 7, 1), + PIN_FIELD_BASE(115, 115, 5, 0x00a0, 0x10, 8, 1), + PIN_FIELD_BASE(116, 116, 5, 0x00a0, 0x10, 9, 1), + PIN_FIELD_BASE(125, 125, 4, 0x00a0, 0x10, 8, 1), + PIN_FIELD_BASE(130, 130, 4, 0x00a0, 0x10, 9, 1), + PIN_FIELD_BASE(131, 131, 4, 0x00a0, 0x10, 10, 1), + PIN_FIELD_BASE(132, 132, 4, 0x00a0, 0x10, 11, 1), + PIN_FIELD_BASE(133, 133, 4, 0x00a0, 0x10, 12, 1), + PIN_FIELD_BASE(138, 138, 3, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(139, 139, 3, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(140, 140, 3, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(141, 141, 3, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(142, 142, 3, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(143, 143, 3, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(144, 144, 3, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(145, 145, 3, 0x00c0, 0x10, 7, 1), + PIN_FIELD_BASE(146, 146, 3, 0x00c0, 0x10, 8, 1), + PIN_FIELD_BASE(147, 147, 3, 0x00c0, 0x10, 9, 1), + PIN_FIELD_BASE(148, 148, 3, 0x00c0, 0x10, 10, 1), + PIN_FIELD_BASE(149, 149, 3, 0x00c0, 0x10, 11, 1), + PIN_FIELD_BASE(150, 150, 4, 0x00a0, 0x10, 13, 1), + PIN_FIELD_BASE(151, 151, 4, 0x00a0, 0x10, 18, 1), + PIN_FIELD_BASE(152, 152, 4, 0x00a0, 0x10, 17, 1), + PIN_FIELD_BASE(153, 153, 4, 0x00a0, 0x10, 16, 1), + PIN_FIELD_BASE(154, 154, 4, 0x00a0, 0x10, 14, 1), + PIN_FIELD_BASE(155, 155, 4, 0x00a0, 0x10, 15, 1), + PIN_FIELD_BASE(156, 156, 4, 0x00a0, 0x10, 0, 1), + PIN_FIELD_BASE(157, 157, 4, 0x00a0, 0x10, 1, 1), + PIN_FIELD_BASE(158, 158, 4, 0x00a0, 0x10, 2, 1), + PIN_FIELD_BASE(159, 159, 4, 0x00a0, 0x10, 3, 1), + PIN_FIELD_BASE(160, 160, 4, 0x00a0, 0x10, 4, 1), + PIN_FIELD_BASE(161, 161, 4, 0x00a0, 0x10, 5, 1), + PIN_FIELD_BASE(162, 162, 4, 0x00a0, 0x10, 6, 1), + PIN_FIELD_BASE(163, 163, 4, 0x00a0, 0x10, 7, 1), + PIN_FIELD_BASE(164, 164, 5, 0x00a0, 0x10, 2, 1), + PIN_FIELD_BASE(167, 167, 5, 0x00a0, 0x10, 0, 1), + PIN_FIELD_BASE(168, 168, 5, 0x00a0, 0x10, 3, 1), + PIN_FIELD_BASE(169, 169, 5, 0x00a0, 0x10, 4, 1), + PIN_FIELD_BASE(170, 170, 5, 0x00a0, 0x10, 1, 1), + PIN_FIELD_BASE(171, 171, 6, 0x0080, 0x10, 0, 1), + PIN_FIELD_BASE(172, 172, 6, 0x0080, 0x10, 1, 1), + PIN_FIELD_BASE(173, 173, 6, 0x0080, 0x10, 2, 1), + PIN_FIELD_BASE(174, 174, 6, 0x0080, 0x10, 3, 1), + PIN_FIELD_BASE(175, 175, 6, 0x0080, 0x10, 4, 1), + PIN_FIELD_BASE(176, 176, 6, 0x0080, 0x10, 5, 1), + PIN_FIELD_BASE(177, 177, 6, 0x0080, 0x10, 6, 1), + PIN_FIELD_BASE(178, 178, 6, 0x0080, 0x10, 7, 1), + PIN_FIELD_BASE(179, 179, 6, 0x0080, 0x10, 8, 1), + PIN_FIELD_BASE(180, 180, 6, 0x0080, 0x10, 9, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_r0_range[] = { + PIN_FIELD_BASE(0, 0, 8, 0x0090, 0x10, 0, 1), + PIN_FIELD_BASE(1, 1, 8, 0x0090, 0x10, 1, 1), + PIN_FIELD_BASE(2, 2, 8, 0x0090, 0x10, 2, 1), + PIN_FIELD_BASE(3, 3, 8, 0x0090, 0x10, 3, 1), + PIN_FIELD_BASE(4, 4, 8, 0x0090, 0x10, 4, 1), + PIN_FIELD_BASE(5, 5, 8, 0x0090, 0x10, 5, 1), + PIN_FIELD_BASE(6, 6, 8, 0x0090, 0x10, 6, 1), + PIN_FIELD_BASE(7, 7, 8, 0x0090, 0x10, 7, 1), + PIN_FIELD_BASE(14, 14, 1, 0x00e0, 0x10, 14, 1), + PIN_FIELD_BASE(15, 15, 1, 0x00e0, 0x10, 16, 1), + PIN_FIELD_BASE(16, 16, 1, 0x00e0, 0x10, 17, 1), + PIN_FIELD_BASE(19, 19, 1, 0x00e0, 0x10, 21, 1), + PIN_FIELD_BASE(20, 20, 1, 0x00e0, 0x10, 22, 1), + PIN_FIELD_BASE(21, 21, 1, 0x00e0, 0x10, 20, 1), + PIN_FIELD_BASE(22, 22, 1, 0x00e0, 0x10, 19, 1), + PIN_FIELD_BASE(23, 23, 1, 0x00e0, 0x10, 0, 1), + PIN_FIELD_BASE(24, 24, 1, 0x00e0, 0x10, 1, 1), + PIN_FIELD_BASE(25, 25, 1, 0x00e0, 0x10, 2, 1), + PIN_FIELD_BASE(26, 26, 1, 0x00e0, 0x10, 3, 1), + PIN_FIELD_BASE(27, 27, 1, 0x00e0, 0x10, 4, 1), + PIN_FIELD_BASE(28, 28, 1, 0x00e0, 0x10, 5, 1), + PIN_FIELD_BASE(29, 29, 1, 0x00e0, 0x10, 6, 1), + PIN_FIELD_BASE(30, 30, 1, 0x00e0, 0x10, 7, 1), + PIN_FIELD_BASE(31, 31, 1, 0x00e0, 0x10, 8, 1), + PIN_FIELD_BASE(32, 32, 9, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(33, 33, 1, 0x00e0, 0x10, 9, 1), + PIN_FIELD_BASE(34, 34, 1, 0x00e0, 0x10, 10, 1), + PIN_FIELD_BASE(35, 35, 1, 0x00e0, 0x10, 11, 1), + PIN_FIELD_BASE(36, 36, 9, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(37, 37, 2, 0x00d0, 0x10, 10, 1), + PIN_FIELD_BASE(38, 38, 2, 0x00d0, 0x10, 11, 1), + PIN_FIELD_BASE(39, 39, 1, 0x00e0, 0x10, 12, 1), + PIN_FIELD_BASE(40, 40, 2, 0x00d0, 0x10, 0, 1), + PIN_FIELD_BASE(41, 41, 2, 0x00d0, 0x10, 1, 1), + PIN_FIELD_BASE(42, 42, 2, 0x00d0, 0x10, 2, 1), + PIN_FIELD_BASE(43, 43, 2, 0x00d0, 0x10, 3, 1), + PIN_FIELD_BASE(44, 44, 2, 0x00d0, 0x10, 4, 1), + PIN_FIELD_BASE(45, 45, 2, 0x00d0, 0x10, 5, 1), + PIN_FIELD_BASE(46, 46, 1, 0x00e0, 0x10, 13, 1), + PIN_FIELD_BASE(47, 47, 1, 0x00e0, 0x10, 15, 1), + PIN_FIELD_BASE(48, 48, 2, 0x00d0, 0x10, 6, 1), + PIN_FIELD_BASE(49, 49, 2, 0x00d0, 0x10, 7, 1), + PIN_FIELD_BASE(50, 50, 2, 0x00d0, 0x10, 8, 1), + PIN_FIELD_BASE(51, 51, 2, 0x00d0, 0x10, 9, 1), + PIN_FIELD_BASE(58, 58, 1, 0x00e0, 0x10, 18, 1), + PIN_FIELD_BASE(62, 62, 9, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(63, 63, 9, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(64, 64, 9, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(65, 65, 9, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(68, 68, 9, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(74, 74, 7, 0x00d0, 0x10, 0, 1), + PIN_FIELD_BASE(75, 75, 7, 0x00d0, 0x10, 1, 1), + PIN_FIELD_BASE(76, 76, 7, 0x00d0, 0x10, 2, 1), + PIN_FIELD_BASE(77, 77, 7, 0x00d0, 0x10, 3, 1), + PIN_FIELD_BASE(78, 78, 7, 0x00d0, 0x10, 4, 1), + PIN_FIELD_BASE(79, 79, 7, 0x00d0, 0x10, 5, 1), + PIN_FIELD_BASE(80, 80, 7, 0x00d0, 0x10, 6, 1), + PIN_FIELD_BASE(81, 81, 7, 0x00d0, 0x10, 7, 1), + PIN_FIELD_BASE(82, 82, 7, 0x00d0, 0x10, 8, 1), + PIN_FIELD_BASE(83, 83, 7, 0x00d0, 0x10, 9, 1), + PIN_FIELD_BASE(84, 84, 7, 0x00d0, 0x10, 10, 1), + PIN_FIELD_BASE(85, 85, 7, 0x00d0, 0x10, 11, 1), + PIN_FIELD_BASE(86, 86, 7, 0x00d0, 0x10, 12, 1), + PIN_FIELD_BASE(87, 87, 7, 0x00d0, 0x10, 13, 1), + PIN_FIELD_BASE(90, 90, 7, 0x00d0, 0x10, 14, 1), + PIN_FIELD_BASE(91, 91, 7, 0x00d0, 0x10, 15, 1), + PIN_FIELD_BASE(94, 94, 3, 0x00e0, 0x10, 12, 1), + PIN_FIELD_BASE(95, 95, 3, 0x00e0, 0x10, 13, 1), + PIN_FIELD_BASE(96, 96, 3, 0x00e0, 0x10, 14, 1), + PIN_FIELD_BASE(97, 97, 3, 0x00e0, 0x10, 15, 1), + PIN_FIELD_BASE(98, 98, 3, 0x00e0, 0x10, 16, 1), + PIN_FIELD_BASE(99, 99, 3, 0x00e0, 0x10, 17, 1), + PIN_FIELD_BASE(100, 100, 3, 0x00e0, 0x10, 18, 1), + PIN_FIELD_BASE(101, 101, 3, 0x00e0, 0x10, 19, 1), + PIN_FIELD_BASE(102, 102, 3, 0x00e0, 0x10, 20, 1), + PIN_FIELD_BASE(103, 103, 3, 0x00e0, 0x10, 21, 1), + PIN_FIELD_BASE(104, 104, 3, 0x00e0, 0x10, 22, 1), + PIN_FIELD_BASE(105, 105, 3, 0x00e0, 0x10, 23, 1), + PIN_FIELD_BASE(106, 106, 8, 0x0090, 0x10, 8, 1), + PIN_FIELD_BASE(107, 107, 8, 0x0090, 0x10, 9, 1), + PIN_FIELD_BASE(108, 108, 8, 0x0090, 0x10, 10, 1), + PIN_FIELD_BASE(109, 109, 8, 0x0090, 0x10, 11, 1), + PIN_FIELD_BASE(110, 110, 8, 0x0090, 0x10, 12, 1), + PIN_FIELD_BASE(111, 111, 8, 0x0090, 0x10, 13, 1), + PIN_FIELD_BASE(112, 112, 5, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(113, 113, 5, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(114, 114, 5, 0x00c0, 0x10, 7, 1), + PIN_FIELD_BASE(115, 115, 5, 0x00c0, 0x10, 8, 1), + PIN_FIELD_BASE(116, 116, 5, 0x00c0, 0x10, 9, 1), + PIN_FIELD_BASE(125, 125, 4, 0x00c0, 0x10, 8, 1), + PIN_FIELD_BASE(130, 130, 4, 0x00c0, 0x10, 9, 1), + PIN_FIELD_BASE(131, 131, 4, 0x00c0, 0x10, 10, 1), + PIN_FIELD_BASE(132, 132, 4, 0x00c0, 0x10, 11, 1), + PIN_FIELD_BASE(133, 133, 4, 0x00c0, 0x10, 12, 1), + PIN_FIELD_BASE(138, 138, 3, 0x00e0, 0x10, 0, 1), + PIN_FIELD_BASE(139, 139, 3, 0x00e0, 0x10, 1, 1), + PIN_FIELD_BASE(140, 140, 3, 0x00e0, 0x10, 2, 1), + PIN_FIELD_BASE(141, 141, 3, 0x00e0, 0x10, 3, 1), + PIN_FIELD_BASE(142, 142, 3, 0x00e0, 0x10, 4, 1), + PIN_FIELD_BASE(143, 143, 3, 0x00e0, 0x10, 5, 1), + PIN_FIELD_BASE(144, 144, 3, 0x00e0, 0x10, 6, 1), + PIN_FIELD_BASE(145, 145, 3, 0x00e0, 0x10, 7, 1), + PIN_FIELD_BASE(146, 146, 3, 0x00e0, 0x10, 8, 1), + PIN_FIELD_BASE(147, 147, 3, 0x00e0, 0x10, 9, 1), + PIN_FIELD_BASE(148, 148, 3, 0x00e0, 0x10, 10, 1), + PIN_FIELD_BASE(149, 149, 3, 0x00e0, 0x10, 11, 1), + PIN_FIELD_BASE(150, 150, 4, 0x00c0, 0x10, 13, 1), + PIN_FIELD_BASE(151, 151, 4, 0x00c0, 0x10, 18, 1), + PIN_FIELD_BASE(152, 152, 4, 0x00c0, 0x10, 17, 1), + PIN_FIELD_BASE(153, 153, 4, 0x00c0, 0x10, 16, 1), + PIN_FIELD_BASE(154, 154, 4, 0x00c0, 0x10, 14, 1), + PIN_FIELD_BASE(155, 155, 4, 0x00c0, 0x10, 15, 1), + PIN_FIELD_BASE(156, 156, 4, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(157, 157, 4, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(158, 158, 4, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(159, 159, 4, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(160, 160, 4, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(161, 161, 4, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(162, 162, 4, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(163, 163, 4, 0x00c0, 0x10, 7, 1), + PIN_FIELD_BASE(164, 164, 5, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(167, 167, 5, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(168, 168, 5, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(169, 169, 5, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(170, 170, 5, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(171, 171, 6, 0x0090, 0x10, 0, 1), + PIN_FIELD_BASE(172, 172, 6, 0x0090, 0x10, 1, 1), + PIN_FIELD_BASE(173, 173, 6, 0x0090, 0x10, 2, 1), + PIN_FIELD_BASE(174, 174, 6, 0x0090, 0x10, 3, 1), + PIN_FIELD_BASE(175, 175, 6, 0x0090, 0x10, 4, 1), + PIN_FIELD_BASE(176, 176, 6, 0x0090, 0x10, 5, 1), + PIN_FIELD_BASE(177, 177, 6, 0x0090, 0x10, 6, 1), + PIN_FIELD_BASE(178, 178, 6, 0x0090, 0x10, 7, 1), + PIN_FIELD_BASE(179, 179, 6, 0x0090, 0x10, 8, 1), + PIN_FIELD_BASE(180, 180, 6, 0x0090, 0x10, 9, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_r1_range[] = { + PIN_FIELD_BASE(0, 0, 8, 0x00a0, 0x10, 0, 1), + PIN_FIELD_BASE(1, 1, 8, 0x00a0, 0x10, 1, 1), + PIN_FIELD_BASE(2, 2, 8, 0x00a0, 0x10, 2, 1), + PIN_FIELD_BASE(3, 3, 8, 0x00a0, 0x10, 3, 1), + PIN_FIELD_BASE(4, 4, 8, 0x00a0, 0x10, 4, 1), + PIN_FIELD_BASE(5, 5, 8, 0x00a0, 0x10, 5, 1), + PIN_FIELD_BASE(6, 6, 8, 0x00a0, 0x10, 6, 1), + PIN_FIELD_BASE(7, 7, 8, 0x00a0, 0x10, 7, 1), + PIN_FIELD_BASE(14, 14, 1, 0x00f0, 0x10, 14, 1), + PIN_FIELD_BASE(15, 15, 1, 0x00f0, 0x10, 16, 1), + PIN_FIELD_BASE(16, 16, 1, 0x00f0, 0x10, 17, 1), + PIN_FIELD_BASE(19, 19, 1, 0x00f0, 0x10, 21, 1), + PIN_FIELD_BASE(20, 20, 1, 0x00f0, 0x10, 22, 1), + PIN_FIELD_BASE(21, 21, 1, 0x00f0, 0x10, 20, 1), + PIN_FIELD_BASE(22, 22, 1, 0x00f0, 0x10, 19, 1), + PIN_FIELD_BASE(23, 23, 1, 0x00f0, 0x10, 0, 1), + PIN_FIELD_BASE(24, 24, 1, 0x00f0, 0x10, 1, 1), + PIN_FIELD_BASE(25, 25, 1, 0x00f0, 0x10, 2, 1), + PIN_FIELD_BASE(26, 26, 1, 0x00f0, 0x10, 3, 1), + PIN_FIELD_BASE(27, 27, 1, 0x00f0, 0x10, 4, 1), + PIN_FIELD_BASE(28, 28, 1, 0x00f0, 0x10, 5, 1), + PIN_FIELD_BASE(29, 29, 1, 0x00f0, 0x10, 6, 1), + PIN_FIELD_BASE(30, 30, 1, 0x00f0, 0x10, 7, 1), + PIN_FIELD_BASE(31, 31, 1, 0x00f0, 0x10, 8, 1), + PIN_FIELD_BASE(32, 32, 9, 0x00d0, 0x10, 0, 1), + PIN_FIELD_BASE(33, 33, 1, 0x00f0, 0x10, 9, 1), + PIN_FIELD_BASE(34, 34, 1, 0x00f0, 0x10, 10, 1), + PIN_FIELD_BASE(35, 35, 1, 0x00f0, 0x10, 11, 1), + PIN_FIELD_BASE(36, 36, 9, 0x00d0, 0x10, 6, 1), + PIN_FIELD_BASE(37, 37, 2, 0x00e0, 0x10, 10, 1), + PIN_FIELD_BASE(38, 38, 2, 0x00e0, 0x10, 11, 1), + PIN_FIELD_BASE(39, 39, 1, 0x00f0, 0x10, 12, 1), + PIN_FIELD_BASE(40, 40, 2, 0x00e0, 0x10, 0, 1), + PIN_FIELD_BASE(41, 41, 2, 0x00e0, 0x10, 1, 1), + PIN_FIELD_BASE(42, 42, 2, 0x00e0, 0x10, 2, 1), + PIN_FIELD_BASE(43, 43, 2, 0x00e0, 0x10, 3, 1), + PIN_FIELD_BASE(44, 44, 2, 0x00e0, 0x10, 4, 1), + PIN_FIELD_BASE(45, 45, 2, 0x00e0, 0x10, 5, 1), + PIN_FIELD_BASE(46, 46, 1, 0x00f0, 0x10, 13, 1), + PIN_FIELD_BASE(47, 47, 1, 0x00f0, 0x10, 15, 1), + PIN_FIELD_BASE(48, 48, 2, 0x00e0, 0x10, 6, 1), + PIN_FIELD_BASE(49, 49, 2, 0x00e0, 0x10, 7, 1), + PIN_FIELD_BASE(50, 50, 2, 0x00e0, 0x10, 8, 1), + PIN_FIELD_BASE(51, 51, 2, 0x00e0, 0x10, 9, 1), + PIN_FIELD_BASE(58, 58, 1, 0x00f0, 0x10, 18, 1), + PIN_FIELD_BASE(62, 62, 9, 0x00d0, 0x10, 1, 1), + PIN_FIELD_BASE(63, 63, 9, 0x00d0, 0x10, 2, 1), + PIN_FIELD_BASE(64, 64, 9, 0x00d0, 0x10, 3, 1), + PIN_FIELD_BASE(65, 65, 9, 0x00d0, 0x10, 4, 1), + PIN_FIELD_BASE(68, 68, 9, 0x00d0, 0x10, 5, 1), + PIN_FIELD_BASE(74, 74, 7, 0x00e0, 0x10, 0, 1), + PIN_FIELD_BASE(75, 75, 7, 0x00e0, 0x10, 1, 1), + PIN_FIELD_BASE(76, 76, 7, 0x00e0, 0x10, 2, 1), + PIN_FIELD_BASE(77, 77, 7, 0x00e0, 0x10, 3, 1), + PIN_FIELD_BASE(78, 78, 7, 0x00e0, 0x10, 4, 1), + PIN_FIELD_BASE(79, 79, 7, 0x00e0, 0x10, 5, 1), + PIN_FIELD_BASE(80, 80, 7, 0x00e0, 0x10, 6, 1), + PIN_FIELD_BASE(81, 81, 7, 0x00e0, 0x10, 7, 1), + PIN_FIELD_BASE(82, 82, 7, 0x00e0, 0x10, 8, 1), + PIN_FIELD_BASE(83, 83, 7, 0x00e0, 0x10, 9, 1), + PIN_FIELD_BASE(84, 84, 7, 0x00e0, 0x10, 10, 1), + PIN_FIELD_BASE(85, 85, 7, 0x00e0, 0x10, 11, 1), + PIN_FIELD_BASE(86, 86, 7, 0x00e0, 0x10, 12, 1), + PIN_FIELD_BASE(87, 87, 7, 0x00e0, 0x10, 13, 1), + PIN_FIELD_BASE(90, 90, 7, 0x00e0, 0x10, 14, 1), + PIN_FIELD_BASE(91, 91, 7, 0x00e0, 0x10, 15, 1), + PIN_FIELD_BASE(94, 94, 3, 0x00f0, 0x10, 12, 1), + PIN_FIELD_BASE(95, 95, 3, 0x00f0, 0x10, 13, 1), + PIN_FIELD_BASE(96, 96, 3, 0x00f0, 0x10, 14, 1), + PIN_FIELD_BASE(97, 97, 3, 0x00f0, 0x10, 15, 1), + PIN_FIELD_BASE(98, 98, 3, 0x00f0, 0x10, 16, 1), + PIN_FIELD_BASE(99, 99, 3, 0x00f0, 0x10, 17, 1), + PIN_FIELD_BASE(100, 100, 3, 0x00f0, 0x10, 18, 1), + PIN_FIELD_BASE(101, 101, 3, 0x00f0, 0x10, 19, 1), + PIN_FIELD_BASE(102, 102, 3, 0x00f0, 0x10, 20, 1), + PIN_FIELD_BASE(103, 103, 3, 0x00f0, 0x10, 21, 1), + PIN_FIELD_BASE(104, 104, 3, 0x00f0, 0x10, 22, 1), + PIN_FIELD_BASE(105, 105, 3, 0x00f0, 0x10, 23, 1), + PIN_FIELD_BASE(106, 106, 8, 0x00a0, 0x10, 8, 1), + PIN_FIELD_BASE(107, 107, 8, 0x00a0, 0x10, 9, 1), + PIN_FIELD_BASE(108, 108, 8, 0x00a0, 0x10, 10, 1), + PIN_FIELD_BASE(109, 109, 8, 0x00a0, 0x10, 11, 1), + PIN_FIELD_BASE(110, 110, 8, 0x00a0, 0x10, 12, 1), + PIN_FIELD_BASE(111, 111, 8, 0x00a0, 0x10, 13, 1), + PIN_FIELD_BASE(112, 112, 5, 0x00d0, 0x10, 5, 1), + PIN_FIELD_BASE(113, 113, 5, 0x00d0, 0x10, 6, 1), + PIN_FIELD_BASE(114, 114, 5, 0x00d0, 0x10, 7, 1), + PIN_FIELD_BASE(115, 115, 5, 0x00d0, 0x10, 8, 1), + PIN_FIELD_BASE(116, 116, 5, 0x00d0, 0x10, 9, 1), + PIN_FIELD_BASE(125, 125, 4, 0x00d0, 0x10, 8, 1), + PIN_FIELD_BASE(130, 130, 4, 0x00d0, 0x10, 9, 1), + PIN_FIELD_BASE(131, 131, 4, 0x00d0, 0x10, 10, 1), + PIN_FIELD_BASE(132, 132, 4, 0x00d0, 0x10, 11, 1), + PIN_FIELD_BASE(133, 133, 4, 0x00d0, 0x10, 12, 1), + PIN_FIELD_BASE(138, 138, 3, 0x00f0, 0x10, 0, 1), + PIN_FIELD_BASE(139, 139, 3, 0x00f0, 0x10, 1, 1), + PIN_FIELD_BASE(140, 140, 3, 0x00f0, 0x10, 2, 1), + PIN_FIELD_BASE(141, 141, 3, 0x00f0, 0x10, 3, 1), + PIN_FIELD_BASE(142, 142, 3, 0x00f0, 0x10, 4, 1), + PIN_FIELD_BASE(143, 143, 3, 0x00f0, 0x10, 5, 1), + PIN_FIELD_BASE(144, 144, 3, 0x00f0, 0x10, 6, 1), + PIN_FIELD_BASE(145, 145, 3, 0x00f0, 0x10, 7, 1), + PIN_FIELD_BASE(146, 146, 3, 0x00f0, 0x10, 8, 1), + PIN_FIELD_BASE(147, 147, 3, 0x00f0, 0x10, 9, 1), + PIN_FIELD_BASE(148, 148, 3, 0x00f0, 0x10, 10, 1), + PIN_FIELD_BASE(149, 149, 3, 0x00f0, 0x10, 11, 1), + PIN_FIELD_BASE(150, 150, 4, 0x00d0, 0x10, 13, 1), + PIN_FIELD_BASE(151, 151, 4, 0x00d0, 0x10, 18, 1), + PIN_FIELD_BASE(152, 152, 4, 0x00d0, 0x10, 17, 1), + PIN_FIELD_BASE(153, 153, 4, 0x00d0, 0x10, 16, 1), + PIN_FIELD_BASE(154, 154, 4, 0x00d0, 0x10, 14, 1), + PIN_FIELD_BASE(155, 155, 4, 0x00d0, 0x10, 15, 1), + PIN_FIELD_BASE(156, 156, 4, 0x00d0, 0x10, 0, 1), + PIN_FIELD_BASE(157, 157, 4, 0x00d0, 0x10, 1, 1), + PIN_FIELD_BASE(158, 158, 4, 0x00d0, 0x10, 2, 1), + PIN_FIELD_BASE(159, 159, 4, 0x00d0, 0x10, 3, 1), + PIN_FIELD_BASE(160, 160, 4, 0x00d0, 0x10, 4, 1), + PIN_FIELD_BASE(161, 161, 4, 0x00d0, 0x10, 5, 1), + PIN_FIELD_BASE(162, 162, 4, 0x00d0, 0x10, 6, 1), + PIN_FIELD_BASE(163, 163, 4, 0x00d0, 0x10, 7, 1), + PIN_FIELD_BASE(164, 164, 5, 0x00d0, 0x10, 2, 1), + PIN_FIELD_BASE(167, 167, 5, 0x00d0, 0x10, 0, 1), + PIN_FIELD_BASE(168, 168, 5, 0x00d0, 0x10, 3, 1), + PIN_FIELD_BASE(169, 169, 5, 0x00d0, 0x10, 4, 1), + PIN_FIELD_BASE(170, 170, 5, 0x00d0, 0x10, 1, 1), + PIN_FIELD_BASE(171, 171, 6, 0x00a0, 0x10, 0, 1), + PIN_FIELD_BASE(172, 172, 6, 0x00a0, 0x10, 1, 1), + PIN_FIELD_BASE(173, 173, 6, 0x00a0, 0x10, 2, 1), + PIN_FIELD_BASE(174, 174, 6, 0x00a0, 0x10, 3, 1), + PIN_FIELD_BASE(175, 175, 6, 0x00a0, 0x10, 4, 1), + PIN_FIELD_BASE(176, 176, 6, 0x00a0, 0x10, 5, 1), + PIN_FIELD_BASE(177, 177, 6, 0x00a0, 0x10, 6, 1), + PIN_FIELD_BASE(178, 178, 6, 0x00a0, 0x10, 7, 1), + PIN_FIELD_BASE(179, 179, 6, 0x00a0, 0x10, 8, 1), + PIN_FIELD_BASE(180, 180, 6, 0x00a0, 0x10, 9, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_pu_range[] = { + PIN_FIELD_BASE(8, 8, 2, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(9, 9, 2, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(10, 10, 2, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(11, 11, 2, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(12, 12, 1, 0x00d0, 0x10, 0, 1), + PIN_FIELD_BASE(13, 13, 1, 0x00d0, 0x10, 1, 1), + PIN_FIELD_BASE(17, 17, 1, 0x00d0, 0x10, 2, 1), + PIN_FIELD_BASE(18, 18, 1, 0x00d0, 0x10, 3, 1), + PIN_FIELD_BASE(52, 52, 2, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(53, 53, 2, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(54, 54, 5, 0x00b0, 0x10, 8, 1), + PIN_FIELD_BASE(55, 55, 5, 0x00b0, 0x10, 9, 1), + PIN_FIELD_BASE(56, 56, 1, 0x00d0, 0x10, 4, 1), + PIN_FIELD_BASE(57, 57, 1, 0x00d0, 0x10, 5, 1), + PIN_FIELD_BASE(59, 59, 2, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(60, 60, 9, 0x00b0, 0x10, 0, 1), + PIN_FIELD_BASE(61, 61, 9, 0x00b0, 0x10, 1, 1), + PIN_FIELD_BASE(66, 66, 5, 0x00b0, 0x10, 0, 1), + PIN_FIELD_BASE(67, 67, 5, 0x00b0, 0x10, 1, 1), + PIN_FIELD_BASE(69, 69, 7, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(70, 70, 7, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(71, 71, 7, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(72, 72, 7, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(73, 73, 7, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(88, 88, 7, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(89, 89, 7, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(92, 92, 3, 0x00d0, 0x10, 4, 1), + PIN_FIELD_BASE(93, 93, 3, 0x00d0, 0x10, 5, 1), + PIN_FIELD_BASE(117, 117, 4, 0x00b0, 0x10, 0, 1), + PIN_FIELD_BASE(118, 118, 4, 0x00b0, 0x10, 1, 1), + PIN_FIELD_BASE(119, 119, 4, 0x00b0, 0x10, 2, 1), + PIN_FIELD_BASE(120, 120, 4, 0x00b0, 0x10, 3, 1), + PIN_FIELD_BASE(121, 121, 4, 0x00b0, 0x10, 4, 1), + PIN_FIELD_BASE(122, 122, 4, 0x00b0, 0x10, 5, 1), + PIN_FIELD_BASE(123, 123, 4, 0x00b0, 0x10, 6, 1), + PIN_FIELD_BASE(124, 124, 4, 0x00b0, 0x10, 7, 1), + PIN_FIELD_BASE(126, 126, 5, 0x00b0, 0x10, 4, 1), + PIN_FIELD_BASE(127, 127, 5, 0x00b0, 0x10, 5, 1), + PIN_FIELD_BASE(128, 128, 5, 0x00b0, 0x10, 6, 1), + PIN_FIELD_BASE(129, 129, 5, 0x00b0, 0x10, 7, 1), + PIN_FIELD_BASE(134, 134, 3, 0x00d0, 0x10, 0, 1), + PIN_FIELD_BASE(135, 135, 3, 0x00d0, 0x10, 1, 1), + PIN_FIELD_BASE(136, 136, 3, 0x00d0, 0x10, 2, 1), + PIN_FIELD_BASE(137, 137, 3, 0x00d0, 0x10, 3, 1), + PIN_FIELD_BASE(165, 165, 5, 0x00b0, 0x10, 2, 1), + PIN_FIELD_BASE(166, 166, 5, 0x00b0, 0x10, 3, 1), + PIN_FIELD_BASE(181, 181, 10, 0x0060, 0x10, 0, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_pd_range[] = { + PIN_FIELD_BASE(8, 8, 2, 0x00a0, 0x10, 3, 1), + PIN_FIELD_BASE(9, 9, 2, 0x00a0, 0x10, 4, 1), + PIN_FIELD_BASE(10, 10, 2, 0x00a0, 0x10, 5, 1), + PIN_FIELD_BASE(11, 11, 2, 0x00a0, 0x10, 6, 1), + PIN_FIELD_BASE(12, 12, 1, 0x00b0, 0x10, 0, 1), + PIN_FIELD_BASE(13, 13, 1, 0x00b0, 0x10, 1, 1), + PIN_FIELD_BASE(17, 17, 1, 0x00b0, 0x10, 2, 1), + PIN_FIELD_BASE(18, 18, 1, 0x00b0, 0x10, 3, 1), + PIN_FIELD_BASE(52, 52, 2, 0x00a0, 0x10, 1, 1), + PIN_FIELD_BASE(53, 53, 2, 0x00a0, 0x10, 2, 1), + PIN_FIELD_BASE(54, 54, 5, 0x0090, 0x10, 8, 1), + PIN_FIELD_BASE(55, 55, 5, 0x0090, 0x10, 9, 1), + PIN_FIELD_BASE(56, 56, 1, 0x00b0, 0x10, 4, 1), + PIN_FIELD_BASE(57, 57, 1, 0x00b0, 0x10, 5, 1), + PIN_FIELD_BASE(59, 59, 2, 0x00a0, 0x10, 0, 1), + PIN_FIELD_BASE(60, 60, 9, 0x0090, 0x10, 0, 1), + PIN_FIELD_BASE(61, 61, 9, 0x0090, 0x10, 1, 1), + PIN_FIELD_BASE(66, 66, 5, 0x0090, 0x10, 0, 1), + PIN_FIELD_BASE(67, 67, 5, 0x0090, 0x10, 1, 1), + PIN_FIELD_BASE(69, 69, 7, 0x00a0, 0x10, 0, 1), + PIN_FIELD_BASE(70, 70, 7, 0x00a0, 0x10, 1, 1), + PIN_FIELD_BASE(71, 71, 7, 0x00a0, 0x10, 2, 1), + PIN_FIELD_BASE(72, 72, 7, 0x00a0, 0x10, 3, 1), + PIN_FIELD_BASE(73, 73, 7, 0x00a0, 0x10, 4, 1), + PIN_FIELD_BASE(88, 88, 7, 0x00a0, 0x10, 5, 1), + PIN_FIELD_BASE(89, 89, 7, 0x00a0, 0x10, 6, 1), + PIN_FIELD_BASE(92, 92, 3, 0x00b0, 0x10, 4, 1), + PIN_FIELD_BASE(93, 93, 3, 0x00b0, 0x10, 5, 1), + PIN_FIELD_BASE(117, 117, 4, 0x0090, 0x10, 0, 1), + PIN_FIELD_BASE(118, 118, 4, 0x0090, 0x10, 1, 1), + PIN_FIELD_BASE(119, 119, 4, 0x0090, 0x10, 2, 1), + PIN_FIELD_BASE(120, 120, 4, 0x0090, 0x10, 3, 1), + PIN_FIELD_BASE(121, 121, 4, 0x0090, 0x10, 4, 1), + PIN_FIELD_BASE(122, 122, 4, 0x0090, 0x10, 5, 1), + PIN_FIELD_BASE(123, 123, 4, 0x0090, 0x10, 6, 1), + PIN_FIELD_BASE(124, 124, 4, 0x0090, 0x10, 7, 1), + PIN_FIELD_BASE(126, 126, 5, 0x0090, 0x10, 4, 1), + PIN_FIELD_BASE(127, 127, 5, 0x0090, 0x10, 5, 1), + PIN_FIELD_BASE(128, 128, 5, 0x0090, 0x10, 6, 1), + PIN_FIELD_BASE(129, 129, 5, 0x0090, 0x10, 7, 1), + PIN_FIELD_BASE(134, 134, 3, 0x00b0, 0x10, 0, 1), + PIN_FIELD_BASE(135, 135, 3, 0x00b0, 0x10, 1, 1), + PIN_FIELD_BASE(136, 136, 3, 0x00b0, 0x10, 2, 1), + PIN_FIELD_BASE(137, 137, 3, 0x00b0, 0x10, 3, 1), + PIN_FIELD_BASE(165, 165, 5, 0x0090, 0x10, 2, 1), + PIN_FIELD_BASE(166, 166, 5, 0x0090, 0x10, 3, 1), + PIN_FIELD_BASE(181, 181, 10, 0x0050, 0x10, 0, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_drv_range[] = { + PIN_FIELD_BASE(0, 0, 8, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(1, 1, 8, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(2, 2, 8, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(3, 3, 8, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(4, 4, 8, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(5, 5, 8, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(6, 6, 8, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(7, 7, 8, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(8, 8, 2, 0x0010, 0x10, 9, 3), + PIN_FIELD_BASE(9, 9, 2, 0x0010, 0x10, 12, 3), + PIN_FIELD_BASE(10, 10, 2, 0x0010, 0x10, 15, 3), + PIN_FIELD_BASE(11, 11, 2, 0x0010, 0x10, 18, 3), + PIN_FIELD_BASE(12, 12, 1, 0x0010, 0x10, 21, 3), + PIN_FIELD_BASE(13, 13, 1, 0x0010, 0x10, 24, 3), + PIN_FIELD_BASE(14, 14, 1, 0x0010, 0x10, 9, 3), + PIN_FIELD_BASE(15, 15, 1, 0x0010, 0x10, 15, 3), + PIN_FIELD_BASE(16, 16, 1, 0x0010, 0x10, 27, 3), + PIN_FIELD_BASE(17, 17, 1, 0x0020, 0x10, 0, 3), + PIN_FIELD_BASE(18, 18, 1, 0x0020, 0x10, 3, 3), + PIN_FIELD_BASE(19, 19, 1, 0x0020, 0x10, 21, 3), + PIN_FIELD_BASE(20, 20, 1, 0x0020, 0x10, 24, 3), + PIN_FIELD_BASE(21, 21, 1, 0x0020, 0x10, 18, 3), + PIN_FIELD_BASE(22, 22, 1, 0x0020, 0x10, 15, 3), + PIN_FIELD_BASE(23, 23, 1, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(24, 24, 1, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(25, 25, 1, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(26, 26, 1, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(27, 27, 1, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(28, 28, 1, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(29, 29, 1, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(30, 30, 1, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(31, 31, 1, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(32, 32, 9, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(33, 33, 1, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(34, 34, 1, 0x0010, 0x10, 0, 3), + PIN_FIELD_BASE(35, 35, 1, 0x0010, 0x10, 3, 3), + PIN_FIELD_BASE(36, 36, 9, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(37, 37, 2, 0x0010, 0x10, 21, 3), + PIN_FIELD_BASE(38, 38, 2, 0x0010, 0x10, 24, 3), + PIN_FIELD_BASE(39, 39, 1, 0x0010, 0x10, 6, 3), + PIN_FIELD_BASE(40, 40, 2, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(41, 41, 2, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(42, 42, 2, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(43, 43, 2, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(44, 44, 2, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(45, 45, 2, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(46, 46, 1, 0x0010, 0x10, 12, 3), + PIN_FIELD_BASE(47, 47, 1, 0x0010, 0x10, 18, 3), + PIN_FIELD_BASE(48, 48, 2, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(49, 49, 2, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(50, 50, 2, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(51, 51, 2, 0x0010, 0x10, 0, 3), + PIN_FIELD_BASE(52, 52, 2, 0x0010, 0x10, 3, 3), + PIN_FIELD_BASE(53, 53, 2, 0x0010, 0x10, 6, 3), + PIN_FIELD_BASE(54, 54, 5, 0x0010, 0x10, 0, 3), + PIN_FIELD_BASE(55, 55, 5, 0x0010, 0x10, 3, 3), + PIN_FIELD_BASE(56, 56, 1, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(57, 57, 1, 0x0020, 0x10, 9, 3), + PIN_FIELD_BASE(58, 58, 1, 0x0020, 0x10, 12, 3), + PIN_FIELD_BASE(59, 59, 2, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(60, 60, 9, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(61, 61, 9, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(62, 62, 9, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(63, 63, 9, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(64, 64, 9, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(65, 65, 9, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(66, 66, 5, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(67, 67, 5, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(68, 68, 9, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(69, 69, 7, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(70, 70, 7, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(71, 71, 7, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(72, 72, 7, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(73, 73, 7, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(74, 74, 7, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(75, 75, 7, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(76, 76, 7, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(77, 77, 7, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(78, 78, 7, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(79, 79, 7, 0x0010, 0x10, 0, 3), + PIN_FIELD_BASE(80, 80, 7, 0x0010, 0x10, 3, 3), + PIN_FIELD_BASE(81, 81, 7, 0x0010, 0x10, 6, 3), + PIN_FIELD_BASE(82, 82, 7, 0x0010, 0x10, 9, 3), + PIN_FIELD_BASE(83, 83, 7, 0x0010, 0x10, 12, 3), + PIN_FIELD_BASE(84, 84, 7, 0x0010, 0x10, 15, 3), + PIN_FIELD_BASE(85, 85, 7, 0x0010, 0x10, 18, 3), + PIN_FIELD_BASE(86, 86, 7, 0x0010, 0x10, 21, 3), + PIN_FIELD_BASE(87, 87, 7, 0x0010, 0x10, 24, 3), + PIN_FIELD_BASE(88, 88, 7, 0x0010, 0x10, 27, 3), + PIN_FIELD_BASE(89, 89, 7, 0x0020, 0x10, 0, 3), + PIN_FIELD_BASE(90, 90, 7, 0x0020, 0x10, 3, 3), + PIN_FIELD_BASE(91, 91, 7, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(92, 92, 3, 0x0010, 0x10, 18, 3), + PIN_FIELD_BASE(93, 93, 3, 0x0010, 0x10, 21, 3), + PIN_FIELD_BASE(94, 94, 3, 0x0010, 0x10, 24, 3), + PIN_FIELD_BASE(95, 95, 3, 0x0010, 0x10, 27, 3), + PIN_FIELD_BASE(96, 96, 3, 0x0020, 0x10, 0, 3), + PIN_FIELD_BASE(97, 97, 3, 0x0020, 0x10, 3, 3), + PIN_FIELD_BASE(98, 98, 3, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(99, 99, 3, 0x0020, 0x10, 9, 3), + PIN_FIELD_BASE(100, 100, 3, 0x0020, 0x10, 12, 3), + PIN_FIELD_BASE(101, 101, 3, 0x0020, 0x10, 15, 3), + PIN_FIELD_BASE(102, 102, 3, 0x0020, 0x10, 18, 3), + PIN_FIELD_BASE(103, 103, 3, 0x0020, 0x10, 21, 3), + PIN_FIELD_BASE(104, 104, 3, 0x0020, 0x10, 24, 3), + PIN_FIELD_BASE(105, 105, 3, 0x0020, 0x10, 27, 3), + PIN_FIELD_BASE(106, 106, 8, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(107, 107, 8, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(108, 108, 8, 0x0010, 0x10, 0, 3), + PIN_FIELD_BASE(109, 109, 8, 0x0010, 0x10, 3, 3), + PIN_FIELD_BASE(110, 110, 8, 0x0010, 0x10, 6, 3), + PIN_FIELD_BASE(111, 111, 8, 0x0010, 0x10, 9, 3), + PIN_FIELD_BASE(112, 112, 5, 0x0010, 0x10, 15, 3), + PIN_FIELD_BASE(113, 113, 5, 0x0010, 0x10, 18, 3), + PIN_FIELD_BASE(114, 114, 5, 0x0010, 0x10, 21, 3), + PIN_FIELD_BASE(115, 115, 5, 0x0010, 0x10, 24, 3), + PIN_FIELD_BASE(116, 116, 5, 0x0010, 0x10, 27, 3), + PIN_FIELD_BASE(117, 117, 4, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(118, 118, 4, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(119, 119, 4, 0x0010, 0x10, 0, 3), + PIN_FIELD_BASE(120, 120, 4, 0x0010, 0x10, 3, 3), + PIN_FIELD_BASE(121, 121, 4, 0x0010, 0x10, 6, 3), + PIN_FIELD_BASE(122, 122, 4, 0x0010, 0x10, 9, 3), + PIN_FIELD_BASE(123, 123, 4, 0x0010, 0x10, 12, 3), + PIN_FIELD_BASE(124, 124, 4, 0x0010, 0x10, 15, 3), + PIN_FIELD_BASE(125, 125, 4, 0x0010, 0x10, 18, 3), + PIN_FIELD_BASE(126, 126, 5, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(127, 127, 5, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(128, 128, 5, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(129, 129, 5, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(130, 130, 4, 0x0010, 0x10, 21, 3), + PIN_FIELD_BASE(131, 131, 4, 0x0010, 0x10, 24, 3), + PIN_FIELD_BASE(132, 132, 4, 0x0010, 0x10, 27, 3), + PIN_FIELD_BASE(133, 133, 4, 0x0020, 0x10, 0, 3), + PIN_FIELD_BASE(134, 134, 3, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(135, 135, 3, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(136, 136, 3, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(137, 137, 3, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(138, 138, 3, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(139, 139, 3, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(140, 140, 3, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(141, 141, 3, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(142, 142, 3, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(143, 143, 3, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(144, 144, 3, 0x0010, 0x10, 0, 3), + PIN_FIELD_BASE(145, 145, 3, 0x0010, 0x10, 3, 3), + PIN_FIELD_BASE(146, 146, 3, 0x0010, 0x10, 6, 3), + PIN_FIELD_BASE(147, 147, 3, 0x0010, 0x10, 9, 3), + PIN_FIELD_BASE(148, 148, 3, 0x0010, 0x10, 12, 3), + PIN_FIELD_BASE(149, 149, 3, 0x0010, 0x10, 15, 3), + PIN_FIELD_BASE(150, 150, 4, 0x0020, 0x10, 3, 3), + PIN_FIELD_BASE(151, 151, 4, 0x0020, 0x10, 18, 3), + PIN_FIELD_BASE(152, 152, 4, 0x0020, 0x10, 15, 3), + PIN_FIELD_BASE(153, 153, 4, 0x0020, 0x10, 12, 3), + PIN_FIELD_BASE(154, 154, 4, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(155, 155, 4, 0x0020, 0x10, 9, 3), + PIN_FIELD_BASE(156, 156, 4, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(157, 157, 4, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(158, 158, 4, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(159, 159, 4, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(160, 160, 4, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(161, 161, 4, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(162, 162, 4, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(163, 163, 4, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(164, 164, 5, 0x0010, 0x10, 6, 3), + PIN_FIELD_BASE(165, 165, 5, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(166, 166, 5, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(167, 167, 5, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(168, 168, 5, 0x0010, 0x10, 9, 3), + PIN_FIELD_BASE(169, 169, 5, 0x0010, 0x10, 12, 3), + PIN_FIELD_BASE(170, 170, 5, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(171, 171, 6, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(172, 172, 6, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(173, 173, 6, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(174, 174, 6, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(175, 175, 6, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(176, 176, 6, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(177, 177, 6, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(178, 178, 6, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(179, 179, 6, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(180, 180, 6, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(181, 181, 10, 0x0000, 0x10, 0, 3), +}; + +static const struct mtk_pin_field_calc mt8901_pin_drv_adv_range[] = { + PIN_FIELD_BASE(8, 8, 2, 0x0030, 0x10, 6, 3), + PIN_FIELD_BASE(9, 9, 2, 0x0030, 0x10, 9, 3), + PIN_FIELD_BASE(10, 10, 2, 0x0030, 0x10, 12, 3), + PIN_FIELD_BASE(11, 11, 2, 0x0030, 0x10, 15, 3), + PIN_FIELD_BASE(12, 12, 1, 0x0040, 0x10, 0, 3), + PIN_FIELD_BASE(13, 13, 1, 0x0040, 0x10, 3, 3), + PIN_FIELD_BASE(17, 17, 1, 0x0040, 0x10, 6, 3), + PIN_FIELD_BASE(18, 18, 1, 0x0040, 0x10, 9, 3), + PIN_FIELD_BASE(52, 52, 2, 0x0030, 0x10, 0, 3), + PIN_FIELD_BASE(53, 53, 2, 0x0030, 0x10, 3, 3), + PIN_FIELD_BASE(54, 54, 5, 0x0030, 0x10, 24, 3), + PIN_FIELD_BASE(55, 55, 5, 0x0030, 0x10, 27, 3), + PIN_FIELD_BASE(56, 56, 1, 0x0040, 0x10, 12, 3), + PIN_FIELD_BASE(57, 57, 1, 0x0040, 0x10, 15, 3), + PIN_FIELD_BASE(60, 60, 9, 0x0020, 0x10, 0, 3), + PIN_FIELD_BASE(61, 61, 9, 0x0020, 0x10, 3, 3), + PIN_FIELD_BASE(66, 66, 5, 0x0030, 0x10, 0, 3), + PIN_FIELD_BASE(67, 67, 5, 0x0030, 0x10, 3, 3), + PIN_FIELD_BASE(70, 70, 7, 0x0030, 0x10, 0, 3), + PIN_FIELD_BASE(71, 71, 7, 0x0030, 0x10, 3, 3), + PIN_FIELD_BASE(72, 72, 7, 0x0030, 0x10, 6, 3), + PIN_FIELD_BASE(73, 73, 7, 0x0030, 0x10, 9, 3), + PIN_FIELD_BASE(88, 88, 7, 0x0030, 0x10, 12, 3), + PIN_FIELD_BASE(89, 89, 7, 0x0030, 0x10, 15, 3), + PIN_FIELD_BASE(92, 92, 3, 0x0040, 0x10, 12, 3), + PIN_FIELD_BASE(93, 93, 3, 0x0040, 0x10, 15, 3), + PIN_FIELD_BASE(117, 117, 4, 0x0030, 0x10, 0, 3), + PIN_FIELD_BASE(118, 118, 4, 0x0030, 0x10, 3, 3), + PIN_FIELD_BASE(119, 119, 4, 0x0030, 0x10, 6, 3), + PIN_FIELD_BASE(120, 120, 4, 0x0030, 0x10, 9, 3), + PIN_FIELD_BASE(121, 121, 4, 0x0030, 0x10, 12, 3), + PIN_FIELD_BASE(122, 122, 4, 0x0030, 0x10, 15, 3), + PIN_FIELD_BASE(123, 123, 4, 0x0030, 0x10, 18, 3), + PIN_FIELD_BASE(124, 124, 4, 0x0030, 0x10, 21, 3), + PIN_FIELD_BASE(126, 126, 5, 0x0030, 0x10, 12, 3), + PIN_FIELD_BASE(127, 127, 5, 0x0030, 0x10, 15, 3), + PIN_FIELD_BASE(128, 128, 5, 0x0030, 0x10, 18, 3), + PIN_FIELD_BASE(129, 129, 5, 0x0030, 0x10, 21, 3), + PIN_FIELD_BASE(134, 134, 3, 0x0040, 0x10, 0, 3), + PIN_FIELD_BASE(135, 135, 3, 0x0040, 0x10, 3, 3), + PIN_FIELD_BASE(136, 136, 3, 0x0040, 0x10, 6, 3), + PIN_FIELD_BASE(137, 137, 3, 0x0040, 0x10, 9, 3), + PIN_FIELD_BASE(165, 165, 5, 0x0030, 0x10, 6, 3), + PIN_FIELD_BASE(166, 166, 5, 0x0030, 0x10, 9, 3), +}; + +static const struct mtk_pin_field_calc mt8901_pin_rsel_range[] = { + PIN_FIELD_BASE(8, 8, 2, 0x0110, 0x10, 6, 3), + PIN_FIELD_BASE(9, 9, 2, 0x0110, 0x10, 9, 3), + PIN_FIELD_BASE(10, 10, 2, 0x0110, 0x10, 12, 3), + PIN_FIELD_BASE(11, 11, 2, 0x0110, 0x10, 15, 3), + PIN_FIELD_BASE(12, 12, 1, 0x0130, 0x10, 0, 1), + PIN_FIELD_BASE(13, 13, 1, 0x0130, 0x10, 1, 1), + PIN_FIELD_BASE(17, 17, 1, 0x0130, 0x10, 2, 1), + PIN_FIELD_BASE(18, 18, 1, 0x0130, 0x10, 3, 1), + PIN_FIELD_BASE(52, 52, 2, 0x0110, 0x10, 0, 3), + PIN_FIELD_BASE(53, 53, 2, 0x0110, 0x10, 3, 3), + PIN_FIELD_BASE(54, 54, 5, 0x0110, 0x10, 12, 3), + PIN_FIELD_BASE(55, 55, 5, 0x0110, 0x10, 15, 3), + PIN_FIELD_BASE(56, 56, 1, 0x0130, 0x10, 4, 1), + PIN_FIELD_BASE(57, 57, 1, 0x0130, 0x10, 5, 1), + PIN_FIELD_BASE(60, 60, 9, 0x00f0, 0x10, 0, 3), + PIN_FIELD_BASE(61, 61, 9, 0x00f0, 0x10, 3, 3), + PIN_FIELD_BASE(66, 66, 5, 0x0110, 0x10, 0, 1), + PIN_FIELD_BASE(67, 67, 5, 0x0110, 0x10, 1, 1), + PIN_FIELD_BASE(70, 70, 7, 0x0100, 0x10, 0, 3), + PIN_FIELD_BASE(71, 71, 7, 0x0100, 0x10, 3, 3), + PIN_FIELD_BASE(72, 72, 7, 0x0100, 0x10, 6, 3), + PIN_FIELD_BASE(73, 73, 7, 0x0100, 0x10, 9, 3), + PIN_FIELD_BASE(88, 88, 7, 0x0100, 0x10, 12, 3), + PIN_FIELD_BASE(89, 89, 7, 0x0100, 0x10, 15, 3), + PIN_FIELD_BASE(92, 92, 3, 0x0120, 0x10, 12, 3), + PIN_FIELD_BASE(93, 93, 3, 0x0120, 0x10, 15, 3), + PIN_FIELD_BASE(117, 117, 4, 0x0100, 0x10, 0, 3), + PIN_FIELD_BASE(118, 118, 4, 0x0100, 0x10, 3, 3), + PIN_FIELD_BASE(119, 119, 4, 0x0100, 0x10, 6, 3), + PIN_FIELD_BASE(120, 120, 4, 0x0100, 0x10, 9, 3), + PIN_FIELD_BASE(121, 121, 4, 0x0100, 0x10, 12, 3), + PIN_FIELD_BASE(122, 122, 4, 0x0100, 0x10, 15, 3), + PIN_FIELD_BASE(123, 123, 4, 0x0100, 0x10, 18, 3), + PIN_FIELD_BASE(124, 124, 4, 0x0100, 0x10, 21, 3), + PIN_FIELD_BASE(126, 126, 5, 0x0110, 0x10, 8, 1), + PIN_FIELD_BASE(127, 127, 5, 0x0110, 0x10, 9, 1), + PIN_FIELD_BASE(128, 128, 5, 0x0110, 0x10, 10, 1), + PIN_FIELD_BASE(129, 129, 5, 0x0110, 0x10, 11, 1), + PIN_FIELD_BASE(134, 134, 3, 0x0120, 0x10, 0, 3), + PIN_FIELD_BASE(135, 135, 3, 0x0120, 0x10, 3, 3), + PIN_FIELD_BASE(136, 136, 3, 0x0120, 0x10, 6, 3), + PIN_FIELD_BASE(137, 137, 3, 0x0120, 0x10, 9, 3), + PIN_FIELD_BASE(165, 165, 5, 0x0110, 0x10, 2, 3), + PIN_FIELD_BASE(166, 166, 5, 0x0110, 0x10, 5, 3), +}; + +static const struct mtk_pin_rsel mt8901_pin_rsel_val_range[] = { + 0 +}; + +static const unsigned int mt8901_pull_type[] = { + MTK_PULL_PUPD_R1R0_TYPE, /*0*/ + MTK_PULL_PUPD_R1R0_TYPE, /*1*/ + MTK_PULL_PUPD_R1R0_TYPE, /*2*/ + MTK_PULL_PUPD_R1R0_TYPE, /*3*/ + MTK_PULL_PUPD_R1R0_TYPE, /*4*/ + MTK_PULL_PUPD_R1R0_TYPE, /*5*/ + MTK_PULL_PUPD_R1R0_TYPE, /*6*/ + MTK_PULL_PUPD_R1R0_TYPE, /*7*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*8*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*9*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*10*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*11*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*12*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*13*/ + MTK_PULL_PUPD_R1R0_TYPE, /*14*/ + MTK_PULL_PUPD_R1R0_TYPE, /*15*/ + MTK_PULL_PUPD_R1R0_TYPE, /*16*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*17*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*18*/ + MTK_PULL_PUPD_R1R0_TYPE, /*19*/ + MTK_PULL_PUPD_R1R0_TYPE, /*20*/ + MTK_PULL_PUPD_R1R0_TYPE, /*21*/ + MTK_PULL_PUPD_R1R0_TYPE, /*22*/ + MTK_PULL_PUPD_R1R0_TYPE, /*23*/ + MTK_PULL_PUPD_R1R0_TYPE, /*24*/ + MTK_PULL_PUPD_R1R0_TYPE, /*25*/ + MTK_PULL_PUPD_R1R0_TYPE, /*26*/ + MTK_PULL_PUPD_R1R0_TYPE, /*27*/ + MTK_PULL_PUPD_R1R0_TYPE, /*28*/ + MTK_PULL_PUPD_R1R0_TYPE, /*29*/ + MTK_PULL_PUPD_R1R0_TYPE, /*30*/ + MTK_PULL_PUPD_R1R0_TYPE, /*31*/ + MTK_PULL_PUPD_R1R0_TYPE, /*32*/ + MTK_PULL_PUPD_R1R0_TYPE, /*33*/ + MTK_PULL_PUPD_R1R0_TYPE, /*34*/ + MTK_PULL_PUPD_R1R0_TYPE, /*35*/ + MTK_PULL_PUPD_R1R0_TYPE, /*36*/ + MTK_PULL_PUPD_R1R0_TYPE, /*37*/ + MTK_PULL_PUPD_R1R0_TYPE, /*38*/ + MTK_PULL_PUPD_R1R0_TYPE, /*39*/ + MTK_PULL_PUPD_R1R0_TYPE, /*40*/ + MTK_PULL_PUPD_R1R0_TYPE, /*41*/ + MTK_PULL_PUPD_R1R0_TYPE, /*42*/ + MTK_PULL_PUPD_R1R0_TYPE, /*43*/ + MTK_PULL_PUPD_R1R0_TYPE, /*44*/ + MTK_PULL_PUPD_R1R0_TYPE, /*45*/ + MTK_PULL_PUPD_R1R0_TYPE, /*46*/ + MTK_PULL_PUPD_R1R0_TYPE, /*47*/ + MTK_PULL_PUPD_R1R0_TYPE, /*48*/ + MTK_PULL_PUPD_R1R0_TYPE, /*49*/ + MTK_PULL_PUPD_R1R0_TYPE, /*50*/ + MTK_PULL_PUPD_R1R0_TYPE, /*51*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*52*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*53*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*54*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*55*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*56*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*57*/ + MTK_PULL_PUPD_R1R0_TYPE, /*58*/ + MTK_PULL_PU_PD_TYPE, /*59*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*60*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*61*/ + MTK_PULL_PUPD_R1R0_TYPE, /*62*/ + MTK_PULL_PUPD_R1R0_TYPE, /*63*/ + MTK_PULL_PUPD_R1R0_TYPE, /*64*/ + MTK_PULL_PUPD_R1R0_TYPE, /*65*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*66*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*67*/ + MTK_PULL_PUPD_R1R0_TYPE, /*68*/ + MTK_PULL_PU_PD_TYPE, /*69*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*70*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*71*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*72*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*73*/ + MTK_PULL_PUPD_R1R0_TYPE, /*74*/ + MTK_PULL_PUPD_R1R0_TYPE, /*75*/ + MTK_PULL_PUPD_R1R0_TYPE, /*76*/ + MTK_PULL_PUPD_R1R0_TYPE, /*77*/ + MTK_PULL_PUPD_R1R0_TYPE, /*78*/ + MTK_PULL_PUPD_R1R0_TYPE, /*79*/ + MTK_PULL_PUPD_R1R0_TYPE, /*80*/ + MTK_PULL_PUPD_R1R0_TYPE, /*81*/ + MTK_PULL_PUPD_R1R0_TYPE, /*82*/ + MTK_PULL_PUPD_R1R0_TYPE, /*83*/ + MTK_PULL_PUPD_R1R0_TYPE, /*84*/ + MTK_PULL_PUPD_R1R0_TYPE, /*85*/ + MTK_PULL_PUPD_R1R0_TYPE, /*86*/ + MTK_PULL_PUPD_R1R0_TYPE, /*87*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*88*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*89*/ + MTK_PULL_PUPD_R1R0_TYPE, /*90*/ + MTK_PULL_PUPD_R1R0_TYPE, /*91*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*92*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*93*/ + MTK_PULL_PUPD_R1R0_TYPE, /*94*/ + MTK_PULL_PUPD_R1R0_TYPE, /*95*/ + MTK_PULL_PUPD_R1R0_TYPE, /*96*/ + MTK_PULL_PUPD_R1R0_TYPE, /*97*/ + MTK_PULL_PUPD_R1R0_TYPE, /*98*/ + MTK_PULL_PUPD_R1R0_TYPE, /*99*/ + MTK_PULL_PUPD_R1R0_TYPE, /*100*/ + MTK_PULL_PUPD_R1R0_TYPE, /*101*/ + MTK_PULL_PUPD_R1R0_TYPE, /*102*/ + MTK_PULL_PUPD_R1R0_TYPE, /*103*/ + MTK_PULL_PUPD_R1R0_TYPE, /*104*/ + MTK_PULL_PUPD_R1R0_TYPE, /*105*/ + MTK_PULL_PUPD_R1R0_TYPE, /*106*/ + MTK_PULL_PUPD_R1R0_TYPE, /*107*/ + MTK_PULL_PUPD_R1R0_TYPE, /*108*/ + MTK_PULL_PUPD_R1R0_TYPE, /*109*/ + MTK_PULL_PUPD_R1R0_TYPE, /*110*/ + MTK_PULL_PUPD_R1R0_TYPE, /*111*/ + MTK_PULL_PUPD_R1R0_TYPE, /*112*/ + MTK_PULL_PUPD_R1R0_TYPE, /*113*/ + MTK_PULL_PUPD_R1R0_TYPE, /*114*/ + MTK_PULL_PUPD_R1R0_TYPE, /*115*/ + MTK_PULL_PUPD_R1R0_TYPE, /*116*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*117*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*118*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*119*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*120*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*121*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*122*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*123*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*124*/ + MTK_PULL_PUPD_R1R0_TYPE, /*125*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*126*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*127*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*128*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*129*/ + MTK_PULL_PUPD_R1R0_TYPE, /*130*/ + MTK_PULL_PUPD_R1R0_TYPE, /*131*/ + MTK_PULL_PUPD_R1R0_TYPE, /*132*/ + MTK_PULL_PUPD_R1R0_TYPE, /*133*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*134*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*135*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*136*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*137*/ + MTK_PULL_PUPD_R1R0_TYPE, /*138*/ + MTK_PULL_PUPD_R1R0_TYPE, /*139*/ + MTK_PULL_PUPD_R1R0_TYPE, /*140*/ + MTK_PULL_PUPD_R1R0_TYPE, /*141*/ + MTK_PULL_PUPD_R1R0_TYPE, /*142*/ + MTK_PULL_PUPD_R1R0_TYPE, /*143*/ + MTK_PULL_PUPD_R1R0_TYPE, /*144*/ + MTK_PULL_PUPD_R1R0_TYPE, /*145*/ + MTK_PULL_PUPD_R1R0_TYPE, /*146*/ + MTK_PULL_PUPD_R1R0_TYPE, /*147*/ + MTK_PULL_PUPD_R1R0_TYPE, /*148*/ + MTK_PULL_PUPD_R1R0_TYPE, /*149*/ + MTK_PULL_PUPD_R1R0_TYPE, /*150*/ + MTK_PULL_PUPD_R1R0_TYPE, /*151*/ + MTK_PULL_PUPD_R1R0_TYPE, /*152*/ + MTK_PULL_PUPD_R1R0_TYPE, /*153*/ + MTK_PULL_PUPD_R1R0_TYPE, /*154*/ + MTK_PULL_PUPD_R1R0_TYPE, /*155*/ + MTK_PULL_PUPD_R1R0_TYPE, /*156*/ + MTK_PULL_PUPD_R1R0_TYPE, /*157*/ + MTK_PULL_PUPD_R1R0_TYPE, /*158*/ + MTK_PULL_PUPD_R1R0_TYPE, /*159*/ + MTK_PULL_PUPD_R1R0_TYPE, /*160*/ + MTK_PULL_PUPD_R1R0_TYPE, /*161*/ + MTK_PULL_PUPD_R1R0_TYPE, /*162*/ + MTK_PULL_PUPD_R1R0_TYPE, /*163*/ + MTK_PULL_PUPD_R1R0_TYPE, /*164*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*165*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*166*/ + MTK_PULL_PUPD_R1R0_TYPE, /*167*/ + MTK_PULL_PUPD_R1R0_TYPE, /*168*/ + MTK_PULL_PUPD_R1R0_TYPE, /*169*/ + MTK_PULL_PUPD_R1R0_TYPE, /*170*/ + MTK_PULL_PUPD_R1R0_TYPE, /*171*/ + MTK_PULL_PUPD_R1R0_TYPE, /*172*/ + MTK_PULL_PUPD_R1R0_TYPE, /*173*/ + MTK_PULL_PUPD_R1R0_TYPE, /*174*/ + MTK_PULL_PUPD_R1R0_TYPE, /*175*/ + MTK_PULL_PUPD_R1R0_TYPE, /*176*/ + MTK_PULL_PUPD_R1R0_TYPE, /*177*/ + MTK_PULL_PUPD_R1R0_TYPE, /*178*/ + MTK_PULL_PUPD_R1R0_TYPE, /*179*/ + MTK_PULL_PUPD_R1R0_TYPE, /*180*/ + MTK_PULL_PU_PD_TYPE, /*181*/ +}; + +static const struct mtk_pin_reg_calc mt8901_reg_cals[PINCTRL_PIN_REG_MAX] = { + [PINCTRL_PIN_REG_MODE] = MTK_RANGE(mt8901_pin_mode_range), + [PINCTRL_PIN_REG_DIR] = MTK_RANGE(mt8901_pin_dir_range), + [PINCTRL_PIN_REG_DI] = MTK_RANGE(mt8901_pin_di_range), + [PINCTRL_PIN_REG_DO] = MTK_RANGE(mt8901_pin_do_range), + [PINCTRL_PIN_REG_SMT] = MTK_RANGE(mt8901_pin_smt_range), + [PINCTRL_PIN_REG_IES] = MTK_RANGE(mt8901_pin_ies_range), + [PINCTRL_PIN_REG_PUPD] = MTK_RANGE(mt8901_pin_pupd_range), + [PINCTRL_PIN_REG_R0] = MTK_RANGE(mt8901_pin_r0_range), + [PINCTRL_PIN_REG_R1] = MTK_RANGE(mt8901_pin_r1_range), + [PINCTRL_PIN_REG_PU] = MTK_RANGE(mt8901_pin_pu_range), + [PINCTRL_PIN_REG_PD] = MTK_RANGE(mt8901_pin_pd_range), + [PINCTRL_PIN_REG_DRV] = MTK_RANGE(mt8901_pin_drv_range), + [PINCTRL_PIN_REG_DRV_ADV] = MTK_RANGE(mt8901_pin_drv_adv_range), + [PINCTRL_PIN_REG_RSEL] = MTK_RANGE(mt8901_pin_rsel_range), +}; + +static const char * const mt8901_pinctrl_register_base_name[] = { + "iocfg0", "iocfg_lt2", "iocfg_lt3", "iocfg_rt1", "iocfg_rt2", "iocfg_rt3", + "iocfg_tr", "iocfg_rt0", "iocfg_lt1", "iocfg_lb", "iocfg_rb", +}; + +static const struct mtk_eint_hw mt8901_eint_hw = { + .port_mask = 0xf, + .ports = 7, + .ap_num = 209, + .db_cnt = 32, + .db_time = debounce_time_mt8901, +}; + +static const struct mtk_pin_soc mt8901_data = { + .reg_cal = mt8901_reg_cals, + .pins = mtk_pins_mt8901, + .npins = ARRAY_SIZE(mtk_pins_mt8901), + .ngrps = ARRAY_SIZE(mtk_pins_mt8901), + .eint_hw = &mt8901_eint_hw, + .eint_pin = eint_pins_mt8901, + .nfuncs = 8, + .gpio_m = 0, + .base_names = mt8901_pinctrl_register_base_name, + .nbase_names = ARRAY_SIZE(mt8901_pinctrl_register_base_name), + .pull_type = mt8901_pull_type, + .pin_rsel = mt8901_pin_rsel_val_range, + .npin_rsel = ARRAY_SIZE(mt8901_pin_rsel_val_range), /*numsel*/ + .bias_set_combo = mtk_pinconf_bias_set_combo, + .bias_get_combo = mtk_pinconf_bias_get_combo, + .drive_set = mtk_pinconf_drive_set_rev1, + .drive_get = mtk_pinconf_drive_get_rev1, + .adv_drive_set = mtk_pinconf_adv_drive_set_raw, + .adv_drive_get = mtk_pinconf_adv_drive_get_raw, +}; + +static const struct acpi_device_id mt8901_pinctrl_acpi_match[] = { + {"NVDA9221", (kernel_ulong_t)&mt8901_data }, + { } +}; +MODULE_DEVICE_TABLE(acpi, mt8901_pinctrl_acpi_match); + +static struct platform_driver mt8901_pinctrl_driver = { + .driver = { + .name = "mt8901-pinctrl", + .acpi_match_table = ACPI_PTR(mt8901_pinctrl_acpi_match), + .pm = pm_sleep_ptr(&mtk_paris_pinctrl_pm_ops) + }, + .probe = mtk_paris_pinctrl_probe, +}; + +static int __init mt8901_pinctrl_init(void) +{ + return platform_driver_register(&mt8901_pinctrl_driver); +} + +arch_initcall(mt8901_pinctrl_init); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("MediaTek MT8901 Pinctrl Driver"); diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-mt8901.h b/drivers/pinctrl/mediatek/pinctrl-mtk-mt8901.h new file mode 100644 index 0000000000000..fc64fc6ff5f9b --- /dev/null +++ b/drivers/pinctrl/mediatek/pinctrl-mtk-mt8901.h @@ -0,0 +1,2130 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2025 MediaTek Inc. + * + */ + +#ifndef __PINCTRL_MTK_MT8901_H +#define __PINCTRL_MTK_MT8901_H + +#include "pinctrl-paris.h" + +#define INVALID_BASE 0xFF + +static const struct mtk_pin_desc mtk_pins_mt8901[] = { + MTK_PIN( + 0, "GPIO0", + MTK_EINT_FUNCTION(0, 0), + DRV_GRP4, + MTK_FUNCTION(0, "B:GPIO0"), + MTK_FUNCTION(1, "O:ESPI_SCK") + ), + MTK_PIN( + 1, "GPIO1", + MTK_EINT_FUNCTION(0, 1), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO1"), + MTK_FUNCTION(1, "B1_ESPI_IO0") + ), + MTK_PIN( + 2, "GPIO2", + MTK_EINT_FUNCTION(0, 2), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO2"), + MTK_FUNCTION(1, "B1_ESPI_IO1") + ), + MTK_PIN( + 3, "GPIO3", + MTK_EINT_FUNCTION(0, 3), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO3"), + MTK_FUNCTION(1, "B1_ESPI_IO2") + ), + MTK_PIN( + 4, "GPIO4", + MTK_EINT_FUNCTION(0, 4), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO4"), + MTK_FUNCTION(1, "B1_ESPI_IO3") + ), + MTK_PIN( + 5, "GPIO5", + MTK_EINT_FUNCTION(0, 5), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO5"), + MTK_FUNCTION(1, "O_ESPI_CSN") + ), + MTK_PIN( + 6, "GPIO6", + MTK_EINT_FUNCTION(0, 6), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO6"), + MTK_FUNCTION(1, "O_ESPI_RESET_O"), + MTK_FUNCTION(2, "I1_ESPI_RESET_I") + ), + MTK_PIN( + 7, "GPIO7", + MTK_EINT_FUNCTION(0, 7), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO7"), + MTK_FUNCTION(1, "I1_ESPI_ALERT") + ), + MTK_PIN( + 8, "GPIO8", + MTK_EINT_FUNCTION(0, 8), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO8"), + MTK_FUNCTION(1, "B1_I2C_SCL3"), + MTK_FUNCTION(2, "B1_DISP_SCL2"), + MTK_FUNCTION(4, "O_PMSR_SMAP"), + MTK_FUNCTION(6, "O_MD32_0_TXD"), + MTK_FUNCTION(7, "O_MD32_1_TXD") + ), + MTK_PIN( + 9, "GPIO9", + MTK_EINT_FUNCTION(0, 9), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO9"), + MTK_FUNCTION(1, "B1_I2C_SDA3"), + MTK_FUNCTION(2, "B1_DISP_SDA2"), + MTK_FUNCTION(4, "O_PMSR_SMAP_MAX"), + MTK_FUNCTION(6, "I1_MD32_0_RXD"), + MTK_FUNCTION(7, "I1_MD32_1_RXD") + ), + MTK_PIN( + 10, "GPIO10", + MTK_EINT_FUNCTION(0, 10), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO10"), + MTK_FUNCTION(1, "B1_I2C_SCL4"), + MTK_FUNCTION(2, "B1_DISP_SCL2"), + MTK_FUNCTION(4, "O_PMSR_SMAP_MAX_W"), + MTK_FUNCTION(6, "O_MD32_0_GPIO0"), + MTK_FUNCTION(7, "O_MD32_1_GPIO0") + ), + MTK_PIN( + 11, "GPIO11", + MTK_EINT_FUNCTION(0, 11), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO11"), + MTK_FUNCTION(1, "B1_I2C_SDA4"), + MTK_FUNCTION(2, "B1_DISP_SDA2") + ), + MTK_PIN( + 12, "GPIO12", + MTK_EINT_FUNCTION(0, 12), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO12"), + MTK_FUNCTION(1, "B0_SPMI_M_SCL"), + MTK_FUNCTION(2, "B0_TP_GPIO31_AO") + ), + MTK_PIN( + 13, "GPIO13", + MTK_EINT_FUNCTION(0, 13), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO13"), + MTK_FUNCTION(1, "B0_SPMI_M_SDA"), + MTK_FUNCTION(2, "B0_TP_GPIO6_AO") + ), + MTK_PIN( + 14, "GPIO14", + MTK_EINT_FUNCTION(0, 14), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO14"), + MTK_FUNCTION(1, "I0_DPAUX_HPD_IN_2"), + MTK_FUNCTION(7, "O_DBG_MON_A0") + ), + MTK_PIN( + 15, "GPIO15", + MTK_EINT_FUNCTION(0, 15), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO15"), + MTK_FUNCTION(1, "I0_DPAUX_HPD_IN_3"), + MTK_FUNCTION(2, "B0_TP_GPIO25_AO"), + MTK_FUNCTION(7, "O_DBG_MON_A1") + ), + MTK_PIN( + 16, "GPIO16", + MTK_EINT_FUNCTION(0, 16), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO16"), + MTK_FUNCTION(1, "O_USB4_L_TCPC_RESET"), + MTK_FUNCTION(7, "O_DBG_MON_A18") + ), + MTK_PIN( + 17, "GPIO17", + MTK_EINT_FUNCTION(0, 17), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO17"), + MTK_FUNCTION(1, "B0_SPMI_P_SCL") + ), + MTK_PIN( + 18, "GPIO18", + MTK_EINT_FUNCTION(0, 18), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO18"), + MTK_FUNCTION(1, "B0_SPMI_P_SDA") + ), + MTK_PIN( + 19, "GPIO19", + MTK_EINT_FUNCTION(0, 19), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO19"), + MTK_FUNCTION(1, "B0_TP_GPIO29_AO") + ), + MTK_PIN( + 20, "GPIO20", + MTK_EINT_FUNCTION(0, 20), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO20"), + MTK_FUNCTION(1, "B0_TP_GPIO30_AO") + ), + MTK_PIN( + 21, "GPIO21", + MTK_EINT_FUNCTION(0, 21), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO21"), + MTK_FUNCTION(1, "B1_PROCHOT") + ), + MTK_PIN( + 22, "GPIO22", + MTK_EINT_FUNCTION(0, 22), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO22"), + MTK_FUNCTION(1, "I0_RTC32K_CK") + ), + MTK_PIN( + 23, "GPIO23", + MTK_EINT_FUNCTION(0, 23), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO23"), + MTK_FUNCTION(1, "B0_TP_GPIO0_AO") + ), + MTK_PIN( + 24, "GPIO24", + MTK_EINT_FUNCTION(0, 24), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO24"), + MTK_FUNCTION(2, "B0_TP_GPIO1_AO"), + MTK_FUNCTION(3, "O_CMMCLK1"), + MTK_FUNCTION(4, "O_SROOT_GPIO_O"), + MTK_FUNCTION(5, "O_MD32_10_TXD"), + MTK_FUNCTION(6, "O_MD32_11_TXD"), + MTK_FUNCTION(7, "O_DBG_MON_A3") + ), + MTK_PIN( + 25, "GPIO25", + MTK_EINT_FUNCTION(0, 25), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO25"), + MTK_FUNCTION(1, "B0_TP_GPIO2_AO"), + MTK_FUNCTION(2, "I0_VBUSVALID_0P"), + MTK_FUNCTION(4, "I0_SROOT_GPIO_I"), + MTK_FUNCTION(5, "I1_MD32_10_RXD"), + MTK_FUNCTION(6, "I1_MD32_11_RXD"), + MTK_FUNCTION(7, "O_DBG_MON_A4") + ), + MTK_PIN( + 26, "GPIO26", + MTK_EINT_FUNCTION(0, 26), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO26"), + MTK_FUNCTION(1, "B0_TP_GPIO3_AO"), + MTK_FUNCTION(5, "O_MD32_12_TXD"), + MTK_FUNCTION(6, "O_MD32_13_TXD"), + MTK_FUNCTION(7, "O_DBG_MON_A5") + ), + MTK_PIN( + 27, "GPIO27", + MTK_EINT_FUNCTION(0, 27), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO27"), + MTK_FUNCTION(1, "B0_TP_GPIO4_AO"), + MTK_FUNCTION(5, "I1_MD32_12_RXD"), + MTK_FUNCTION(6, "I1_MD32_13_RXD"), + MTK_FUNCTION(7, "O_DBG_MON_A6") + ), + MTK_PIN( + 28, "GPIO28", + MTK_EINT_FUNCTION(0, 28), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO28"), + MTK_FUNCTION(1, "B0_TP_GPIO5_AO"), + MTK_FUNCTION(5, "O_MD32_12_GPIO0"), + MTK_FUNCTION(6, "O_MD32_13_GPIO0"), + MTK_FUNCTION(7, "O_DBG_MON_A7") + ), + MTK_PIN( + 29, "GPIO29", + MTK_EINT_FUNCTION(0, 29), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO29"), + MTK_FUNCTION(1, "B1_THERMTRIP") + ), + MTK_PIN( + 30, "GPIO30", + MTK_EINT_FUNCTION(0, 30), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO30"), + MTK_FUNCTION(1, "B0_TP_GPIO7_AO"), + MTK_FUNCTION(2, "O_CMMCLK0"), + MTK_FUNCTION(4, "I0_CLUSTER0_SLV_CPUEB_JTAG_TRSTN"), + MTK_FUNCTION(5, "I0_CLUSTER1_SLV_CPUEB_JTAG_TRSTN"), + MTK_FUNCTION(6, "I0_OSROOT_GPIO_I"), + MTK_FUNCTION(7, "O_DBG_MON_A8") + ), + MTK_PIN( + 31, "GPIO31", + MTK_EINT_FUNCTION(0, 31), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO31"), + MTK_FUNCTION(1, "B0_TP_GPIO8_AO"), + MTK_FUNCTION(2, "O_CMMCLK1"), + MTK_FUNCTION(4, "I1_CLUSTER0_SLV_CPUEB_JTAG_TMS"), + MTK_FUNCTION(5, "I1_CLUSTER1_SLV_CPUEB_JTAG_TMS"), + MTK_FUNCTION(6, "O_OSROOT_GPIO_O"), + MTK_FUNCTION(7, "O_DBG_MON_A9") + ), + MTK_PIN( + 32, "GPIO32", + MTK_EINT_FUNCTION(0, 32), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO32"), + MTK_FUNCTION(1, "B0_TP_GPIO9_AO"), + MTK_FUNCTION(5, "O_SROOT_UTX"), + MTK_FUNCTION(6, "I1_TP_UCTS1_VLP"), + MTK_FUNCTION(7, "O_DBG_MON_A10") + ), + MTK_PIN( + 33, "GPIO33", + MTK_EINT_FUNCTION(0, 33), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO33"), + MTK_FUNCTION(1, "B0_TP_GPIO10_AO"), + MTK_FUNCTION(2, "O_CMMCLK2"), + MTK_FUNCTION(3, "I0_VBUSVALID_1P"), + MTK_FUNCTION(4, "I1_CLUSTER0_SLV_CPUEB_JTAG_TCK"), + MTK_FUNCTION(5, "I1_CLUSTER1_SLV_CPUEB_JTAG_TCK"), + MTK_FUNCTION(6, "I0_SROOT_TCK"), + MTK_FUNCTION(7, "I0_OSROOT_TCK") + ), + MTK_PIN( + 34, "GPIO34", + MTK_EINT_FUNCTION(0, 34), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO34"), + MTK_FUNCTION(1, "B0_TP_GPIO11_AO"), + MTK_FUNCTION(2, "O_CMMCLK4"), + MTK_FUNCTION(3, "I0_VBUSVALID_3P"), + MTK_FUNCTION(4, "I1_CLUSTER0_SLV_CPUEB_JTAG_TDI"), + MTK_FUNCTION(5, "I1_CLUSTER1_SLV_CPUEB_JTAG_TDI"), + MTK_FUNCTION(6, "I0_SROOT_TDI"), + MTK_FUNCTION(7, "I0_OSROOT_TDI") + ), + MTK_PIN( + 35, "GPIO35", + MTK_EINT_FUNCTION(0, 35), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO35"), + MTK_FUNCTION(1, "B0_TP_GPIO12_AO"), + MTK_FUNCTION(2, "O_SCP_PWM_1_VLP"), + MTK_FUNCTION(4, "O_CLUSTER0_SLV_CPUEB_JTAG_TDO"), + MTK_FUNCTION(5, "O_CLUSTER1_SLV_CPUEB_JTAG_TDO"), + MTK_FUNCTION(6, "O_SROOT_TDO"), + MTK_FUNCTION(7, "O_OSROOT_TDO") + ), + MTK_PIN( + 36, "GPIO36", + MTK_EINT_FUNCTION(0, 36), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO36"), + MTK_FUNCTION(1, "B0_TP_GPIO22_AO"), + MTK_FUNCTION(5, "I1_SROOT_URX"), + MTK_FUNCTION(6, "O_TP_URTS1_VLP"), + MTK_FUNCTION(7, "O_DBG_MON_A31") + ), + MTK_PIN( + 37, "GPIO37", + MTK_EINT_FUNCTION(0, 37), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO37"), + MTK_FUNCTION(1, "B0_TP_GPIO23_AO"), + MTK_FUNCTION(2, "O_CMMCLK3"), + MTK_FUNCTION(4, "O_SCP_PWM_2_VLP"), + MTK_FUNCTION(6, "O_MD32_5_GPIO0"), + MTK_FUNCTION(7, "O_DBG_MON_A11") + ), + MTK_PIN( + 38, "GPIO38", + MTK_EINT_FUNCTION(0, 38), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO38"), + MTK_FUNCTION(1, "B0_TP_GPIO24_AO"), + MTK_FUNCTION(2, "O_SCP_VREQ_VAO") + ), + MTK_PIN( + 39, "GPIO39", + MTK_EINT_FUNCTION(0, 39), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO39"), + MTK_FUNCTION(1, "B0_TP_GPIO13_AO"), + MTK_FUNCTION(2, "O_SCP_PWM_2_VLP"), + MTK_FUNCTION(3, "O_CMMCLK0"), + MTK_FUNCTION(4, "I0_VBUSVALID_2P"), + MTK_FUNCTION(5, "O_MD32_10_GPIO0"), + MTK_FUNCTION(6, "I0_SROOT_TMS"), + MTK_FUNCTION(7, "I0_OSROOT_TMS") + ), + MTK_PIN( + 40, "GPIO40", + MTK_EINT_FUNCTION(0, 40), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO40"), + MTK_FUNCTION(1, "B0_TP_GPIO14_AO"), + MTK_FUNCTION(2, "I0_VBUSVALID_1P"), + MTK_FUNCTION(3, "O_URTS2"), + MTK_FUNCTION(4, "O_TP_URTS2_VLP"), + MTK_FUNCTION(5, "O_SPMI_P_TRIG_FLAG"), + MTK_FUNCTION(6, "I1_MD32_5_RXD"), + MTK_FUNCTION(7, "O_DBG_MON_A13") + ), + MTK_PIN( + 41, "GPIO41", + MTK_EINT_FUNCTION(0, 41), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO41"), + MTK_FUNCTION(1, "B0_TP_GPIO15_AO"), + MTK_FUNCTION(2, "I0_VBUSVALID_0P"), + MTK_FUNCTION(3, "I1_UCTS2"), + MTK_FUNCTION(4, "I1_TP_UCTS2_VLP"), + MTK_FUNCTION(5, "O_SPMI_S_TRIG_FLAG"), + MTK_FUNCTION(6, "O_MD32_5_TXD"), + MTK_FUNCTION(7, "O_DBG_MON_A12") + ), + MTK_PIN( + 42, "GPIO42", + MTK_EINT_FUNCTION(0, 42), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO42"), + MTK_FUNCTION(1, "B0_TP_GPIO16_AO"), + MTK_FUNCTION(2, "O_CMMCLK3"), + MTK_FUNCTION(3, "O_UTXD2"), + MTK_FUNCTION(4, "O_TP_UTXD2_VLP"), + MTK_FUNCTION(5, "O_SPMI_M_TRIG_FLAG"), + MTK_FUNCTION(6, "I0_SROOT_NTRST"), + MTK_FUNCTION(7, "I0_OSROOT_NTRST") + ), + MTK_PIN( + 43, "GPIO43", + MTK_EINT_FUNCTION(0, 43), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO43"), + MTK_FUNCTION(1, "B0_TP_GPIO17_AO"), + MTK_FUNCTION(2, "O_CMMCLK4"), + MTK_FUNCTION(3, "I1_URXD2"), + MTK_FUNCTION(4, "I1_TP_URXD2_VLP"), + MTK_FUNCTION(5, "O_MD32_4_TXD"), + MTK_FUNCTION(6, "O_MD32PCM_UTXD_AO_VLP") + ), + MTK_PIN( + 44, "GPIO44", + MTK_EINT_FUNCTION(0, 44), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO44"), + MTK_FUNCTION(1, "B0_TP_GPIO18_AO"), + MTK_FUNCTION(2, "I1_TP_UCTS2_VLP"), + MTK_FUNCTION(3, "I1_UCTS2"), + MTK_FUNCTION(5, "I1_MD32_4_RXD"), + MTK_FUNCTION(6, "I1_MD32PCM_URXD_AO_VLP") + ), + MTK_PIN( + 45, "GPIO45", + MTK_EINT_FUNCTION(0, 45), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO45"), + MTK_FUNCTION(1, "B0_TP_GPIO19_AO"), + MTK_FUNCTION(2, "O_TP_URTS2_VLP"), + MTK_FUNCTION(3, "O_URTS2"), + MTK_FUNCTION(5, "O_MD32_4_GPIO0"), + MTK_FUNCTION(6, "O_MD32_11_GPIO0") + ), + MTK_PIN( + 46, "GPIO46", + MTK_EINT_FUNCTION(0, 46), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO46"), + MTK_FUNCTION(1, "B0_TP_GPIO20_AO"), + MTK_FUNCTION(2, "I0_VBUSVALID_2P"), + MTK_FUNCTION(3, "O_SCP_VREQ_VAO"), + MTK_FUNCTION(4, "O_SCP_PWM_1_VLP"), + MTK_FUNCTION(5, "O_SROOT_GPIO_O"), + MTK_FUNCTION(6, "O_OSROOT_GPIO_O") + ), + MTK_PIN( + 47, "GPIO47", + MTK_EINT_FUNCTION(0, 47), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO47"), + MTK_FUNCTION(1, "B0_TP_GPIO21_AO"), + MTK_FUNCTION(2, "I0_VBUSVALID_3P"), + MTK_FUNCTION(3, "O_CMMCLK2"), + MTK_FUNCTION(5, "I0_SROOT_GPIO_I"), + MTK_FUNCTION(6, "I0_OSROOT_GPIO_I"), + MTK_FUNCTION(7, "O_DBG_MON_A2") + ), + MTK_PIN( + 48, "GPIO48", + MTK_EINT_FUNCTION(0, 48), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO48"), + MTK_FUNCTION(1, "O_UTXD0"), + MTK_FUNCTION(2, "O_TP_UTXD1_VLP"), + MTK_FUNCTION(6, "O_ADSP_UTXD0"), + MTK_FUNCTION(7, "O_DBG_MON_A19") + ), + MTK_PIN( + 49, "GPIO49", + MTK_EINT_FUNCTION(0, 49), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO49"), + MTK_FUNCTION(1, "I1_URXD0"), + MTK_FUNCTION(2, "I1_TP_URXD1_VLP"), + MTK_FUNCTION(6, "I1_ADSP_URXD0"), + MTK_FUNCTION(7, "O_DBG_MON_A20") + ), + MTK_PIN( + 50, "GPIO50", + MTK_EINT_FUNCTION(0, 50), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO50"), + MTK_FUNCTION(1, "O_TP_UTXD2_VLP"), + MTK_FUNCTION(2, "O_UTXD2"), + MTK_FUNCTION(4, "B0_TP_GPIO26_AO"), + MTK_FUNCTION(5, "O_TP_UTXD1_VLP"), + MTK_FUNCTION(7, "O_SROOT_UTX") + ), + MTK_PIN( + 51, "GPIO51", + MTK_EINT_FUNCTION(0, 51), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO51"), + MTK_FUNCTION(1, "I1_TP_URXD2_VLP"), + MTK_FUNCTION(2, "I1_URXD2"), + MTK_FUNCTION(4, "B0_TP_GPIO27_AO"), + MTK_FUNCTION(5, "I1_TP_URXD1_VLP"), + MTK_FUNCTION(7, "I1_SROOT_URX") + ), + MTK_PIN( + 52, "GPIO52", + MTK_EINT_FUNCTION(0, 52), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO52"), + MTK_FUNCTION(1, "B1_USB4_L_PD_SCL"), + MTK_FUNCTION(7, "O_ADSP_UTXD0") + ), + MTK_PIN( + 53, "GPIO53", + MTK_EINT_FUNCTION(0, 53), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO53"), + MTK_FUNCTION(1, "B1_USB4_L_PD_SDA"), + MTK_FUNCTION(5, "O_MD32_7_TXD"), + MTK_FUNCTION(6, "O_MD32_6_TXD"), + MTK_FUNCTION(7, "I1_ADSP_URXD0") + ), + MTK_PIN( + 54, "GPIO54", + MTK_EINT_FUNCTION(0, 54), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO54"), + MTK_FUNCTION(1, "B1_USB4_R_PD_SCL"), + MTK_FUNCTION(4, "I1_CKM_SCL"), + MTK_FUNCTION(5, "I1_MD32_7_RXD"), + MTK_FUNCTION(6, "I1_USB4_L_PAR_SCL"), + MTK_FUNCTION(7, "O_PBUD_CTRL_UTXD_AO_VLP") + ), + MTK_PIN( + 55, "GPIO55", + MTK_EINT_FUNCTION(0, 55), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO55"), + MTK_FUNCTION(1, "B1_USB4_R_PD_SDA"), + MTK_FUNCTION(4, "B1_CKM_SDA"), + MTK_FUNCTION(5, "O_MD32_7_GPIO0"), + MTK_FUNCTION(6, "B1_USB4_L_PAR_SDA"), + MTK_FUNCTION(7, "I1_PBUD_CTRL_URXD_AO_VLP") + ), + MTK_PIN( + 56, "GPIO56", + MTK_EINT_FUNCTION(0, 56), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO56"), + MTK_FUNCTION(1, "B0_SPMI_S_SCL") + ), + MTK_PIN( + 57, "GPIO57", + MTK_EINT_FUNCTION(0, 57), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO57"), + MTK_FUNCTION(1, "B0_SPMI_S_SDA") + ), + MTK_PIN( + 58, "GPIO58", + MTK_EINT_FUNCTION(0, 58), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO58"), + MTK_FUNCTION(1, "O_WATCHDOG") + ), + MTK_PIN( + 59, "GPIO59", + MTK_EINT_FUNCTION(0, 59), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO59"), + MTK_FUNCTION(1, "B0_PAD_RESET_DRAM_0") + ), + MTK_PIN( + 60, "GPIO60", + MTK_EINT_FUNCTION(0, 60), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO60"), + MTK_FUNCTION(1, "B1_I2C_SCL0"), + MTK_FUNCTION(5, "O_MD32_2_TXD"), + MTK_FUNCTION(6, "O_MD32_3_TXD") + ), + MTK_PIN( + 61, "GPIO61", + MTK_EINT_FUNCTION(0, 61), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO61"), + MTK_FUNCTION(1, "B1_I2C_SDA0"), + MTK_FUNCTION(5, "I1_MD32_2_RXD"), + MTK_FUNCTION(6, "I1_MD32_3_RXD") + ), + MTK_PIN( + 62, "GPIO62", + MTK_EINT_FUNCTION(0, 62), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO62"), + MTK_FUNCTION(1, "I0_DMIC0_DAT"), + MTK_FUNCTION(6, "O_TP_UTXD1_VLP"), + MTK_FUNCTION(7, "O_DBG_MON_B0") + ), + MTK_PIN( + 63, "GPIO63", + MTK_EINT_FUNCTION(0, 63), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO63"), + MTK_FUNCTION(1, "O_DMIC0_CLK"), + MTK_FUNCTION(6, "I1_TP_URXD1_VLP"), + MTK_FUNCTION(7, "O_DBG_MON_B1") + ), + MTK_PIN( + 64, "GPIO64", + MTK_EINT_FUNCTION(0, 64), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO64"), + MTK_FUNCTION(1, "I0_DMIC1_DAT"), + MTK_FUNCTION(5, "O_MD32_2_GPIO0"), + MTK_FUNCTION(6, "O_MD32_3_GPIO0"), + MTK_FUNCTION(7, "O_DBG_MON_B2") + ), + MTK_PIN( + 65, "GPIO65", + MTK_EINT_FUNCTION(0, 65), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO65"), + MTK_FUNCTION(1, "O_DMIC1_CLK"), + MTK_FUNCTION(7, "O_DBG_MON_B3") + ), + MTK_PIN( + 66, "GPIO66", + MTK_EINT_FUNCTION(0, 66), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO66"), + MTK_FUNCTION(1, "O_SOUNDWIRE0_CK"), + MTK_FUNCTION(3, "I1_TP_UCTS0_VLP"), + MTK_FUNCTION(4, "O_SPI_HID_IRQ_S_MON0"), + MTK_FUNCTION(7, "O_VADSP_UTXD0") + ), + MTK_PIN( + 67, "GPIO67", + MTK_EINT_FUNCTION(0, 67), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO67"), + MTK_FUNCTION(1, "B0_SOUNDWIRE0_D0"), + MTK_FUNCTION(3, "O_TP_URTS0_VLP"), + MTK_FUNCTION(7, "I1_VADSP_URXD0") + ), + MTK_PIN( + 68, "GPIO68", + MTK_EINT_FUNCTION(0, 68), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO68"), + MTK_FUNCTION(1, "O_SCP_PWM_0_VLP"), + MTK_FUNCTION(2, "O_PWM_VLP"), + MTK_FUNCTION(3, "B0_TP_GPIO28_AO") + ), + MTK_PIN( + 69, "GPIO69", + MTK_EINT_FUNCTION(0, 69), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO69"), + MTK_FUNCTION(1, "B0_PAD_RESET_DRAM_8") + ), + MTK_PIN( + 70, "GPIO70", + MTK_EINT_FUNCTION(0, 70), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO70"), + MTK_FUNCTION(1, "B1_SCP_SCL0"), + MTK_FUNCTION(4, "I0_VADSP_JTAG0_TCK"), + MTK_FUNCTION(5, "I1_PCIE4_USB3_PAR_SCL"), + MTK_FUNCTION(6, "I1_SCP_JTAG0_TCK_VLP"), + MTK_FUNCTION(7, "B1_SROOT_SCL") + ), + MTK_PIN( + 71, "GPIO71", + MTK_EINT_FUNCTION(0, 71), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO71"), + MTK_FUNCTION(1, "B1_SCP_SDA0"), + MTK_FUNCTION(4, "I1_VADSP_JTAG0_TMS"), + MTK_FUNCTION(5, "B1_PCIE4_USB3_PAR_SDA"), + MTK_FUNCTION(6, "B1_SCP_JTAG0_TMS_VLP"), + MTK_FUNCTION(7, "B1_SROOT_SDA") + ), + MTK_PIN( + 72, "GPIO72", + MTK_EINT_FUNCTION(0, 72), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO72"), + MTK_FUNCTION(1, "B1_SCP_SCL2"), + MTK_FUNCTION(2, "B1_I3C_HCI_AO_SCL"), + MTK_FUNCTION(4, "I1_VADSP_JTAG0_TDI"), + MTK_FUNCTION(5, "I1_PCIE5_PAR_SCL"), + MTK_FUNCTION(6, "I1_SCP_JTAG0_TDI_VLP"), + MTK_FUNCTION(7, "O_OSROOT_UTX") + ), + MTK_PIN( + 73, "GPIO73", + MTK_EINT_FUNCTION(0, 73), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO73"), + MTK_FUNCTION(1, "B1_SCP_SDA2"), + MTK_FUNCTION(2, "B1_I3C_HCI_AO_SDA"), + MTK_FUNCTION(4, "O_VADSP_JTAG0_TDO"), + MTK_FUNCTION(5, "B1_PCIE5_PAR_SDA"), + MTK_FUNCTION(6, "O_SCP_JTAG0_TDO_VLP"), + MTK_FUNCTION(7, "I1_OSROOT_URX") + ), + MTK_PIN( + 74, "GPIO74", + MTK_EINT_FUNCTION(0, 74), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO74"), + MTK_FUNCTION(1, "I0_SCP_SPIS0_SCL"), + MTK_FUNCTION(2, "O_SPI2_CLK"), + MTK_FUNCTION(3, "O_SCP_SPIM0_CK"), + MTK_FUNCTION(4, "I1_SPM_JTAG_TCK_VLP"), + MTK_FUNCTION(5, "I1_SSPM_JTAG_TCK_VLP"), + MTK_FUNCTION(6, "I1_PBUD_CTRL_JTAG_TCK_VLP"), + MTK_FUNCTION(7, "I0_OSROOT_TCK") + ), + MTK_PIN( + 75, "GPIO75", + MTK_EINT_FUNCTION(0, 75), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO75"), + MTK_FUNCTION(1, "B0_SCP_SPIS0_SIO0"), + MTK_FUNCTION(2, "B0_SPI2_MI"), + MTK_FUNCTION(3, "B0_SCP_SPIM0_SIO0"), + MTK_FUNCTION(4, "I1_SPM_JTAG_TDI_VLP"), + MTK_FUNCTION(5, "I1_SSPM_JTAG_TDI_VLP"), + MTK_FUNCTION(6, "I1_PBUD_CTRL_JTAG_TDI_VLP"), + MTK_FUNCTION(7, "I0_OSROOT_TDI") + ), + MTK_PIN( + 76, "GPIO76", + MTK_EINT_FUNCTION(0, 76), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO76"), + MTK_FUNCTION(1, "B0_SCP_SPIS0_SIO1"), + MTK_FUNCTION(2, "B0_SPI2_MO"), + MTK_FUNCTION(3, "B0_SCP_SPIM0_SIO1"), + MTK_FUNCTION(4, "B1_SPM_JTAG_TDO_VLP"), + MTK_FUNCTION(5, "O_SSPM_JTAG_TDO_VLP"), + MTK_FUNCTION(6, "O_PBUD_CTRL_JTAG_TDO_VLP"), + MTK_FUNCTION(7, "O_OSROOT_TDO") + ), + MTK_PIN( + 77, "GPIO77", + MTK_EINT_FUNCTION(0, 77), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO77"), + MTK_FUNCTION(1, "B0_SCP_SPIS0_SIO2"), + MTK_FUNCTION(2, "B0_SPI2_WP"), + MTK_FUNCTION(3, "B0_SCP_SPIM0_SIO2"), + MTK_FUNCTION(4, "I1_SPM_JTAG_TMS_VLP"), + MTK_FUNCTION(5, "I1_SSPM_JTAG_TMS_VLP"), + MTK_FUNCTION(6, "I1_PBUD_CTRL_JTAG_TMS_VLP"), + MTK_FUNCTION(7, "I0_OSROOT_TMS") + ), + MTK_PIN( + 78, "GPIO78", + MTK_EINT_FUNCTION(0, 78), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO78"), + MTK_FUNCTION(1, "B0_SCP_SPIS0_SIO3"), + MTK_FUNCTION(2, "B0_SPI2_HOLD"), + MTK_FUNCTION(3, "B0_SCP_SPIM0_SIO3"), + MTK_FUNCTION(4, "I0_SPM_JTAG_TRSTN_VLP"), + MTK_FUNCTION(5, "I0_SSPM_JTAG_TRSTN_VLP"), + MTK_FUNCTION(6, "I0_PBUD_CTRL_JTAG_TRSTN_VLP"), + MTK_FUNCTION(7, "I0_OSROOT_NTRST") + ), + MTK_PIN( + 79, "GPIO79", + MTK_EINT_FUNCTION(0, 79), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO79"), + MTK_FUNCTION(1, "I1_SCP_SPIS0_CS"), + MTK_FUNCTION(2, "O_SPI2_CSB"), + MTK_FUNCTION(3, "O_SCP_SPIM0_CS"), + MTK_FUNCTION(4, "I1_VADSP_JTAG0_TRSTN"), + MTK_FUNCTION(6, "I0_SCP_JTAG0_TRSTN_VLP") + ), + MTK_PIN( + 80, "GPIO80", + MTK_EINT_FUNCTION(0, 80), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO80"), + MTK_FUNCTION(1, "O_SPI0_CLK"), + MTK_FUNCTION(2, "B0_SPI0_OSROOT_CLK"), + MTK_FUNCTION(4, "I1_SSPM_JTAG_TCK_VLP"), + MTK_FUNCTION(5, "I0_VADSP_JTAG0_TCK"), + MTK_FUNCTION(6, "I1_SPM_JTAG_TCK_VLP"), + MTK_FUNCTION(7, "I1_SCP_JTAG0_TCK_VLP") + ), + MTK_PIN( + 81, "GPIO81", + MTK_EINT_FUNCTION(0, 81), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO81"), + MTK_FUNCTION(1, "B0_SPI0_MI"), + MTK_FUNCTION(2, "I0_SPI0_OSROOT_MI"), + MTK_FUNCTION(4, "I1_SSPM_JTAG_TDI_VLP"), + MTK_FUNCTION(5, "I1_VADSP_JTAG0_TDI"), + MTK_FUNCTION(6, "I1_SPM_JTAG_TDI_VLP"), + MTK_FUNCTION(7, "I1_SCP_JTAG0_TDI_VLP") + ), + MTK_PIN( + 82, "GPIO82", + MTK_EINT_FUNCTION(0, 82), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO82"), + MTK_FUNCTION(1, "B0_SPI0_MO"), + MTK_FUNCTION(2, "O_SPI0_OSROOT_MO"), + MTK_FUNCTION(4, "O_SSPM_JTAG_TDO_VLP"), + MTK_FUNCTION(5, "O_VADSP_JTAG0_TDO"), + MTK_FUNCTION(6, "B1_SPM_JTAG_TDO_VLP"), + MTK_FUNCTION(7, "O_SCP_JTAG0_TDO_VLP") + ), + MTK_PIN( + 83, "GPIO83", + MTK_EINT_FUNCTION(0, 83), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO83"), + MTK_FUNCTION(1, "O_SPI0_CSB0"), + MTK_FUNCTION(2, "O_SPI0_OSROOT_CSB"), + MTK_FUNCTION(4, "I1_SSPM_JTAG_TMS_VLP"), + MTK_FUNCTION(5, "I1_VADSP_JTAG0_TMS"), + MTK_FUNCTION(6, "I1_SPM_JTAG_TMS_VLP"), + MTK_FUNCTION(7, "B1_SCP_JTAG0_TMS_VLP") + ), + MTK_PIN( + 84, "GPIO84", + MTK_EINT_FUNCTION(0, 84), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO84"), + MTK_FUNCTION(1, "O_SPI0_CSB1"), + MTK_FUNCTION(4, "I0_SSPM_JTAG_TRSTN_VLP"), + MTK_FUNCTION(5, "I1_VADSP_JTAG0_TRSTN"), + MTK_FUNCTION(6, "I0_SPM_JTAG_TRSTN_VLP"), + MTK_FUNCTION(7, "I0_SCP_JTAG0_TRSTN_VLP") + ), + MTK_PIN( + 85, "GPIO85", + MTK_EINT_FUNCTION(0, 85), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO85"), + MTK_FUNCTION(1, "O_DISP_PWM"), + MTK_FUNCTION(4, "B1_U4CP_JTAG_TMS"), + MTK_FUNCTION(5, "I0_CLUSTER0_UDI_TDI_1"), + MTK_FUNCTION(6, "I0_CLUSTER1_UDI_TDI_1"), + MTK_FUNCTION(7, "O_DBG_MON_A21") + ), + MTK_PIN( + 86, "GPIO86", + MTK_EINT_FUNCTION(0, 86), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO86"), + MTK_FUNCTION(1, "O_DISP_BL_EN"), + MTK_FUNCTION(4, "I1_U4CP_JTAG_TDI"), + MTK_FUNCTION(5, "O_CLUSTER0_UDI_TDO_1"), + MTK_FUNCTION(6, "O_CLUSTER1_UDI_TDO_1"), + MTK_FUNCTION(7, "O_DBG_MON_A22") + ), + MTK_PIN( + 87, "GPIO87", + MTK_EINT_FUNCTION(0, 87), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO87"), + MTK_FUNCTION(1, "B0_DISP_GPIO_N_1"), + MTK_FUNCTION(4, "O_U4CP_JTAG_TDO"), + MTK_FUNCTION(5, "I0_CLUSTER0_UDI_TDI_2"), + MTK_FUNCTION(6, "I0_CLUSTER1_UDI_TDI_2"), + MTK_FUNCTION(7, "O_DBG_MON_A23") + ), + MTK_PIN( + 88, "GPIO88", + MTK_EINT_FUNCTION(0, 88), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO88"), + MTK_FUNCTION(1, "B1_DISP_SCL1"), + MTK_FUNCTION(4, "I1_U4CP_JTAG_TCK"), + MTK_FUNCTION(5, "I1_EDP0_SCL"), + MTK_FUNCTION(6, "I1_HDMITX_DBG_I2C_SCL") + ), + MTK_PIN( + 89, "GPIO89", + MTK_EINT_FUNCTION(0, 89), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO89"), + MTK_FUNCTION(1, "B1_DISP_SDA1"), + MTK_FUNCTION(4, "I0_U4CP_JTAG_TRSTN"), + MTK_FUNCTION(5, "B1_EDP0_SDA"), + MTK_FUNCTION(6, "B1_HDMITX_DBG_I2C_SDA") + ), + MTK_PIN( + 90, "GPIO90", + MTK_EINT_FUNCTION(0, 90), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO90"), + MTK_FUNCTION(1, "B0_DISP_GPIO_N2"), + MTK_FUNCTION(4, "O_CLKM0_C"), + MTK_FUNCTION(5, "O_CLUSTER0_UDI_TDO_2"), + MTK_FUNCTION(6, "O_CLUSTER1_UDI_TDO_2"), + MTK_FUNCTION(7, "O_DBG_MON_A24") + ), + MTK_PIN( + 91, "GPIO91", + MTK_EINT_FUNCTION(0, 91), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO91"), + MTK_FUNCTION(1, "I0_DPAUX_HPD_IN_4"), + MTK_FUNCTION(4, "O_CLKM1_C"), + MTK_FUNCTION(5, "I0_CLUSTER0_UDI_TDI_3"), + MTK_FUNCTION(6, "I0_CLUSTER1_UDI_TDI_3"), + MTK_FUNCTION(7, "O_DBG_MON_A25") + ), + MTK_PIN( + 92, "GPIO92", + MTK_EINT_FUNCTION(0, 92), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO92"), + MTK_FUNCTION(1, "B1_I2C_SCL5"), + MTK_FUNCTION(2, "O_TP_UTXD0_VLP"), + MTK_FUNCTION(3, "O_SSPM_UTXD_AO_VLP"), + MTK_FUNCTION(4, "O_TSFDC_FOUT"), + MTK_FUNCTION(5, "O_CLUSTER0_UDI_TDO_3"), + MTK_FUNCTION(6, "O_CLUSTER1_UDI_TDO_3"), + MTK_FUNCTION(7, "O_DBG_MON_A26") + ), + MTK_PIN( + 93, "GPIO93", + MTK_EINT_FUNCTION(0, 93), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO93"), + MTK_FUNCTION(1, "B1_I2C_SDA5"), + MTK_FUNCTION(2, "I1_TP_URXD0_VLP"), + MTK_FUNCTION(3, "I1_SSPM_URXD_AO_VLP"), + MTK_FUNCTION(4, "O_TSFDC_SDO"), + MTK_FUNCTION(5, "I0_CLUSTER0_UDI_TDI_4"), + MTK_FUNCTION(6, "I0_CLUSTER1_UDI_TDI_4"), + MTK_FUNCTION(7, "O_DBG_MON_A27") + ), + MTK_PIN( + 94, "GPIO94", + MTK_EINT_FUNCTION(0, 94), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO94"), + MTK_FUNCTION(1, "O_CMFLASH0"), + MTK_FUNCTION(4, "I0_TSFDC_26M"), + MTK_FUNCTION(5, "O_CLUSTER0_UDI_TDO_4"), + MTK_FUNCTION(6, "O_CLUSTER1_UDI_TDO_4"), + MTK_FUNCTION(7, "O_DBG_MON_A28") + ), + MTK_PIN( + 95, "GPIO95", + MTK_EINT_FUNCTION(0, 95), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO95"), + MTK_FUNCTION(1, "O_CMFLASH1"), + MTK_FUNCTION(4, "I0_TSFDC_SCF"), + MTK_FUNCTION(5, "I0_CLUSTER0_UDI_TDI_5"), + MTK_FUNCTION(6, "I0_CLUSTER1_UDI_TDI_5"), + MTK_FUNCTION(7, "O_DBG_MON_A29") + ), + MTK_PIN( + 96, "GPIO96", + MTK_EINT_FUNCTION(0, 96), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO96"), + MTK_FUNCTION(1, "O_CMVREF0"), + MTK_FUNCTION(2, "O_CMFLASH1"), + MTK_FUNCTION(4, "I0_TSFDC_SCK"), + MTK_FUNCTION(5, "O_CLUSTER0_UDI_TDO_5"), + MTK_FUNCTION(6, "O_CLUSTER1_UDI_TDO_5"), + MTK_FUNCTION(7, "O_DBG_MON_A30") + ), + MTK_PIN( + 97, "GPIO97", + MTK_EINT_FUNCTION(0, 97), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO97"), + MTK_FUNCTION(1, "O_CMVREF1"), + MTK_FUNCTION(2, "O_CMFLASH0"), + MTK_FUNCTION(4, "I0_TSFDC_SDI"), + MTK_FUNCTION(5, "I0_CLUSTER0_UDI_TDI_6"), + MTK_FUNCTION(6, "I0_CLUSTER1_UDI_TDI_6"), + MTK_FUNCTION(7, "O_U4CP_UTXD") + ), + MTK_PIN( + 98, "GPIO98", + MTK_EINT_FUNCTION(0, 98), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO98"), + MTK_FUNCTION(2, "O_CMFLASH2"), + MTK_FUNCTION(4, "I0_RG_TSFDC_LDO_EN"), + MTK_FUNCTION(5, "O_CLUSTER0_UDI_TDO_6"), + MTK_FUNCTION(6, "O_CLUSTER1_UDI_TDO_6"), + MTK_FUNCTION(7, "I1_U4CP_URXD") + ), + MTK_PIN( + 99, "GPIO99", + MTK_EINT_FUNCTION(0, 99), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO99"), + MTK_FUNCTION(1, "I0_MCU_M_PMIC_POC_I"), + MTK_FUNCTION(4, "I0_DA_TSFDC_LDO_MODE"), + MTK_FUNCTION(5, "I0_CLUSTER0_UDI_TDI_7"), + MTK_FUNCTION(6, "I0_CLUSTER1_UDI_TDI_7"), + MTK_FUNCTION(7, "O_U4CP_URTS") + ), + MTK_PIN( + 100, "GPIO100", + MTK_EINT_FUNCTION(0, 100), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO100"), + MTK_FUNCTION(1, "I0_MCU_B_PMIC_POC_I"), + MTK_FUNCTION(4, "I0_RG_TSFDC_LDO_REFSEL1"), + MTK_FUNCTION(5, "O_CLUSTER0_UDI_TDO_7"), + MTK_FUNCTION(6, "O_CLUSTER1_UDI_TDO_7"), + MTK_FUNCTION(7, "I1_U4CP_UCTS") + ), + MTK_PIN( + 101, "GPIO101", + MTK_EINT_FUNCTION(0, 101), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO101"), + MTK_FUNCTION(1, "O_CMFLASH2"), + MTK_FUNCTION(2, "O_CMVREF1"), + MTK_FUNCTION(3, "I1_UCTS0"), + MTK_FUNCTION(4, "I0_RG_TSFDC_LDO_REFSEL0"), + MTK_FUNCTION(5, "I1_U4CP_JTAG_TCK"), + MTK_FUNCTION(6, "I1_PBUD_CTRL_JTAG_TCK_VCORE"), + MTK_FUNCTION(7, "O_CLKM0_A") + ), + MTK_PIN( + 102, "GPIO102", + MTK_EINT_FUNCTION(0, 102), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO102"), + MTK_FUNCTION(1, "O_CMFLASH3"), + MTK_FUNCTION(2, "O_CMVREF0"), + MTK_FUNCTION(3, "O_URTS0"), + MTK_FUNCTION(4, "O_TSFDC_BG_COMP"), + MTK_FUNCTION(5, "B1_U4CP_JTAG_TMS"), + MTK_FUNCTION(6, "I1_PBUD_CTRL_JTAG_TMS_VCORE"), + MTK_FUNCTION(7, "O_CLKM1_A") + ), + MTK_PIN( + 103, "GPIO103", + MTK_EINT_FUNCTION(0, 103), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO103"), + MTK_FUNCTION(1, "O_CMVREF2"), + MTK_FUNCTION(2, "O_UTXD0"), + MTK_FUNCTION(4, "O_CLKM2_B"), + MTK_FUNCTION(5, "I1_U4CP_JTAG_TDI"), + MTK_FUNCTION(6, "I1_PBUD_CTRL_JTAG_TDI_VCORE"), + MTK_FUNCTION(7, "O_CLKM2_A") + ), + MTK_PIN( + 104, "GPIO104", + MTK_EINT_FUNCTION(0, 104), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO104"), + MTK_FUNCTION(1, "O_CMVREF3"), + MTK_FUNCTION(2, "I1_URXD0"), + MTK_FUNCTION(4, "O_CLKM3_B"), + MTK_FUNCTION(5, "O_U4CP_JTAG_TDO"), + MTK_FUNCTION(6, "O_PBUD_CTRL_JTAG_TDO_VCORE"), + MTK_FUNCTION(7, "O_CLKM3_A") + ), + MTK_PIN( + 105, "GPIO105", + MTK_EINT_FUNCTION(0, 105), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO105"), + MTK_FUNCTION(2, "O_CMFLASH3"), + MTK_FUNCTION(4, "O_CLKM0_B"), + MTK_FUNCTION(5, "I0_U4CP_JTAG_TRSTN"), + MTK_FUNCTION(6, "I0_PBUD_CTRL_JTAG_TRSTN_VCORE"), + MTK_FUNCTION(7, "O_PMSR_SMAP") + ), + MTK_PIN( + 106, "GPIO106", + MTK_EINT_FUNCTION(0, 106), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO106"), + MTK_FUNCTION(1, "B0_SPINOR_CK") + ), + MTK_PIN( + 107, "GPIO107", + MTK_EINT_FUNCTION(0, 107), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO107"), + MTK_FUNCTION(1, "B0_SPINOR_IO0") + ), + MTK_PIN( + 108, "GPIO108", + MTK_EINT_FUNCTION(0, 108), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO108"), + MTK_FUNCTION(1, "B0_SPINOR_IO1") + ), + MTK_PIN( + 109, "GPIO109", + MTK_EINT_FUNCTION(0, 109), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO109"), + MTK_FUNCTION(1, "B0_SPINOR_IO2") + ), + MTK_PIN( + 110, "GPIO110", + MTK_EINT_FUNCTION(0, 110), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO110"), + MTK_FUNCTION(1, "B0_SPINOR_IO3") + ), + MTK_PIN( + 111, "GPIO111", + MTK_EINT_FUNCTION(0, 111), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO111"), + MTK_FUNCTION(1, "B1_SPINOR_CS") + ), + MTK_PIN( + 112, "GPIO112", + MTK_EINT_FUNCTION(0, 112), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO112"), + MTK_FUNCTION(1, "O_SPI1_CLK"), + MTK_FUNCTION(5, "I1_HFRP_JTAG1_TCK") + ), + MTK_PIN( + 113, "GPIO113", + MTK_EINT_FUNCTION(0, 113), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO113"), + MTK_FUNCTION(1, "B0_SPI1_MI"), + MTK_FUNCTION(5, "I1_HFRP_JTAG1_TMS") + ), + MTK_PIN( + 114, "GPIO114", + MTK_EINT_FUNCTION(0, 114), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO114"), + MTK_FUNCTION(1, "B0_SPI1_MO"), + MTK_FUNCTION(5, "I1_HFRP_JTAG1_TDI") + ), + MTK_PIN( + 115, "GPIO115", + MTK_EINT_FUNCTION(0, 115), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO115"), + MTK_FUNCTION(1, "O_SPI1_CSB0"), + MTK_FUNCTION(5, "O_HFRP_JTAG1_TDO") + ), + MTK_PIN( + 116, "GPIO116", + MTK_EINT_FUNCTION(0, 116), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO116"), + MTK_FUNCTION(1, "O_SPI1_CSB1"), + MTK_FUNCTION(5, "I0_HFRP_JTAG1_TRSTN") + ), + MTK_PIN( + 117, "GPIO117", + MTK_EINT_FUNCTION(0, 117), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO117"), + MTK_FUNCTION(1, "B1_I2C_SCL1"), + MTK_FUNCTION(2, "B1_OSROOT_SCL"), + MTK_FUNCTION(4, "O_SPI_CS_S_MON0"), + MTK_FUNCTION(5, "I1_USB4_L_PAR_SCL"), + MTK_FUNCTION(6, "I1_CKM_SCL"), + MTK_FUNCTION(7, "I1_USB4_R_PAR_SCL") + ), + MTK_PIN( + 118, "GPIO118", + MTK_EINT_FUNCTION(0, 118), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO118"), + MTK_FUNCTION(1, "B1_I2C_SDA1"), + MTK_FUNCTION(2, "B1_OSROOT_SDA"), + MTK_FUNCTION(4, "O_SPI_SCL_S_MON0"), + MTK_FUNCTION(5, "B1_USB4_L_PAR_SDA"), + MTK_FUNCTION(6, "B1_CKM_SDA"), + MTK_FUNCTION(7, "B1_USB4_R_PAR_SDA") + ), + MTK_PIN( + 119, "GPIO119", + MTK_EINT_FUNCTION(0, 119), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO119"), + MTK_FUNCTION(1, "B1_I2C_SCL2"), + MTK_FUNCTION(4, "I1_HDMITX_DBG_I2C_SCL"), + MTK_FUNCTION(5, "O_CLUSTER0_MBISTREADEN_TRIGGER"), + MTK_FUNCTION(6, "O_CLUSTER1_MBISTREADEN_TRIGGER"), + MTK_FUNCTION(7, "O_PMSR_SMAP_MAX") + ), + MTK_PIN( + 120, "GPIO120", + MTK_EINT_FUNCTION(0, 120), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO120"), + MTK_FUNCTION(1, "B1_I2C_SDA2+J130_S133"), + MTK_FUNCTION(4, "B1_HDMITX_DBG_I2C_SDA"), + MTK_FUNCTION(5, "O_CLUSTER0_MBISTWRITEEN_TRIGGER"), + MTK_FUNCTION(6, "O_CLUSTER1_MBISTWRITEEN_TRIGGER"), + MTK_FUNCTION(7, "O_PMSR_SMAP_MAX_W") + ), + MTK_PIN( + 121, "GPIO121", + MTK_EINT_FUNCTION(0, 121), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO121"), + MTK_FUNCTION(1, "B1_I3C_SCL0"), + MTK_FUNCTION(2, "B1_I3C_HCI_0_AO_SCL"), + MTK_FUNCTION(4, "I1_PCIE5_PAR_SCL"), + MTK_FUNCTION(5, "O_CLUSTER0_AD_ILDO_DTEST0"), + MTK_FUNCTION(6, "O_CLUSTER1_AD_ILDO_DTEST0"), + MTK_FUNCTION(7, "I1_USB4_R_PAR_SCL") + ), + MTK_PIN( + 122, "GPIO122", + MTK_EINT_FUNCTION(0, 122), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO122"), + MTK_FUNCTION(1, "B1_I3C_SDA0"), + MTK_FUNCTION(2, "B1_I3C_HCI_0_AO_SDA"), + MTK_FUNCTION(4, "B1_PCIE5_PAR_SDA"), + MTK_FUNCTION(5, "O_CLUSTER0_AD_ILDO_DTEST1"), + MTK_FUNCTION(6, "O_CLUSTER1_AD_ILDO_DTEST1"), + MTK_FUNCTION(7, "B1_USB4_R_PAR_SDA") + ), + MTK_PIN( + 123, "GPIO123", + MTK_EINT_FUNCTION(0, 123), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO123"), + MTK_FUNCTION(1, "B1_I3C_SCL1"), + MTK_FUNCTION(2, "B1_I3C_HCI_1_AO_SCL"), + MTK_FUNCTION(4, "I1_PCIE4_USB3_PAR_SCL"), + MTK_FUNCTION(5, "O_CLUSTER0_AD_ILDO_DTEST2"), + MTK_FUNCTION(6, "O_CLUSTER1_AD_ILDO_DTEST2"), + MTK_FUNCTION(7, "O_VADSP_UTXD0") + ), + MTK_PIN( + 124, "GPIO124", + MTK_EINT_FUNCTION(0, 124), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO124"), + MTK_FUNCTION(1, "B1_I3C_SDA1"), + MTK_FUNCTION(2, "B1_I3C_HCI_1_AO_SDA"), + MTK_FUNCTION(4, "B1_PCIE4_USB3_PAR_SDA"), + MTK_FUNCTION(5, "O_CLUSTER0_AD_ILDO_DTEST3"), + MTK_FUNCTION(6, "O_CLUSTER1_AD_ILDO_DTEST3"), + MTK_FUNCTION(7, "I1_VADSP_URXD0") + ), + MTK_PIN( + 125, "GPIO125", + MTK_EINT_FUNCTION(0, 125), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO125"), + MTK_FUNCTION(1, "O_I2SIN_1_MCK"), + MTK_FUNCTION(4, "I0_TSFDC_SCK"), + MTK_FUNCTION(6, "O_CLKM2_C") + ), + MTK_PIN( + 126, "GPIO126", + MTK_EINT_FUNCTION(0, 126), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO126"), + MTK_FUNCTION(1, "O_SOUNDWIRE1_CK"), + MTK_FUNCTION(2, "O_I2SIN_1_BCK"), + MTK_FUNCTION(4, "O_CLUSTER0_AD_ILDO_DTEST4"), + MTK_FUNCTION(5, "O_CLUSTER1_AD_ILDO_DTEST4"), + MTK_FUNCTION(6, "I0_ADSP_JTAG0_TCK"), + MTK_FUNCTION(7, "I1_HFRP_JTAG0_TCK") + ), + MTK_PIN( + 127, "GPIO127", + MTK_EINT_FUNCTION(0, 127), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO127"), + MTK_FUNCTION(1, "B0_SOUNDWIRE1_D0"), + MTK_FUNCTION(2, "O_I2SOUT1_DO"), + MTK_FUNCTION(4, "O_CLUSTER0_AD_ILDO_DTEST5"), + MTK_FUNCTION(5, "O_CLUSTER1_AD_ILDO_DTEST5"), + MTK_FUNCTION(6, "I1_ADSP_JTAG0_TMS"), + MTK_FUNCTION(7, "B1_HFRP_JTAG0_TMS") + ), + MTK_PIN( + 128, "GPIO128", + MTK_EINT_FUNCTION(0, 128), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO128"), + MTK_FUNCTION(1, "B0_SOUNDWIRE1_D1"), + MTK_FUNCTION(2, "I0_I2SIN_1_DI"), + MTK_FUNCTION(4, "O_CLUSTER0_AD_ILDO_DTEST6"), + MTK_FUNCTION(5, "O_CLUSTER1_AD_ILDO_DTEST6"), + MTK_FUNCTION(6, "I1_ADSP_JTAG0_TDI"), + MTK_FUNCTION(7, "I1_HFRP_JTAG0_TDI") + ), + MTK_PIN( + 129, "GPIO129", + MTK_EINT_FUNCTION(0, 129), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO129"), + MTK_FUNCTION(1, "B0_SOUNDWIRE1_D2"), + MTK_FUNCTION(2, "O_I2SIN_1_LRCK"), + MTK_FUNCTION(4, "O_CLUSTER0_AD_ILDO_DTEST7"), + MTK_FUNCTION(5, "O_CLUSTER1_AD_ILDO_DTEST7"), + MTK_FUNCTION(6, "O_ADSP_JTAG0_TDO"), + MTK_FUNCTION(7, "O_HFRP_JTAG0_TDO") + ), + MTK_PIN( + 130, "GPIO130", + MTK_EINT_FUNCTION(0, 130), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO130"), + MTK_FUNCTION(1, "B0_I2SIN0_BCK"), + MTK_FUNCTION(3, "O_DISP_CLKM0"), + MTK_FUNCTION(4, "I0_TSFDC_26M"), + MTK_FUNCTION(5, "I0_RG_TSFDC_LDO_EN"), + MTK_FUNCTION(6, "O_CCU0_URTS"), + MTK_FUNCTION(7, "O_DBG_MON_B4") + ), + MTK_PIN( + 131, "GPIO131", + MTK_EINT_FUNCTION(0, 131), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO131"), + MTK_FUNCTION(1, "I0_I2SIN0_DI"), + MTK_FUNCTION(3, "O_DISP_CLKM1"), + MTK_FUNCTION(4, "O_TSFDC_FOUT"), + MTK_FUNCTION(5, "I0_DA_TSFDC_LDO_MODE"), + MTK_FUNCTION(6, "I1_CCU0_UCTS"), + MTK_FUNCTION(7, "O_DBG_MON_B5") + ), + MTK_PIN( + 132, "GPIO132", + MTK_EINT_FUNCTION(0, 132), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO132"), + MTK_FUNCTION(1, "O_I2SOUT0_DO"), + MTK_FUNCTION(3, "O_DISP_CLKM2"), + MTK_FUNCTION(4, "O_TSFDC_SDO"), + MTK_FUNCTION(5, "I0_RG_TSFDC_LDO_REFSEL1"), + MTK_FUNCTION(6, "O_CCU1_URTS"), + MTK_FUNCTION(7, "O_DBG_MON_B6") + ), + MTK_PIN( + 133, "GPIO133", + MTK_EINT_FUNCTION(0, 133), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO133"), + MTK_FUNCTION(1, "B0_I2SIN0_LRCK"), + MTK_FUNCTION(3, "O_DISP_CLKM3"), + MTK_FUNCTION(4, "I0_TSFDC_SCF"), + MTK_FUNCTION(5, "I0_RG_TSFDC_LDO_REFSEL0"), + MTK_FUNCTION(6, "I1_CCU1_UCTS"), + MTK_FUNCTION(7, "O_DBG_MON_B7") + ), + MTK_PIN( + 134, "GPIO134", + MTK_EINT_FUNCTION(0, 134), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO134"), + MTK_FUNCTION(1, "B1_SCP_SCL1"), + MTK_FUNCTION(2, "B1_VADSP_SCL0"), + MTK_FUNCTION(3, "B1_SROOT_SCL"), + MTK_FUNCTION(4, "O_SSPM_UTXD_AO_VLP"), + MTK_FUNCTION(5, "O_SPI_HID_IRQ_S_MON0"), + MTK_FUNCTION(6, "I1_ADSP_JTAG0_TRSTN"), + MTK_FUNCTION(7, "O_DBG_MON_B8") + ), + MTK_PIN( + 135, "GPIO135", + MTK_EINT_FUNCTION(0, 135), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO135"), + MTK_FUNCTION(1, "B1_SCP_SDA1"), + MTK_FUNCTION(2, "B1_VADSP_SDA0"), + MTK_FUNCTION(3, "B1_SROOT_SDA"), + MTK_FUNCTION(4, "I1_SSPM_URXD_AO_VLP"), + MTK_FUNCTION(7, "O_DBG_MON_B9") + ), + MTK_PIN( + 136, "GPIO136", + MTK_EINT_FUNCTION(0, 136), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO136"), + MTK_FUNCTION(1, "O_CMVREF2"), + MTK_FUNCTION(2, "B1_SCP_SCL3"), + MTK_FUNCTION(3, "B1_VADSP_SCL0"), + MTK_FUNCTION(4, "B1_SCP_SCL1"), + MTK_FUNCTION(5, "I1_MD32_8_RXD"), + MTK_FUNCTION(6, "I1_MD32_9_RXD"), + MTK_FUNCTION(7, "O_DBG_MON_B10") + ), + MTK_PIN( + 137, "GPIO137", + MTK_EINT_FUNCTION(0, 137), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO137"), + MTK_FUNCTION(1, "O_CMVREF3"), + MTK_FUNCTION(2, "B1_SCP_SDA3"), + MTK_FUNCTION(3, "B1_VADSP_SDA0"), + MTK_FUNCTION(4, "B1_SCP_SDA1"), + MTK_FUNCTION(5, "O_MD32_8_GPIO0"), + MTK_FUNCTION(6, "O_MD32_9_GPIO0"), + MTK_FUNCTION(7, "O_DBG_MON_B11") + ), + MTK_PIN( + 138, "GPIO138", + MTK_EINT_FUNCTION(0, 138), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO138"), + MTK_FUNCTION(1, "B0_DISP_GPIO_N3"), + MTK_FUNCTION(2, "I0_DISP_LSPII"), + MTK_FUNCTION(4, "O_CLKM1_B"), + MTK_FUNCTION(5, "O_MD32_8_TXD"), + MTK_FUNCTION(6, "O_MD32_9_TXD"), + MTK_FUNCTION(7, "O_DBG_MON_B12") + ), + MTK_PIN( + 139, "GPIO139", + MTK_EINT_FUNCTION(0, 139), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO139"), + MTK_FUNCTION(1, "B0_DISP_GPIO_N4"), + MTK_FUNCTION(2, "O_DISP_HSYNC0"), + MTK_FUNCTION(3, "O_DISP_HSYNC1"), + MTK_FUNCTION(4, "O_CLKM0_A"), + MTK_FUNCTION(5, "O_CLKM0_B"), + MTK_FUNCTION(6, "O_CLKM0_C"), + MTK_FUNCTION(7, "O_DBG_MON_B13") + ), + MTK_PIN( + 140, "GPIO140", + MTK_EINT_FUNCTION(0, 140), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO140"), + MTK_FUNCTION(1, "B0_DISP_GPIO_N5"), + MTK_FUNCTION(2, "O_DISP_VSYNC0"), + MTK_FUNCTION(3, "O_DISP_VSYNC1"), + MTK_FUNCTION(4, "O_CLKM1_A"), + MTK_FUNCTION(5, "O_CLKM1_B"), + MTK_FUNCTION(6, "O_CLKM1_C"), + MTK_FUNCTION(7, "O_DBG_MON_A14") + ), + MTK_PIN( + 141, "GPIO141", + MTK_EINT_FUNCTION(0, 141), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO141"), + MTK_FUNCTION(1, "B0_DISP_GPIO_N6"), + MTK_FUNCTION(2, "O_DISP_HSYNC2"), + MTK_FUNCTION(3, "O_DISP_HSYNC3"), + MTK_FUNCTION(4, "O_CLKM2_A"), + MTK_FUNCTION(5, "O_CLKM2_B"), + MTK_FUNCTION(6, "O_CLKM2_C"), + MTK_FUNCTION(7, "O_DBG_MON_A15") + ), + MTK_PIN( + 142, "GPIO142", + MTK_EINT_FUNCTION(0, 142), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO142"), + MTK_FUNCTION(1, "B0_DISP_GPIO_N7"), + MTK_FUNCTION(2, "O_DISP_VSYNC2"), + MTK_FUNCTION(3, "O_DISP_VSYNC3"), + MTK_FUNCTION(4, "O_CLKM3_A"), + MTK_FUNCTION(5, "O_CLKM3_B"), + MTK_FUNCTION(6, "O_CLKM3_C"), + MTK_FUNCTION(7, "O_DBG_MON_A16") + ), + MTK_PIN( + 143, "GPIO143", + MTK_EINT_FUNCTION(0, 143), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO143"), + MTK_FUNCTION(1, "I0_MCU_M_PMIC_POC_I"), + MTK_FUNCTION(2, "I0_JTCK_SEL1"), + MTK_FUNCTION(3, "O_JTAGAP_JTCK"), + MTK_FUNCTION(4, "I0_ADSP_JTAG1_TCK"), + MTK_FUNCTION(5, "I0_ADSP_JTAG0_TCK"), + MTK_FUNCTION(6, "I0_CLUSTER0_UDI_TCK"), + MTK_FUNCTION(7, "I0_CLUSTER1_UDI_TCK") + ), + MTK_PIN( + 144, "GPIO144", + MTK_EINT_FUNCTION(0, 144), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO144"), + MTK_FUNCTION(1, "I0_MCU_B_PMIC_POC_I"), + MTK_FUNCTION(2, "B1_JTMS_SEL1"), + MTK_FUNCTION(3, "O_JTAGAP_JTMS"), + MTK_FUNCTION(4, "I1_ADSP_JTAG1_TMS"), + MTK_FUNCTION(5, "I1_ADSP_JTAG0_TMS"), + MTK_FUNCTION(6, "I0_CLUSTER0_UDI_TMS"), + MTK_FUNCTION(7, "I0_CLUSTER1_UDI_TMS") + ), + MTK_PIN( + 145, "GPIO145", + MTK_EINT_FUNCTION(0, 145), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO145"), + MTK_FUNCTION(1, "I1_UCTS0"), + MTK_FUNCTION(2, "I1_JTDI_SEL1"), + MTK_FUNCTION(3, "O_JTAGAP_JTDI"), + MTK_FUNCTION(4, "I1_ADSP_JTAG1_TDI"), + MTK_FUNCTION(5, "I1_ADSP_JTAG0_TDI"), + MTK_FUNCTION(6, "I0_CLUSTER0_UDI_TDI_0"), + MTK_FUNCTION(7, "I0_CLUSTER1_UDI_TDI_0") + ), + MTK_PIN( + 146, "GPIO146", + MTK_EINT_FUNCTION(0, 146), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO146"), + MTK_FUNCTION(1, "O_URTS0"), + MTK_FUNCTION(2, "O_JTDO_SEL1"), + MTK_FUNCTION(3, "I0_JTAGAP_JTDO"), + MTK_FUNCTION(4, "O_ADSP_JTAG1_TDO"), + MTK_FUNCTION(5, "O_ADSP_JTAG0_TDO"), + MTK_FUNCTION(6, "O_CLUSTER0_UDI_TDO_0"), + MTK_FUNCTION(7, "O_CLUSTER1_UDI_TDO_0") + ), + MTK_PIN( + 147, "GPIO147", + MTK_EINT_FUNCTION(0, 147), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO147"), + MTK_FUNCTION(2, "I1_JTRSTn_SEL1"), + MTK_FUNCTION(3, "O_JTAGAP_JTRSTn"), + MTK_FUNCTION(4, "I1_ADSP_JTAG1_TRSTN"), + MTK_FUNCTION(5, "I1_ADSP_JTAG0_TRSTN"), + MTK_FUNCTION(6, "I0_CLUSTER0_UDI_NTRST"), + MTK_FUNCTION(7, "I0_CLUSTER1_UDI_NTRST") + ), + MTK_PIN( + 148, "GPIO148", + MTK_EINT_FUNCTION(0, 148), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO148"), + MTK_FUNCTION(1, "O_SRCLKENA0") + ), + MTK_PIN( + 149, "GPIO149", + MTK_EINT_FUNCTION(0, 149), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO149"), + MTK_FUNCTION(1, "O_SRCLKENA1") + ), + MTK_PIN( + 150, "GPIO150", + MTK_EINT_FUNCTION(0, 150), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO150"), + MTK_FUNCTION(2, "O_NVJTAG_SEL"), + MTK_FUNCTION(4, "I0_TSFDC_SDI"), + MTK_FUNCTION(5, "O_TSFDC_BG_COMP"), + MTK_FUNCTION(6, "O_CLKM3_C"), + MTK_FUNCTION(7, "I0_HFRP_JTAG0_TRSTN") + ), + MTK_PIN( + 151, "GPIO151", + MTK_EINT_FUNCTION(0, 151), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO151"), + MTK_FUNCTION(1, "I1_JTRSTn_SEL1"), + MTK_FUNCTION(2, "O_JTAGAP_JTRSTn"), + MTK_FUNCTION(6, "I0_HFRP_JTAG1_TRSTN"), + MTK_FUNCTION(7, "I1_ADSP_JTAG1_TRSTN") + ), + MTK_PIN( + 152, "GPIO152", + MTK_EINT_FUNCTION(0, 152), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO152"), + MTK_FUNCTION(1, "B1_JTMS_SEL1"), + MTK_FUNCTION(2, "O_JTAGAP_JTMS"), + MTK_FUNCTION(6, "I1_HFRP_JTAG1_TMS"), + MTK_FUNCTION(7, "I1_ADSP_JTAG1_TMS") + ), + MTK_PIN( + 153, "GPIO153", + MTK_EINT_FUNCTION(0, 153), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO153"), + MTK_FUNCTION(1, "O_JTDO_SEL1"), + MTK_FUNCTION(2, "I0_JTAGAP_JTDO"), + MTK_FUNCTION(6, "O_HFRP_JTAG1_TDO"), + MTK_FUNCTION(7, "O_ADSP_JTAG1_TDO") + ), + MTK_PIN( + 154, "GPIO154", + MTK_EINT_FUNCTION(0, 154), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO154"), + MTK_FUNCTION(1, "I0_JTCK_SEL1"), + MTK_FUNCTION(2, "O_JTAGAP_JTCK"), + MTK_FUNCTION(6, "I1_HFRP_JTAG1_TCK"), + MTK_FUNCTION(7, "I0_ADSP_JTAG1_TCK") + ), + MTK_PIN( + 155, "GPIO155", + MTK_EINT_FUNCTION(0, 155), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO155"), + MTK_FUNCTION(1, "I1_JTDI_SEL1"), + MTK_FUNCTION(2, "O_JTAGAP_JTDI"), + MTK_FUNCTION(6, "I1_HFRP_JTAG1_TDI"), + MTK_FUNCTION(7, "I1_ADSP_JTAG1_TDI") + ), + MTK_PIN( + 156, "GPIO156", + MTK_EINT_FUNCTION(0, 156), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO156"), + MTK_FUNCTION(1, "O_TP_UTXD0_VLP"), + MTK_FUNCTION(2, "O_UTXD3"), + MTK_FUNCTION(3, "O_VADSP_UTXD1"), + MTK_FUNCTION(4, "O_SPI_SIO0_S_MON0"), + MTK_FUNCTION(5, "O_ADSP_UTXD1"), + MTK_FUNCTION(6, "I0_SROOT_TCK"), + MTK_FUNCTION(7, "O_OSROOT_UTX") + ), + MTK_PIN( + 157, "GPIO157", + MTK_EINT_FUNCTION(0, 157), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO157"), + MTK_FUNCTION(1, "I1_TP_URXD0_VLP"), + MTK_FUNCTION(2, "I1_URXD3"), + MTK_FUNCTION(3, "I1_VADSP_URXD1"), + MTK_FUNCTION(4, "O_SPI_SIO1_S_MON0"), + MTK_FUNCTION(5, "I1_ADSP_URXD1"), + MTK_FUNCTION(6, "I0_SROOT_TDI"), + MTK_FUNCTION(7, "I1_OSROOT_URX") + ), + MTK_PIN( + 158, "GPIO158", + MTK_EINT_FUNCTION(0, 158), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO158"), + MTK_FUNCTION(1, "O_TP_URTS0_VLP"), + MTK_FUNCTION(2, "O_URTS3"), + MTK_FUNCTION(3, "O_VADSP_URTS1"), + MTK_FUNCTION(4, "O_SPI_SIO2_S_MON0"), + MTK_FUNCTION(5, "O_ADSP_URTX1"), + MTK_FUNCTION(6, "O_SROOT_TDO"), + MTK_FUNCTION(7, "O_PBUD_CTRL_UTXD_AO_VLP") + ), + MTK_PIN( + 159, "GPIO159", + MTK_EINT_FUNCTION(0, 159), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO159"), + MTK_FUNCTION(1, "I1_TP_UCTS0_VLP"), + MTK_FUNCTION(2, "I1_UCTS3"), + MTK_FUNCTION(3, "I1_VADSP_UCTS1"), + MTK_FUNCTION(4, "O_SPI_SIO3_S_MON0"), + MTK_FUNCTION(5, "I1_ADSP_UCTS1"), + MTK_FUNCTION(6, "I0_SROOT_TMS"), + MTK_FUNCTION(7, "I1_PBUD_CTRL_URXD_AO_VLP") + ), + MTK_PIN( + 160, "GPIO160", + MTK_EINT_FUNCTION(0, 160), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO160"), + MTK_FUNCTION(1, "O_UTXD1"), + MTK_FUNCTION(2, "O_ADSP_UTXD1"), + MTK_FUNCTION(3, "O_HFRP_UTXD1"), + MTK_FUNCTION(4, "O_CCU1_UTXD"), + MTK_FUNCTION(5, "O_PBUD_CTRL_UTXD_AO_VCORE"), + MTK_FUNCTION(6, "O_U4CP_UTXD"), + MTK_FUNCTION(7, "O_DBG_MON_B14") + ), + MTK_PIN( + 161, "GPIO161", + MTK_EINT_FUNCTION(0, 161), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO161"), + MTK_FUNCTION(1, "I1_URXD1"), + MTK_FUNCTION(2, "I1_ADSP_URXD1"), + MTK_FUNCTION(3, "I1_HFRP_URXD1"), + MTK_FUNCTION(4, "I1_CCU1_URXD"), + MTK_FUNCTION(5, "I1_PBUD_CTRL_URXD_AO_VCORE"), + MTK_FUNCTION(6, "I1_U4CP_URXD"), + MTK_FUNCTION(7, "O_DBG_MON_B15") + ), + MTK_PIN( + 162, "GPIO162", + MTK_EINT_FUNCTION(0, 162), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO162"), + MTK_FUNCTION(1, "O_URTS1"), + MTK_FUNCTION(2, "O_ADSP_URTX1"), + MTK_FUNCTION(3, "O_HFRP_URTS1"), + MTK_FUNCTION(4, "O_CCU0_UTXD"), + MTK_FUNCTION(5, "O_PBUD_CTRL_UTXD_AO_VCORE"), + MTK_FUNCTION(6, "O_U4CP_URTS"), + MTK_FUNCTION(7, "O_DBG_MON_B16") + ), + MTK_PIN( + 163, "GPIO163", + MTK_EINT_FUNCTION(0, 163), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO163"), + MTK_FUNCTION(1, "I1_UCTS1"), + MTK_FUNCTION(2, "I1_ADSP_UCTS1"), + MTK_FUNCTION(3, "I1_HFRP_UCTS1"), + MTK_FUNCTION(4, "I1_CCU0_URXD"), + MTK_FUNCTION(5, "I1_PBUD_CTRL_URXD_AO_VCORE"), + MTK_FUNCTION(6, "I1_U4CP_UCTS"), + MTK_FUNCTION(7, "O_DBG_MON_B17") + ), + MTK_PIN( + 164, "GPIO164", + MTK_EINT_FUNCTION(0, 164), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO164"), + MTK_FUNCTION(1, "O_HDMITX_DC_CTRL"), + MTK_FUNCTION(6, "I0_SROOT_NTRST"), + MTK_FUNCTION(7, "O_DBG_MON_B18") + ), + MTK_PIN( + 165, "GPIO165", + MTK_EINT_FUNCTION(0, 165), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO165"), + MTK_FUNCTION(1, "B1_DISP_SCL0"), + MTK_FUNCTION(4, "I1_EDP0_SCL"), + MTK_FUNCTION(5, "O_CCU0_URTS"), + MTK_FUNCTION(6, "O_CCU1_URTS"), + MTK_FUNCTION(7, "I1_MD32_6_RXD") + ), + MTK_PIN( + 166, "GPIO166", + MTK_EINT_FUNCTION(0, 166), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO166"), + MTK_FUNCTION(1, "B1_DISP_SDA0"), + MTK_FUNCTION(4, "B1_EDP0_SDA"), + MTK_FUNCTION(5, "I1_CCU0_UCTS"), + MTK_FUNCTION(6, "I1_CCU1_UCTS"), + MTK_FUNCTION(7, "O_MD32_6_GPIO0") + ), + MTK_PIN( + 167, "GPIO167", + MTK_EINT_FUNCTION(0, 167), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO167"), + MTK_FUNCTION(1, "B0_DISP_GPIO_N0"), + MTK_FUNCTION(5, "O_CLUSTER0_AD_ILDO_DTEST0"), + MTK_FUNCTION(6, "O_CLUSTER1_AD_ILDO_DTEST0"), + MTK_FUNCTION(7, "O_DBG_MON_B19") + ), + MTK_PIN( + 168, "GPIO168", + MTK_EINT_FUNCTION(0, 168), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO168"), + MTK_FUNCTION(1, "I0_DPAUX_HPD_IN_0"), + MTK_FUNCTION(5, "O_CLUSTER0_AD_ILDO_DTEST1"), + MTK_FUNCTION(6, "O_CLUSTER1_AD_ILDO_DTEST1"), + MTK_FUNCTION(7, "O_DBG_MON_B20") + ), + MTK_PIN( + 169, "GPIO169", + MTK_EINT_FUNCTION(0, 169), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO169"), + MTK_FUNCTION(1, "I0_DPAUX_HPD_IN_1"), + MTK_FUNCTION(5, "O_CLUSTER0_AD_ILDO_DTEST3"), + MTK_FUNCTION(6, "O_CLUSTER1_AD_ILDO_DTEST3"), + MTK_FUNCTION(7, "O_DBG_MON_B21") + ), + MTK_PIN( + 170, "GPIO170", + MTK_EINT_FUNCTION(0, 170), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO170"), + MTK_FUNCTION(1, "O_USB4_R_TCPC_RESET"), + MTK_FUNCTION(5, "O_CLUSTER0_AD_ILDO_DTEST2"), + MTK_FUNCTION(6, "O_CLUSTER1_AD_ILDO_DTEST2"), + MTK_FUNCTION(7, "O_DBG_MON_A17") + ), + MTK_PIN( + 171, "GPIO171", + MTK_EINT_FUNCTION(0, 171), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO171"), + MTK_FUNCTION(1, "O_PCIE_PERSTN"), + MTK_FUNCTION(4, "O_MD32_14_GPIO0"), + MTK_FUNCTION(7, "O_DBG_MON_B22") + ), + MTK_PIN( + 172, "GPIO172", + MTK_EINT_FUNCTION(0, 172), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO172"), + MTK_FUNCTION(1, "B1_PCIE_CLKREQN_0P"), + MTK_FUNCTION(2, "I1_PCIE_PRSNT_0P"), + MTK_FUNCTION(5, "O_HFRP_UTXD1"), + MTK_FUNCTION(6, "O_CCU0_UTXD"), + MTK_FUNCTION(7, "O_DBG_MON_B23") + ), + MTK_PIN( + 173, "GPIO173", + MTK_EINT_FUNCTION(0, 173), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO173"), + MTK_FUNCTION(1, "B1_PCIE_CLKREQN_1P"), + MTK_FUNCTION(2, "I1_PCIE_PRSNT_1P"), + MTK_FUNCTION(5, "I1_HFRP_URXD1"), + MTK_FUNCTION(6, "I1_CCU0_URXD"), + MTK_FUNCTION(7, "O_DBG_MON_B24") + ), + MTK_PIN( + 174, "GPIO174", + MTK_EINT_FUNCTION(0, 174), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO174"), + MTK_FUNCTION(1, "B1_PCIE_CLKREQN_2P"), + MTK_FUNCTION(2, "I1_PCIE_PRSNT_2P"), + MTK_FUNCTION(4, "O_MD32PCM_UTXD_AO_VLP"), + MTK_FUNCTION(5, "O_HFRP_URTS1"), + MTK_FUNCTION(6, "O_CCU1_UTXD"), + MTK_FUNCTION(7, "O_DBG_MON_B25") + ), + MTK_PIN( + 175, "GPIO175", + MTK_EINT_FUNCTION(0, 175), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO175"), + MTK_FUNCTION(1, "B1_PCIE_CLKREQN_3P"), + MTK_FUNCTION(2, "I1_PCIE_PRSNT_3P"), + MTK_FUNCTION(4, "I1_MD32PCM_URXD_AO_VLP"), + MTK_FUNCTION(5, "I1_HFRP_UCTS1"), + MTK_FUNCTION(6, "I1_CCU1_URXD"), + MTK_FUNCTION(7, "O_DBG_MON_B26") + ), + MTK_PIN( + 176, "GPIO176", + MTK_EINT_FUNCTION(0, 176), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO176"), + MTK_FUNCTION(1, "B1_PCIE_CLKREQN_4P"), + MTK_FUNCTION(2, "I1_PCIE_PRSNT_4P"), + MTK_FUNCTION(4, "O_MD32_14_TXD"), + MTK_FUNCTION(7, "O_DBG_MON_B27") + ), + MTK_PIN( + 177, "GPIO177", + MTK_EINT_FUNCTION(0, 177), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO177"), + MTK_FUNCTION(1, "B1_PCIE_CLKREQN_5P"), + MTK_FUNCTION(2, "I1_PCIE_PRSNT_5P"), + MTK_FUNCTION(4, "I1_MD32_14_RXD"), + MTK_FUNCTION(7, "O_DBG_MON_B28") + ), + MTK_PIN( + 178, "GPIO178", + MTK_EINT_FUNCTION(0, 178), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO178"), + MTK_FUNCTION(1, "B1_PCIE_CLKREQN_6P"), + MTK_FUNCTION(2, "I1_PCIE_PRSNT_6P"), + MTK_FUNCTION(4, "O_MD32_15_TXD"), + MTK_FUNCTION(7, "O_DBG_MON_B29") + ), + MTK_PIN( + 179, "GPIO179", + MTK_EINT_FUNCTION(0, 179), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO179"), + MTK_FUNCTION(1, "B1_PCIE_CLKREQN_7P"), + MTK_FUNCTION(2, "I1_PCIE_PRSNT_7P"), + MTK_FUNCTION(4, "I1_MD32_15_RXD"), + MTK_FUNCTION(7, "O_DBG_MON_B30") + ), + MTK_PIN( + 180, "GPIO180", + MTK_EINT_FUNCTION(0, 180), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO180"), + MTK_FUNCTION(1, "I1_PCIE_WAKEN"), + MTK_FUNCTION(4, "O_MD32_15_GPIO0"), + MTK_FUNCTION(7, "O_DBG_MON_B31") + ), + MTK_PIN( + 181, "GPIO181", + MTK_EINT_FUNCTION(0, 181), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO181"), + MTK_FUNCTION(1, "O_GPU_PWRGOOD") + ), +}; + +static struct mtk_eint_pin eint_pins_mt8901[] = { + MTK_EINT_PIN(0, 0, 16, 0), + MTK_EINT_PIN(1, 0, 17, 0), + MTK_EINT_PIN(2, 0, 18, 0), + MTK_EINT_PIN(3, 0, 19, 0), + MTK_EINT_PIN(4, 0, 20, 0), + MTK_EINT_PIN(5, 0, 21, 0), + MTK_EINT_PIN(6, 0, 22, 0), + MTK_EINT_PIN(7, 0, 23, 0), + MTK_EINT_PIN(8, 0, 24, 0), + MTK_EINT_PIN(9, 0, 25, 0), + MTK_EINT_PIN(10, 0, 26, 0), + MTK_EINT_PIN(11, 0, 27, 0), + MTK_EINT_PIN(12, INVALID_BASE, 0, 0), + MTK_EINT_PIN(13, INVALID_BASE, 0, 0), + MTK_EINT_PIN(14, 0, 28, 0), + MTK_EINT_PIN(15, 0, 29, 0), + MTK_EINT_PIN(16, 0, 30, 0), + MTK_EINT_PIN(17, INVALID_BASE, 0, 0), + MTK_EINT_PIN(18, INVALID_BASE, 0, 0), + MTK_EINT_PIN(19, 0, 31, 0), + MTK_EINT_PIN(20, 0, 0, 1), + MTK_EINT_PIN(21, 0, 1, 1), + MTK_EINT_PIN(22, INVALID_BASE, 0, 0), + MTK_EINT_PIN(23, 0, 2, 1), + MTK_EINT_PIN(24, 0, 3, 1), + MTK_EINT_PIN(25, 0, 4, 1), + MTK_EINT_PIN(26, 0, 5, 1), + MTK_EINT_PIN(27, 0, 6, 1), + MTK_EINT_PIN(28, 0, 7, 1), + MTK_EINT_PIN(29, 0, 8, 1), + MTK_EINT_PIN(30, 0, 9, 1), + MTK_EINT_PIN(31, 0, 10, 1), + MTK_EINT_PIN(32, 1, 0, 1), + MTK_EINT_PIN(33, 0, 32, 0), + MTK_EINT_PIN(34, 0, 33, 0), + MTK_EINT_PIN(35, 0, 34, 0), + MTK_EINT_PIN(36, 1, 1, 1), + MTK_EINT_PIN(37, 0, 11, 1), + MTK_EINT_PIN(38, 0, 12, 1), + MTK_EINT_PIN(39, 0, 35, 0), + MTK_EINT_PIN(40, 0, 36, 0), + MTK_EINT_PIN(41, 0, 37, 0), + MTK_EINT_PIN(42, 0, 13, 1), + MTK_EINT_PIN(43, 0, 14, 1), + MTK_EINT_PIN(44, 0, 38, 0), + MTK_EINT_PIN(45, 0, 39, 0), + MTK_EINT_PIN(46, 0, 40, 0), + MTK_EINT_PIN(47, 0, 15, 1), + MTK_EINT_PIN(48, 0, 41, 0), + MTK_EINT_PIN(49, 0, 42, 0), + MTK_EINT_PIN(50, 0, 43, 0), + MTK_EINT_PIN(51, 0, 44, 0), + MTK_EINT_PIN(52, 0, 45, 0), + MTK_EINT_PIN(53, 0, 46, 0), + MTK_EINT_PIN(54, 2, 13, 0), + MTK_EINT_PIN(55, 2, 14, 0), + MTK_EINT_PIN(56, INVALID_BASE, 0, 0), + MTK_EINT_PIN(57, INVALID_BASE, 0, 0), + MTK_EINT_PIN(58, INVALID_BASE, 0, 0), + MTK_EINT_PIN(59, INVALID_BASE, 0, 0), + MTK_EINT_PIN(60, 1, 3, 0), + MTK_EINT_PIN(61, 1, 4, 0), + MTK_EINT_PIN(62, 1, 5, 0), + MTK_EINT_PIN(63, 1, 6, 0), + MTK_EINT_PIN(64, 1, 7, 0), + MTK_EINT_PIN(65, 1, 8, 0), + MTK_EINT_PIN(66, 2, 15, 0), + MTK_EINT_PIN(67, 2, 16, 0), + MTK_EINT_PIN(68, 1, 2, 1), + MTK_EINT_PIN(69, INVALID_BASE, 0, 0), + MTK_EINT_PIN(70, 2, 17, 0), + MTK_EINT_PIN(71, 2, 18, 0), + MTK_EINT_PIN(72, 2, 19, 0), + MTK_EINT_PIN(73, 2, 20, 0), + MTK_EINT_PIN(74, 2, 21, 0), + MTK_EINT_PIN(75, 2, 22, 0), + MTK_EINT_PIN(76, 2, 23, 0), + MTK_EINT_PIN(77, 2, 24, 0), + MTK_EINT_PIN(78, 2, 25, 0), + MTK_EINT_PIN(79, 2, 26, 0), + MTK_EINT_PIN(80, 2, 27, 0), + MTK_EINT_PIN(81, 2, 28, 0), + MTK_EINT_PIN(82, 2, 29, 1), + MTK_EINT_PIN(83, 2, 30, 1), + MTK_EINT_PIN(84, 2, 31, 1), + MTK_EINT_PIN(85, 2, 32, 1), + MTK_EINT_PIN(86, 2, 33, 0), + MTK_EINT_PIN(87, 2, 34, 0), + MTK_EINT_PIN(88, 2, 35, 0), + MTK_EINT_PIN(89, 2, 36, 0), + MTK_EINT_PIN(90, 2, 37, 0), + MTK_EINT_PIN(91, 2, 38, 0), + MTK_EINT_PIN(92, 2, 39, 0), + MTK_EINT_PIN(93, 2, 40, 0), + MTK_EINT_PIN(94, 2, 0, 1), + MTK_EINT_PIN(95, 2, 1, 1), + MTK_EINT_PIN(96, 2, 2, 1), + MTK_EINT_PIN(97, 2, 3, 1), + MTK_EINT_PIN(98, 2, 41, 0), + MTK_EINT_PIN(99, 2, 4, 1), + MTK_EINT_PIN(100, 2, 5, 1), + MTK_EINT_PIN(101, 2, 6, 1), + MTK_EINT_PIN(102, 2, 7, 1), + MTK_EINT_PIN(103, 2, 8, 1), + MTK_EINT_PIN(104, 2, 9, 1), + MTK_EINT_PIN(105, 2, 10, 1), + MTK_EINT_PIN(106, 0, 47, 0), + MTK_EINT_PIN(107, 0, 48, 0), + MTK_EINT_PIN(108, 0, 49, 0), + MTK_EINT_PIN(109, 0, 50, 0), + MTK_EINT_PIN(110, 0, 51, 0), + MTK_EINT_PIN(111, 0, 52, 0), + MTK_EINT_PIN(112, 2, 42, 0), + MTK_EINT_PIN(113, 2, 43, 0), + MTK_EINT_PIN(114, 2, 44, 0), + MTK_EINT_PIN(115, 2, 45, 0), + MTK_EINT_PIN(116, 2, 46, 0), + MTK_EINT_PIN(117, 2, 47, 0), + MTK_EINT_PIN(118, 2, 48, 0), + MTK_EINT_PIN(119, 2, 49, 0), + MTK_EINT_PIN(120, 2, 50, 0), + MTK_EINT_PIN(121, 2, 51, 0), + MTK_EINT_PIN(122, 2, 52, 0), + MTK_EINT_PIN(123, 2, 53, 0), + MTK_EINT_PIN(124, 2, 54, 0), + MTK_EINT_PIN(125, 2, 55, 0), + MTK_EINT_PIN(126, 2, 56, 0), + MTK_EINT_PIN(127, 2, 57, 0), + MTK_EINT_PIN(128, 2, 58, 0), + MTK_EINT_PIN(129, 2, 59, 0), + MTK_EINT_PIN(130, 2, 60, 0), + MTK_EINT_PIN(131, 2, 61, 0), + MTK_EINT_PIN(132, 2, 62, 0), + MTK_EINT_PIN(133, 2, 63, 0), + MTK_EINT_PIN(134, 2, 64, 0), + MTK_EINT_PIN(135, 2, 65, 0), + MTK_EINT_PIN(136, 2, 66, 0), + MTK_EINT_PIN(137, 2, 67, 0), + MTK_EINT_PIN(138, 2, 11, 1), + MTK_EINT_PIN(139, 2, 12, 1), + MTK_EINT_PIN(140, 2, 68, 0), + MTK_EINT_PIN(141, 2, 69, 0), + MTK_EINT_PIN(142, 2, 70, 0), + MTK_EINT_PIN(143, 2, 71, 0), + MTK_EINT_PIN(144, 2, 72, 0), + MTK_EINT_PIN(145, 2, 73, 0), + MTK_EINT_PIN(146, 2, 74, 0), + MTK_EINT_PIN(147, 2, 75, 0), + MTK_EINT_PIN(148, INVALID_BASE, 0, 0), + MTK_EINT_PIN(149, INVALID_BASE, 0, 0), + MTK_EINT_PIN(150, 2, 76, 0), + MTK_EINT_PIN(151, 2, 77, 0), + MTK_EINT_PIN(152, 2, 78, 0), + MTK_EINT_PIN(153, 2, 79, 0), + MTK_EINT_PIN(154, 2, 80, 0), + MTK_EINT_PIN(155, 2, 81, 0), + MTK_EINT_PIN(156, 2, 82, 0), + MTK_EINT_PIN(157, 2, 83, 0), + MTK_EINT_PIN(158, 2, 84, 0), + MTK_EINT_PIN(159, 2, 85, 0), + MTK_EINT_PIN(160, 2, 86, 0), + MTK_EINT_PIN(161, 2, 87, 0), + MTK_EINT_PIN(162, 2, 88, 0), + MTK_EINT_PIN(163, 2, 89, 0), + MTK_EINT_PIN(164, 2, 90, 0), + MTK_EINT_PIN(165, 2, 91, 0), + MTK_EINT_PIN(166, 2, 92, 0), + MTK_EINT_PIN(167, 2, 93, 0), + MTK_EINT_PIN(168, 2, 94, 0), + MTK_EINT_PIN(169, 2, 95, 0), + MTK_EINT_PIN(170, 2, 96, 0), + MTK_EINT_PIN(171, 2, 97, 0), + MTK_EINT_PIN(172, 2, 98, 0), + MTK_EINT_PIN(173, 2, 99, 0), + MTK_EINT_PIN(174, 2, 100, 0), + MTK_EINT_PIN(175, 2, 101, 0), + MTK_EINT_PIN(176, 2, 102, 0), + MTK_EINT_PIN(177, 2, 103, 0), + MTK_EINT_PIN(178, 2, 104, 0), + MTK_EINT_PIN(179, 2, 105, 0), + MTK_EINT_PIN(180, 2, 106, 0), + MTK_EINT_PIN(181, 3, 0, 0), + MTK_EINT_PIN(182, 3, 1, 0), + MTK_EINT_PIN(183, 3, 2, 0), + MTK_EINT_PIN(184, 3, 3, 0), + MTK_EINT_PIN(185, 3, 4, 0), + MTK_EINT_PIN(186, 3, 5, 0), + MTK_EINT_PIN(187, 3, 6, 0), + MTK_EINT_PIN(188, 3, 7, 0), + MTK_EINT_PIN(189, 3, 8, 0), + MTK_EINT_PIN(190, 3, 9, 0), + MTK_EINT_PIN(191, 3, 10, 0), + MTK_EINT_PIN(192, 3, 11, 0), + MTK_EINT_PIN(193, 3, 12, 0), + MTK_EINT_PIN(194, 3, 13, 0), + MTK_EINT_PIN(195, 3, 14, 0), + MTK_EINT_PIN(196, 3, 15, 0), + MTK_EINT_PIN(197, 3, 16, 0), + MTK_EINT_PIN(198, 3, 17, 0), + MTK_EINT_PIN(199, 3, 18, 0), + MTK_EINT_PIN(200, 3, 19, 0), + MTK_EINT_PIN(201, 3, 20, 0), + MTK_EINT_PIN(202, 3, 21, 0), + MTK_EINT_PIN(203, 3, 22, 0), + MTK_EINT_PIN(204, 3, 23, 0), + MTK_EINT_PIN(205, 3, 24, 0), + MTK_EINT_PIN(206, 3, 25, 0), + MTK_EINT_PIN(207, 3, 26, 0), + MTK_EINT_PIN(208, 3, 27, 0), +}; + +#endif /* __PINCTRL__MTK_MT8901_H */ From 8a2f86eaaa3900251e7808687258673837510ccc Mon Sep 17 00:00:00 2001 From: Abhishek Sahu Date: Tue, 22 Jul 2025 14:08:34 +0000 Subject: [PATCH 053/311] UBUNTU: [Config] nvidia: Update annotations to enable CONFIG_PINCTRL_MT8901 BugLink: https://bugs.launchpad.net/bugs/2117784 Signed-off-by: Abhishek Sahu Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Acked-by: nvmochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 0bd85d02570017780577755049df70668266ad78) (cherry picked from commit 0bd85d025700 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 94b2089f6ff5607413bf98b24c5bd62a70d87290 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- debian.nvidia/config/annotations | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index 8404149f7f758..d6b264f2030fe 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -162,6 +162,9 @@ CONFIG_NVIDIA_FFA_EC note<'LP: #2114230'> CONFIG_PID_IN_CONTEXTIDR policy<{'arm64': 'y'}> CONFIG_PID_IN_CONTEXTIDR note<'Required for Grace enablement'> +CONFIG_PINCTRL_MT8901 policy<{'arm64': 'y'}> +CONFIG_PINCTRL_MT8901 note<'LP: #2117784'> + CONFIG_R8127 policy<{'amd64': 'n', 'arm64': 'm'}> CONFIG_R8127 note<'LP: #2109730'> From 16d5a2fa1743756eb9c985c055fbd504a37fe537 Mon Sep 17 00:00:00 2001 From: Abhishek Sahu Date: Tue, 22 Jul 2025 13:45:33 +0000 Subject: [PATCH 054/311] NVIDIA: SAUCE: Fix FFH data response length BugLink: https://bugs.launchpad.net/bugs/2118357 commit d0038ee1df2b ("NVIDIA: SAUCE: Add support for EC secure service communication") added nvidia_ffh_handler() function. While copying the data back into ACPI FFH packet, it uses the request length. The response data can be larger than request length. The response length can't be fetched in the linux FFH handler function. We can copy all the bytes from ffa_data.data. The ACPI AML code will only use the required number bytes from this. Normally we don't need response length to be known. The ACPI table are not using that. It is parsing response data directly. In the latest revision of spec, the length field itself has been removed https://github.com/OpenDevicePartnership/documentation/blob/b23acb09f7cf03a5c3167509533f396d547e6291/guide_book/src/specs/ec_interface/secure-ec-services-overview.md#operation-region-definition For DIGITS GB10, it is using older revision of spec and the launch is planned with older revision of spec. When we move to latest revision, then we need to copy all data bytes for both request and response. The info->length is corresponding to FFH buffer length in ACPI table. Following is the code in ACPI table Name (_HID, "MSFT000C") // _HID: Hardware ID OperationRegion (AFFH, FFixedHW, 0x04, 0x90) info->length will be 0x90 (144) bytes. ffa_packet->length in the older revision is valid data bytes (https://github.com/OpenDevicePartnership/documentation/blob/45ad9b30be0f40e229deed2fef7a60d0b0b591f5/bookshelf/Shelf%204%20Specifications/EC%20Interface/src/secure-ec-services-overview.md) struct nvidia_ec_ffa_packet *ffa_packet = (struct nvidia_ec_ffa_packet *)value; This value buffer length should be info->length. We are taking minimum of sizeof(ffa_data.data) = 112 and (info->length = 144) - (offsetof(struct nvidia_ec_ffa_packet, rawdata) = 18) = 126, so ffh_copy_len will be 112 for the current DIGITS ACPI implementation. In the latest revision, this length mismatch is also fixed. Raw data will start at offset 32, so there both will come as 112. Fixes: d0038ee1df2b ("NVIDIA: SAUCE: Add support for EC secure service communication") Signed-off-by: Abhishek Sahu Acked-by: Carol L Soto Acked-by: Matthew R. Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 141bd5652ecc noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 0477ce56148b65bbaa946135a4791fe7d89b7e8d noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/platform/arm64/nvidia-ffa-ec.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/platform/arm64/nvidia-ffa-ec.c b/drivers/platform/arm64/nvidia-ffa-ec.c index 78068f1237b57..d9e8b7fdda30c 100644 --- a/drivers/platform/arm64/nvidia-ffa-ec.c +++ b/drivers/platform/arm64/nvidia-ffa-ec.c @@ -530,6 +530,7 @@ static int nvidia_ffh_handler(struct acpi_ffh_info *info, acpi_integer *value, v struct nvidia_ec_ffa_packet *ffa_packet = (struct nvidia_ec_ffa_packet *)value; struct nvidia_ec_ffa_device *cur, *ec_dev = NULL; int ret; + unsigned int ffh_copy_len; uuid_t uuid; /* Only offset 4 is supported */ @@ -592,9 +593,16 @@ static int nvidia_ffh_handler(struct acpi_ffh_info *info, acpi_integer *value, v /* Set the status as success */ ffa_packet->status = 0; - /* Copy the ACPI FFA data back into ACPI FFH packet */ - memcpy(ffa_packet->rawdata, ffa_data.data, ffa_packet->length); + /* + * Copy the ACPI FFA data back into ACPI FFH packet. + * + * ACPI FFH packet raw data length can't be fetched here, so copy + * all bytes from ffa_data.data + */ + ffh_copy_len = min(sizeof(ffa_data.data), + info->length - offsetof(struct nvidia_ec_ffa_packet, rawdata)); + memcpy(ffa_packet->rawdata, ffa_data.data, ffh_copy_len); return 0; } From f68287fc25efafded11865c44791435ebc4d63b7 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Mon, 4 Dec 2023 22:38:25 +0000 Subject: [PATCH 055/311] NVIDIA: SAUCE: arm64: configs: Build NVGRACE_GPU_VFIO_PCI as LKM BugLink: https://bugs.launchpad.net/bugs/2119656 Signed-off-by: Nicolin Chen Signed-off-by: Ankit Agrawal Signed-off-by: Matthew R. Ochs Acked-by: Kai-Heng Feng Acked-by: Koba Ko Signed-off-by: Matthew R. Ochs (cherry picked from commit 9433fd4ac5f0d1a63feed968a8b16261fcd7d808 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.8-next) Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit a1bdf88a26695bd4a255bdad7c263fe9d6d2ab58 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 60f9b04529b6 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 13f98cc45ca813feca952ff9c61bd3129338a115 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- arch/arm64/configs/defconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index b67d5b1fc45b0..4ea813d63f65f 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1957,3 +1957,4 @@ CONFIG_CORESIGHT_STM=m CONFIG_CORESIGHT_CPU_DEBUG=m CONFIG_CORESIGHT_CTI=m CONFIG_MEMTEST=y +CONFIG_NVGRACE_GPU_VFIO_PCI=m From 04297fdb389fcd4fb693610c034f8dbaca164c2a Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Tue, 7 Nov 2023 04:07:47 -0800 Subject: [PATCH 056/311] NVIDIA: SAUCE: arm64: configs: Enable IOMMUFD and VFIO_DEVICE_CDEV BugLink: https://bugs.launchpad.net/bugs/2119656 Signed-off-by: Nicolin Chen Signed-off-by: Matthew R. Ochs Acked-by: Kai-Heng Feng Acked-by: Koba Ko Signed-off-by: Matthew R. Ochs (cherry picked from commit 3eff6df2e892f9ea4a564ac27ae8fc005ad054be https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.8-next) Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit 6c6e8936e0f502f9225ad38070f90d259266ffb8 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit a6a3ccc38385 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit ecc87b5069293ce3a37fa15c4d870cb7b4abaa8e noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- arch/arm64/configs/defconfig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 4ea813d63f65f..678b993e77828 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1958,3 +1958,8 @@ CONFIG_CORESIGHT_CPU_DEBUG=m CONFIG_CORESIGHT_CTI=m CONFIG_MEMTEST=y CONFIG_NVGRACE_GPU_VFIO_PCI=m +CONFIG_VFIO_DEVICE_CDEV=y +CONFIG_FAULT_INJECTION=y +CONFIG_IOMMUFD_DRIVER=y +CONFIG_IOMMUFD=y +CONFIG_IOMMUFD_TEST=y From bcfcef95ff7a687786adcd46483bf3e6e730f47a Mon Sep 17 00:00:00 2001 From: Ankit Agrawal Date: Thu, 29 Aug 2024 08:15:40 +0000 Subject: [PATCH 057/311] NVIDIA: SAUCE: vfio/nvgrace-egm: Introduce module to manage EGM BugLink: https://bugs.launchpad.net/bugs/2119656 The Extended GPU Memory (EGM) feature enables the GPU access to the system memory across sockets and nodes. In this mode, the physical memory can be allocated for GPU usage from anywhere in a multi-node system. The feature is being extended to virtualization. EGM when enabled in the virtualization stack, the host memory is partitioned into 2: One partition for the Host OS usage, and a second EGM region. The EGM region essentially becomes the system memory of the VM. The following figure shows the memory map in the virtualization environment. |---- Sysmem ----| |--- GPU mem ---| VM Memory Map | | | | | | | | |------ EGM -----|--Host Mem----| |--- GPU mem ---| Host Memory Map The EGM region is not available to the host memory for its usage as it is not added to the kernel. Its base HPA and the length is communicated through the DSDT entries. A linear mapping between the VM IPA and system HPA is a requirement for EGM support. The EGM region is thus assigned to a VM by mapping the QEMU VMA to a linearly increasing HPA of the EGM region using remap_pfn_range(). Introduce a new nvgrace-egm helper module to nvgrace-gpu to manage the EGM/VM region for the VM. nvgrace-egm module handles the following: 1. Fetch the EGM memory properties (base HPA, length, proximity domain). 2. Create a char device that can be used as memory-backend-file by Qemu for the VM and implement file operations. The char device is /dev/egmX, where X is the PXM node ID of the EGM being mapped fetched in 1. 3. Zero the EGM memory on first device open(). 4. Map the QEMU VMA to the EGM region using remap_pfn_range. 5. Cleaning up state and destroying the chardev on device unbind. Signed-off-by: Ankit Agrawal Signed-off-by: Matthew R. Ochs Acked-by: Kai-Heng Feng Acked-by: Koba Ko Signed-off-by: Matthew R. Ochs (cherry picked from commit 892ac2417c614969ff215ad75c0249af6073ffb9 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.8-next) Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit 3a1b8196060afeaec7b37a1300706d59642e8212 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off-by: Brad Figg (cherry picked from commit 8807f4b90409 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (backported from commit fa304984adc4ac07d2fd10d68c1cea99e2b7c12f noble:linux-nvidia-6.17) [jacobmartin: adjust patch context to align with upstream commit e5f19b619fa0 ("vfio/nvgrace-gpu: register device memory for poison handling"), as opposed to the original SAUCE version of the same patch.] Signed-off-by: Jacob Martin --- drivers/vfio/pci/nvgrace-gpu/Kconfig | 11 ++ drivers/vfio/pci/nvgrace-gpu/Makefile | 3 + drivers/vfio/pci/nvgrace-gpu/egm.c | 235 ++++++++++++++++++++++++++ drivers/vfio/pci/nvgrace-gpu/egm.h | 12 ++ drivers/vfio/pci/nvgrace-gpu/main.c | 31 +++- 5 files changed, 291 insertions(+), 1 deletion(-) create mode 100644 drivers/vfio/pci/nvgrace-gpu/egm.c create mode 100644 drivers/vfio/pci/nvgrace-gpu/egm.h diff --git a/drivers/vfio/pci/nvgrace-gpu/Kconfig b/drivers/vfio/pci/nvgrace-gpu/Kconfig index a7f624b37e410..d5773bbd22f5e 100644 --- a/drivers/vfio/pci/nvgrace-gpu/Kconfig +++ b/drivers/vfio/pci/nvgrace-gpu/Kconfig @@ -1,8 +1,19 @@ # SPDX-License-Identifier: GPL-2.0-only +config NVGRACE_EGM + tristate "EGM driver for NVIDIA Grace Hopper and Blackwell Superchip" + depends on ARM64 || (COMPILE_TEST && 64BIT) + help + Extended GPU Memory (EGM) support for the GPU in the NVIDIA Grace + based chips required to avail the CPU memory as additional + cross-node/cross-socket memory for GPU using KVM/qemu. + + If you don't know what to do here, say N. + config NVGRACE_GPU_VFIO_PCI tristate "VFIO support for the GPU in the NVIDIA Grace Hopper Superchip" depends on ARM64 || (COMPILE_TEST && 64BIT) select VFIO_PCI_CORE + select NVGRACE_EGM help VFIO support for the GPU in the NVIDIA Grace Hopper Superchip is required to assign the GPU device to userspace using KVM/qemu/etc. diff --git a/drivers/vfio/pci/nvgrace-gpu/Makefile b/drivers/vfio/pci/nvgrace-gpu/Makefile index 3ca8c187897a9..c99b04a94e770 100644 --- a/drivers/vfio/pci/nvgrace-gpu/Makefile +++ b/drivers/vfio/pci/nvgrace-gpu/Makefile @@ -1,3 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_NVGRACE_GPU_VFIO_PCI) += nvgrace-gpu-vfio-pci.o nvgrace-gpu-vfio-pci-y := main.o + +obj-$(CONFIG_NVGRACE_EGM) += nvgrace-egm.o +nvgrace-egm-y := egm.o diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.c b/drivers/vfio/pci/nvgrace-gpu/egm.c new file mode 100644 index 0000000000000..f3c22a9dfecb9 --- /dev/null +++ b/drivers/vfio/pci/nvgrace-gpu/egm.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved + */ + +#include +#include "egm.h" + +#define MAX_EGM_NODES 256 + +struct egm_region { + struct list_head list; + int egmpxm; + atomic_t open_count; + phys_addr_t egmphys; + size_t egmlength; + struct device device; + struct cdev cdev; +}; + +static dev_t dev; +static struct class *class; +static struct list_head egm_list; + +static int nvgrace_egm_open(struct inode *inode, struct file *file) +{ + void *memaddr; + struct egm_region *region = container_of(inode->i_cdev, + struct egm_region, cdev); + + if (!region) + return -EINVAL; + + if (atomic_inc_return(®ion->open_count) > 1) + return 0; + + memaddr = memremap(region->egmphys, region->egmlength, MEMREMAP_WB); + if (!memaddr) { + atomic_dec(®ion->open_count); + return -EINVAL; + } + + memset((u8 *)memaddr, 0, region->egmlength); + memunmap(memaddr); + file->private_data = region; + + return 0; +} + +static int nvgrace_egm_release(struct inode *inode, struct file *file) +{ + struct egm_region *region = container_of(inode->i_cdev, + struct egm_region, cdev); + + if (!region) + return -EINVAL; + + if (atomic_dec_and_test(®ion->open_count)) + file->private_data = NULL; + + return 0; +} + +static int nvgrace_egm_mmap(struct file *file, struct vm_area_struct *vma) +{ + int ret = 0; + struct egm_region *region = file->private_data; + + if (!region) + return -EINVAL; + + ret = remap_pfn_range(vma, vma->vm_start, + PHYS_PFN(region->egmphys), + (vma->vm_end - vma->vm_start), + vma->vm_page_prot); + return ret; +} + +static const struct file_operations file_ops = { + .owner = THIS_MODULE, + .open = nvgrace_egm_open, + .release = nvgrace_egm_release, + .mmap = nvgrace_egm_mmap, +}; + +static int setup_egm_chardev(struct egm_region *region) +{ + int ret = 0; + + device_initialize(®ion->device); + + /* + * Use the proximity domain number as the device minor + * number. So the EGM corresponding to node X would be + * /dev/egmX. + */ + region->device.devt = MKDEV(MAJOR(dev), region->egmpxm); + region->device.class = class; + cdev_init(®ion->cdev, &file_ops); + region->cdev.owner = THIS_MODULE; + + ret = dev_set_name(®ion->device, "egm%d", region->egmpxm); + if (ret) + return ret; + + ret = cdev_device_add(®ion->cdev, ®ion->device); + + return ret; +} + +static int +nvgrace_gpu_fetch_egm_property(struct pci_dev *pdev, u64 *pegmphys, + u64 *pegmlength, u64 *pegmpxm) +{ + int ret; + + /* + * The memory information is present in the system ACPI tables as DSD + * properties nvidia,egm-base-pa and nvidia,egmm-size. + */ + ret = device_property_read_u64(&pdev->dev, "nvidia,egm-size", + pegmlength); + if (ret) + return ret; + + if (*pegmlength > type_max(size_t)) + return -EOVERFLOW; + + ret = device_property_read_u64(&pdev->dev, "nvidia,egm-base-pa", + pegmphys); + if (ret) + return ret; + + if (*pegmphys > type_max(phys_addr_t)) + return -EOVERFLOW; + + ret = device_property_read_u64(&pdev->dev, "nvidia,egm-pxm", + pegmpxm); + + if (*pegmpxm > type_max(phys_addr_t)) + return -EOVERFLOW; + + return ret; +} + +int register_egm_node(struct pci_dev *pdev) +{ + struct egm_region *region = NULL; + u64 egmphys, egmlength, egmpxm; + int ret; + + ret = nvgrace_gpu_fetch_egm_property(pdev, &egmphys, &egmlength, &egmpxm); + if (ret) + return ret; + + list_for_each_entry(region, &egm_list, list) { + if (region->egmphys == egmphys) + return 0; + } + + region = kvzalloc(sizeof(*region), GFP_KERNEL); + region->egmphys = egmphys; + region->egmlength = egmlength; + region->egmpxm = egmpxm; + + atomic_set(®ion->open_count, 0); + + list_add_tail(®ion->list, &egm_list); + + setup_egm_chardev(region); + + return 0; +} +EXPORT_SYMBOL_GPL(register_egm_node); + +static void destroy_egm_chardev(struct egm_region *region) +{ + cdev_device_del(®ion->cdev, ®ion->device); +} + +void unregister_egm_node(int egm_node) +{ + struct egm_region *region, *temp_region; + + list_for_each_entry_safe(region, temp_region, &egm_list, list) { + if (egm_node == region->egmpxm) { + destroy_egm_chardev(region); + list_del(®ion->list); + } + } +} +EXPORT_SYMBOL_GPL(unregister_egm_node); + +static char *egm_devnode(const struct device *device, umode_t *mode) +{ + if (mode) + *mode = 0600; + + return NULL; +} + +static int __init nvgrace_egm_init(void) +{ + int ret; + + ret = alloc_chrdev_region(&dev, + 0, MAX_EGM_NODES, "egm"); + if (ret < 0) + return ret; + + class = class_create("egm"); + if (IS_ERR(class)) { + unregister_chrdev_region(dev, MAX_EGM_NODES); + return PTR_ERR(class); + } + + class->devnode = egm_devnode; + + INIT_LIST_HEAD(&egm_list); + + return 0; +} + +static void __exit nvgrace_egm_cleanup(void) +{ + class_destroy(class); + unregister_chrdev_region(dev, MAX_EGM_NODES); +} + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Ankit Agrawal "); +MODULE_DESCRIPTION("NVGRACE EGM - Helper module of NVGRACE GPU to support Extended GPU Memory"); + +module_init(nvgrace_egm_init); +module_exit(nvgrace_egm_cleanup); diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.h b/drivers/vfio/pci/nvgrace-gpu/egm.h new file mode 100644 index 0000000000000..28cc59e04a0b0 --- /dev/null +++ b/drivers/vfio/pci/nvgrace-gpu/egm.h @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved + */ + +#ifndef NVGRACE_EGM_H +#define NVGRACE_EGM_H + +int register_egm_node(struct pci_dev *pdev); +void unregister_egm_node(int egm_node); + +#endif /* NVGRACE_EGM_H */ diff --git a/drivers/vfio/pci/nvgrace-gpu/main.c b/drivers/vfio/pci/nvgrace-gpu/main.c index fa056b69f899a..fe2ab87a1aed8 100644 --- a/drivers/vfio/pci/nvgrace-gpu/main.c +++ b/drivers/vfio/pci/nvgrace-gpu/main.c @@ -10,6 +10,7 @@ #include #include #include +#include "egm.h" /* * The device memory usable to the workloads running in the VM is cached @@ -64,8 +65,11 @@ struct nvgrace_gpu_pci_core_device { bool has_mig_hw_bug; /* GPU has just been reset */ bool reset_done; + int egm_node; }; +static bool egm_enabled; + static void nvgrace_gpu_init_fake_bar_emu_regs(struct vfio_device *core_vdev) { struct nvgrace_gpu_pci_core_device *nvdev = @@ -1012,6 +1016,13 @@ nvgrace_gpu_fetch_memory_property(struct pci_dev *pdev, return ret; } +static int +nvgrace_gpu_has_egm_property(struct pci_dev *pdev, u64 *pegmpxm) +{ + return device_property_read_u64(&pdev->dev, "nvidia,egm-pxm", + pegmpxm); +} + static int nvgrace_gpu_init_nvdev_struct(struct pci_dev *pdev, struct nvgrace_gpu_pci_core_device *nvdev, @@ -1181,6 +1192,7 @@ static int nvgrace_gpu_probe(struct pci_dev *pdev, const struct vfio_device_ops *ops = &nvgrace_gpu_pci_core_ops; struct nvgrace_gpu_pci_core_device *nvdev; u64 memphys, memlength; + u64 egmpxm; int ret; ret = nvgrace_gpu_probe_check_device_ready(pdev); @@ -1188,9 +1200,14 @@ static int nvgrace_gpu_probe(struct pci_dev *pdev, return ret; ret = nvgrace_gpu_fetch_memory_property(pdev, &memphys, &memlength); - if (!ret) + if (!ret) { ops = &nvgrace_gpu_pci_ops; + ret = nvgrace_gpu_has_egm_property(pdev, &egmpxm); + if (!ret) + egm_enabled = true; + } + nvdev = vfio_alloc_device(nvgrace_gpu_pci_core_device, core_device.vdev, &pdev->dev, ops); if (IS_ERR(nvdev)) @@ -1210,6 +1227,12 @@ static int nvgrace_gpu_probe(struct pci_dev *pdev, if (ret) goto out_put_vdev; nvdev->core_device.pci_ops = &nvgrace_gpu_pci_dev_ops; + + if (egm_enabled) { + register_egm_node(pdev); + nvdev->egm_node = egmpxm; + } + } else { nvdev->core_device.pci_ops = &nvgrace_gpu_pci_dev_core_ops; } @@ -1228,6 +1251,12 @@ static int nvgrace_gpu_probe(struct pci_dev *pdev, static void nvgrace_gpu_remove(struct pci_dev *pdev) { struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev); + struct nvgrace_gpu_pci_core_device *nvdev = + container_of(core_device, struct nvgrace_gpu_pci_core_device, + core_device); + + if (egm_enabled) + unregister_egm_node(nvdev->egm_node); vfio_pci_core_unregister_device(core_device); vfio_put_device(&core_device->vdev); From f12fc5417d1a317e0f041dd222b79696feed9ffd Mon Sep 17 00:00:00 2001 From: Ankit Agrawal Date: Thu, 29 Aug 2024 08:15:41 +0000 Subject: [PATCH 058/311] NVIDIA: SAUCE: vfio/nvgrace-egm: Handle pages with ECC errors on the EGM BugLink: https://bugs.launchpad.net/bugs/2119656 It is possible for some system memory pages on the EGM to have uncorrectable ECC errors. A list of pages known with such errors (referred as retired pages) are maintained by the Host UEFI. The Host UEFI populates such list in a reserved region. It communicates the SPA of this region through a ACPI DSDT property. nvgrace-egm module is responsible to store the list of retired page offsets to be made available for usermode processes. The module: 1. Get the reserved memory region SPA and maps to it to fetch the list of bad pages. 2. Calculate the retired page offsets in the EGM and stores it. 3. Expose an ioctl to allow querying of the offsets. The ioctl is called by usermode apps such as QEMU to get the retired page offsets. The usermode apps are expected to take appropriate action to communicate the list to the VM. Signed-off-by: Ankit Agrawal Signed-off-by: Matthew R. Ochs Acked-by: Kai-Heng Feng Acked-by: Koba Ko Signed-off-by: Matthew R. Ochs (cherry picked from commit be54641b9f3e52a471e9d02aa12723bfb47a7060 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.8-next) Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit c4cb1930d93ac2c7bb4f0cfba0a9e3e4ff180879 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 6b0a6d6644e3 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 64942451faf3b08f9f3014d70c871fe0ffcf041f noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/vfio/pci/nvgrace-gpu/egm.c | 126 +++++++++++++++++++++++++++++ include/uapi/linux/egm.h | 26 ++++++ 2 files changed, 152 insertions(+) create mode 100644 include/uapi/linux/egm.h diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.c b/drivers/vfio/pci/nvgrace-gpu/egm.c index f3c22a9dfecb9..8c9ff6313e9f4 100644 --- a/drivers/vfio/pci/nvgrace-gpu/egm.c +++ b/drivers/vfio/pci/nvgrace-gpu/egm.c @@ -4,6 +4,8 @@ */ #include +#include +#include #include "egm.h" #define MAX_EGM_NODES 256 @@ -16,6 +18,12 @@ struct egm_region { size_t egmlength; struct device device; struct cdev cdev; + DECLARE_HASHTABLE(htbl, 0x10); +}; + +struct h_node { + unsigned long mem_offset; + struct hlist_node node; }; static dev_t dev; @@ -76,11 +84,80 @@ static int nvgrace_egm_mmap(struct file *file, struct vm_area_struct *vma) return ret; } +static long nvgrace_egm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + unsigned long minsz = offsetofend(struct egm_bad_pages_list, count); + struct egm_bad_pages_list info; + void __user *uarg = (void __user *)arg; + struct egm_region *region = file->private_data; + + if (copy_from_user(&info, uarg, minsz)) + return -EFAULT; + + if (info.argsz < minsz) + return -EINVAL; + + if (!region) + return -EINVAL; + + switch (cmd) { + case EGM_BAD_PAGES_LIST: + int ret; + unsigned long bad_page_struct_size = sizeof(struct egm_bad_pages_info); + struct egm_bad_pages_info tmp; + struct h_node *cur_page; + struct hlist_node *tmp_node; + unsigned long bkt; + int count = 0, index = 0; + + hash_for_each_safe(region->htbl, bkt, tmp_node, cur_page, node) + count++; + + if (info.argsz < (minsz + count * bad_page_struct_size)) { + info.argsz = minsz + count * bad_page_struct_size; + info.count = 0; + goto done; + } else { + hash_for_each_safe(region->htbl, bkt, tmp_node, cur_page, node) { + /* + * This check fails if there was an ECC error + * after the usermode app read the count of + * bad pages through this ioctl. + */ + if (minsz + index * bad_page_struct_size >= info.argsz) { + info.argsz = minsz + index * bad_page_struct_size; + info.count = index; + goto done; + } + + tmp.offset = cur_page->mem_offset; + tmp.size = PAGE_SIZE; + + ret = copy_to_user(uarg + minsz + + index * bad_page_struct_size, + &tmp, bad_page_struct_size); + if (ret) + return ret; + index++; + } + + info.count = index; + } + break; + default: + return -EINVAL; + } + +done: + return copy_to_user(uarg, &info, minsz) ? -EFAULT : 0; +} + static const struct file_operations file_ops = { .owner = THIS_MODULE, .open = nvgrace_egm_open, .release = nvgrace_egm_release, .mmap = nvgrace_egm_mmap, + .unlocked_ioctl = nvgrace_egm_ioctl, }; static int setup_egm_chardev(struct egm_region *region) @@ -143,6 +220,45 @@ nvgrace_gpu_fetch_egm_property(struct pci_dev *pdev, u64 *pegmphys, return ret; } +static void nvgrace_egm_fetch_bad_pages(struct pci_dev *pdev, + struct egm_region *region) +{ + u64 retiredpagesphys, count; + void *memaddr; + int index; + + if (device_property_read_u64(&pdev->dev, + "nvidia,egm-retired-pages-data-base", + &retiredpagesphys)) + return; + + memaddr = memremap(retiredpagesphys, PAGE_SIZE, MEMREMAP_WB); + if (!memaddr) + return; + + count = *(u64 *)memaddr; + + hash_init(region->htbl); + + for (index = 0; index < count; index++) { + struct h_node *retired_page; + + /* + * Since the EGM is linearly mapped, the offset in the + * carveout is the same offset in the VM system memory. + * + * Calculate the offset to communicate to the usermode + * apps. + */ + retired_page = (struct h_node *)(vzalloc(sizeof(struct h_node))); + retired_page->mem_offset = *((u64 *)memaddr + index + 1) - + region->egmphys; + hash_add(region->htbl, &retired_page->node, retired_page->mem_offset); + } + + memunmap(memaddr); +} + int register_egm_node(struct pci_dev *pdev) { struct egm_region *region = NULL; @@ -165,6 +281,8 @@ int register_egm_node(struct pci_dev *pdev) atomic_set(®ion->open_count, 0); + nvgrace_egm_fetch_bad_pages(pdev, region); + list_add_tail(®ion->list, &egm_list); setup_egm_chardev(region); @@ -181,9 +299,17 @@ static void destroy_egm_chardev(struct egm_region *region) void unregister_egm_node(int egm_node) { struct egm_region *region, *temp_region; + struct h_node *cur_page; + unsigned long bkt; + struct hlist_node *temp_node; list_for_each_entry_safe(region, temp_region, &egm_list, list) { if (egm_node == region->egmpxm) { + hash_for_each_safe(region->htbl, bkt, temp_node, cur_page, node) { + hash_del(&cur_page->node); + vfree(cur_page); + } + destroy_egm_chardev(region); list_del(®ion->list); } diff --git a/include/uapi/linux/egm.h b/include/uapi/linux/egm.h new file mode 100644 index 0000000000000..8a808e45c2052 --- /dev/null +++ b/include/uapi/linux/egm.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved + */ + +#ifndef _UAPIEGM_H +#define _UAPIEGM_H + +#define EGM_TYPE ('E') + +struct egm_bad_pages_info { + __aligned_u64 offset; + __aligned_u64 size; +}; + +struct egm_bad_pages_list { + __u32 argsz; + /* out */ + __u32 count; + /* out */ + struct egm_bad_pages_info bad_pages[]; +}; + +#define EGM_BAD_PAGES_LIST _IO(EGM_TYPE, 100) + +#endif /* _UAPIEGM_H */ From e6da4774861b02ec1726731f3f1a5a22131bded7 Mon Sep 17 00:00:00 2001 From: "Matthew R. Ochs" Date: Thu, 29 Aug 2024 18:49:03 -0700 Subject: [PATCH 059/311] NVIDIA: SAUCE: arm64: configs: Build CONFIG_NVGRACE_EGM as LKM BugLink: https://bugs.launchpad.net/bugs/2119656 Signed-off-by: Matthew R. Ochs Acked-by: Kai-Heng Feng Acked-by: Koba Ko Signed-off-by: Matthew R. Ochs (cherry picked from commit 5bb23c179220ec77ac9fb2ed610618ce1a902bd4 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.8-next) Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit 7d2ea5531c96fb9acd5704b6bec20aa29ca1fd39 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 077c8340953f noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit f5a03d00aed1fb83a77abd723dee5a8a79392f3b noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- arch/arm64/configs/defconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 678b993e77828..269e811132f9b 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1958,6 +1958,7 @@ CONFIG_CORESIGHT_CPU_DEBUG=m CONFIG_CORESIGHT_CTI=m CONFIG_MEMTEST=y CONFIG_NVGRACE_GPU_VFIO_PCI=m +CONFIG_NVGRACE_EGM=m CONFIG_VFIO_DEVICE_CDEV=y CONFIG_FAULT_INJECTION=y CONFIG_IOMMUFD_DRIVER=y From 05fb6aeb902f18ae68a3147440568a32b7fbb84e Mon Sep 17 00:00:00 2001 From: Ankit Agrawal Date: Sun, 13 Oct 2024 04:53:38 +0000 Subject: [PATCH 060/311] NVIDIA: SAUCE: vfio/nvgrace-egm: Move the egm header file to include BugLink: https://bugs.launchpad.net/bugs/2119656 nvgrace-egm exposes the API register_egm_node & unregister_egm_node to manage EGM (Extended GPU Memory) present on the system. To allow out-of-tree driver such as nvidia-vgpu-vfio make use of them, move the declaration to a new nvgrace-egm.h in include. Signed-off-by: Ankit Agrawal Signed-off-by: Matthew R. Ochs Acked-by: Kai-Heng Feng Acked-by: Koba Ko Signed-off-by: Matthew R. Ochs (cherry picked from commit bed340f2023f22192893e9121834ee3ce252edd1 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.8-next) Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit a9616639ce81799f5b3133c47c25f8d875728f4f https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 020c46c87e7a noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 739457a5ff1b0b30f904d2c1ab28eebcd670a628 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/vfio/pci/nvgrace-gpu/egm.c | 2 +- drivers/vfio/pci/nvgrace-gpu/main.c | 2 +- .../pci/nvgrace-gpu/egm.h => include/linux/nvgrace-egm.h | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) rename drivers/vfio/pci/nvgrace-gpu/egm.h => include/linux/nvgrace-egm.h (55%) diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.c b/drivers/vfio/pci/nvgrace-gpu/egm.c index 8c9ff6313e9f4..598a1d07d00b7 100644 --- a/drivers/vfio/pci/nvgrace-gpu/egm.c +++ b/drivers/vfio/pci/nvgrace-gpu/egm.c @@ -6,7 +6,7 @@ #include #include #include -#include "egm.h" +#include #define MAX_EGM_NODES 256 diff --git a/drivers/vfio/pci/nvgrace-gpu/main.c b/drivers/vfio/pci/nvgrace-gpu/main.c index fe2ab87a1aed8..cb3eeaa3f560f 100644 --- a/drivers/vfio/pci/nvgrace-gpu/main.c +++ b/drivers/vfio/pci/nvgrace-gpu/main.c @@ -10,7 +10,7 @@ #include #include #include -#include "egm.h" +#include /* * The device memory usable to the workloads running in the VM is cached diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.h b/include/linux/nvgrace-egm.h similarity index 55% rename from drivers/vfio/pci/nvgrace-gpu/egm.h rename to include/linux/nvgrace-egm.h index 28cc59e04a0b0..48add892aa5bf 100644 --- a/drivers/vfio/pci/nvgrace-gpu/egm.h +++ b/include/linux/nvgrace-egm.h @@ -1,12 +1,12 @@ -// SPDX-License-Identifier: GPL-2.0-only +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved */ -#ifndef NVGRACE_EGM_H -#define NVGRACE_EGM_H +#ifndef _NVGRACE_EGM_H +#define _NVGRACE_EGM_H int register_egm_node(struct pci_dev *pdev); void unregister_egm_node(int egm_node); -#endif /* NVGRACE_EGM_H */ +#endif /* _NVGRACE_EGM_H */ From afd2b1b332d6948b2951c195e6bbe462345e7146 Mon Sep 17 00:00:00 2001 From: "Matthew R. Ochs" Date: Thu, 7 Nov 2024 15:06:57 -0800 Subject: [PATCH 061/311] NVIDIA: SAUCE: vfio/nvgrace-egm: Free region memory during unregistration BugLink: https://bugs.launchpad.net/bugs/2119656 Free the kmalloc'd region when the EGM is unregistered. Signed-off-by: Matthew R. Ochs Acked-by: Kai-Heng Feng Acked-by: Carol L. Soto Acked-by: Koba Ko Signed-off-by: Matthew R. Ochs (cherry picked from commit fc592b9b4f8b455205abd2b2395671a831bb942e https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.8-next) Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit f24760ccecb8c5517fca6791082ab89cf94b9f9f https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 374b166787e0 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 8f781d07d28638ab3c31c46ed79c0fdc9711a9c4 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/vfio/pci/nvgrace-gpu/egm.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.c b/drivers/vfio/pci/nvgrace-gpu/egm.c index 598a1d07d00b7..06f41049275bd 100644 --- a/drivers/vfio/pci/nvgrace-gpu/egm.c +++ b/drivers/vfio/pci/nvgrace-gpu/egm.c @@ -312,6 +312,7 @@ void unregister_egm_node(int egm_node) destroy_egm_chardev(region); list_del(®ion->list); + kfree(region); } } } From 14f29cf62f8725369e6341b47fde42fad94420ae Mon Sep 17 00:00:00 2001 From: "Matthew R. Ochs" Date: Thu, 7 Nov 2024 15:38:11 -0800 Subject: [PATCH 062/311] NVIDIA: SAUCE: vfio/nvgrace-egm: Move region hash initialization BugLink: https://bugs.launchpad.net/bugs/2119656 Move region hash initiaization alongside the other region initialization statements to avoid situations where the hash table was not properly initialized. Signed-off-by: Matthew R. Ochs Acked-by: Kai-Heng Feng Acked-by: Carol L. Soto Acked-by: Koba Ko Signed-off-by: Matthew R. Ochs (cherry picked from commit 8021c1d2b1c73015102bc69eda0029114989dd1f https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.8-next) Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit e1264a62e8841fd5332f7f02a921242ff1b51dfa https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 0f8a09890f67 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 22f790add5eb4d1b17cd3e056d8f783af25b3f72 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/vfio/pci/nvgrace-gpu/egm.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.c b/drivers/vfio/pci/nvgrace-gpu/egm.c index 06f41049275bd..621d046084a18 100644 --- a/drivers/vfio/pci/nvgrace-gpu/egm.c +++ b/drivers/vfio/pci/nvgrace-gpu/egm.c @@ -238,8 +238,6 @@ static void nvgrace_egm_fetch_bad_pages(struct pci_dev *pdev, count = *(u64 *)memaddr; - hash_init(region->htbl); - for (index = 0; index < count; index++) { struct h_node *retired_page; @@ -279,6 +277,7 @@ int register_egm_node(struct pci_dev *pdev) region->egmlength = egmlength; region->egmpxm = egmpxm; + hash_init(region->htbl); atomic_set(®ion->open_count, 0); nvgrace_egm_fetch_bad_pages(pdev, region); From 1222f904702369f5e120f252fcc95cce32e0a619 Mon Sep 17 00:00:00 2001 From: "Matthew R. Ochs" Date: Thu, 7 Nov 2024 15:48:47 -0800 Subject: [PATCH 063/311] NVIDIA: SAUCE: vfio/nvgrace-egm: Handle and convey EGM registration errors BugLink: https://bugs.launchpad.net/bugs/2119656 Update error handling within EGM regiration routine to catch and return errors to the caller. Signed-off-by: Matthew R. Ochs Acked-by: Kai-Heng Feng Acked-by: Carol L. Soto Acked-by: Koba Ko Signed-off-by: Matthew R. Ochs (cherry picked from commit a57210c88c1c3693a24684c967c0858d75cabd32 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.8-next) Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit a706ff8c445abed002e0b9493dfc9c664b1ffd57 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit edc0ac06e8e9 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit e7a177e2e55f20b7508e3017be2a5259a91e38d5 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/vfio/pci/nvgrace-gpu/egm.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.c b/drivers/vfio/pci/nvgrace-gpu/egm.c index 621d046084a18..140f0f10f2c3e 100644 --- a/drivers/vfio/pci/nvgrace-gpu/egm.c +++ b/drivers/vfio/pci/nvgrace-gpu/egm.c @@ -273,6 +273,9 @@ int register_egm_node(struct pci_dev *pdev) } region = kvzalloc(sizeof(*region), GFP_KERNEL); + if (!region) + return -ENOMEM; + region->egmphys = egmphys; region->egmlength = egmlength; region->egmpxm = egmpxm; @@ -282,11 +285,16 @@ int register_egm_node(struct pci_dev *pdev) nvgrace_egm_fetch_bad_pages(pdev, region); - list_add_tail(®ion->list, &egm_list); + ret = setup_egm_chardev(region); + if (ret) + goto err; - setup_egm_chardev(region); + list_add_tail(®ion->list, &egm_list); return 0; +err: + kfree(region); + return ret; } EXPORT_SYMBOL_GPL(register_egm_node); From e3d7a6b56ff80e42ede629825c7a40f691a15a1c Mon Sep 17 00:00:00 2001 From: "Matthew R. Ochs" Date: Thu, 7 Nov 2024 15:55:58 -0800 Subject: [PATCH 064/311] NVIDIA: SAUCE: vfio/nvgrace-gpu: Handle EGM registration failure BugLink: https://bugs.launchpad.net/bugs/2119656 Detect and handle a failure from the EGM registration service. Signed-off-by: Matthew R. Ochs Acked-by: Kai-Heng Feng Acked-by: Carol L. Soto Acked-by: Koba Ko Signed-off-by: Matthew R. Ochs (cherry picked from commit f18eee3bbdea77a9b525c0665d7ebe1992bb00b2 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.8-next) Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit 8371b68c33cc03a7ea6dfd7bdfc0fe9d47ec64fb https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit be5ae8ffa6ef noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 2dd59038740de38b4b2400930b05a49fd6051446 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/vfio/pci/nvgrace-gpu/main.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/vfio/pci/nvgrace-gpu/main.c b/drivers/vfio/pci/nvgrace-gpu/main.c index cb3eeaa3f560f..5b26235e32cad 100644 --- a/drivers/vfio/pci/nvgrace-gpu/main.c +++ b/drivers/vfio/pci/nvgrace-gpu/main.c @@ -1229,7 +1229,10 @@ static int nvgrace_gpu_probe(struct pci_dev *pdev, nvdev->core_device.pci_ops = &nvgrace_gpu_pci_dev_ops; if (egm_enabled) { - register_egm_node(pdev); + ret = register_egm_node(pdev); + if (ret) + goto out_put_vdev; + nvdev->egm_node = egmpxm; } @@ -1239,10 +1242,13 @@ static int nvgrace_gpu_probe(struct pci_dev *pdev, ret = vfio_pci_core_register_device(&nvdev->core_device); if (ret) - goto out_put_vdev; + goto out_egm_unreg; return ret; +out_egm_unreg: + if (egm_enabled) + unregister_egm_node(nvdev->egm_node); out_put_vdev: vfio_put_device(&nvdev->core_device.vdev); return ret; From 155c2c4d85331f29691d7c477472116116d6dfb9 Mon Sep 17 00:00:00 2001 From: "Matthew R. Ochs" Date: Thu, 7 Nov 2024 16:07:26 -0800 Subject: [PATCH 065/311] NVIDIA: SAUCE: vfio/nvgrace-egm: Address sparse errors BugLink: https://bugs.launchpad.net/bugs/2119656 Fix minor syntax errors from sparse. Signed-off-by: Matthew R. Ochs Acked-by: Kai-Heng Feng Acked-by: Carol L. Soto Acked-by: Koba Ko Signed-off-by: Matthew R. Ochs (cherry picked from commit bbb64e63a0b5e8c8eeec52b1e901745ba64b96d3 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.8-next) Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit fe7819421a04be2b2405376da3550beb03986b6c https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit b19296004d0d noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit c1d3f2196850eef4c2eb77326d9475252e7a4087 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/vfio/pci/nvgrace-gpu/egm.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.c b/drivers/vfio/pci/nvgrace-gpu/egm.c index 140f0f10f2c3e..33ed9a1f1a03f 100644 --- a/drivers/vfio/pci/nvgrace-gpu/egm.c +++ b/drivers/vfio/pci/nvgrace-gpu/egm.c @@ -102,6 +102,7 @@ static long nvgrace_egm_ioctl(struct file *file, unsigned int cmd, unsigned long switch (cmd) { case EGM_BAD_PAGES_LIST: + { int ret; unsigned long bad_page_struct_size = sizeof(struct egm_bad_pages_info); struct egm_bad_pages_info tmp; @@ -144,6 +145,7 @@ static long nvgrace_egm_ioctl(struct file *file, unsigned int cmd, unsigned long info.count = index; } break; + } default: return -EINVAL; } From 5ddf25903b0a910f72e8b95cb0759bff5e0b5eb0 Mon Sep 17 00:00:00 2001 From: "Matthew R. Ochs" Date: Thu, 7 Nov 2024 20:09:38 -0800 Subject: [PATCH 066/311] NVIDIA: SAUCE: vfio/nvgrace-gpu: Address smatch errors BugLink: https://bugs.launchpad.net/bugs/2119656 Use the correct macro and types for overflow checking. Signed-off-by: Matthew R. Ochs Acked-by: Kai-Heng Feng Acked-by: Carol L. Soto Acked-by: Koba Ko Signed-off-by: Matthew R. Ochs (cherry picked from commit afa8f63898cf65cf0d9cf3209ef486daa11a42c7 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.8-next) Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit d110330e4b93894a6234b6c4036f8422883fab90 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit a6c050804fa4 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 8ec6da274a04af027be4532917932380e09d7d97 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/vfio/pci/nvgrace-gpu/main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/vfio/pci/nvgrace-gpu/main.c b/drivers/vfio/pci/nvgrace-gpu/main.c index 5b26235e32cad..d812cbbf84512 100644 --- a/drivers/vfio/pci/nvgrace-gpu/main.c +++ b/drivers/vfio/pci/nvgrace-gpu/main.c @@ -995,7 +995,7 @@ nvgrace_gpu_fetch_memory_property(struct pci_dev *pdev, if (ret) return ret; - if (*pmemphys > type_max(phys_addr_t)) + if (overflows_type(*pmemphys, phys_addr_t)) return -EOVERFLOW; ret = device_property_read_u64(&pdev->dev, "nvidia,gpu-mem-size", @@ -1003,7 +1003,7 @@ nvgrace_gpu_fetch_memory_property(struct pci_dev *pdev, if (ret) return ret; - if (*pmemlength > type_max(size_t)) + if (overflows_type(*pmemlength, size_t)) return -EOVERFLOW; /* From b281da4a5386146cb15f32875a3851a28606d6cd Mon Sep 17 00:00:00 2001 From: "Matthew R. Ochs" Date: Fri, 22 Nov 2024 15:48:10 -0800 Subject: [PATCH 067/311] NVIDIA: SAUCE: vfio/nvgrace-egm: Ensure ACPI value reads are successful BugLink: https://bugs.launchpad.net/bugs/2119656 Ensure ACPI table reads are successful prior to using the value. Signed-off-by: Matthew R. Ochs Acked-by: Kai-Heng Feng Acked-by: Carol L. Soto Acked-by: Koba Ko Signed-off-by: Matthew R. Ochs (cherry picked from commit b2947b075de6c887660cc8bc23ab5f0b6e7bfd17 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.8-next) Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit 92583550c3b22d1d00bfc6f59f3fc943cbd3a29e https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 2c5b472932c1 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 3f72f24ead3380c02907ee5d145f1b7a8e053d96 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/vfio/pci/nvgrace-gpu/egm.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.c b/drivers/vfio/pci/nvgrace-gpu/egm.c index 33ed9a1f1a03f..9388bdefe09aa 100644 --- a/drivers/vfio/pci/nvgrace-gpu/egm.c +++ b/drivers/vfio/pci/nvgrace-gpu/egm.c @@ -215,11 +215,13 @@ nvgrace_gpu_fetch_egm_property(struct pci_dev *pdev, u64 *pegmphys, ret = device_property_read_u64(&pdev->dev, "nvidia,egm-pxm", pegmpxm); + if (ret) + return ret; if (*pegmpxm > type_max(phys_addr_t)) return -EOVERFLOW; - return ret; + return 0; } static void nvgrace_egm_fetch_bad_pages(struct pci_dev *pdev, From f51e2fb91d9e8042dc4ab3314f36afa88cc634bd Mon Sep 17 00:00:00 2001 From: "Matthew R. Ochs" Date: Thu, 14 Nov 2024 08:12:22 -0800 Subject: [PATCH 068/311] NVIDIA: SAUCE: vfio/nvgrace-egm: Avoid invalid retired pages base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BugLink: https://bugs.launchpad.net/bugs/2119656 Some environments may provide a "nvidia,egm-retired-pages-data-base” but fail to populate it with a base address, leaving it NULL. Mapping this invalid value results in a synchronous exception when the region is first touched. Detect a NULL value, generate a warning to draw attention to the firmware bug, and return without mapping. INFO: th500_ras_intr_handler: External Abort reason=1 syndrome=0x92000410 flags=0x1 [ 82.104493] Internal error: synchronous external abort: 0000000096000410 [#1] SMP [ 82.114898] Modules linked in: nvgrace_gpu_vfio_pci(E) nvgrace_egm(E) [ 82.257218] CPU: 0 PID: 10 Comm: kworker/0:1 Tainted: G OE 6.8.12+ #5 [ 82.265135] Hardware name: NVIDIA GH200 P5042, BIOS 24103110 20241031 [ 82.271720] Workqueue: events work_for_cpu_fn [ 82.276180] pstate: 03400009 (nzcv daif +PAN -UAO +TCO +DIT -SSBS BTYPE=--) [ 82.283298] pc : register_egm_node+0x2cc/0x440 [nvgrace_egm] [ 82.289087] lr : register_egm_node+0x2c4/0x440 [nvgrace_egm] [ 82.294872] sp : ffff8000802ebc30 [ 82.298254] x29: ffff8000802ebc60 x28: 00000000000000ff x27: 0000000000000000 [ 82.305550] x26: ffff000087a320c8 x25: ffff0000a5700000 x24: ffff000087a32000 [ 82.312846] x23: ffffa77cd758e368 x22: 0000000000000000 x21: ffffa77cd758c640 [ 82.320141] x20: ffffa77cd758e170 x19: ffff800081e7d000 x18: ffff800080293038 [ 82.327437] x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000 [ 82.334732] x14: 0000000000000000 x13: 65203a65646f6e5f x12: 0000000000000000 [ 82.342027] x11: 0000000000000000 x10: 0000000000000000 x9 : 0000000000000000 [ 82.349322] x8 : 0000000000000000 x7 : 0000000000000000 x6 : 0000000000000000 [ 82.356618] x5 : 0000000000000000 x4 : 0000000000000000 x3 : 0000000000000000 [ 82.363913] x2 : 0000000000000000 x1 : 0000000000000000 x0 : ffff800081e7d000 [ 82.371210] Call trace: [ 82.373705] register_egm_node+0x2cc/0x440 [nvgrace_egm] [ 82.379135] nvgrace_gpu_probe+0x2ac/0x528 [nvgrace_gpu_vfio_pci] [ 82.385366] local_pci_probe+0x4c/0xe0 [ 82.389198] work_for_cpu_fn+0x28/0x58 [ 82.393026] process_one_work+0x168/0x3f0 [ 82.397123] worker_thread+0x360/0x480 [ 82.400952] kthread+0x11c/0x128 [ 82.404248] ret_from_fork+0x10/0x20 [ 82.407906] Code: d2820001 940002b3 aa0003f3 b4fffac0 (f9400017) [ 82.414134] ---[ end trace 0000000000000000 ]--- Signed-off-by: Matthew R. Ochs Acked-by: Kai-Heng Feng Acked-by: Carol L. Soto Acked-by: Koba Ko Signed-off-by: Matthew R. Ochs (cherry picked from commit 7ba29302925c6f2e1b9825d06f7468acc175ab85 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.8-next) Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit 349fb1c23faef926f3bdbc479b088c7b6b66853f https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 6e9c94a06e83 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit c5992d5bdc0402f5c327a5fe65791cb5df278de9 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/vfio/pci/nvgrace-gpu/egm.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.c b/drivers/vfio/pci/nvgrace-gpu/egm.c index 9388bdefe09aa..2ffae71f7f458 100644 --- a/drivers/vfio/pci/nvgrace-gpu/egm.c +++ b/drivers/vfio/pci/nvgrace-gpu/egm.c @@ -236,6 +236,10 @@ static void nvgrace_egm_fetch_bad_pages(struct pci_dev *pdev, &retiredpagesphys)) return; + /* Catch firmware bug and avoid a crash */ + if (WARN_ON_ONCE(retiredpagesphys == 0)) + return; + memaddr = memremap(retiredpagesphys, PAGE_SIZE, MEMREMAP_WB); if (!memaddr) return; From 8f65e20d8858c4880d629ddf774982164164b5de Mon Sep 17 00:00:00 2001 From: "Matthew R. Ochs" Date: Thu, 23 Jan 2025 12:07:12 -0800 Subject: [PATCH 069/311] NVIDIA: SAUCE: vfio/nvgrace-egm: Update EGM unregistration API BugLink: https://bugs.launchpad.net/bugs/2119656 In an effort to simplify the programming model, use a symmetrical model for the the EGM regsiration APIs. This avoids the caller needing to keep a cookie or even have knowlege of if EGM is supported. Update the EGM unregisration API to use the PCI device as its parameter. Signed-off-by: Matthew R. Ochs (cherry picked from commit d8903ecbf6ae94cbf67b8492996021cd2488033c https://github.com/nvmochs/NV-Kernels/tree/vegm_01232025) Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit 5839fc506349c858a90a19e713c46fce025b2ec6 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit f6fb40e917fd noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 0e607bc7591e5b9e161ae98dfe9e42c02864b402 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/vfio/pci/nvgrace-gpu/egm.c | 10 ++++++++-- drivers/vfio/pci/nvgrace-gpu/main.c | 4 ++-- include/linux/nvgrace-egm.h | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.c b/drivers/vfio/pci/nvgrace-gpu/egm.c index 2ffae71f7f458..1545ac695ad77 100644 --- a/drivers/vfio/pci/nvgrace-gpu/egm.c +++ b/drivers/vfio/pci/nvgrace-gpu/egm.c @@ -311,15 +311,21 @@ static void destroy_egm_chardev(struct egm_region *region) cdev_device_del(®ion->cdev, ®ion->device); } -void unregister_egm_node(int egm_node) +void unregister_egm_node(struct pci_dev *pdev) { struct egm_region *region, *temp_region; struct h_node *cur_page; unsigned long bkt; struct hlist_node *temp_node; + u64 egmphys, egmlength, egmpxm; + int ret; + + ret = nvgrace_gpu_fetch_egm_property(pdev, &egmphys, &egmlength, &egmpxm); + if (ret) + return; list_for_each_entry_safe(region, temp_region, &egm_list, list) { - if (egm_node == region->egmpxm) { + if (egmpxm == region->egmpxm) { hash_for_each_safe(region->htbl, bkt, temp_node, cur_page, node) { hash_del(&cur_page->node); vfree(cur_page); diff --git a/drivers/vfio/pci/nvgrace-gpu/main.c b/drivers/vfio/pci/nvgrace-gpu/main.c index d812cbbf84512..967313b9e0029 100644 --- a/drivers/vfio/pci/nvgrace-gpu/main.c +++ b/drivers/vfio/pci/nvgrace-gpu/main.c @@ -1248,7 +1248,7 @@ static int nvgrace_gpu_probe(struct pci_dev *pdev, out_egm_unreg: if (egm_enabled) - unregister_egm_node(nvdev->egm_node); + unregister_egm_node(pdev); out_put_vdev: vfio_put_device(&nvdev->core_device.vdev); return ret; @@ -1262,7 +1262,7 @@ static void nvgrace_gpu_remove(struct pci_dev *pdev) core_device); if (egm_enabled) - unregister_egm_node(nvdev->egm_node); + unregister_egm_node(pdev); vfio_pci_core_unregister_device(core_device); vfio_put_device(&core_device->vdev); diff --git a/include/linux/nvgrace-egm.h b/include/linux/nvgrace-egm.h index 48add892aa5bf..4bbd383a02732 100644 --- a/include/linux/nvgrace-egm.h +++ b/include/linux/nvgrace-egm.h @@ -7,6 +7,6 @@ #define _NVGRACE_EGM_H int register_egm_node(struct pci_dev *pdev); -void unregister_egm_node(int egm_node); +void unregister_egm_node(struct pci_dev *pdev); #endif /* _NVGRACE_EGM_H */ From b82121b47275e77351f4278335deb4988c88257f Mon Sep 17 00:00:00 2001 From: Ankit Agrawal Date: Tue, 6 May 2025 09:38:38 -0500 Subject: [PATCH 070/311] NVIDIA: SAUCE: vfio/nvgrace-egm: track GPUs associated with the EGM regions BugLink: https://bugs.launchpad.net/bugs/2119656 GB200 systems could have multiple GPUs associated with an EGM region. For proper EGM functionality the host topology in terms of GPU affinity has to be replicated in the VM. Hence the EGM region structure must track the GPU devices belonging to the same socket. On the device probe, the device pci_dev struct is added to a linked list of the appropriate EGM region. Similarly on device remove, the pci_dev struct for the GPU is removed from the EGM region. Signed-off-by: Ankit Agrawal Ref: sj24: /home/nvidia/ankita/kernel_patches/0001_vfio_nvgrace-egm_track_GPUs_associated_with_the_EGM_regions.patch (koba: Enhance error handling, Remove egm_node from unregister_egm_node and move destroy_egm_chardev a little forward) Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit 0222c35fb26285ee1a6185ef50414093850ea352 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 5ba1a1f84f9d noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit c167095eba967b7ae64023927a015df0643c7460 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/vfio/pci/nvgrace-gpu/egm.c | 68 ++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 9 deletions(-) diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.c b/drivers/vfio/pci/nvgrace-gpu/egm.c index 1545ac695ad77..67cc5254f681b 100644 --- a/drivers/vfio/pci/nvgrace-gpu/egm.c +++ b/drivers/vfio/pci/nvgrace-gpu/egm.c @@ -10,6 +10,11 @@ #define MAX_EGM_NODES 256 +struct gpu_node { + struct list_head list; + struct pci_dev *pdev; +}; + struct egm_region { struct list_head list; int egmpxm; @@ -18,6 +23,7 @@ struct egm_region { size_t egmlength; struct device device; struct cdev cdev; + struct list_head gpus; DECLARE_HASHTABLE(htbl, 0x10); }; @@ -187,6 +193,11 @@ static int setup_egm_chardev(struct egm_region *region) return ret; } +static void destroy_egm_chardev(struct egm_region *region) +{ + cdev_device_del(®ion->cdev, ®ion->device); +} + static int nvgrace_gpu_fetch_egm_property(struct pci_dev *pdev, u64 *pegmphys, u64 *pegmlength, u64 *pegmpxm) @@ -265,6 +276,32 @@ static void nvgrace_egm_fetch_bad_pages(struct pci_dev *pdev, memunmap(memaddr); } +static int add_gpu(struct egm_region *region, struct pci_dev *pdev) +{ + struct gpu_node *node; + + node = kvzalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return -ENOMEM; + + node->pdev = pdev; + + list_add_tail(&node->list, ®ion->gpus); + return 0; +} + +static void remove_gpu(struct egm_region *region, struct pci_dev *pdev) +{ + struct gpu_node *node, *tmp; + + list_for_each_entry_safe(node, tmp, ®ion->gpus, list) { + if (node->pdev == pdev) { + list_del(&node->list); + kvfree(node); + } + } +} + int register_egm_node(struct pci_dev *pdev) { struct egm_region *region = NULL; @@ -275,11 +312,15 @@ int register_egm_node(struct pci_dev *pdev) if (ret) return ret; + /* Check if region already exists */ list_for_each_entry(region, &egm_list, list) { - if (region->egmphys == egmphys) - return 0; + if (region->egmphys == egmphys) { + /* Add GPU to existing region */ + return add_gpu(region, pdev); + } } + /* Create new region */ region = kvzalloc(sizeof(*region), GFP_KERNEL); if (!region) return -ENOMEM; @@ -289,28 +330,33 @@ int register_egm_node(struct pci_dev *pdev) region->egmpxm = egmpxm; hash_init(region->htbl); + INIT_LIST_HEAD(®ion->gpus); + atomic_set(®ion->open_count, 0); nvgrace_egm_fetch_bad_pages(pdev, region); ret = setup_egm_chardev(region); if (ret) - goto err; + goto err_free_region; list_add_tail(®ion->list, &egm_list); + ret = add_gpu(region, pdev); + if (ret) + goto err_remove_from_list; + return 0; -err: + +err_remove_from_list: + list_del(®ion->list); + destroy_egm_chardev(region); +err_free_region: kfree(region); return ret; } EXPORT_SYMBOL_GPL(register_egm_node); -static void destroy_egm_chardev(struct egm_region *region) -{ - cdev_device_del(®ion->cdev, ®ion->device); -} - void unregister_egm_node(struct pci_dev *pdev) { struct egm_region *region, *temp_region; @@ -326,6 +372,10 @@ void unregister_egm_node(struct pci_dev *pdev) list_for_each_entry_safe(region, temp_region, &egm_list, list) { if (egmpxm == region->egmpxm) { + remove_gpu(region, pdev); + if (!list_empty(®ion->gpus)) + break; + hash_for_each_safe(region->htbl, bkt, temp_node, cur_page, node) { hash_del(&cur_page->node); vfree(cur_page); From bf7b7e6e5d46a0e3d77700be85346b3302090d15 Mon Sep 17 00:00:00 2001 From: Ankit Agrawal Date: Tue, 6 May 2025 09:39:33 -0500 Subject: [PATCH 071/311] NVIDIA: SAUCE: vfio/nvgrace-egm: list gpus through sysfs BugLink: https://bugs.launchpad.net/bugs/2119656 To replicate the host EGM topology in the VM in terms of the GPU affinity, the userspace need to be aware of which GPUs belong to the same socket as the EGM region. Expose the list of GPUs associated with an EGM region through sysfs. The list can be queried from the location /sys/devices/virtual/egm/egmX/gpu_devices. Signed-off-by: Ankit Agrawal Ref: sj24: /home/nvidia/ankita/kernel_patches/0002_vfio_nvgrace-egm_list_gpus_through_sysfs.patch (koba: Enchance error handling for sysfs_create_group) Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit fec2356d20f7054c0c89b1d32e7862bba34bda54 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 5dde2f0e0bb5 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit a5284ca673f5a3732431c7e1cb49a48c347a0087 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/vfio/pci/nvgrace-gpu/egm.c | 41 +++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.c b/drivers/vfio/pci/nvgrace-gpu/egm.c index 67cc5254f681b..2988d55208bf4 100644 --- a/drivers/vfio/pci/nvgrace-gpu/egm.c +++ b/drivers/vfio/pci/nvgrace-gpu/egm.c @@ -276,6 +276,38 @@ static void nvgrace_egm_fetch_bad_pages(struct pci_dev *pdev, memunmap(memaddr); } +static ssize_t gpu_devices_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct egm_region *region = + container_of(dev, struct egm_region, device); + struct gpu_node *node, *temp_node; + int len = 0; + + list_for_each_entry_safe(node, temp_node, ®ion->gpus, list) { + struct pci_dev *pdev = node->pdev; + + len += sysfs_emit_at(buf, len, "%04x:%02x:%02x.%x\n", + pci_domain_nr(pdev->bus), + pdev->bus->number, + PCI_SLOT(pdev->devfn), + PCI_FUNC(pdev->devfn)); + } + + return len; +} + +static DEVICE_ATTR_RO(gpu_devices); + +static struct attribute *attrs[] = { + &dev_attr_gpu_devices.attr, + NULL, +}; + +static struct attribute_group attr_group = { + .attrs = attrs, +}; + static int add_gpu(struct egm_region *region, struct pci_dev *pdev) { struct gpu_node *node; @@ -342,12 +374,18 @@ int register_egm_node(struct pci_dev *pdev) list_add_tail(®ion->list, &egm_list); - ret = add_gpu(region, pdev); + ret = sysfs_create_group(®ion->device.kobj, &attr_group); if (ret) goto err_remove_from_list; + ret = add_gpu(region, pdev); + if (ret) + goto err_remove_sysfs; + return 0; +err_remove_sysfs: + sysfs_remove_group(®ion->device.kobj, &attr_group); err_remove_from_list: list_del(®ion->list); destroy_egm_chardev(region); @@ -381,6 +419,7 @@ void unregister_egm_node(struct pci_dev *pdev) vfree(cur_page); } + sysfs_remove_group(®ion->device.kobj, &attr_group); destroy_egm_chardev(region); list_del(®ion->list); kfree(region); From 36e294f8b4b79811027c82273e938b626fe179b1 Mon Sep 17 00:00:00 2001 From: Ankit Agrawal Date: Tue, 6 May 2025 09:40:16 -0500 Subject: [PATCH 072/311] NVIDIA: SAUCE: vfio/nvgrace-egm: expose the egm size through sysfs BugLink: https://bugs.launchpad.net/bugs/2119656 To allocate the EGM, the userspace need to know it's size. Currently, there is no easy way for the userspace to determine that. Make nvgrace-egm expose the size through sysfs that can be queried by the userspace from /sys/devices/virtual/egm/egmX/egm_size. Signed-off-by: Ankit Agrawal Ref: sj24: /home/nvidia/ankita/kernel_patches/0003_vfio_nvgrace-egm_expose_the_egm_size_through_sysfs.patch Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit dcdcef245e8d648d38ef75f1023c7437b5639ddf https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 994015745197 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit e025c29ebeb237a2e85b86e5c6b1f83602e7f8bd noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/vfio/pci/nvgrace-gpu/egm.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.c b/drivers/vfio/pci/nvgrace-gpu/egm.c index 2988d55208bf4..1e8f2f10b06f9 100644 --- a/drivers/vfio/pci/nvgrace-gpu/egm.c +++ b/drivers/vfio/pci/nvgrace-gpu/egm.c @@ -299,8 +299,19 @@ static ssize_t gpu_devices_show(struct device *dev, struct device_attribute *att static DEVICE_ATTR_RO(gpu_devices); +static ssize_t egm_size_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct egm_region *region = + container_of(dev, struct egm_region, device); + return sysfs_emit(buf, "0x%lx\n", region->egmlength); +} + +static DEVICE_ATTR_RO(egm_size); + static struct attribute *attrs[] = { &dev_attr_gpu_devices.attr, + &dev_attr_egm_size.attr, NULL, }; From 438251fd53a133c7a1447d9512becaf1c2863e6d Mon Sep 17 00:00:00 2001 From: Ankit Agrawal Date: Sun, 18 Jan 2026 02:03:13 +0000 Subject: [PATCH 073/311] NVIDIA: SAUCE: vfio/nvgrace-egm: register EGM PFNMAP range with memory_failure BugLink: https://bugs.launchpad.net/bugs/2138892 EGM carveout memory is mapped directly into userspace (QEMU) and is not added to the kernel. It is not managed by the kernel page allocator and has no struct pages. The module can thus utilize the Linux memory manager's memory_failure mechanism for regions with no struct pages. The Linux MM code exposes register/unregister APIs allowing modules to register such memory regions for memory_failure handling. Register the EGM PFN range with the MM memory_failure infrastructure on open, and unregister it on the last close. Provide a PFN-to-VMA offset callback that validates the PFN is within the EGM region and the VMA, then converts it to a file offset and records the poisoned offset in the existing hashtable for reporting to userspace. Signed-off-by: Ankit Agrawal Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Acked-by: Jacob Martin Acked-by: Noah Wager Signed-off-by: Brad Figg (cherry picked from commit 3fde504ffb6ab8def2971607833069daa29835c4 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/vfio/pci/nvgrace-gpu/egm.c | 100 ++++++++++++++++++++++++++++- 1 file changed, 99 insertions(+), 1 deletion(-) diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.c b/drivers/vfio/pci/nvgrace-gpu/egm.c index 1e8f2f10b06f9..9e47813f4ecbe 100644 --- a/drivers/vfio/pci/nvgrace-gpu/egm.c +++ b/drivers/vfio/pci/nvgrace-gpu/egm.c @@ -7,6 +7,8 @@ #include #include #include +#include +#include #define MAX_EGM_NODES 256 @@ -25,6 +27,7 @@ struct egm_region { struct cdev cdev; struct list_head gpus; DECLARE_HASHTABLE(htbl, 0x10); + struct pfn_address_space pfn_address_space; }; struct h_node { @@ -36,11 +39,97 @@ static dev_t dev; static struct class *class; static struct list_head egm_list; +static int pfn_memregion_offset(struct egm_region *region, + unsigned long pfn, + pgoff_t *pfn_offset_in_region) +{ + unsigned long start_pfn, num_pages; + + start_pfn = PHYS_PFN(region->egmphys); + num_pages = region->egmlength >> PAGE_SHIFT; + + if (pfn < start_pfn || pfn >= start_pfn + num_pages) + return -EFAULT; + + *pfn_offset_in_region = pfn - start_pfn; + + return 0; +} + +static int track_ecc_offset(struct egm_region *region, + unsigned long mem_offset) +{ + struct h_node *cur_page, *ecc_page; + unsigned long bkt; + + hash_for_each(region->htbl, bkt, cur_page, node) { + if (cur_page->mem_offset == mem_offset) + return 0; + } + + ecc_page = (struct h_node *)(vzalloc(sizeof(struct h_node))); + if (!ecc_page) + return -ENOMEM; + + ecc_page->mem_offset = mem_offset; + + hash_add(region->htbl, &ecc_page->node, ecc_page->mem_offset); + + return 0; +} + +static int nvgrace_egm_pfn_to_vma_pgoff(struct vm_area_struct *vma, + unsigned long pfn, + pgoff_t *pgoff) +{ + struct egm_region *region = vma->vm_file->private_data; + pgoff_t vma_offset_in_region = vma->vm_pgoff & + ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); + pgoff_t pfn_offset_in_region; + int ret; + + ret = pfn_memregion_offset(region, pfn, &pfn_offset_in_region); + if (ret) + return ret; + + /* Ensure PFN is not before VMA's start within the region */ + if (pfn_offset_in_region < vma_offset_in_region) + return -EFAULT; + + /* Calculate offset from VMA start */ + *pgoff = vma->vm_pgoff + + (pfn_offset_in_region - vma_offset_in_region); + + /* Track and save the poisoned offset */ + return track_ecc_offset(region, *pgoff << PAGE_SHIFT); +} + +static int +nvgrace_egm_vfio_pci_register_pfn_range(struct inode *inode, + struct egm_region *region) +{ + int ret; + unsigned long pfn, nr_pages; + + pfn = PHYS_PFN(region->egmphys); + nr_pages = region->egmlength >> PAGE_SHIFT; + + region->pfn_address_space.node.start = pfn; + region->pfn_address_space.node.last = pfn + nr_pages - 1; + region->pfn_address_space.mapping = inode->i_mapping; + region->pfn_address_space.pfn_to_vma_pgoff = nvgrace_egm_pfn_to_vma_pgoff; + + ret = register_pfn_address_space(®ion->pfn_address_space); + + return ret; +} + static int nvgrace_egm_open(struct inode *inode, struct file *file) { void *memaddr; struct egm_region *region = container_of(inode->i_cdev, struct egm_region, cdev); + int ret; if (!region) return -EINVAL; @@ -58,6 +147,12 @@ static int nvgrace_egm_open(struct inode *inode, struct file *file) memunmap(memaddr); file->private_data = region; + ret = nvgrace_egm_vfio_pci_register_pfn_range(inode, region); + if (ret && ret != -EOPNOTSUPP) { + file->private_data = NULL; + return ret; + } + return 0; } @@ -69,8 +164,11 @@ static int nvgrace_egm_release(struct inode *inode, struct file *file) if (!region) return -EINVAL; - if (atomic_dec_and_test(®ion->open_count)) + if (atomic_dec_and_test(®ion->open_count)) { + unregister_pfn_address_space(®ion->pfn_address_space); + file->private_data = NULL; + } return 0; } From c8f6ef7685992ba59b0ed21a371984e64ae296f2 Mon Sep 17 00:00:00 2001 From: "Matthew R. Ochs" Date: Thu, 7 Nov 2024 20:03:50 -0800 Subject: [PATCH 074/311] NVIDIA: SAUCE: vfio/nvgrace-egm: Address smatch errors BugLink: https://bugs.launchpad.net/bugs/2119656 Return the intended errno upon a copyout fault, remove unnecessary checks following container_of pointer derivation, and use the correct macro and types for overflow checking. Signed-off-by: Matthew R. Ochs Acked-by: Kai-Heng Feng Acked-by: Carol L. Soto Acked-by: Koba Ko Signed-off-by: Matthew R. Ochs (cherry picked from commit 429910b6fba450a9831f590d9622d16b79006311 https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.8-next) Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit bda63f340176a3a610a64512176f0e133b9efd9f https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit 942bf3b26275 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 63dd05905701cd6f68b3194afafca2bedb344222 noble:linux-nvidia-6.17) Signed-off-by: Matthew R. Ochs --- drivers/vfio/pci/nvgrace-gpu/egm.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.c b/drivers/vfio/pci/nvgrace-gpu/egm.c index 9e47813f4ecbe..aa9e796b6fb4f 100644 --- a/drivers/vfio/pci/nvgrace-gpu/egm.c +++ b/drivers/vfio/pci/nvgrace-gpu/egm.c @@ -131,9 +131,6 @@ static int nvgrace_egm_open(struct inode *inode, struct file *file) struct egm_region, cdev); int ret; - if (!region) - return -EINVAL; - if (atomic_inc_return(®ion->open_count) > 1) return 0; @@ -161,9 +158,6 @@ static int nvgrace_egm_release(struct inode *inode, struct file *file) struct egm_region *region = container_of(inode->i_cdev, struct egm_region, cdev); - if (!region) - return -EINVAL; - if (atomic_dec_and_test(®ion->open_count)) { unregister_pfn_address_space(®ion->pfn_address_space); @@ -242,7 +236,7 @@ static long nvgrace_egm_ioctl(struct file *file, unsigned int cmd, unsigned long index * bad_page_struct_size, &tmp, bad_page_struct_size); if (ret) - return ret; + return -EFAULT; index++; } @@ -311,7 +305,7 @@ nvgrace_gpu_fetch_egm_property(struct pci_dev *pdev, u64 *pegmphys, if (ret) return ret; - if (*pegmlength > type_max(size_t)) + if (overflows_type(*pegmlength, size_t)) return -EOVERFLOW; ret = device_property_read_u64(&pdev->dev, "nvidia,egm-base-pa", @@ -319,7 +313,7 @@ nvgrace_gpu_fetch_egm_property(struct pci_dev *pdev, u64 *pegmphys, if (ret) return ret; - if (*pegmphys > type_max(phys_addr_t)) + if (overflows_type(*pegmphys, phys_addr_t)) return -EOVERFLOW; ret = device_property_read_u64(&pdev->dev, "nvidia,egm-pxm", @@ -327,7 +321,7 @@ nvgrace_gpu_fetch_egm_property(struct pci_dev *pdev, u64 *pegmphys, if (ret) return ret; - if (*pegmpxm > type_max(phys_addr_t)) + if (overflows_type(*pegmpxm, int)) return -EOVERFLOW; return 0; From 5c1b13539cfc6e448226415c9033163c1e38ef13 Mon Sep 17 00:00:00 2001 From: kobakonvidia Date: Mon, 26 May 2025 16:48:35 +0000 Subject: [PATCH 075/311] NVIDIA: SAUCE: vfio/nvgrace-egm: Add null pointer checks after memory allocations BugLink: https://bugs.launchpad.net/bugs/2119656 Add missing null pointer checks after vzalloc() calls in the NVIDIA Grace GPU driver's EGM (External GPU Memory) handling code. This prevents potential null pointer dereferences in the memory failure handling and bad page fetching functions, providing proper error handling for allocation failures. Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L. Soto Signed-off-by: Matthew R. Ochs (cherry picked from commit 63127e2996a244841309ee86b4535f41e2b0de1f https://github.com/NVIDIA/NV-Kernels/tree/24.04_linux-nvidia-adv-6.11-next) Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit e5f0c8d1ba27 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (backported from commit 862ed5a2b5c58c1a1bc5d6a9d012ea82ee177389 noble:linux-nvidia-6.17) [mochs: Addressed collission for a null pointer check that is no longer needed] Signed-off-by: Matthew R. Ochs --- drivers/vfio/pci/nvgrace-gpu/egm.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.c b/drivers/vfio/pci/nvgrace-gpu/egm.c index aa9e796b6fb4f..9ad08c9cb59d0 100644 --- a/drivers/vfio/pci/nvgrace-gpu/egm.c +++ b/drivers/vfio/pci/nvgrace-gpu/egm.c @@ -360,6 +360,8 @@ static void nvgrace_egm_fetch_bad_pages(struct pci_dev *pdev, * apps. */ retired_page = (struct h_node *)(vzalloc(sizeof(struct h_node))); + if (!retired_page) + continue; /* Skip this entry on allocation failure */ retired_page->mem_offset = *((u64 *)memaddr + index + 1) - region->egmphys; hash_add(region->htbl, &retired_page->node, retired_page->mem_offset); From ee813ff447361d7ddedfea3c54832827000d3e95 Mon Sep 17 00:00:00 2001 From: Ankit Agrawal Date: Fri, 13 Feb 2026 04:11:25 +0000 Subject: [PATCH 076/311] NVIDIA: SAUCE: vfio/nvgrace-egm: split zapping EGM into 1GB chunks BugLink: https://bugs.launchpad.net/bugs/2142160 When initializing EGM (Extended GPU Memory) regions, the current implementation performs a single memset operation over the entire memory region. For very large regions, this can result in long-running uninterruptible operations that may cause system responsiveness issues or trigger watchdog timeouts. Split the memset operation into 1GB chunks. Signed-off-by: Ankit Agrawal Acked-by: Carol L Soto Acked-by: Matthew R. Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off-by: Brad Figg (cherry picked from commit 355474478031a909b2588458efb9d2c48e571735 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/vfio/pci/nvgrace-gpu/egm.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.c b/drivers/vfio/pci/nvgrace-gpu/egm.c index 9ad08c9cb59d0..a2e4c05a83c34 100644 --- a/drivers/vfio/pci/nvgrace-gpu/egm.c +++ b/drivers/vfio/pci/nvgrace-gpu/egm.c @@ -140,7 +140,20 @@ static int nvgrace_egm_open(struct inode *inode, struct file *file) return -EINVAL; } - memset((u8 *)memaddr, 0, region->egmlength); + { + size_t remaining = region->egmlength; + u8 *chunk_addr = (u8 *)memaddr; + size_t chunk_size; + + while (remaining > 0) { + chunk_size = min(remaining, SZ_1G); + memset(chunk_addr, 0, chunk_size); + cond_resched(); + chunk_addr += chunk_size; + remaining -= chunk_size; + } + } + memunmap(memaddr); file->private_data = region; From f7b2b52bd5242d52321d0da4428d8c5999d29e2c Mon Sep 17 00:00:00 2001 From: Nirmoy Das Date: Mon, 14 Jul 2025 06:56:53 -0700 Subject: [PATCH 077/311] NVIDIA: SAUCE: arm64: configs: enable NVGRACE_EGM as module BugLink: https://bugs.launchpad.net/bugs/2119656 Add CONFIG_NVGRACE_EGM with policy 'm' for arm64 architecture. Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matt Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off--by: Brad Figg (cherry picked from commit ddf68d0e4cc7 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 9ef26d933e9a9a1e1468393b0d59c0b758b1301a noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- debian.nvidia/config/annotations | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index d6b264f2030fe..76b45d4e6b595 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -156,6 +156,9 @@ CONFIG_NOUVEAU_PLATFORM_DRIVER note<'Disable nouveau for NVIDIA CONFIG_NR_CPUS policy<{'amd64': '8192', 'arm64': '512'}> CONFIG_NR_CPUS note<'LP: #1864198'> +CONFIG_NVGRACE_EGM policy<{'arm64': 'm'}> +CONFIG_NVGRACE_EGM note<'LP: #2119656'> + CONFIG_NVIDIA_FFA_EC policy<{'arm64': 'y'}> CONFIG_NVIDIA_FFA_EC note<'LP: #2114230'> From e520e822940265ce9857fb13b1033f51b6d885fd Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Tue, 26 Aug 2025 16:27:29 -0500 Subject: [PATCH 078/311] UBUNTU: [Config] nvidia: Use performance CPU frequency governor on amd64 BugLink: https://bugs.launchpad.net/bugs/2028576 The bug indicates NVIDIA wanted to enable the performance governor by default on all arches for the NVIDIA kernels. However, this was mistakenly only configured for arm64 systems. Fix this by also using the performance CPU frequency governor as the default on amd64 systems. Signed-off-by: Jacob Martin (cherry picked from commit a5304114539a272f61f5e9085c6c9ab4f1e1c783) (cherry picked from commit a5304114539a noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 93d3ca3c2d83d06cb771abf0c177e88060f02f76 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- debian.nvidia/config/annotations | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index 76b45d4e6b595..20b2f23c27369 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -97,10 +97,13 @@ CONFIG_CORESIGHT_TRBE policy<{'arm64': 'm'}> CONFIG_CORESIGHT_TRBE note<'Required for Grace enablement'> CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND policy<{'arm64': 'n'}> -CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND note<'required for NVIDIA workloads'> +CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND note<'LP: #2028576: Perf governor required for NVIDIA workloads'> -CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE policy<{'amd64': 'n', 'arm64': 'y'}> -CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE note<'required for NVIDIA workloads'> +CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE policy<{'amd64': 'y', 'arm64': 'y'}> +CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE note<'LP: #2028576: Perf governor required for NVIDIA workloads'> + +CONFIG_CPU_FREQ_DEFAULT_GOV_SCHEDUTIL policy<{'amd64': 'n', 'arm64': 'n'}> +CONFIG_CPU_FREQ_DEFAULT_GOV_SCHEDUTIL note<'LP: #2028576: Perf governor required for NVIDIA workloads'> CONFIG_DRM_NOUVEAU policy<{'amd64': 'n', 'arm64': 'n'}> CONFIG_DRM_NOUVEAU note<'Disable nouveau for NVIDIA kernels'> From c96cb83216cfc01bfbcedb0295ee3f44e9b37edf Mon Sep 17 00:00:00 2001 From: Abhishek Sahu Date: Mon, 15 Sep 2025 17:23:32 +0530 Subject: [PATCH 079/311] NVIDIA: SAUCE: Fix FFA notification count initialization BugLink: https://bugs.launchpad.net/bugs/2123861 In nvidia_ffa_create_notifications(), it invokes nvidia_ffa_fill_notification_map(), which fills the virtual notification IDs array supported by the current FFA device. This function updates notification_count after traversing the notification array in the _DSD method. For FFA devices without an entry in the _DSD method, notification_count is assumed to be zero initialized. However, nvidia_ffa_ec_service_probe() uses kmalloc() instead of kzalloc(), so notification_count may contain random values. This causes FFA device probe failures. This patch fixes this by using kzalloc() to zero initialize the nvidia_ec_ffa_device structure. Fixes: ae8718738ca8 ("NVIDIA: SAUCE: Add support for notifications from secure EC services") Signed-off-by: Abhishek Sahu Acked-by: Matthew R. Ochs Acked-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Jacob Martin Acked-by: Abdur Rahman Signed-off--by: Brad Figg (cherry picked from commit 518a89b25556 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit bf0587bed38496ce0affa41f55a51c140a4ef3e0 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/platform/arm64/nvidia-ffa-ec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/platform/arm64/nvidia-ffa-ec.c b/drivers/platform/arm64/nvidia-ffa-ec.c index d9e8b7fdda30c..7a259ba64956c 100644 --- a/drivers/platform/arm64/nvidia-ffa-ec.c +++ b/drivers/platform/arm64/nvidia-ffa-ec.c @@ -617,7 +617,7 @@ static int nvidia_ffa_ec_service_probe(struct ffa_device *ffa_dev) return -ENODEV; } - nvidia_ec_ffa_dev = devm_kmalloc(&ffa_dev->dev, + nvidia_ec_ffa_dev = devm_kzalloc(&ffa_dev->dev, sizeof(*nvidia_ec_ffa_dev), GFP_KERNEL); if (!nvidia_ec_ffa_dev) { From 7f30c8c699505177f4f038a51073e772aedd9bc8 Mon Sep 17 00:00:00 2001 From: Us Chien Date: Mon, 26 May 2025 17:46:21 +0800 Subject: [PATCH 080/311] NVIDIA: SAUCE: MEDIATEK: usb: host: xhci-hub: fix MT89xx SoCs return PORTLI value BugLink: https://bugs.launchpad.net/bugs/2125126 For DIGITS GB10, USB SuperSpeed Plus Gen2x1 device enumeration speed is being shown as Gen2x2 in the dmesg log. usb 4-1: new SuperSpeed Plus Gen 2x2 USB device number 2 using xhci-hcd The USB3 link lanes count is recorded in the read-only TLC and RLC fields of the PORTLI register. In MT89xx SoCs (used by GB10), the lane count can be wrongly set to 2 instead of 1 for USB Gen2x1 devices due to HW Bug. As per Table 7-13 in USB xHCI revision 1.2 specification, the value 0x5 is for SuperSpeedPlus Gen2x1. This patch adds a SW WAR to read the port speed in PORTSC register. If port speed value is 0x5, then the lane count can be updated to 1. Signed-off-by: Us Chien Signed-off-by: Abhishek Sahu Acked-by: Carol L Soto Acked-by: Kai-Heng Feng Acked-by: Matthew R. Ochs Acked-by: Jacob Martin Acked-by: Abdur Rahman Signed-off--by: Brad Figg (cherry picked from commit b5b3a58605e6 noble:linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit c65499578d0c4af75076eea7d9a44ba8476d34a7 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/usb/host/xhci-hub.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/usb/host/xhci-hub.c b/drivers/usb/host/xhci-hub.c index 04cc3d681495f..09a92ab1537b2 100644 --- a/drivers/usb/host/xhci-hub.c +++ b/drivers/usb/host/xhci-hub.c @@ -1290,6 +1290,17 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, } port_li = readl(&port->port_reg->portli); status = xhci_get_ext_port_status(temp, port_li); + + /* + * In MT8901 USB host controller, the lane count can be wrongly set + * to 2 instead of 1 for USB Gen2x1 devices due to a HW Bug. As a SW + * WAR, check if port speed is 0x5 (SuperSpeedPlus Gen2x1) in + * PORTSC register and update the lane count as 1. + */ + if ((xhci->quirks & XHCI_NVIDIA_MT8901_HOST) && + DEV_SUPERSPEEDPLUS(temp)) + status &= ~0xff00; + put_unaligned_le32(status, &buf[4]); } break; From 072524189286352e56f2426039807d7ca33e6ef9 Mon Sep 17 00:00:00 2001 From: Andy Ritger Date: Tue, 20 May 2025 17:11:58 -0700 Subject: [PATCH 081/311] NVIDIA: SAUCE: iommu/io-pgtable-arm: backport contiguous bit support BugLink: https://bugs.launchpad.net/bugs/2112600 iommu/io-pgtable-arm: Support contiguous bit in translation tables The contiguous bit in translation table entries can be used as a hint to SMMU that a group of adjacent translation table entries have consistent attributes and point to a contiguous and properly aligned output address range. This enables SMMU to predict the properties of the remaining translation table entries in the same group without accessing them. It also allows an SMMU implementation to make more efficient use of its TLB by using a single TLB entry to cover all translation table entries in the same group. In the case of 4KB granule size, there are 16 translation table entries in one group. This change sets the contiguous bit for such groups of entries that are completely covered by a single call to map_pages. As it stands, the code wouldn't set the contiguous bit if a group of adjacent descriptors is completed by separate calls to map_pages. Signed-off-by: Daniel Mentz Link: https://lore.kernel.org/linux-iommu/20250430231924.1481493-1-danielmentz@google.com/ [aritger: For the backport: updated the __arm_lpae_init_pte() callsite in arm_lpae_split_blk_unmap() (arm_lpae_split_blk_unmap() is no longer present at top of tree where the original commit was written).] Signed-off-by: Andy Ritger Acked-by: Jamie Nguyen Acked-by: Carol L Soto Acked-by: Abdur Rahman Acked-by: Noah Wager Signed-off--by: Brad Figg (backported from commit 990fa55beac8efffb7a94b26be553d52c232b9f1 linux-nvidia-6.11) [kobak: arm_lpae_split_blk_unmap is removed since https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=33729a5fc0caf7a97d20507acbeee6b012e7e519 iommu/io-pgtable-arm: Remove split on unmap behavior so modify as per functions] Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Jacob Martin Acked-by: Abdur Rahman Signed-off-by: Ian May (cherry picked from commit c601e605745ea9f16834da6e24049b92b759b78f noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/iommu/io-pgtable-arm.c | 52 ++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 0208e5897c299..1ebf9631c1825 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -75,6 +75,7 @@ #define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63) #define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53) +#define ARM_LPAE_PTE_CONT (((arm_lpae_iopte)1) << 52) #define ARM_LPAE_PTE_DBM (((arm_lpae_iopte)1) << 51) #define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10) #define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8) @@ -320,6 +321,27 @@ static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries, sizeof(*ptep) * num_entries, DMA_TO_DEVICE); } +static int arm_lpae_cont_ptes(int lvl, struct arm_lpae_io_pgtable *data) +{ + switch (ARM_LPAE_GRANULE(data)) { + case SZ_4K: + if (lvl >= 1) + return 16; + break; + case SZ_16K: + if (lvl == 2) + return 32; + else if (lvl == 3) + return 128; + break; + case SZ_64K: + if (lvl >= 2) + return 32; + break; + } + return 1; +} + static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg, int num_entries) { for (int i = 0; i < num_entries; i++) @@ -329,13 +351,35 @@ static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cf __arm_lpae_sync_pte(ptep, num_entries, cfg); } +static bool arm_lpae_use_contpte(struct arm_lpae_io_pgtable *data, + unsigned long iova, phys_addr_t paddr, + int lvl, int num_entries, int i) +{ + size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data); + int cont_ptes = arm_lpae_cont_ptes(lvl, data); + int contmask = cont_ptes - 1; + int contpte_addr_mask = sz * cont_ptes - 1; + int map_idx_start, tbl_idx; + + if ((paddr & contpte_addr_mask) != (iova & contpte_addr_mask)) + return false; + + map_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data); + tbl_idx = map_idx_start + i; + if (((tbl_idx & contmask) <= i) && + (tbl_idx < ((map_idx_start + num_entries) & ~contmask))) + return true; + + return false; +} + static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, struct iommu_iotlb_gather *gather, unsigned long iova, size_t size, size_t pgcount, int lvl, arm_lpae_iopte *ptep); static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, - phys_addr_t paddr, arm_lpae_iopte prot, + unsigned long iova, phys_addr_t paddr, arm_lpae_iopte prot, int lvl, int num_entries, arm_lpae_iopte *ptep) { arm_lpae_iopte pte = prot; @@ -349,7 +393,9 @@ static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, pte |= ARM_LPAE_PTE_TYPE_BLOCK; for (i = 0; i < num_entries; i++) - ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data); + ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data) | + (arm_lpae_use_contpte(data, iova, paddr, lvl, num_entries, i) ? + ARM_LPAE_PTE_CONT : 0); if (!cfg->coherent_walk) __arm_lpae_sync_pte(ptep, num_entries, cfg); @@ -383,7 +429,7 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, } } - __arm_lpae_init_pte(data, paddr, prot, lvl, num_entries, ptep); + __arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep); return 0; } From 466c0922814cbe5ae0355eb86f8db95b66854809 Mon Sep 17 00:00:00 2001 From: Leon Yen Date: Fri, 26 Sep 2025 13:34:47 +0800 Subject: [PATCH 082/311] NVIDIA: SAUCE: wifi: mt76: mt7925: introduce CSA support in non-MLO mode BugLink: https://bugs.launchpad.net/bugs/2129209 Add CSA (Channel Switch Announcement) related implementation in collaboration with mac80211 to deal with dynamic channel switching. Signed-off-by: Leon Yen Signed-off-by: Ming Yen Hsieh Conflicts: - drivers/net/wireless/mediatek/mt76/mt7925/main.c Code is different in mt7925_add_interface() - drivers/net/wireless/mediatek/mt76/mt792x_core.c Code organization is different Signed-off-by: Abhishek Sahu [Backported from https://patchwork.kernel.org/project/linux-wireless/patch/20250926053447.4036650-1-mingyen.hsieh@mediatek.com/] Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Acked-by: Abdur Rahman Acked-by: Jacob Martin Signed-off-by: Ian May (cherry picked from commit 060dc92285ea8a49e7ea0a87f028066cd105b62f noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- .../net/wireless/mediatek/mt76/mt7925/main.c | 138 ++++++++++++++++++ .../wireless/mediatek/mt76/mt7925/mt7925.h | 1 + .../net/wireless/mediatek/mt76/mt792x_core.c | 4 +- 3 files changed, 140 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/main.c b/drivers/net/wireless/mediatek/mt76/mt7925/main.c index 2d358a96640c9..45810ff17c019 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/main.c @@ -245,6 +245,7 @@ int mt7925_init_mlo_caps(struct mt792x_phy *phy) { struct wiphy *wiphy = phy->mt76->hw->wiphy; static const u8 ext_capa_sta[] = { + [0] = WLAN_EXT_CAPA1_EXT_CHANNEL_SWITCHING, [2] = WLAN_EXT_CAPA3_MULTI_BSSID_SUPPORT, [7] = WLAN_EXT_CAPA8_OPMODE_NOTIF, }; @@ -438,6 +439,8 @@ mt7925_add_interface(struct ieee80211_hw *hw, struct ieee80211_vif *vif) if (phy->chip_cap & MT792x_CHIP_CAP_RSSI_NOTIFY_EVT_EN) vif->driver_flags |= IEEE80211_VIF_SUPPORTS_CQM_RSSI; + INIT_WORK(&mvif->csa_work, mt7925_csa_work); + timer_setup(&mvif->csa_timer, mt792x_csa_timer, 0); out: mt792x_mutex_release(dev); @@ -1749,6 +1752,10 @@ static int mt7925_add_chanctx(struct ieee80211_hw *hw, struct ieee80211_chanctx_conf *ctx) { + struct mt792x_dev *dev = mt792x_hw_dev(hw); + + dev->new_ctx = ctx; + return 0; } @@ -1756,6 +1763,11 @@ static void mt7925_remove_chanctx(struct ieee80211_hw *hw, struct ieee80211_chanctx_conf *ctx) { + struct mt792x_dev *dev = mt792x_hw_dev(hw); + + if (dev->new_ctx == ctx) + dev->new_ctx = NULL; + } static void @@ -2144,6 +2156,11 @@ static void mt7925_unassign_vif_chanctx(struct ieee80211_hw *hw, mctx->bss_conf = NULL; mconf->mt76.ctx = NULL; mutex_unlock(&dev->mt76.mutex); + + if (link_conf->csa_active) { + timer_delete_sync(&mvif->csa_timer); + cancel_work_sync(&mvif->csa_work); + } } static void mt7925_rfkill_poll(struct ieee80211_hw *hw) @@ -2158,6 +2175,121 @@ static void mt7925_rfkill_poll(struct ieee80211_hw *hw) wiphy_rfkill_set_hw_state(hw->wiphy, ret == 0); } +static int mt7925_switch_vif_chanctx(struct ieee80211_hw *hw, + struct ieee80211_vif_chanctx_switch *vifs, + int n_vifs, + enum ieee80211_chanctx_switch_mode mode) +{ + return mt7925_assign_vif_chanctx(hw, vifs->vif, vifs->link_conf, + vifs->new_ctx); +} + +void mt7925_csa_work(struct work_struct *work) +{ + struct mt792x_vif *mvif; + struct mt792x_dev *dev; + struct ieee80211_vif *vif; + struct ieee80211_bss_conf *link_conf; + struct mt792x_bss_conf *mconf; + u8 link_id, roc_rtype; + int ret = 0; + + mvif = (struct mt792x_vif *)container_of(work, struct mt792x_vif, + csa_work); + dev = mvif->phy->dev; + vif = container_of((void *)mvif, struct ieee80211_vif, drv_priv); + + if (ieee80211_vif_is_mld(vif)) + return; + + if (!dev->new_ctx) + return; + + link_id = 0; + mconf = &mvif->bss_conf; + link_conf = &vif->bss_conf; + roc_rtype = MT7925_ROC_REQ_JOIN; + + mt792x_mutex_acquire(dev); + ret = mt7925_set_roc(mvif->phy, mconf, dev->new_ctx->def.chan, + 4000, roc_rtype); + mt792x_mutex_release(dev); + if (!ret) { + mt792x_mutex_acquire(dev); + ret = mt7925_mcu_set_chctx(mvif->phy->mt76, &mconf->mt76, link_conf, + dev->new_ctx); + mt792x_mutex_release(dev); + + mt7925_abort_roc(mvif->phy, mconf); + } + + ieee80211_chswitch_done(vif, !ret, link_id); +} + +static int mt7925_pre_channel_switch(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_channel_switch *chsw) +{ + if (ieee80211_vif_is_mld(vif)) + return -EOPNOTSUPP; + + if (vif->type != NL80211_IFTYPE_STATION || !vif->cfg.assoc) + return -EOPNOTSUPP; + + if (!cfg80211_chandef_usable(hw->wiphy, &chsw->chandef, + IEEE80211_CHAN_DISABLED)) + return -EOPNOTSUPP; + + return 0; +} + +static void mt7925_channel_switch(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_channel_switch *chsw) +{ + struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + u16 beacon_interval; + + if (ieee80211_vif_is_mld(vif)) + return; + + beacon_interval = vif->bss_conf.beacon_int; + + mvif->csa_timer.expires = TU_TO_EXP_TIME(beacon_interval * chsw->count); + add_timer(&mvif->csa_timer); +} + +static void mt7925_abort_channel_switch(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_bss_conf *link_conf) +{ + struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + + timer_delete_sync(&mvif->csa_timer); + cancel_work_sync(&mvif->csa_work); +} + +static void mt7925_channel_switch_rx_beacon(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_channel_switch *chsw) +{ + struct mt792x_dev *dev = mt792x_hw_dev(hw); + struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + u16 beacon_interval; + + if (ieee80211_vif_is_mld(vif)) + return; + + beacon_interval = vif->bss_conf.beacon_int; + + if (cfg80211_chandef_identical(&chsw->chandef, + &dev->new_ctx->def) && + chsw->count) { + mod_timer(&mvif->csa_timer, + TU_TO_EXP_TIME(beacon_interval * chsw->count)); + } +} + const struct ieee80211_ops mt7925_ops = { .tx = mt792x_tx, .start = mt7925_start, @@ -2221,6 +2353,12 @@ const struct ieee80211_ops mt7925_ops = { .change_vif_links = mt7925_change_vif_links, .change_sta_links = mt7925_change_sta_links, .rfkill_poll = mt7925_rfkill_poll, + + .switch_vif_chanctx = mt7925_switch_vif_chanctx, + .pre_channel_switch = mt7925_pre_channel_switch, + .channel_switch = mt7925_channel_switch, + .abort_channel_switch = mt7925_abort_channel_switch, + .channel_switch_rx_beacon = mt7925_channel_switch_rx_beacon, }; EXPORT_SYMBOL_GPL(mt7925_ops); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mt7925.h b/drivers/net/wireless/mediatek/mt76/mt7925/mt7925.h index 6b9bf1b890320..5030d7714bcf2 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mt7925.h +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mt7925.h @@ -298,6 +298,7 @@ int mt7925_mcu_uni_rx_ba(struct mt792x_dev *dev, void mt7925_mlo_pm_work(struct work_struct *work); void mt7925_scan_work(struct work_struct *work); void mt7925_roc_work(struct work_struct *work); +void mt7925_csa_work(struct work_struct *work); int mt7925_mcu_uni_bss_ps(struct mt792x_dev *dev, struct ieee80211_bss_conf *link_conf); void mt7925_coredump_work(struct work_struct *work); diff --git a/drivers/net/wireless/mediatek/mt76/mt792x_core.c b/drivers/net/wireless/mediatek/mt76/mt792x_core.c index f2ed16feb6c1b..1d8b8c00b7eef 100644 --- a/drivers/net/wireless/mediatek/mt76/mt792x_core.c +++ b/drivers/net/wireless/mediatek/mt76/mt792x_core.c @@ -691,9 +691,7 @@ int mt792x_init_wiphy(struct ieee80211_hw *hw) ieee80211_hw_set(hw, SUPPORTS_MULTI_BSSID); ieee80211_hw_set(hw, SUPPORTS_ONLY_HE_MULTI_BSSID); - if (is_mt7921(&dev->mt76)) { - ieee80211_hw_set(hw, CHANCTX_STA_CSA); - } + ieee80211_hw_set(hw, CHANCTX_STA_CSA); if (dev->pm.enable) ieee80211_hw_set(hw, CONNECTION_MONITOR); From 8fb5682ecdcea271dc3b2c2c8ecdf03ea8c6a005 Mon Sep 17 00:00:00 2001 From: Nirmoy Das Date: Wed, 10 Sep 2025 08:23:52 -0700 Subject: [PATCH 083/311] NVIDIA: SAUCE: iommu/arm-smmu-v3: Set DGX Spark iGPU default domain type to DMA BugLink: https://bugs.launchpad.net/bugs/2129776 Force DGX Spark systems to use DMA translation as current drivers require this. Suggested-by: Jason Gunthorpe Signed-off-by: Nirmoy Das Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Acked-by: Abdur Rahman Acked-by: Jacob Martin Signed-off-by: Ian May (cherry picked from commit 54575cfd0fa45463ba90a504e01fcd27d9dd020f noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 4d00d796f0783..45a3d6d0a29f2 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3730,6 +3730,9 @@ static int arm_smmu_def_domain_type(struct device *dev) if (IS_HISI_PTT_DEVICE(pdev)) return IOMMU_DOMAIN_IDENTITY; + + if (pdev->vendor == PCI_VENDOR_ID_NVIDIA && pdev->device == 0x2E12) + return IOMMU_DOMAIN_DMA; } return 0; From c069047fda987f026e78670059b1c5b44d9705f8 Mon Sep 17 00:00:00 2001 From: Nirmoy Das Date: Fri, 24 Oct 2025 09:44:45 -0700 Subject: [PATCH 084/311] UBUNTU: [Config] nvidia: Update annotations to set CONFIG_IOMMU_DEFAULT_PASSTHROUGH BugLink: https://bugs.launchpad.net/bugs/2129776 Default to CONFIG_IOMMU_DEFAULT_PASSTHROUGH on NVIDIA CPU on kernel above 6.11 as suggested by perf team. x86 always defaults to CONFIG_IOMMU_DEFAULT_DMA_LAZY so remove redundant amd64 setting. Signed-off-by: Nirmoy Das Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Acked-by: Abdur Rahman Acked-by: Jacob Martin Signed-off-by: Ian May (backported from commit e74a7d849ad16ef5a96df087a880c42ce6985a0b noble:linux-nvidia-6.17) [mochs: Addressed minor context collission] Signed-off-by: Matthew R. Ochs --- debian.nvidia/config/annotations | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index 20b2f23c27369..e9093504c2ddd 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -132,6 +132,15 @@ CONFIG_GPIO_AAEON note<'Disable all Ubuntu ODM dri CONFIG_IOMMUFD_VFIO_CONTAINER policy<{'arm64': 'y'}> CONFIG_IOMMUFD_VFIO_CONTAINER note<'LP: #2095028'> +CONFIG_IOMMU_DEFAULT_DMA_LAZY policy<{'amd64': 'y', 'arm64': 'n'}> +CONFIG_IOMMU_DEFAULT_DMA_LAZY note<'On Nvidia CPU passthrough mode is recommend so set passthrough mode as default for better performance'> + +CONFIG_IOMMU_DEFAULT_DMA_STRICT policy<{'amd64': 'n', 'arm64': 'n'}> +CONFIG_IOMMU_DEFAULT_DMA_STRICT note<'On Nvidia CPU passthrough mode is recommend so set passthrough mode as default for better performance'> + +CONFIG_IOMMU_DEFAULT_PASSTHROUGH policy<{'amd64': 'n', 'arm64': 'y'}> +CONFIG_IOMMU_DEFAULT_PASSTHROUGH note<'On Nvidia CPU passthrough mode is recommend so set passthrough mode as default for better performance'> + CONFIG_LEDS_AAEON policy<{'amd64': '-'}> CONFIG_LEDS_AAEON note<'Disable all Ubuntu ODM drivers'> From 53690650fe9c18ef6b76c23e212566a8ca266b69 Mon Sep 17 00:00:00 2001 From: ChunHao Lin Date: Tue, 28 Oct 2025 22:51:44 +0800 Subject: [PATCH 085/311] NVIDIA: SAUCE: r8127: fix a kernel panic when dump all registers BugLink: https://bugs.launchpad.net/bugs/2130445 The call to cat registers2 will acquire rtnl_lock and dump all mapped mmio. Due to acquire rtnl_lock too long, it will cause a kernel panic "not syncing: SBSA Generic Watchdog timeout". Fix this issue by acquiring rtnl_lock every 16 byte when dump all mapped mmio. Signed-off-by: ChunHao Lin Signed-off-by: Abhishek Sahu Acked-by: Carol L Soto Acked-by: Matthew R. Ochs Acked-by: Ian May Acked-by: Noah Wager Acked-by: Abdur Rahman Signed-off--by: Brad Figg (cherry picked from commit 9737b84c973cb1e6b8352d6a679051b4d4b09acc noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/net/ethernet/realtek/r8127/r8127_n.c | 32 ++++++++++++-------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/drivers/net/ethernet/realtek/r8127/r8127_n.c b/drivers/net/ethernet/realtek/r8127/r8127_n.c index 4f83e44869deb..4ec33b3d77d5f 100755 --- a/drivers/net/ethernet/realtek/r8127/r8127_n.c +++ b/drivers/net/ethernet/realtek/r8127/r8127_n.c @@ -1161,17 +1161,21 @@ static int proc_get_registers(struct seq_file *m, void *v) seq_puts(m, "\nDump MAC Registers\n"); seq_puts(m, "Offset\tValue\n------\t-----\n"); - rtnl_lock(); - for (n = 0; n < max;) { seq_printf(m, "\n0x%04x:\t", n); + rtnl_lock(); + for (i = 0; i < 16 && n < max; i++, n++) { byte_rd = readb(ioaddr + n); seq_printf(m, "%02x ", byte_rd); } + + rtnl_unlock(); } + rtnl_lock(); + max = 0xB00; for (n = 0xA00; n < max;) { seq_printf(m, "\n0x%04x:\t", n); @@ -1220,20 +1224,20 @@ static int proc_get_all_registers(struct seq_file *m, void *v) seq_puts(m, "\nDump All MAC Registers\n"); seq_puts(m, "Offset\tValue\n------\t-----\n"); - rtnl_lock(); - max = pci_resource_len(pdev, 2); for (n = 0; n < max;) { seq_printf(m, "\n0x%04x:\t", n); + rtnl_lock(); + for (i = 0; i < 16 && n < max; i++, n++) { byte_rd = readb(ioaddr + n); seq_printf(m, "%02x ", byte_rd); } - } - rtnl_unlock(); + rtnl_unlock(); + } seq_printf(m, "\nTotal length:0x%X", max); @@ -2073,21 +2077,25 @@ static int proc_get_registers(char *page, char **start, "\nDump MAC Registers\n" "Offset\tValue\n------\t-----\n"); - rtnl_lock(); - for (n = 0; n < max;) { len += snprintf(page + len, count - len, "\n0x%04x:\t", n); + rtnl_lock(); + for (i = 0; i < 16 && n < max; i++, n++) { byte_rd = readb(ioaddr + n); len += snprintf(page + len, count - len, "%02x ", byte_rd); } + + rtnl_unlock(); } + rtnl_lock(); + max = 0xB00; for (n = 0xA00; n < max;) { len += snprintf(page + len, count - len, @@ -2154,8 +2162,6 @@ static int proc_get_all_registers(char *page, char **start, "\nDump All MAC Registers\n" "Offset\tValue\n------\t-----\n"); - rtnl_lock(); - max = pci_resource_len(pdev, 2); for (n = 0; n < max;) { @@ -2163,15 +2169,17 @@ static int proc_get_all_registers(char *page, char **start, "\n0x%04x:\t", n); + rtnl_lock(); + for (i = 0; i < 16 && n < max; i++, n++) { byte_rd = readb(ioaddr + n); len += snprintf(page + len, count - len, "%02x ", byte_rd); } - } - rtnl_unlock(); + rtnl_unlock(); + } len += snprintf(page + len, count - len, "\nTotal length:0x%X", max); From 2415881d8727a3dd536fbea2cd32cbdd4cd049a1 Mon Sep 17 00:00:00 2001 From: ChunHao Lin Date: Tue, 28 Oct 2025 23:01:21 +0800 Subject: [PATCH 086/311] NVIDIA: SAUCE: r8127: add support for RTL8127 cable diagnostic test BugLink: https://bugs.launchpad.net/bugs/2130445 Use following command to do the test. cat /proc/net/r8127//test/cdt Signed-off-by: ChunHao Lin Signed-off-by: Abhishek Sahu Acked-by: Carol L Soto Acked-by: Matthew R. Ochs Acked-by: Ian May Acked-by: Noah Wager Acked-by: Abdur Rahman Signed-off--by: Brad Figg (cherry picked from commit aaf12caf97f50f3c310db3d9a2f321e93d8ac62d noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/net/ethernet/realtek/r8127/r8127_n.c | 45 +++++++++----------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/drivers/net/ethernet/realtek/r8127/r8127_n.c b/drivers/net/ethernet/realtek/r8127/r8127_n.c index 4ec33b3d77d5f..2c9c262abe2ea 100755 --- a/drivers/net/ethernet/realtek/r8127/r8127_n.c +++ b/drivers/net/ethernet/realtek/r8127/r8127_n.c @@ -690,6 +690,12 @@ rtl8127_get_sw_tail_ptr(struct rtl8127_tx_ring *ring) } } +static u32 +rtl8127_get_phy_status(struct rtl8127_private *tp) +{ + return RTL_R32(tp, PHYstatus); +} + static bool rtl8127_sysfs_testmode_on(struct rtl8127_private *tp) { @@ -795,20 +801,19 @@ static void rtl8127_get_cp_len(struct rtl8127_private *tp, int cp_len[RTL8127_CP_NUM]) { int i; - u16 status; + u32 status; int tmp_cp_len; - status = RTL_R16(tp, PHYstatus); + status = rtl8127_get_phy_status(tp); if (status & LinkStatus) { if (status & _10bps) { tmp_cp_len = -1; } else if (status & (_100bps | _1000bpsF)) { - rtl8127_mdio_write(tp, 0x1f, 0x0a88); - tmp_cp_len = rtl8127_mdio_read(tp, 0x10); - } else if (status & _2500bpsF) { - rtl8127_mdio_write(tp, 0x1f, 0x0acb); - tmp_cp_len = rtl8127_mdio_read(tp, 0x15); - tmp_cp_len >>= 2; + tmp_cp_len = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA880);; + } else if (status & (_10000bpsF | _10000bpsL | _5000bpsF | + _5000bpsL | _2500bpsF | _2500bpsL)) { + tmp_cp_len = rtl8127_mdio_direct_read_phy_ocp(tp, 0xAC2E);; + tmp_cp_len >>= 5; } else tmp_cp_len = 0; } else @@ -834,12 +839,11 @@ static int __rtl8127_get_cp_status(u16 val) case 0x0060: return rtl8127_cp_normal; case 0x0048: + case 0x0042: return rtl8127_cp_open; case 0x0050: - return rtl8127_cp_short; - case 0x0042: case 0x0044: - return rtl8127_cp_mismatch; + return rtl8127_cp_short; default: return rtl8127_cp_normal; } @@ -853,7 +857,7 @@ static int _rtl8127_get_cp_status(struct rtl8127_private *tp, u8 pair_num) if (pair_num > 3) goto exit; - rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8027 + 4 * pair_num); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8026 + 4 * pair_num); val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA438); cp_status = __rtl8127_get_cp_status(val); @@ -885,7 +889,7 @@ static u16 rtl8127_get_cp_pp(struct rtl8127_private *tp, u8 pair_num) if (pair_num > 3) goto exit; - rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8029 + 4 * pair_num); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8028 + 4 * pair_num); pp = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA438); pp &= 0x3fff; @@ -899,10 +903,10 @@ static void rtl8127_get_cp_status(struct rtl8127_private *tp, int cp_status[RTL8127_CP_NUM], bool poe_mode) { - u16 status; + u32 status; int i; - status = RTL_R16(tp, PHYstatus); + status = rtl8127_get_phy_status(tp); if (status & LinkStatus && !(status & (_10bps | _100bps))) { for (i=0; iprivate; @@ -1452,12 +1456,6 @@ static int _proc_get_cable_info(struct seq_file *m, void *v, bool poe_mode) const char *pair_str[RTL8127_CP_NUM] = {"1-2", "3-6", "4-5", "7-8"}; int ret; - switch (tp->mcfg) { - default: - ret = -EOPNOTSUPP; - goto error_out; - } - rtnl_lock(); if (!rtl8127_sysfs_testmode_on(tp)) { @@ -1474,7 +1472,7 @@ static int _proc_get_cable_info(struct seq_file *m, void *v, bool poe_mode) netif_testing_on(dev); - status = RTL_R16(tp, PHYstatus); + status = rtl8127_get_phy_status(tp); if (status & LinkStatus) seq_printf(m, "\nlink speed:%d", rtl8127_convert_link_speed(status)); @@ -1511,7 +1509,6 @@ static int _proc_get_cable_info(struct seq_file *m, void *v, bool poe_mode) error_unlock: rtnl_unlock(); -error_out: return ret; } From 924a7a94dd9668ba46124046bb98729cc7f393d8 Mon Sep 17 00:00:00 2001 From: Nirmoy Das Date: Thu, 20 Nov 2025 08:10:15 -0800 Subject: [PATCH 087/311] NVIDIA: SAUCE: iommu/arm-smmu-v3: Add two more DGX Spark iGPU IDs for existing iommu quirk BugLink: https://bugs.launchpad.net/bugs/2132033 Add two more device IDs for the existing Spark iommu quirk. Link: https://bugs.launchpad.net/ubuntu/+source/linux-nvidia-6.14/+bug/2132033 Signed-off-by: Nirmoy Das Acked-by: Jamie Nguyen Acked-by: Carol L Soto Acked-by: Abdur Rahman Acked-by: Noah Wager Signed-off--by: Brad Figg (cherry picked from commit ba9315458447d2677fb83131d78e1520f51b6c8a noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 45a3d6d0a29f2..821e7d3da07bb 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3731,7 +3731,9 @@ static int arm_smmu_def_domain_type(struct device *dev) if (IS_HISI_PTT_DEVICE(pdev)) return IOMMU_DOMAIN_IDENTITY; - if (pdev->vendor == PCI_VENDOR_ID_NVIDIA && pdev->device == 0x2E12) + if (pdev->vendor == PCI_VENDOR_ID_NVIDIA && + (pdev->device == 0x2E12 || pdev->device == 0x2E2A || + pdev->device == 0x2E2B)) return IOMMU_DOMAIN_DMA; } From 857693fe6972ac0d4898bfc88058278d9ab2eb4f Mon Sep 17 00:00:00 2001 From: ChunHao Lin Date: Thu, 4 Dec 2025 16:51:16 +0800 Subject: [PATCH 088/311] NVIDIA: SAUCE: r8127: Remove registers2 proc entry BugLink: https://bugs.launchpad.net/bugs/2134991 Remove registers2 proc entry as it is causing system crash on running opensource LTP test suite. Change-Id: I47846bca0401d4403fba026d4a348eef3d454f80 Signed-off-by: ChunHao Lin Acked-by: Jamie Nguyen Acked-by: Carol L Soto Acked-by: Matthew R. Ochs Acked-by: Abdur Rahman Acked-by: Jacob Martin Signed-off-by: Brad Figg (cherry picked from commit 4ff6f9ff6979cd66f18f8c620fc966971fb48470 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/net/ethernet/realtek/r8127/r8127_n.c | 77 -------------------- 1 file changed, 77 deletions(-) diff --git a/drivers/net/ethernet/realtek/r8127/r8127_n.c b/drivers/net/ethernet/realtek/r8127/r8127_n.c index 2c9c262abe2ea..496fec1320d12 100755 --- a/drivers/net/ethernet/realtek/r8127/r8127_n.c +++ b/drivers/net/ethernet/realtek/r8127/r8127_n.c @@ -1216,39 +1216,6 @@ static int proc_get_registers(struct seq_file *m, void *v) return 0; } -static int proc_get_all_registers(struct seq_file *m, void *v) -{ - struct net_device *dev = m->private; - int i, n, max; - u8 byte_rd; - struct rtl8127_private *tp = netdev_priv(dev); - void __iomem *ioaddr = tp->mmio_addr; - struct pci_dev *pdev = tp->pci_dev; - - seq_puts(m, "\nDump All MAC Registers\n"); - seq_puts(m, "Offset\tValue\n------\t-----\n"); - - max = pci_resource_len(pdev, 2); - - for (n = 0; n < max;) { - seq_printf(m, "\n0x%04x:\t", n); - - rtnl_lock(); - - for (i = 0; i < 16 && n < max; i++, n++) { - byte_rd = readb(ioaddr + n); - seq_printf(m, "%02x ", byte_rd); - } - - rtnl_unlock(); - } - - seq_printf(m, "\nTotal length:0x%X", max); - - seq_putc(m, '\n'); - return 0; -} - static int proc_get_pcie_phy(struct seq_file *m, void *v) { struct net_device *dev = m->private; @@ -2143,49 +2110,6 @@ static int proc_get_registers(char *page, char **start, return len; } -static int proc_get_all_registers(char *page, char **start, - off_t offset, int count, - int *eof, void *data) -{ - struct net_device *dev = data; - int i, n, max; - u8 byte_rd; - struct rtl8127_private *tp = netdev_priv(dev); - void __iomem *ioaddr = tp->mmio_addr; - struct pci_dev *pdev = tp->pci_dev; - int len = 0; - - len += snprintf(page + len, count - len, - "\nDump All MAC Registers\n" - "Offset\tValue\n------\t-----\n"); - - max = pci_resource_len(pdev, 2); - - for (n = 0; n < max;) { - len += snprintf(page + len, count - len, - "\n0x%04x:\t", - n); - - rtnl_lock(); - - for (i = 0; i < 16 && n < max; i++, n++) { - byte_rd = readb(ioaddr + n); - len += snprintf(page + len, count - len, - "%02x ", - byte_rd); - } - - rtnl_unlock(); - } - - len += snprintf(page + len, count - len, "\nTotal length:0x%X", max); - - len += snprintf(page + len, count - len, "\n"); - - *eof = 1; - return len; -} - static int proc_get_pcie_phy(char *page, char **start, off_t offset, int count, int *eof, void *data) @@ -2784,7 +2708,6 @@ static const struct rtl8127_proc_file rtl8127_debug_proc_files[] = { { "driver_var", &proc_get_driver_variable }, { "tally", &proc_get_tally_counter }, { "registers", &proc_get_registers }, - { "registers2", &proc_get_all_registers }, { "pcie_phy", &proc_get_pcie_phy }, { "eth_phy", &proc_get_eth_phy }, { "ext_regs", &proc_get_extended_registers }, From 74743883f6279383f07143c312cae71ced76fd21 Mon Sep 17 00:00:00 2001 From: Surabhi Chythanya Kumar Date: Thu, 8 Jan 2026 18:13:30 -0800 Subject: [PATCH 089/311] NVIDIA: SAUCE: MEDIATEK: platform: Add PCIe Hotplug Driver for CX7 on DGX Spark BugLink: https://bugs.launchpad.net/bugs/2138269 This driver manages PCIe link for NVIDIA ConnectX-7 (CX7) hot-plug/unplug on DGX Spark systems with GB10 SoC. It disables the PCIe link on cable removal and enables it on cable insertion. Upstream-friendly improvements over 6.14 driver: - Separated from MTK pinctrl driver into NVIDIA platform driver - Configuration via ACPI (_CRS and _DSD), no hardcoded values - Device-managed resources (devm_*) for automatic cleanup - Thread-safe state management with locking - Enhanced error handling and logging - Uses standard Linux kernel APIs The driver exposes a sysfs interface to emulate cable plug in/out: echo 1 > /sys/devices/platform/MTKP0001:00/pcie_hotplug/debug_state # plug in echo 0 > /sys/devices/platform/MTKP0001:00/pcie_hotplug/debug_state # plug out It also provides a runtime enable/disable switch via sysfs: echo 1 > /sys/devices/platform/MTKP0001:00/pcie_hotplug/hotplug_enabled # Enable echo 0 > /sys/devices/platform/MTKP0001:00/pcie_hotplug/hotplug_enabled # Disable This allows enabling/disabling hotplug functionality. Hotplug is disabled by default and must be explicitly enabled via userspace. It also implements uevent notifications for coordination with userspace: * cable plug-in: Report plug-in uevent (driver) Enable PCIe link (driver) Rescan CX7 devices (application) * cable removal: Report removal uevent (driver) Remove CX7 devices (application) Disable PCIe link (driver) Signed-off-by: Vaibhav Vyas Signed-off-by: Scott Fudally Signed-off-by: Surabhi Chythanya Kumar Acked-by: Jamie Nguyen Acked-by: Carol L Soto Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off-by: Brad Figg (cherry picked from commit 4894eb0dfc2743560465531f17e9070f6cf52017 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- debian.nvidia/config/annotations | 3 + drivers/platform/arm64/Kconfig | 2 + drivers/platform/arm64/Makefile | 1 + drivers/platform/arm64/nvidia/Kconfig | 17 + drivers/platform/arm64/nvidia/Makefile | 9 + .../platform/arm64/nvidia/mtk-pcie-hotplug.c | 2324 +++++++++++++++++ 6 files changed, 2356 insertions(+) create mode 100644 drivers/platform/arm64/nvidia/Kconfig create mode 100644 drivers/platform/arm64/nvidia/Makefile create mode 100644 drivers/platform/arm64/nvidia/mtk-pcie-hotplug.c diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index e9093504c2ddd..4d07edf8ec02a 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -150,6 +150,9 @@ CONFIG_MFD_AAEON note<'Disable all Ubuntu ODM dri CONFIG_MTD policy<{'amd64': 'm', 'arm64': 'y'}> CONFIG_MTD note<'Essential for boot on ARM64'> +CONFIG_MTK_PCIE_HOTPLUG policy<{'arm64': 'm'}> +CONFIG_MTK_PCIE_HOTPLUG note<'CX7 PCIe hotplug driver for NVIDIA DGX Spark systems with GB10 SoC.'> + CONFIG_NOUVEAU_DEBUG policy<{'amd64': '-', 'arm64': '-'}> CONFIG_NOUVEAU_DEBUG note<'Disable nouveau for NVIDIA kernels'> diff --git a/drivers/platform/arm64/Kconfig b/drivers/platform/arm64/Kconfig index 80cefd5772cec..4bbd8eca38a87 100644 --- a/drivers/platform/arm64/Kconfig +++ b/drivers/platform/arm64/Kconfig @@ -115,4 +115,6 @@ config NVIDIA_FFA_EC Say M or Y here to include this support. +source "drivers/platform/arm64/nvidia/Kconfig" + endif # ARM64_PLATFORM_DEVICES diff --git a/drivers/platform/arm64/Makefile b/drivers/platform/arm64/Makefile index c693a0501631b..baa8d477a2cd7 100644 --- a/drivers/platform/arm64/Makefile +++ b/drivers/platform/arm64/Makefile @@ -11,3 +11,4 @@ obj-$(CONFIG_EC_LENOVO_YOGA_C630) += lenovo-yoga-c630.o obj-$(CONFIG_EC_LENOVO_THINKPAD_T14S) += lenovo-thinkpad-t14s.o obj-$(CONFIG_EC_LENOVO_YOGA_SLIM7X) += lenovo-yoga-slim7x.o obj-$(CONFIG_NVIDIA_FFA_EC) += nvidia-ffa-ec.o +obj-y += nvidia/ diff --git a/drivers/platform/arm64/nvidia/Kconfig b/drivers/platform/arm64/nvidia/Kconfig new file mode 100644 index 0000000000000..b12b290f30d4f --- /dev/null +++ b/drivers/platform/arm64/nvidia/Kconfig @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# NVIDIA ARM64 Platform-Specific Device Drivers +# + +config MTK_PCIE_HOTPLUG + tristate "CX7 PCIe Hotplug Driver" + depends on EINT_MTK + depends on PCI && ACPI + help + Say Y here to support PCIe device plug in/out detection. + It will disable PCIe link when plug out and enable + PCIe link after plug in. + + This is particularly useful for GB10 SoC. + + If unsure, say N. diff --git a/drivers/platform/arm64/nvidia/Makefile b/drivers/platform/arm64/nvidia/Makefile new file mode 100644 index 0000000000000..37cfbebb8d1af --- /dev/null +++ b/drivers/platform/arm64/nvidia/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for NVIDIA ARM64 platform-specific drivers +# +# CX7 PCIe Hotplug Driver +# Provides hotplug support for CX7 PCIe devices on GB10 SoC-based systems +# + +obj-$(CONFIG_MTK_PCIE_HOTPLUG) += mtk-pcie-hotplug.o diff --git a/drivers/platform/arm64/nvidia/mtk-pcie-hotplug.c b/drivers/platform/arm64/nvidia/mtk-pcie-hotplug.c new file mode 100644 index 0000000000000..06a84a29aa6fd --- /dev/null +++ b/drivers/platform/arm64/nvidia/mtk-pcie-hotplug.c @@ -0,0 +1,2324 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2014-2025 MediaTek Inc. + * Copyright (c) 2025-2026 NVIDIA Corporation + * + * CX7 PCIe Hotplug Driver + * + * Manages PCIe device hotplug using GPIO interrupts and ACPI resources. + * Supports cable insertion/removal detection and device power management. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HP_PORT_MAX 3 +#define HP_POLL_CNT_MAX 200 +#define MAX_VENDOR_DATA_LEN 16 +#define CX7_HP_MMIO_REGION_COUNT 5 /* TOP, PROTECT, CKM, MAC Port 0, MAC Port 1 */ +#define CX7_HP_MIN_GPIO_COUNT 4 /* Minimum required: BOOT, PRSNT, PERST, EN */ +#define PINCTRL_MAPPING_ENTRY_SIZE 5 /* dev_name, state, ctrl_dev, group, function */ +/* Indices for pinctrl mapping entry strings */ +#define PINCTRL_IDX_DEV_NAME 0 +#define PINCTRL_IDX_STATE 1 +#define PINCTRL_IDX_CTRL_DEV 2 +#define PINCTRL_IDX_GROUP 3 +#define PINCTRL_IDX_FUNCTION 4 + +/* Hardware timing requirements (in microseconds unless noted) */ +#define CX7_HP_DELAY_SHORT_US 10 /* Short delay for register writes */ +#define CX7_HP_DELAY_STANDARD_US 10000 /* Standard delay (10ms) */ +#define CX7_HP_DELAY_BUS_PROTECT_US 5000 /* Bus protection setup delay */ +#define CX7_HP_DELAY_PHY_RESET_US 3000 /* PHY reset delay */ +#define CX7_HP_DELAY_LINK_STABLE_MS 100 /* Link stabilization delay (ms) */ +#define CX7_HP_POLL_SLEEP_US 10000 /* Polling loop sleep interval */ + +#define PLUG_IN_EVT "HOTPLUG_STATE=plugin" +#define REMOVAL_EVT "HOTPLUG_STATE=removal" + +/* Bus protection stages to prevent PCIe core reset glitches */ +#define BUS_PROTECT_INIT 0 +#define BUS_PROTECT_CABLE_REMOVAL 1 +#define BUS_PROTECT_CABLE_PLUGIN 2 +#define BUS_PROTECT_CLEANUP 3 + +enum cx7_hp_state { + STATE_READY = 0, + STATE_PLUG_OUT, /* Cable plug-out */ + STATE_DEV_POWER_OFF, /* Device is powered off */ + STATE_PLUG_IN, /* Cable plug-in detected */ + STATE_DEV_POWER_ON, /* Device is powered on */ + STATE_DEV_FW_START, /* Device firmware is running */ + STATE_RESCAN, /* Device ready, can perform bus rescan */ + STATE_UNKNOWN +}; + +enum pcie_pin_index { + PCIE_PIN_BOOT = 0, /* Device boot status pin */ + PCIE_PIN_PRSNT, /* Presence detection pin */ + PCIE_PIN_PERST, /* PCIe reset pin */ + PCIE_PIN_EN, /* Power enable pin */ + PCIE_PIN_CLQ0, /* Clock request pin 0 */ + PCIE_PIN_CLQ1, /* Clock request pin 1 */ + PCIE_PIN_MAX +}; + +struct pcie_port_info { + int domain; + int bus; + int devfn; +}; + +struct rp_bus_mmio_top { + u32 ctrl; + u32 port_bits[HP_PORT_MAX]; + u32 update_bit; +}; + +struct rp_bus_mmio_protect { + u32 mode; + u32 enable; + u32 port_bits[HP_PORT_MAX]; +}; + +struct rp_bus_mmio_mac { + u32 init_ctrl; + u32 ltssm_bit; + u32 phy_rst_bit; +}; + +struct rp_bus_mmio_ckm { + u32 ctrl; + u32 disable_bit; +}; + +struct rp_bus_mmio_info { + struct rp_bus_mmio_top top; + struct rp_bus_mmio_protect protect; + struct rp_bus_mmio_mac mac; + struct rp_bus_mmio_ckm ckm; +}; + +struct gpio_acpi_context { + struct device *dev; + unsigned int debounce_timeout_us; + int pin; + int wake_capable; + int triggering; + int polarity; + unsigned long irq_flags; + int valid; + unsigned int connection_type; + char vendor_data[MAX_VENDOR_DATA_LEN + 1]; +}; + +struct cx7_hp_dev; + +/** + * struct cx7_hp_plat_data - Platform configuration data parsed from ACPI + * + * Platform-specific configuration parsed from ACPI devices: + * - RES0 device (PNP0C02): PCIe configuration and MMIO register offsets via _DSD + * - PEDE device (MTKP0001): Pinctrl mappings via _DSD + */ +struct cx7_hp_plat_data { + int port_nums; + struct pcie_port_info ports[HP_PORT_MAX]; + u32 vendor_id; + u32 device_id; + int num_devices; + struct rp_bus_mmio_info rp_bus_mmio; + u32 ltssm_reg; + u32 ltssm_l0_state; + int pin_nums; + struct pinctrl_map *parsed_pinmap; +}; + +struct cx7_hp_gpio_ctx { + struct gpio_desc *desc; + struct gpio_acpi_context *ctx; + struct cx7_hp_dev *hp_dev; +}; + +struct acpi_gpio_parse_context { + struct gpio_acpi_context *ctx; + struct cx7_hp_dev *hp_dev; +}; + +struct acpi_gpio_walk_context { + struct device *dev; + struct gpio_info { + unsigned int pin; + unsigned int connection_type; + unsigned int triggering; + unsigned int polarity; + unsigned int debounce_timeout; + unsigned int wake_capable; + char vendor_data[MAX_VENDOR_DATA_LEN + 1]; + char resource_source[16]; + unsigned int resource_source_index; + } gpios[PCIE_PIN_MAX]; + int count; +}; + +struct cx7_hp_acpi_mmio { + struct acpi_resource_fixed_memory32 + mmio_regions[CX7_HP_MMIO_REGION_COUNT]; + int count; + struct device *dev; +}; + +enum cx7_hp_debug_val { + CX7_HP_DEBUG_PLUG_OUT = 0, + CX7_HP_DEBUG_PLUG_IN, + CX7_HP_DEBUG_MAX_VAL +}; + +struct cx7_hp_mmio_runtime { + void __iomem *top_base; + void __iomem *protect_base; + void __iomem *ckm_base; + void __iomem *mac_port_base[HP_PORT_MAX]; +}; + +/** + * cx7_hp_dev - Hotplug device structure + * + * ACPI resource sources: + * - MMIO addresses: RES0 device (PNP0C02) _CRS, stored in mmio field + * - GPIO resources: PEDE device (MTKP0001) _CRS, stored in pins field + */ +struct cx7_hp_dev { + struct cx7_hp_gpio_ctx *pins; + struct cx7_hp_plat_data *pd; + struct platform_device *pdev; + enum cx7_hp_state state; + int gpio_count; + int boot_pin; + int prsnt_pin; + enum cx7_hp_debug_val debug_state; + bool hotplug_enabled; + spinlock_t lock; + struct pci_dev *cached_root_ports[HP_PORT_MAX]; + struct cx7_hp_mmio_runtime mmio; + struct gpio_device *gdev; + struct notifier_block pci_notifier; +}; + +/* ACPI _DSD device properties GUID: daffd814-6eba-4d8c-8a91-bc9bbf4aa301 */ +static const guid_t device_properties_guid = +GUID_INIT(0xdaffd814, 0x6eba, 0x4d8c, + 0x8a, 0x91, 0xbc, 0x9b, + 0xbf, 0x4a, 0xa3, 0x01); + +/** + * cx7_hp_parse_pinctrl_config_dsd - Parse pinctrl configuration from PEDE device _DSD + * @hp_dev: hotplug device + * + * Parses pin-nums and pinctrl-mappings from _DSD. + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_parse_pinctrl_config_dsd(struct cx7_hp_dev *hp_dev) +{ + struct acpi_device *adev; + struct device *dev = &hp_dev->pdev->dev; + const union acpi_object *mappings_pkg, *mapping_entry; + struct pinctrl_map *pinmap; + u32 pin_nums = 0; + int k; + const char *strings[PINCTRL_MAPPING_ENTRY_SIZE]; + + adev = ACPI_COMPANION(dev); + if (!adev) { + dev_err(dev, "Failed to get ACPI companion device\n"); + return -ENODEV; + } + + struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; + acpi_status status; + const union acpi_object *dsd_pkg, *props_pkg = NULL; + int i, j; + + status = acpi_evaluate_object_typed(adev->handle, "_DSD", NULL, &buffer, + ACPI_TYPE_PACKAGE); + if (ACPI_FAILURE(status)) { + dev_err(dev, "Failed to evaluate _DSD: %s\n", + acpi_format_exception(status)); + return -ENODEV; + } + + dsd_pkg = buffer.pointer; + if (!dsd_pkg || dsd_pkg->type != ACPI_TYPE_PACKAGE) { + dev_err(dev, "Invalid _DSD package\n"); + ACPI_FREE(buffer.pointer); + return -EINVAL; + } + /* Find Device Properties GUID package */ + for (i = 0; i + 1 < dsd_pkg->package.count; i += 2) { + const union acpi_object *guid = &dsd_pkg->package.elements[i]; + const union acpi_object *pkg = + &dsd_pkg->package.elements[i + 1]; + + /* Verify GUID matches Device Properties GUID */ + if (guid->type == ACPI_TYPE_BUFFER && guid->buffer.length == 16 && + pkg->type == ACPI_TYPE_PACKAGE && + guid_equal((guid_t *)guid->buffer.pointer, + &device_properties_guid)) { + props_pkg = pkg; + break; + } + } + + if (!props_pkg) { + dev_err(dev, + "Device Properties GUID package not found in _DSD\n"); + ACPI_FREE(buffer.pointer); + return -EINVAL; + } + + for (j = 0; j < props_pkg->package.count; j++) { + const union acpi_object *prop = &props_pkg->package.elements[j]; + + if (prop->type != ACPI_TYPE_PACKAGE || + prop->package.count != 2 || + prop->package.elements[0].type != ACPI_TYPE_STRING) + continue; + + const char *prop_name = + prop->package.elements[0].string.pointer; + const union acpi_object *prop_value = + &prop->package.elements[1]; + + if (!strcmp(prop_name, "pin-nums")) { + if (prop_value->type == ACPI_TYPE_INTEGER) { + pin_nums = prop_value->integer.value; + } + } else if (!strcmp(prop_name, "pinctrl-mappings")) { + if (prop_value->type == ACPI_TYPE_PACKAGE) + mappings_pkg = prop_value; + } + } + + if (pin_nums == 0) { + hp_dev->pd->pin_nums = 0; + ACPI_FREE(buffer.pointer); + return 0; + } + + if (!mappings_pkg) { + dev_err(dev, + "Missing required _DSD property: pinctrl-mappings\n"); + ACPI_FREE(buffer.pointer); + return -EINVAL; + } + + if (mappings_pkg->package.count != pin_nums) { + dev_err(dev, + "pinctrl-mappings count mismatch: expected %u, got %u\n", + pin_nums, mappings_pkg->package.count); + ACPI_FREE(buffer.pointer); + return -EINVAL; + } + + /* Allocate pinmap array */ + pinmap = devm_kcalloc(dev, pin_nums, sizeof(*pinmap), GFP_KERNEL); + if (!pinmap) { + ACPI_FREE(buffer.pointer); + return -ENOMEM; + } + + /* Parse each mapping entry */ + for (k = 0; k < pin_nums; k++) { + mapping_entry = &mappings_pkg->package.elements[k]; + if (mapping_entry->type != ACPI_TYPE_PACKAGE || + mapping_entry->package.count != ARRAY_SIZE(strings)) { + dev_err(dev, + "Invalid pinctrl mapping entry %d: expected Package(%zu), " + "got %s(count=%u)\n", + k, ARRAY_SIZE(strings), + mapping_entry->type == ACPI_TYPE_PACKAGE ? + "Package" : "non-Package", + mapping_entry->type == ACPI_TYPE_PACKAGE ? + mapping_entry->package.count : 0); + ACPI_FREE(buffer.pointer); + return -EINVAL; + } + + /* Extract strings: dev_name, state, ctrl_dev, group, function */ + for (int l = 0; l < ARRAY_SIZE(strings); l++) { + if (mapping_entry->package.elements[l].type != + ACPI_TYPE_STRING) { + dev_err(dev, + "Mapping entry %d element %d is not a string\n", + k, l); + ACPI_FREE(buffer.pointer); + return -EINVAL; + } + strings[l] = + mapping_entry->package.elements[l].string.pointer; + } + + /* Populate pinctrl_map structure */ + pinmap[k].dev_name = + devm_kstrdup(dev, strings[PINCTRL_IDX_DEV_NAME], + GFP_KERNEL); + pinmap[k].name = + devm_kstrdup(dev, strings[PINCTRL_IDX_STATE], GFP_KERNEL); + pinmap[k].type = PIN_MAP_TYPE_MUX_GROUP; + pinmap[k].ctrl_dev_name = + devm_kstrdup(dev, strings[PINCTRL_IDX_CTRL_DEV], + GFP_KERNEL); + pinmap[k].data.mux.group = + devm_kstrdup(dev, strings[PINCTRL_IDX_GROUP], GFP_KERNEL); + pinmap[k].data.mux.function = + devm_kstrdup(dev, strings[PINCTRL_IDX_FUNCTION], + GFP_KERNEL); + + if (!pinmap[k].dev_name || !pinmap[k].name || + !pinmap[k].ctrl_dev_name || !pinmap[k].data.mux.group || + !pinmap[k].data.mux.function) { + dev_err(dev, + "Failed to allocate memory for mapping %d\n", + k); + ACPI_FREE(buffer.pointer); + return -ENOMEM; + } + } + + hp_dev->pd->pin_nums = pin_nums; + hp_dev->pd->parsed_pinmap = pinmap; + ACPI_FREE(buffer.pointer); + dev_dbg(dev, "Successfully parsed %u pinctrl mappings from ACPI\n", + pin_nums); + return 0; +} + +/** + * cx7_hp_pinctrl_init - Register pinctrl mappings for the device + * @hp_dev: hotplug device + * + * Parses pinctrl mappings from _DSD and registers them. + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_pinctrl_init(struct cx7_hp_dev *hp_dev) +{ + int ret; + + ret = cx7_hp_parse_pinctrl_config_dsd(hp_dev); + if (ret) { + dev_err(&hp_dev->pdev->dev, + "Failed to parse pinctrl configuration from ACPI: %d\n", + ret); + return ret; + } + + if (!hp_dev->pd->pin_nums) + return 0; + + ret = + pinctrl_register_mappings(hp_dev->pd->parsed_pinmap, + hp_dev->pd->pin_nums); + if (ret) { + dev_err(&hp_dev->pdev->dev, + "Failed to register pinctrl mappings\n"); + return ret; + } + + dev_dbg(&hp_dev->pdev->dev, "Registered %u pinctrl mappings\n", + hp_dev->pd->pin_nums); + return 0; +} + +/** + * cx7_hp_pinctrl_remove - Unregister pinctrl mappings + * @hp_dev: hotplug device + */ +static void cx7_hp_pinctrl_remove(struct cx7_hp_dev *hp_dev) +{ + if (!hp_dev->pd->pin_nums) + return; + + pinctrl_unregister_mappings(hp_dev->pd->parsed_pinmap); +} + +/** + * cx7_hp_change_pinctrl_state - Change pinctrl state + * @hp_dev: hotplug device + * @new_state: new pinctrl state name + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_change_pinctrl_state(struct cx7_hp_dev *hp_dev, + const char *new_state) +{ + struct pinctrl *pinctrl; + struct pinctrl_state *state; + int ret; + + pinctrl = devm_pinctrl_get(&hp_dev->pdev->dev); + if (IS_ERR(pinctrl)) { + dev_err(&hp_dev->pdev->dev, "Failed to get pinctrl\n"); + return PTR_ERR(pinctrl); + } + + state = pinctrl_lookup_state(pinctrl, new_state); + if (IS_ERR(state)) { + dev_err(&hp_dev->pdev->dev, "Failed to lookup state:%s\n", + new_state); + return PTR_ERR(state); + } + + ret = pinctrl_select_state(pinctrl, state); + if (ret) { + dev_err(&hp_dev->pdev->dev, + "Failed to select pinctrl state:%s\n", new_state); + return ret; + } + + return 0; +} + +/** + * cx7_hp_send_uevent - Send uevent to userspace + * @hp_dev: hotplug device + * @msg: uevent message string + */ +static void cx7_hp_send_uevent(struct cx7_hp_dev *hp_dev, const char *msg) +{ + char *uevent = NULL; + char *envp[2]; + + uevent = kasprintf(GFP_KERNEL, msg); + if (!uevent) { + dev_err(&hp_dev->pdev->dev, + "Failed to allocate uevent string\n"); + return; + } + + envp[0] = uevent; + envp[1] = NULL; + + if (kobject_uevent_env(&hp_dev->pdev->dev.kobj, KOBJ_CHANGE, envp)) + dev_err(&hp_dev->pdev->dev, "Failed to send uevent\n"); + + kfree(uevent); +} + +/** + * cx7_hp_reg_update_bits - Update specific bits in a register + * @base: MMIO base address + * @offset: Register offset + * @mask: Bits to modify + * @set: true to set bits, false to clear bits + */ +static inline void cx7_hp_reg_update_bits(void __iomem *base, u32 offset, + u32 mask, bool set) +{ + u32 val = readl(base + offset); + + if (set) + val |= mask; + else + val &= ~mask; + + writel(val, base + offset); +} + +/** + * cx7_hp_toggle_update_bit - Toggle control register update bit + * @base: MMIO base address + * @ctrl_offset: Control register offset + * @bits: Bits to set/clear before toggling update + * @update_bit: Update bit mask + * @set: true to set bits, false to clear bits + * + * Performs the sequence: modify bits, clear update bit, set update bit + */ +static void cx7_hp_toggle_update_bit(void __iomem *base, u32 ctrl_offset, + u32 bits, u32 update_bit, bool set) +{ + cx7_hp_reg_update_bits(base, ctrl_offset, bits, set); + cx7_hp_reg_update_bits(base, ctrl_offset, update_bit, false); + cx7_hp_reg_update_bits(base, ctrl_offset, update_bit, true); +} + +/** + * cx7_hp_bus_protect_enable - Enable bus protection for a port + * @dev: hotplug device + * @port_idx: Port index + */ +static void cx7_hp_bus_protect_enable(struct cx7_hp_dev *dev, int port_idx) +{ + struct rp_bus_mmio_info *mmio_info = &dev->pd->rp_bus_mmio; + u32 port_bit = mmio_info->protect.port_bits[port_idx]; + + cx7_hp_reg_update_bits(dev->mmio.protect_base, + mmio_info->protect.mode, port_bit, true); + cx7_hp_reg_update_bits(dev->mmio.protect_base, + mmio_info->protect.enable, port_bit, true); +} + +/** + * cx7_hp_bus_protect_disable - Disable bus protection for a port + * @dev: hotplug device + * @port_idx: Port index + */ +static void cx7_hp_bus_protect_disable(struct cx7_hp_dev *dev, int port_idx) +{ + struct rp_bus_mmio_info *mmio_info = &dev->pd->rp_bus_mmio; + u32 port_bit = mmio_info->protect.port_bits[port_idx]; + + cx7_hp_reg_update_bits(dev->mmio.protect_base, + mmio_info->protect.enable, port_bit, false); + cx7_hp_reg_update_bits(dev->mmio.protect_base, + mmio_info->protect.mode, port_bit, false); +} + +/** + * cx7_hp_ckm_control - Control clock module + * @dev: hotplug device + * @disable: true to disable clock, false to enable + */ +static void cx7_hp_ckm_control(struct cx7_hp_dev *dev, bool disable) +{ + struct rp_bus_mmio_info *mmio_info = &dev->pd->rp_bus_mmio; + + if (!dev->mmio.ckm_base) + return; + + cx7_hp_reg_update_bits(dev->mmio.ckm_base, mmio_info->ckm.ctrl, + mmio_info->ckm.disable_bit, disable); +} + +/** + * cx7_hp_parse_mmio_resources - ACPI resource callback for parsing MMIO from _CRS + * @ares: ACPI resource being processed + * @data: pointer to cx7_hp_acpi_mmio structure + * + * Returns: AE_OK to continue iteration, AE_ERROR on error + */ +static acpi_status cx7_hp_parse_mmio_resources(struct acpi_resource *ares, + void *data) +{ + struct cx7_hp_acpi_mmio *parsed = data; + + switch (ares->type) { + case ACPI_RESOURCE_TYPE_FIXED_MEMORY32: + if (parsed->count >= CX7_HP_MMIO_REGION_COUNT) { + dev_warn(parsed->dev, + "More than %d MMIO regions found in platform configuration device, ignoring extras\n", + CX7_HP_MMIO_REGION_COUNT); + break; + } + parsed->mmio_regions[parsed->count] = ares->data.fixed_memory32; + parsed->count++; + break; + default: + break; + } + + return AE_OK; +} + +/** + * cx7_hp_find_pcie_config_device - Find PCIe configuration device by HID + * + * Finds the ACPI device that provides PCIe configuration via _DSD properties + * and MMIO resources via _CRS. + * + * Returns: acpi_device pointer on success (with reference), NULL on failure + */ +static struct acpi_device *cx7_hp_find_pcie_config_device(void) +{ + return acpi_dev_get_first_match_dev("PNP0C02", NULL, -1); +} + +/** + * cx7_hp_parse_pcie_config_dsd - Parse PCIe configuration from _DSD + * @pdev: platform device + * @pd: platform data to populate + * + * Parses PCIe MMIO register offsets, bit positions, port configuration, and PCIe device + * identification from PCIe configuration device _DSD. + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_parse_pcie_config_dsd(struct platform_device *pdev, + struct cx7_hp_plat_data *pd) +{ + struct acpi_device *config_adev; + struct device *dev = &pdev->dev; + u32 val, bit1; + + config_adev = cx7_hp_find_pcie_config_device(); + if (!config_adev) { + dev_err(dev, + "Platform configuration device (PNP0C02) not found - _DSD is required\n"); + return -ENODEV; + } + + if (!acpi_dev_has_props(config_adev)) { + dev_err(dev, + "Platform configuration device has no _DSD properties. Check DSDT.\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "mac-init-ctrl-offset", &val)) { + dev_err(dev, + "Missing required _DSD property: mac-init-ctrl-offset\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.mac.init_ctrl = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "mac-ltssm-bit", &val)) { + dev_err(dev, "Missing required _DSD property: mac-ltssm-bit\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.mac.ltssm_bit = BIT(val); + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "mac-phy-rst-bit", &val)) { + dev_err(dev, + "Missing required _DSD property: mac-phy-rst-bit\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.mac.phy_rst_bit = BIT(val); + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "top-ctrl-offset", &val)) { + dev_err(dev, + "Missing required _DSD property: top-ctrl-offset\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.top.ctrl = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "top-update-bit", &val)) { + dev_err(dev, + "Missing required _DSD property: top-update-bit\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.top.update_bit = BIT(val); + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "top-port0-bit", &val)) { + dev_err(dev, "Missing required _DSD property: top-port0-bit\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.top.port_bits[0] = BIT(val); + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "top-port1-bit", &val)) { + dev_err(dev, "Missing required _DSD property: top-port1-bit\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.top.port_bits[1] = BIT(val); + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "protect-mode-offset", &val)) { + dev_err(dev, + "Missing required _DSD property: protect-mode-offset\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.protect.mode = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "protect-enable-offset", &val)) { + dev_err(dev, + "Missing required _DSD property: protect-enable-offset\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.protect.enable = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "protect-port0-bit", &val)) { + dev_err(dev, + "Missing required _DSD property: protect-port0-bit\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.protect.port_bits[0] = BIT(val); + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "protect-port1-bit", &val)) { + dev_err(dev, + "Missing required _DSD property: protect-port1-bit\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.protect.port_bits[1] = BIT(val); + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "ckm-ctrl-offset", &val)) { + dev_err(dev, + "Missing required _DSD property: ckm-ctrl-offset\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.ckm.ctrl = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "ckm-disable-bit0", &val)) { + dev_err(dev, + "Missing required _DSD property: ckm-disable-bit0\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "ckm-disable-bit1", &bit1)) { + dev_err(dev, + "Missing required _DSD property: ckm-disable-bit1\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.ckm.disable_bit = BIT(val) | BIT(bit1); + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "ltssm-reg-offset", &val)) { + dev_err(dev, + "Missing required _DSD property: ltssm-reg-offset\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->ltssm_reg = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "ltssm-l0-state", &val)) { + dev_err(dev, + "Missing required _DSD property: ltssm-l0-state\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->ltssm_l0_state = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "port-nums", &val)) { + dev_err(dev, "Missing required _DSD property: port-nums\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + if (val == 0 || val > HP_PORT_MAX) { + dev_err(dev, + "Invalid _DSD property port-nums: %u (must be 1-%d)\n", + val, HP_PORT_MAX); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->port_nums = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "port0-domain", &val)) { + dev_err(dev, "Missing required _DSD property: port0-domain\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->ports[0].domain = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "port0-bus", &val)) { + dev_err(dev, "Missing required _DSD property: port0-bus\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->ports[0].bus = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "port0-devfn", &val)) { + dev_err(dev, "Missing required _DSD property: port0-devfn\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->ports[0].devfn = val; + + if (pd->port_nums >= 2) { + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "port1-domain", &val)) { + dev_err(dev, + "Missing required _DSD property: port1-domain\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->ports[1].domain = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "port1-bus", &val)) { + dev_err(dev, + "Missing required _DSD property: port1-bus\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->ports[1].bus = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "port1-devfn", &val)) { + dev_err(dev, + "Missing required _DSD property: port1-devfn\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->ports[1].devfn = val; + } + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "vendor-id", &val)) { + dev_err(dev, "Missing required _DSD property: vendor-id\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->vendor_id = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "device-id", &val)) { + dev_err(dev, "Missing required _DSD property: device-id\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->device_id = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "num-devices", &val)) { + dev_err(dev, "Missing required _DSD property: num-devices\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->num_devices = val; + + dev_dbg(dev, "Successfully parsed all required _DSD properties\n"); + + acpi_dev_put(config_adev); + return 0; +} + +/** + * cx7_hp_parse_mmio_resources_from_acpi - Parse MMIO regions from _CRS + * @dev: hotplug device + * @parsed: pointer to parsed MMIO structure + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_parse_mmio_resources_from_acpi(struct cx7_hp_dev *dev, + struct cx7_hp_acpi_mmio + *parsed) +{ + struct acpi_device *config_adev; + acpi_status status; + int ret = 0; + + if (!dev || !dev->pdev) { + return -EINVAL; + } + + config_adev = cx7_hp_find_pcie_config_device(); + if (!config_adev) + return -ENODEV; + + parsed->count = 0; + memset(parsed->mmio_regions, 0, sizeof(parsed->mmio_regions)); + + status = + acpi_walk_resources(config_adev->handle, METHOD_NAME__CRS, + cx7_hp_parse_mmio_resources, parsed); + if (ACPI_FAILURE(status)) { + dev_err(&dev->pdev->dev, + "Failed to walk platform configuration resources: %s\n", + acpi_format_exception(status)); + ret = -ENODEV; + goto out; + } + + if (parsed->count < CX7_HP_MMIO_REGION_COUNT) { + dev_warn(&dev->pdev->dev, + "Expected %d MMIO regions from platform configuration device, found %d\n", + CX7_HP_MMIO_REGION_COUNT, parsed->count); + ret = -ENODEV; + goto out; + } + +out: + acpi_dev_put(config_adev); + return ret; +} + +/** + * cx7_hp_map_mmio_resources - Map all MMIO regions from ACPI _CRS + * @dev: hotplug device + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_map_mmio_resources(struct cx7_hp_dev *dev) +{ + struct platform_device *pdev = dev->pdev; + struct cx7_hp_acpi_mmio parsed = {.count = 0, .dev = &pdev->dev }; + int ret; + int i; + + ret = cx7_hp_parse_mmio_resources_from_acpi(dev, &parsed); + if (ret) { + dev_err(&pdev->dev, + "Failed to get MMIO regions from platform configuration device\n"); + return ret; + } + + dev_dbg(&pdev->dev, "Found %d MMIO regions in _CRS, mapping...\n", + parsed.count); + + int mapped_count = 0; + for (i = 0; i < parsed.count; i++) { + void __iomem *base = NULL; + u32 addr = parsed.mmio_regions[i].address; + u32 size = parsed.mmio_regions[i].address_length; + + switch (i) { + case 0: + if (dev->pd->port_nums >= 1) { + base = devm_ioremap(&pdev->dev, addr, size); + if (!base) { + dev_err(&pdev->dev, + "Failed to map MAC Port 0 region (0x%08x)\n", + addr); + return -ENOMEM; + } + dev->mmio.mac_port_base[0] = base; + mapped_count++; + } + break; + case 1: + if (dev->pd->port_nums >= 2) { + base = devm_ioremap(&pdev->dev, addr, size); + if (!base) { + dev_err(&pdev->dev, + "Failed to map MAC Port 1 region (0x%08x)\n", + addr); + return -ENOMEM; + } + dev->mmio.mac_port_base[1] = base; + mapped_count++; + } + break; + case 2: + base = devm_ioremap(&pdev->dev, addr, size); + if (!base) { + dev_err(&pdev->dev, + "Failed to map TOP region (0x%08x)\n", + addr); + return -ENOMEM; + } + dev->mmio.top_base = base; + mapped_count++; + break; + case 3: + base = devm_ioremap(&pdev->dev, addr, size); + if (!base) { + dev_err(&pdev->dev, + "Failed to map PROTECT region (0x%08x)\n", + addr); + return -ENOMEM; + } + dev->mmio.protect_base = base; + mapped_count++; + break; + case 4: + base = devm_ioremap(&pdev->dev, addr, size); + if (!base) { + dev_err(&pdev->dev, + "Failed to map CKM region (0x%08x)\n", + addr); + return -ENOMEM; + } + dev->mmio.ckm_base = base; + mapped_count++; + break; + default: + dev_warn(&pdev->dev, + "Unexpected MMIO region at 0x%08x (size 0x%x), skipping\n", + addr, size); + break; + } + } + + if (!dev->mmio.top_base || !dev->mmio.protect_base + || !dev->mmio.ckm_base || (dev->pd->port_nums >= 1 + && !dev->mmio.mac_port_base[0]) + || (dev->pd->port_nums >= 2 && !dev->mmio.mac_port_base[1])) { + dev_err(&pdev->dev, + "Required MMIO regions not mapped from ACPI _CRS (mapped %d)\n", + mapped_count); + if (!dev->mmio.top_base) + dev_err(&pdev->dev, " Missing: TOP\n"); + if (!dev->mmio.protect_base) + dev_err(&pdev->dev, " Missing: PROTECT\n"); + if (!dev->mmio.ckm_base) + dev_err(&pdev->dev, " Missing: CKM\n"); + if (dev->pd->port_nums >= 1 && !dev->mmio.mac_port_base[0]) + dev_err(&pdev->dev, + " Missing: MAC Port 0 (port_nums=%d)\n", + dev->pd->port_nums); + if (dev->pd->port_nums >= 2 && !dev->mmio.mac_port_base[1]) + dev_err(&pdev->dev, + " Missing: MAC Port 1 (port_nums=%d)\n", + dev->pd->port_nums); + dev->mmio.top_base = NULL; + dev->mmio.protect_base = NULL; + dev->mmio.ckm_base = NULL; + for (i = 0; i < HP_PORT_MAX; i++) + dev->mmio.mac_port_base[i] = NULL; + return -ENODEV; + } + + dev_dbg(&pdev->dev, + "Successfully mapped all MMIO regions from ACPI _CRS\n"); + return 0; +} + +/** + * cx7_hp_rp_bus_protect - Bus protection handler + * @dev: hotplug device + * @port_idx: port index (0-based) + * @stage: protection stage (BUS_PROTECT_INIT, BUS_PROTECT_CLEANUP, etc.) + */ +static void cx7_hp_rp_bus_protect(struct cx7_hp_dev *dev, int port_idx, + int stage) +{ + switch (stage) { + case BUS_PROTECT_INIT: + { + int ret; + + ret = cx7_hp_map_mmio_resources(dev); + if (ret) { + dev_err(&dev->pdev->dev, + "Failed to map MMIO resources during bus init: %d\n", + ret); + return; + } + } + return; + + case BUS_PROTECT_CLEANUP: + { + int i; + + for (i = 0; i < HP_PORT_MAX; i++) { + if (dev->mmio.mac_port_base[i]) + dev->mmio.mac_port_base[i] = NULL; + } + if (dev->mmio.top_base) + dev->mmio.top_base = NULL; + if (dev->mmio.protect_base) + dev->mmio.protect_base = NULL; + if (dev->mmio.ckm_base) + dev->mmio.ckm_base = NULL; + } + return; + + case BUS_PROTECT_CABLE_REMOVAL: + case BUS_PROTECT_CABLE_PLUGIN: + { + struct rp_bus_mmio_info *mmio_info = + &dev->pd->rp_bus_mmio; + void __iomem *mac_base; + + if (port_idx >= dev->pd->port_nums) + return; + + mac_base = dev->mmio.mac_port_base[port_idx]; + if (!mac_base) + return; + + if (stage == BUS_PROTECT_CABLE_REMOVAL) { + cx7_hp_reg_update_bits(mac_base, + mmio_info->mac.init_ctrl, + mmio_info->mac.ltssm_bit, + false); + cx7_hp_reg_update_bits(mac_base, + mmio_info->mac.init_ctrl, + mmio_info->mac. + phy_rst_bit, false); + return; + } + + cx7_hp_toggle_update_bit(dev->mmio.top_base, + mmio_info->top.ctrl, + mmio_info->top. + port_bits[port_idx], + mmio_info->top.update_bit, + false); + udelay(CX7_HP_DELAY_SHORT_US); + + cx7_hp_bus_protect_enable(dev, port_idx); + usleep_range(CX7_HP_DELAY_BUS_PROTECT_US, + CX7_HP_DELAY_BUS_PROTECT_US + 1000); + + cx7_hp_reg_update_bits(mac_base, + mmio_info->mac.init_ctrl, + mmio_info->mac.phy_rst_bit, + true); + cx7_hp_reg_update_bits(mac_base, + mmio_info->mac.init_ctrl, + mmio_info->mac.ltssm_bit, true); + usleep_range(CX7_HP_DELAY_PHY_RESET_US, + CX7_HP_DELAY_PHY_RESET_US + 1000); + + cx7_hp_bus_protect_disable(dev, port_idx); + + cx7_hp_toggle_update_bit(dev->mmio.top_base, + mmio_info->top.ctrl, + mmio_info->top. + port_bits[port_idx], + mmio_info->top.update_bit, + true); + } + break; + + default: + dev_warn(&dev->pdev->dev, "Unknown bus protect stage: %d\n", + stage); + break; + } +} + +/** + * retrain_pcie_link - Retrain PCIe link + * @dev: PCI device + */ +static void retrain_pcie_link(struct pci_dev *dev) +{ + u16 link_control, lnksta; + int pos, i = 0; + + pos = pci_find_capability(dev, PCI_CAP_ID_EXP); + if (!pos) { + dev_err(&dev->dev, "PCIe capability not found\n"); + return; + } + + pci_read_config_word(dev, pos + PCI_EXP_LNKCTL, &link_control); + link_control |= PCI_EXP_LNKCTL_RL; + + pci_write_config_word(dev, pos + PCI_EXP_LNKCTL, link_control); + + while (i < HP_POLL_CNT_MAX) { + i++; + pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta); + if (lnksta & PCI_EXP_LNKSTA_DLLLA) + break; + usleep_range(CX7_HP_POLL_SLEEP_US, CX7_HP_POLL_SLEEP_US + 1000); + } + + pcie_capability_write_word(dev, PCI_EXP_LNKSTA, PCI_EXP_LNKSTA_LBMS); +} + +/** + * get_port_root_port - Get PCI root port device for a port + * @hp_dev: hotplug device + * @port_idx: port index + * + * Returns cached or newly found root port, or NULL if not found. + */ +static struct pci_dev *get_port_root_port(struct cx7_hp_dev *hp_dev, + int port_idx) +{ + struct pcie_port_info *port; + + if (!hp_dev->pd || port_idx >= hp_dev->pd->port_nums) + return NULL; + + port = &hp_dev->pd->ports[port_idx]; + + if (!hp_dev->cached_root_ports[port_idx]) { + hp_dev->cached_root_ports[port_idx] = + pci_get_domain_bus_and_slot(port->domain, + port->bus, port->devfn); + if (!hp_dev->cached_root_ports[port_idx]) { + dev_warn(&hp_dev->pdev->dev, + "Root port not found for domain %d bus %d\n", + port->domain, port->bus); + return NULL; + } + } + + return hp_dev->cached_root_ports[port_idx]; +} + +/** + * remove_device - Remove PCIe devices and power down hardware + * @dev: hotplug device + */ +static void remove_device(struct cx7_hp_dev *dev) +{ + int i; + + dev_info(&dev->pdev->dev, "Cable removal\n"); + + for (i = 0; i < dev->pd->port_nums; i++) + cx7_hp_rp_bus_protect(dev, i, BUS_PROTECT_CABLE_REMOVAL); + + gpiod_set_value(dev->pins[PCIE_PIN_PERST].desc, 0); + cx7_hp_change_pinctrl_state(dev, "default"); + cx7_hp_ckm_control(dev, true); + gpiod_set_value(dev->pins[PCIE_PIN_EN].desc, 0); +} + +/** + * polling_link_to_l0 - Poll until all PCIe ports reach L0 state + * @dev: hotplug device + * + * Returns: 0 on success, negative error code on failure + */ +static int polling_link_to_l0(struct cx7_hp_dev *dev) +{ + struct pci_dev *pci_dev; + u32 ltssm_reg; + u32 l0_state; + u32 ltssm_vals[HP_PORT_MAX] = { 0 }; + int count = 0; + int i; + bool all_l0; + + ltssm_reg = dev->pd->ltssm_reg; + l0_state = dev->pd->ltssm_l0_state; + + if (!ltssm_reg || !l0_state) + return 0; /* Skip if not configured */ + + /* Poll until all ports reach L0 state */ + all_l0 = false; + while (!all_l0) { + all_l0 = true; + + for (i = 0; i < dev->pd->port_nums; i++) { + pci_dev = get_port_root_port(dev, i); + if (!pci_dev) { + all_l0 = false; + continue; + } + + pci_read_config_dword(pci_dev, ltssm_reg, + <ssm_vals[i]); + if ((ltssm_vals[i] & l0_state) != l0_state) + all_l0 = false; + } + + if (all_l0) + break; + + usleep_range(CX7_HP_POLL_SLEEP_US, CX7_HP_POLL_SLEEP_US + 1000); + count++; + + if (count > HP_POLL_CNT_MAX) { + dev_err(&dev->pdev->dev, + "Timeout waiting for link to reach L0 (reached max count)\n"); + break; + } + } + + if (count > HP_POLL_CNT_MAX) { + return -ETIMEDOUT; + } + + return 0; +} + +/** + * rescan_device - Rescan PCIe bus to discover devices + * @dev: hotplug device + * + * Returns: 0 on success, negative error code on failure + */ +static int rescan_device(struct cx7_hp_dev *dev) +{ + struct pci_dev *pci_dev; + int i, err; + + err = cx7_hp_change_pinctrl_state(dev, "clkreqn"); + if (err) + return err; + + cx7_hp_ckm_control(dev, false); + usleep_range(CX7_HP_DELAY_STANDARD_US, CX7_HP_DELAY_STANDARD_US + 1000); + + for (i = 0; i < dev->pd->port_nums; i++) { + pci_dev = get_port_root_port(dev, i); + if (!pci_dev) + continue; + + err = pm_runtime_resume_and_get(&pci_dev->dev); + if (err < 0) { + dev_err(&dev->pdev->dev, + "Runtime resume failed for %s: %d\n", + pci_name(pci_dev), err); + } + } + + gpiod_set_value(dev->pins[PCIE_PIN_PERST].desc, 1); + + for (i = 0; i < dev->pd->port_nums; i++) + cx7_hp_rp_bus_protect(dev, i, BUS_PROTECT_CABLE_PLUGIN); + + err = polling_link_to_l0(dev); + if (err) + return err; + + for (i = 0; i < dev->pd->port_nums; i++) { + pci_dev = get_port_root_port(dev, i); + if (pci_dev) + retrain_pcie_link(pci_dev); + } + + msleep(CX7_HP_DELAY_LINK_STABLE_MS); + + return 0; +} + +/** + * cx7_hp_work - Work queue handler for hotplug state machine + * @irq: interrupt number + * @dev_id: GPIO context pointer + * + * Processes hotplug state transitions based on current state. + */ +static irqreturn_t cx7_hp_work(int irq, void *dev_id) +{ + struct cx7_hp_gpio_ctx *app_ctx = dev_id; + struct cx7_hp_dev *hp_dev; + enum cx7_hp_state state; + unsigned long flags; + int ret; + + if (!app_ctx || !app_ctx->hp_dev) + return IRQ_NONE; + + hp_dev = app_ctx->hp_dev; + + spin_lock_irqsave(&hp_dev->lock, flags); + if (!hp_dev->hotplug_enabled) { + spin_unlock_irqrestore(&hp_dev->lock, flags); + return IRQ_HANDLED; + } + state = hp_dev->state; + spin_unlock_irqrestore(&hp_dev->lock, flags); + + switch (state) { + case STATE_PLUG_OUT: + remove_device(hp_dev); + break; + case STATE_PLUG_IN: + dev_info(&hp_dev->pdev->dev, "Cable plugin\n"); + gpiod_set_value(hp_dev->pins[PCIE_PIN_EN].desc, 1); + break; + case STATE_DEV_POWER_OFF: + case STATE_DEV_POWER_ON: + case STATE_DEV_FW_START: + break; + case STATE_RESCAN: + ret = rescan_device(hp_dev); + spin_lock_irqsave(&hp_dev->lock, flags); + if (ret) + dev_err(app_ctx->ctx->dev, "Rescan failed: %d\n", ret); + else + hp_dev->state = STATE_READY; + spin_unlock_irqrestore(&hp_dev->lock, flags); + break; + default: + dev_err(app_ctx->ctx->dev, "Unknown state: %d\n", state); + break; + } + + return IRQ_HANDLED; +} + +/** + * hotplug_irq_handler - GPIO interrupt handler for hotplug events + * @irq: interrupt number + * @dev_id: GPIO context pointer + * + * Handles presence detection and boot status GPIO interrupts. + */ +static irqreturn_t hotplug_irq_handler(int irq, void *dev_id) +{ + struct cx7_hp_gpio_ctx *app_ctx = dev_id; + struct cx7_hp_dev *hp_dev = app_ctx->hp_dev; + struct gpio_acpi_context *gpio_ctx = app_ctx->ctx; + unsigned long flags; + int value; + enum cx7_hp_state state; + + value = gpiod_get_value(app_ctx->desc); + + if (gpio_ctx->pin == hp_dev->prsnt_pin) { + if (value) { + cx7_hp_send_uevent(hp_dev, REMOVAL_EVT); + } else { + cx7_hp_send_uevent(hp_dev, PLUG_IN_EVT); + } + return IRQ_HANDLED; + } + + spin_lock_irqsave(&hp_dev->lock, flags); + if (!hp_dev->hotplug_enabled) { + spin_unlock_irqrestore(&hp_dev->lock, flags); + return IRQ_HANDLED; + } + state = hp_dev->state; + + if (gpio_ctx->pin == hp_dev->boot_pin) { + if (value && state == STATE_PLUG_IN) { + hp_dev->state = STATE_DEV_POWER_ON; + } else if (value && state == STATE_DEV_FW_START) { + hp_dev->state = STATE_RESCAN; + } else if (!value && state == STATE_DEV_POWER_ON) { + hp_dev->state = STATE_DEV_FW_START; + } else if (!value && state == STATE_PLUG_OUT) { + hp_dev->state = STATE_DEV_POWER_OFF; + } else { + spin_unlock_irqrestore(&hp_dev->lock, flags); + return IRQ_HANDLED; + } + spin_unlock_irqrestore(&hp_dev->lock, flags); + return IRQ_WAKE_THREAD; + } + + dev_err(gpio_ctx->dev, + "Unknown GPIO pin event: pin=%d irq=%d value=%d\n", + gpio_ctx->pin, irq, value); + spin_unlock_irqrestore(&hp_dev->lock, flags); + return IRQ_HANDLED; +} + +/** + * acpi_gpio_collect_handler - ACPI resource handler to collect all GPIO resources + * @ares: ACPI resource structure + * @context: Pointer to acpi_gpio_walk_context + * + * Returns: AE_OK to continue iteration + */ +static acpi_status acpi_gpio_collect_handler(struct acpi_resource *ares, + void *context) +{ + struct acpi_gpio_walk_context *walk_ctx = context; + struct acpi_resource_gpio *agpio; + int length; + + if (ares->type != ACPI_RESOURCE_TYPE_GPIO) + return AE_OK; + + if (walk_ctx->count >= PCIE_PIN_MAX) { + dev_warn(walk_ctx->dev, + "Too many GPIO resources, truncating at %d\n", + PCIE_PIN_MAX); + return AE_OK; + } + + agpio = &ares->data.gpio; + + if (!agpio->pin_table || agpio->pin_table_length == 0) { + dev_warn(walk_ctx->dev, "GPIO resource has no pin table\n"); + return AE_OK; + } + + walk_ctx->gpios[walk_ctx->count].pin = agpio->pin_table[0]; + walk_ctx->gpios[walk_ctx->count].connection_type = + agpio->connection_type; + walk_ctx->gpios[walk_ctx->count].triggering = agpio->triggering; + walk_ctx->gpios[walk_ctx->count].polarity = agpio->polarity; + walk_ctx->gpios[walk_ctx->count].debounce_timeout = + agpio->debounce_timeout; + walk_ctx->gpios[walk_ctx->count].wake_capable = agpio->wake_capable; + + if (agpio->vendor_length && agpio->vendor_data) { + length = min_t(int, agpio->vendor_length, MAX_VENDOR_DATA_LEN); + memcpy(walk_ctx->gpios[walk_ctx->count].vendor_data, + agpio->vendor_data, length); + walk_ctx->gpios[walk_ctx->count].vendor_data[length] = '\0'; + } else { + walk_ctx->gpios[walk_ctx->count].vendor_data[0] = '\0'; + } + + if (agpio->resource_source.string_ptr) { + length = min_t(int, agpio->resource_source.string_length, 15); + memcpy(walk_ctx->gpios[walk_ctx->count].resource_source, + agpio->resource_source.string_ptr, length); + walk_ctx->gpios[walk_ctx->count].resource_source[length] = '\0'; + } else { + walk_ctx->gpios[walk_ctx->count].resource_source[0] = '\0'; + } + walk_ctx->gpios[walk_ctx->count].resource_source_index = + agpio->resource_source.index; + walk_ctx->count++; + return AE_OK; +} + +/** + * cx7_hp_walk_acpi_gpios - Walk ACPI _CRS to collect all GPIO resources + * @pdev: Platform device + * @walk_ctx: Context structure to fill with GPIO information + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_walk_acpi_gpios(struct platform_device *pdev, + struct acpi_gpio_walk_context *walk_ctx) +{ + struct acpi_device *adev; + acpi_status status; + + adev = ACPI_COMPANION(&pdev->dev); + if (!adev) { + dev_err(&pdev->dev, "Failed to get ACPI companion device\n"); + return -ENODEV; + } + + memset(walk_ctx, 0, sizeof(*walk_ctx)); + walk_ctx->dev = &pdev->dev; + + status = acpi_walk_resources(adev->handle, METHOD_NAME__CRS, + acpi_gpio_collect_handler, walk_ctx); + if (ACPI_FAILURE(status)) { + dev_err(&pdev->dev, "Failed to walk ACPI GPIO resources: %s\n", + acpi_format_exception(status)); + return -EIO; + } + + dev_dbg(&pdev->dev, "Found %d GPIO resources via ACPI walk\n", + walk_ctx->count); + + if (walk_ctx->count == 0) { + dev_err(&pdev->dev, "No GPIO resources found in ACPI _CRS\n"); + return -ENODEV; + } + + return 0; +} + +/** + * acpi_gpio_lookup_handler - ACPI resource handler to look up a specific GPIO pin + * @ares: ACPI resource being processed + * @context: Pointer to acpi_gpio_parse_context + * + * Returns: AE_OK to continue iteration + */ +static acpi_status acpi_gpio_lookup_handler(struct acpi_resource *ares, + void *context) +{ + struct acpi_gpio_parse_context *parse_ctx = context; + struct gpio_acpi_context *ctx = parse_ctx->ctx; + struct cx7_hp_dev *hp_dev = parse_ctx->hp_dev; + struct acpi_resource_gpio *agpio; + int length; + + if (ares->type != ACPI_RESOURCE_TYPE_GPIO) + return AE_OK; + + agpio = &ares->data.gpio; + + if (ctx->pin != agpio->pin_table[0]) + return AE_OK; + + ctx->valid = 1; + ctx->debounce_timeout_us = agpio->debounce_timeout * 10; + ctx->wake_capable = agpio->wake_capable; + ctx->triggering = agpio->triggering; + ctx->polarity = agpio->polarity; + ctx->connection_type = agpio->connection_type; + + if (agpio->vendor_length && agpio->vendor_data && hp_dev) { + length = min_t(int, agpio->vendor_length, MAX_VENDOR_DATA_LEN); + memcpy(&ctx->vendor_data[0], agpio->vendor_data, length); + ctx->vendor_data[length] = '\0'; + + if (!strncmp("BOOT", ctx->vendor_data, strlen("BOOT"))) + hp_dev->boot_pin = ctx->pin; + else if (!strncmp("PRSNT", ctx->vendor_data, strlen("PRSNT"))) + hp_dev->prsnt_pin = ctx->pin; + } + + if (agpio->triggering == ACPI_EDGE_SENSITIVE) { + if (agpio->polarity == ACPI_ACTIVE_LOW) + ctx->irq_flags = IRQF_TRIGGER_FALLING; + else if (agpio->polarity == ACPI_ACTIVE_HIGH) + ctx->irq_flags = IRQF_TRIGGER_RISING; + else + ctx->irq_flags = + (IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING); + } else { + if (agpio->polarity == ACPI_ACTIVE_LOW) + ctx->irq_flags = IRQF_TRIGGER_LOW; + else + ctx->irq_flags = IRQF_TRIGGER_HIGH; + } + + return AE_OK; +} + +/** + * pci_devices_present_on_domain() - Check if PCI devices exist on a domain + * @domain: PCI domain number to check + * + * Returns: true if any PCI devices are present on the specified domain, + * false otherwise. This is used as a safety check before hardware shutdown. + */ +static bool pci_devices_present_on_domain(int domain) +{ + struct pci_bus *bus; + struct pci_dev *dev; + bool has_endpoint_devices = false; + + bus = pci_find_bus(domain, 1); + if (!bus) + return false; + + list_for_each_entry(dev, &bus->devices, bus_list) { + has_endpoint_devices = true; + break; + } + + return has_endpoint_devices; +} + +static ssize_t debug_state_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cx7_hp_dev *hp_dev = dev_get_drvdata(dev); + + if (!hp_dev) + return -EINVAL; + + return scnprintf(buf, PAGE_SIZE, "%d\n", hp_dev->debug_state); +} + +static ssize_t debug_state_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct cx7_hp_dev *hp_dev = dev_get_drvdata(dev); + unsigned long val, flags; + int err, i; + + if (!hp_dev || !hp_dev->pd) + return -EINVAL; + + err = kstrtoul(buf, 10, &val); + if (err) + return err; + + spin_lock_irqsave(&hp_dev->lock, flags); + if (!hp_dev->hotplug_enabled) { + spin_unlock_irqrestore(&hp_dev->lock, flags); + dev_info(dev, "Hotplug is disabled.\n"); + return -EPERM; + } + spin_unlock_irqrestore(&hp_dev->lock, flags); + + switch (val) { + case CX7_HP_DEBUG_PLUG_OUT: + /* Safety check: Verify no devices on the bus before hardware shutdown. */ + for (i = 0; i < hp_dev->pd->port_nums; i++) { + if (pci_devices_present_on_domain + (hp_dev->pd->ports[i].domain)) { + dev_err(dev, + "PCI devices still present, remove them first\n"); + return -EBUSY; + } + } + + spin_lock_irqsave(&hp_dev->lock, flags); + hp_dev->state = STATE_PLUG_OUT; + hp_dev->debug_state = val; + spin_unlock_irqrestore(&hp_dev->lock, flags); + remove_device(hp_dev); + return count; + + case CX7_HP_DEBUG_PLUG_IN: + for (i = 0; i < hp_dev->pd->port_nums; i++) { + if (pci_devices_present_on_domain + (hp_dev->pd->ports[i].domain)) { + dev_err(dev, + "PCI devices already present, cannot reinitialize hardware\n"); + return -EBUSY; + } + } + + spin_lock_irqsave(&hp_dev->lock, flags); + hp_dev->state = STATE_PLUG_IN; + hp_dev->debug_state = val; + spin_unlock_irqrestore(&hp_dev->lock, flags); + dev_info(dev, "Cable plugin\n"); + gpiod_set_value(hp_dev->pins[PCIE_PIN_EN].desc, 1); + return count; + + default: + return -EINVAL; + } + + return count; +} + +DEVICE_ATTR_RW(debug_state); + +static ssize_t hotplug_enabled_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cx7_hp_dev *hp_dev = dev_get_drvdata(dev); + + if (!hp_dev) + return -EINVAL; + + return scnprintf(buf, PAGE_SIZE, "%d\n", hp_dev->hotplug_enabled ? 1 : 0); +} + +static ssize_t hotplug_enabled_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct cx7_hp_dev *hp_dev = dev_get_drvdata(dev); + unsigned long val; + int err; + + if (!hp_dev) + return -EINVAL; + + err = kstrtoul(buf, 10, &val); + if (err) + return err; + + hp_dev->hotplug_enabled = (val != 0); + dev_info(dev, "Hotplug %s\n", hp_dev->hotplug_enabled ? "enabled" : "disabled"); + + return count; +} + +DEVICE_ATTR_RW(hotplug_enabled); + +static struct attribute *cx7_hp_attrs[] = { + &dev_attr_debug_state.attr, + &dev_attr_hotplug_enabled.attr, + NULL +}; + +static const struct attribute_group cx7_hp_attr_group = { + .name = "pcie_hotplug", + .attrs = cx7_hp_attrs +}; + +/** + * gpio_acpi_setup - Setup GPIO ACPI context from _CRS + * @pdev: platform device + * @desc: GPIO descriptor + * @hp_dev: hotplug device + * @gpio_index: GPIO index + * + * Returns: GPIO ACPI context on success, NULL on failure + */ +static struct gpio_acpi_context *gpio_acpi_setup(struct platform_device *pdev, + struct gpio_desc *desc, + struct cx7_hp_dev *hp_dev, + int gpio_index) +{ + struct acpi_gpio_parse_context parse_ctx; + struct gpio_acpi_context *ctx; + struct acpi_device *adev; + acpi_status status; + + adev = ACPI_COMPANION(&pdev->dev); + if (!adev) { + dev_err(&pdev->dev, "Failed to get ACPI companion device\n"); + return NULL; + } + + ctx = devm_kzalloc(&pdev->dev, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return NULL; + + ctx->pin = + desc_to_gpio(desc) - + gpio_device_get_base(gpiod_to_gpio_device(desc)); + ctx->dev = &pdev->dev; + + parse_ctx.ctx = ctx; + parse_ctx.hp_dev = hp_dev; + + status = acpi_walk_resources(adev->handle, METHOD_NAME__CRS, + acpi_gpio_lookup_handler, &parse_ctx); + if (ACPI_FAILURE(status)) { + devm_kfree(&pdev->dev, ctx); + return NULL; + } + + if (ctx->valid) { + if (gpio_index == PCIE_PIN_BOOT && hp_dev->boot_pin == -1) { + hp_dev->boot_pin = ctx->pin; + } else if (gpio_index == PCIE_PIN_PRSNT + && hp_dev->prsnt_pin == -1) { + hp_dev->prsnt_pin = ctx->pin; + } + return ctx; + } + + devm_kfree(&pdev->dev, ctx); + return NULL; +} + +/** + * cx7_hp_setup_irq - Setup IRQ for GPIO + * @app_ctx: GPIO context + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_setup_irq(struct cx7_hp_gpio_ctx *app_ctx) +{ + struct gpio_acpi_context *ctx = app_ctx->ctx; + int irq, ret; + + irq = gpiod_to_irq(app_ctx->desc); + if (irq < 0) { + dev_err(ctx->dev, "Failed to get IRQ for GPIO\n"); + return irq; + } + + if (ctx->wake_capable) + enable_irq_wake(irq); + + ret = devm_request_threaded_irq(ctx->dev, irq, + hotplug_irq_handler, cx7_hp_work, + ctx->irq_flags | IRQF_ONESHOT, + "pcie_hotplug", app_ctx); + if (ret) + dev_err(ctx->dev, "Failed to request IRQ %d: %d\n", irq, ret); + + return ret; +} + +/** + * cx7_hp_put_gpio_device - Release GPIO device reference + * @data: GPIO device pointer + */ +static void cx7_hp_put_gpio_device(void *data) +{ + struct gpio_device *gdev = data; + + gpio_device_put(gdev); +} + +/** + * cx7_hp_discover_pcie_devices - Discover existing PCI devices on managed ports + * @pdev: platform device + * @pd: platform data + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_discover_pcie_devices(struct platform_device *pdev, + struct cx7_hp_plat_data *pd) +{ + struct pci_dev *pci_dev = NULL; + int device_count = 0; + int i; + + if (!pd->vendor_id || !pd->device_id) + return 0; + + while ((pci_dev = pci_get_device(pd->vendor_id, + pd->device_id, pci_dev)) != NULL) { + if (!pci_dev->state_saved) { + pci_dev_put(pci_dev); + return -EPROBE_DEFER; + } + + for (i = 0; i < pd->port_nums; i++) { + if (pci_domain_nr(pci_dev->bus) == pd->ports[i].domain) + break; + } + + if (i == pd->port_nums) { + dev_err(&pdev->dev, + "Device %s found on unexpected domain %d\n", + pci_name(pci_dev), pci_domain_nr(pci_dev->bus)); + pci_dev_put(pci_dev); + return -ENODEV; + } + + device_count++; + } + + if (pd->num_devices && device_count != pd->num_devices) { + dev_err(&pdev->dev, + "Required number of devices not found. Expected=%d Actual=%d\n", + pd->num_devices, device_count); + return -ENODEV; + } + + return 0; +} + +/** + * cx7_hp_init_pcie_data - Initialize PCIe data from _DSD and discover devices + * @pdev: platform device + * @pd: platform data to populate + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_init_pcie_data(struct platform_device *pdev, + struct cx7_hp_plat_data *pd) +{ + int ret; + + ret = cx7_hp_parse_pcie_config_dsd(pdev, pd); + if (ret) { + dev_err(&pdev->dev, + "Failed to parse PCIe configuration _DSD properties: %d\n", + ret); + return ret; + } + + if (pd->port_nums == 0 || pd->port_nums >= HP_PORT_MAX) { + dev_err(&pdev->dev, + "Invalid port count from _DSD: %d (must be 1-%d)\n", + pd->port_nums, HP_PORT_MAX - 1); + return -EINVAL; + } + + ret = cx7_hp_discover_pcie_devices(pdev, pd); + if (ret) { + dev_dbg(&pdev->dev, "Device discovery failed: %d\n", ret); + return ret; + } + + return 0; +} + +/** + * cx7_hp_enumerate_gpios - Enumerate GPIOs from ACPI + * @pdev: Platform device + * @hp_dev: Hotplug device structure + * + * Returns: Number of GPIOs found, or negative error code + */ +static int cx7_hp_enumerate_gpios(struct platform_device *pdev, + struct cx7_hp_dev *hp_dev) +{ + struct acpi_gpio_walk_context walk_ctx; + struct fwnode_handle *gpio_fwnode = NULL; + struct acpi_device *gpio_adev = NULL; + acpi_handle gpio_handle; + acpi_status status; + int ret, i; + + ret = cx7_hp_walk_acpi_gpios(pdev, &walk_ctx); + if (ret) { + dev_err(&pdev->dev, "Failed to walk ACPI GPIO resources: %d\n", + ret); + return ret; + } + + if (walk_ctx.count < CX7_HP_MIN_GPIO_COUNT) { + dev_err(&pdev->dev, + "Insufficient GPIOs from ACPI: required at least %d, got %d\n", + CX7_HP_MIN_GPIO_COUNT, walk_ctx.count); + return -ENODEV; + } + + /* Find GPIO device using resource_source from first GPIO */ + if (walk_ctx.count == 0 || walk_ctx.gpios[0].resource_source[0] == '\0') { + dev_err(&pdev->dev, + "No resource_source in ACPI GPIO resources\n"); + return -ENODEV; + } + + status = + acpi_get_handle(NULL, walk_ctx.gpios[0].resource_source, + &gpio_handle); + if (ACPI_FAILURE(status)) { + dev_err(&pdev->dev, + "Failed to get ACPI handle for GPIO controller %s\n", + walk_ctx.gpios[0].resource_source); + return -ENODEV; + } + + gpio_adev = acpi_fetch_acpi_dev(gpio_handle); + if (!gpio_adev) { + dev_err(&pdev->dev, + "Failed to get ACPI device for GPIO controller %s\n", + walk_ctx.gpios[0].resource_source); + return -ENODEV; + } + + gpio_fwnode = acpi_fwnode_handle(gpio_adev); + hp_dev->gdev = gpio_device_find_by_fwnode(gpio_fwnode); + if (!hp_dev->gdev) { + return dev_err_probe(&pdev->dev, -EPROBE_DEFER, + "GPIO controller not available\n"); + } + + /* Successfully found GPIO device - manage reference */ + ret = devm_add_action_or_reset(&pdev->dev, cx7_hp_put_gpio_device, + hp_dev->gdev); + if (ret) { + gpio_device_put(hp_dev->gdev); + hp_dev->gdev = NULL; + dev_err(&pdev->dev, "Failed to register GPIO device cleanup\n"); + return ret; + } + + hp_dev->gpio_count = walk_ctx.count; + + hp_dev->pins = devm_kzalloc(&pdev->dev, + sizeof(struct cx7_hp_gpio_ctx) * + hp_dev->gpio_count, GFP_KERNEL); + if (!hp_dev->pins) { + dev_err(&pdev->dev, "Failed to allocate memory for GPIOs\n"); + return -ENOMEM; + } + + for (i = 0; i < hp_dev->gpio_count; i++) { + struct cx7_hp_gpio_ctx *app_ctx = &hp_dev->pins[i]; + + app_ctx->desc = + gpio_device_get_desc(hp_dev->gdev, walk_ctx.gpios[i].pin); + if (IS_ERR(app_ctx->desc)) { + dev_err(&pdev->dev, + "Failed to get GPIO descriptor for ACPI pin %u (index %d): %ld\n", + walk_ctx.gpios[i].pin, i, + PTR_ERR(app_ctx->desc)); + return PTR_ERR(app_ctx->desc); + } + + app_ctx->hp_dev = hp_dev; + } + + return hp_dev->gpio_count; +} + +/** + * cx7_hp_pci_notifier - PCI bus notifier to configure MPS for CX7 devices + * @nb: notifier block + * @action: bus notification action + * @data: pointer to device being added/removed + * + * Returns: NOTIFY_OK on success, NOTIFY_DONE if not a CX7 device + */ +static int cx7_hp_pci_notifier(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct device *dev = data; + struct pci_dev *pdev = to_pci_dev(dev); + struct cx7_hp_dev *hp_dev; + unsigned long flags; + + if (action != BUS_NOTIFY_ADD_DEVICE) + return NOTIFY_DONE; + + hp_dev = container_of(nb, struct cx7_hp_dev, pci_notifier); + if (!hp_dev || !hp_dev->pd) + return NOTIFY_DONE; + + spin_lock_irqsave(&hp_dev->lock, flags); + if (!hp_dev->hotplug_enabled) { + spin_unlock_irqrestore(&hp_dev->lock, flags); + return NOTIFY_DONE; + } + spin_unlock_irqrestore(&hp_dev->lock, flags); + + if (!pdev || !hp_dev->pd->vendor_id || !hp_dev->pd->device_id) + return NOTIFY_DONE; + + if (pdev->vendor != hp_dev->pd->vendor_id || + pdev->device != hp_dev->pd->device_id) + return NOTIFY_DONE; + + if (pdev->bus) + pcie_bus_configure_settings(pdev->bus); + + return NOTIFY_OK; +} + +/** + * cx7_hp_probe - Platform device probe function + * @pdev: platform device + * + * Initializes the PCIe hotplug driver, parses ACPI resources, and sets up + * GPIO interrupts and sysfs interface. + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_probe(struct platform_device *pdev) +{ + struct cx7_hp_plat_data *pd; + struct cx7_hp_gpio_ctx *app_ctx; + struct cx7_hp_dev *hp_dev; + int ret, i; + + pd = devm_kzalloc(&pdev->dev, sizeof(*pd), GFP_KERNEL); + if (!pd) { + dev_err(&pdev->dev, + "Failed to allocate memory for platform data\n"); + return -ENOMEM; + } + + ret = cx7_hp_init_pcie_data(pdev, pd); + if (ret) + return ret; + + hp_dev = devm_kzalloc(&pdev->dev, sizeof(*hp_dev), GFP_KERNEL); + if (!hp_dev) { + dev_err(&pdev->dev, + "Failed to allocate memory for hotplug device\n"); + return -ENOMEM; + } + + hp_dev->pdev = pdev; + hp_dev->pd = pd; + hp_dev->state = STATE_READY; + hp_dev->boot_pin = -1; + hp_dev->prsnt_pin = -1; + hp_dev->hotplug_enabled = false; + spin_lock_init(&hp_dev->lock); + + for (i = 0; i < HP_PORT_MAX; i++) + hp_dev->cached_root_ports[i] = NULL; + + ret = cx7_hp_enumerate_gpios(pdev, hp_dev); + if (ret < 0) { + dev_err(&pdev->dev, "Failed to enumerate GPIOs from ACPI: %d\n", + ret); + return ret; + } + + for (i = 0; i < hp_dev->gpio_count; i++) { + app_ctx = &hp_dev->pins[i]; + + app_ctx->ctx = gpio_acpi_setup(pdev, app_ctx->desc, hp_dev, i); + if (!app_ctx->ctx) { + dev_err(&pdev->dev, "Failed to setup GPIO %d\n", i); + return -ENODEV; + } + + gpiod_set_debounce(app_ctx->desc, + app_ctx->ctx->debounce_timeout_us); + + if (app_ctx->ctx->connection_type == + ACPI_RESOURCE_GPIO_TYPE_INT) { + ret = cx7_hp_setup_irq(app_ctx); + if (ret) { + dev_err(&pdev->dev, + "Failed to setup IRQ for GPIO %d\n", i); + return ret; + } + } + } + + platform_set_drvdata(pdev, hp_dev); + + ret = cx7_hp_pinctrl_init(hp_dev); + if (ret) { + dev_err(&pdev->dev, "Pinmux init failed, ret: %d\n", ret); + return ret; + } + + ret = sysfs_create_group(&pdev->dev.kobj, &cx7_hp_attr_group); + if (ret) { + dev_err(&pdev->dev, "Sysfs creation failed: %d\n", ret); + goto pinctrl_remove; + } + + cx7_hp_rp_bus_protect(hp_dev, 0, BUS_PROTECT_INIT); + + hp_dev->pci_notifier.notifier_call = cx7_hp_pci_notifier; + ret = bus_register_notifier(&pci_bus_type, &hp_dev->pci_notifier); + if (ret) { + dev_err(&pdev->dev, "Failed to register PCI bus notifier: %d\n", + ret); + goto sysfs_remove; + } + + if (gpiod_get_value(hp_dev->pins[PCIE_PIN_PRSNT].desc)) { + hp_dev->debug_state = CX7_HP_DEBUG_PLUG_OUT; + cx7_hp_send_uevent(hp_dev, REMOVAL_EVT); + } else { + hp_dev->debug_state = CX7_HP_DEBUG_PLUG_IN; + cx7_hp_send_uevent(hp_dev, PLUG_IN_EVT); + } + + dev_info(&pdev->dev, "PCIe hotplug driver initialized successfully\n"); + return 0; + +sysfs_remove: + sysfs_remove_group(&pdev->dev.kobj, &cx7_hp_attr_group); +pinctrl_remove: + cx7_hp_pinctrl_remove(hp_dev); + return ret; +} + +/** + * cx7_hp_remove - Platform device remove function + * @pdev: platform device + * + * Cleans up GPIO pins, pinctrl, sysfs interface, and bus protection. + */ +static void cx7_hp_remove(struct platform_device *pdev) +{ + struct cx7_hp_dev *hp_dev = platform_get_drvdata(pdev); + int i; + + if (!hp_dev) + return; + + sysfs_remove_group(&pdev->dev.kobj, &cx7_hp_attr_group); + + bus_unregister_notifier(&pci_bus_type, &hp_dev->pci_notifier); + + cx7_hp_rp_bus_protect(hp_dev, 0, BUS_PROTECT_CLEANUP); + + cx7_hp_pinctrl_remove(hp_dev); + + for (i = 0; i < hp_dev->pd->port_nums; i++) { + if (hp_dev->cached_root_ports[i]) + pci_dev_put(hp_dev->cached_root_ports[i]); + } + + platform_set_drvdata(pdev, NULL); +} + +static const struct acpi_device_id cx7_hp_acpi_match[] = { + {"MTKP0001", 0}, + {} +}; + +MODULE_DEVICE_TABLE(acpi, cx7_hp_acpi_match); + +static struct platform_driver cx7_hp_driver = { + .probe = cx7_hp_probe, + .remove = cx7_hp_remove, + .driver = { + .name = "cx7-pcie-hotplug", + .acpi_match_table = ACPI_PTR(cx7_hp_acpi_match), + }, +}; + +module_platform_driver(cx7_hp_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("CX7 PCIe Hotplug Driver for NVIDIA DGX Systems"); From b793ffb49958910566262c460da0ff1e3c45abe7 Mon Sep 17 00:00:00 2001 From: Brad Figg Date: Fri, 29 Mar 2024 13:31:34 -0700 Subject: [PATCH 090/311] NVIDIA: [Packaging] Add nvidia-fs build dependencies BugLink: https://bugs.launchpad.net/bugs/2059814 Signed-off-by: Brad Figg Acked-by: Brad Figg Acked-by: Ian May Signed-off-by: Ian May Signed-off-by: Jacob Martin (cherry picked from commit a64b5977c0cb9bb66af5f3d9fd7ed2a7eaebc131 linux-nvidia-6.14) Signed-off-by: Abdur Rahman (cherry picked from commit 67713ae343c95f540b3f881160dee756ea132a3c noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- debian.nvidia/control.stub.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debian.nvidia/control.stub.in b/debian.nvidia/control.stub.in index 2528e3ae03ab7..310e68d312c8e 100644 --- a/debian.nvidia/control.stub.in +++ b/debian.nvidia/control.stub.in @@ -53,6 +53,8 @@ Build-Depends: uuid-dev , zstd , bpftool:native [amd64 arm64] , + nvidia-dkms-kernel [amd64 arm64] , + nvidia-kernel-source [amd64 arm64] , Build-Depends-Indep: asciidoc , bzip2 , From 0159756995697ec0fbe3299a1723f444a39282a7 Mon Sep 17 00:00:00 2001 From: Leon Yen Date: Thu, 11 Dec 2025 20:38:36 +0800 Subject: [PATCH 091/311] NVIDIA: SAUCE: wifi: mt76: mt7925: Fix incorrect MLO mode in firmware control BugLink: https://bugs.launchpad.net/bugs/2138755 The selection of MLO mode should depend on the capabilities of the STA rather than those of the peer AP to avoid compatibility issues with certain APs, such as Xiaomi BE5000 WiFi7 router. Fixes: 69acd6d910b0c ("wifi: mt76: mt7925: add mt7925_change_vif_links") Signed-off-by: Leon Yen (backported from https://lore.kernel.org/all/20251211123836.4169436-1-leon.yen@mediatek.com/) Signed-off-by: Muteeb Akram Acked-by: Jamie Nguyen Acked-by: Carol L Soto Acked-by: Jacob Martin Acked-by: Abdur Rahman Signed-off-by: Brad Figg (cherry picked from commit 023106d93f530dd449780db368915eb048abfb56 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/net/wireless/mediatek/mt76/mt7925/mcu.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c index cf0fdea45cf73..279ace8ab883d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c @@ -1330,6 +1330,8 @@ int mt7925_mcu_set_mlo_roc(struct mt792x_bss_conf *mconf, u16 sel_links, .roc[1].len = cpu_to_le16(sizeof(struct roc_acquire_tlv)) }; + struct wiphy *wiphy = mvif->phy->mt76->hw->wiphy; + if (!mconf || hweight16(vif->valid_links) < 2 || hweight16(sel_links) != 2) return -EPERM; @@ -1352,7 +1354,8 @@ int mt7925_mcu_set_mlo_roc(struct mt792x_bss_conf *mconf, u16 sel_links, is_AG_band |= links[i].chan->band == NL80211_BAND_2GHZ; } - if (vif->cfg.eml_cap & IEEE80211_EML_CAP_EMLSR_SUPP) + if (!(wiphy->iftype_ext_capab[0].mld_capa_and_ops & + IEEE80211_MLD_CAP_OP_MAX_SIMUL_LINKS)) type = is_AG_band ? MT7925_ROC_REQ_MLSR_AG : MT7925_ROC_REQ_MLSR_AA; else From 02a113d6cc7042f4bead0370ce90be5cb1133eef Mon Sep 17 00:00:00 2001 From: Muteeb Akram Date: Tue, 6 Jan 2026 01:58:48 +0000 Subject: [PATCH 092/311] NVIDIA: SAUCE: r8127: print GPL_CLAIM with KERN_INFO BugLink: https://bugs.launchpad.net/bugs/2137588 Add KERN_INFO log level for GPL_CLAIM to downgrade warning messages Signed-off-by: ChunHao Lin Signed-off-by: Revanth Kumar Uppala Signed-off-by: Muteeb Akram Acked-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Jacob Martin Acked-by: Noah Wager Signed-off-by: Brad Figg (cherry picked from commit 3c12da38a1f4ad9723a55cc8aeb0f800ad6e995e noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/net/ethernet/realtek/r8127/r8127_n.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/realtek/r8127/r8127_n.c b/drivers/net/ethernet/realtek/r8127/r8127_n.c index 496fec1320d12..9e39016ea2c80 100755 --- a/drivers/net/ethernet/realtek/r8127/r8127_n.c +++ b/drivers/net/ethernet/realtek/r8127/r8127_n.c @@ -14298,7 +14298,7 @@ rtl8127_init_one(struct pci_dev *pdev, rtl8127_sysfs_init(dev); #endif /* ENABLE_R8127_SYSFS */ - printk("%s", GPL_CLAIM); + printk(KERN_INFO "%s", GPL_CLAIM); out: return rc; From ebfb836a56b7a67353c5c72752e861afcff3eacb Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Fri, 9 Jan 2026 11:49:19 -0600 Subject: [PATCH 093/311] NVIDIA: SAUCE: vfio: Fix missing prototype warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BugLink: https://bugs.launchpad.net/bugs/2138132 Fix this warning about missing prototype: ../drivers/vfio/vfio_main.c:1369:21: warning: no previous prototype for ‘vfio_device_from_file’ [-Wmissing-prototypes] 1369 | struct vfio_device *vfio_device_from_file(struct file *file) | ^~~~~~~~~~~~~~~~~~~~~ Add the declaration in the header since it's no different than e.g. vfio_file_is_valid() that is there and remove the extern in the .c. Signed-off-by: Lucas De Marchi Acked-by: Jamie Nguyen Acked-by: Nirmoy Das Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off-by: Brad Figg (backported from commit 006a8656e31361ea8425d655ceef919fd7814d31 noble:linux-nvidia-6.17) [jacobmartin: upstream did not have the extern in nvgrace-gpu/main.c] Signed-off-by: Jacob Martin --- include/linux/vfio.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/vfio.h b/include/linux/vfio.h index e90859956514a..591b4a148bc70 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -337,6 +337,7 @@ static inline bool vfio_file_has_dev(struct file *file, struct vfio_device *devi return false; } #endif +struct vfio_device *vfio_device_from_file(struct file *file); bool vfio_file_is_valid(struct file *file); bool vfio_file_enforced_coherent(struct file *file); void vfio_file_set_kvm(struct file *file, struct kvm *kvm); From 591107a2b8b84268f7a658563f54236649732454 Mon Sep 17 00:00:00 2001 From: Jeremy Szu Date: Tue, 3 Feb 2026 18:49:43 +0800 Subject: [PATCH 094/311] UBUNTU: [Packaging] Enable coresight in Perf if arm64 BugLink: https://bugs.launchpad.net/bugs/2093957 Signed-off-by: Jeremy Szu Acked-by: Nirmoy Das Acked-by: Matthew R. Ochs Acked-by: Abdur Rahman Acked-by: Noah Wager Signed-off-by: Brad Figg (cherry picked from commit a15be6bb7b879f3060e315a69c8ea649d8f3d28d noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- debian/rules.d/2-binary-arch.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/rules.d/2-binary-arch.mk b/debian/rules.d/2-binary-arch.mk index 7a8bb73afd78b..564923f55af31 100644 --- a/debian/rules.d/2-binary-arch.mk +++ b/debian/rules.d/2-binary-arch.mk @@ -625,7 +625,7 @@ ifeq ($(do_tools_cpupower),true) endif ifeq ($(do_tools_perf),true) cd $(builddirpa)/tools/perf && \ - LLVM_CONFIG=llvm-config-$(LLVM_VERSION) $(kmake) prefix=/usr HAVE_CPLUS_DEMANGLE_SUPPORT=1 CROSS_COMPILE=$(CROSS_COMPILE) NO_LIBPERL=1 WERROR=0 + LLVM_CONFIG=llvm-config-$(LLVM_VERSION) $(kmake) prefix=/usr HAVE_CPLUS_DEMANGLE_SUPPORT=1 CROSS_COMPILE=$(CROSS_COMPILE) NO_LIBPERL=1 WERROR=0 $(if $(filter arm64,$(build_arch)),CORESIGHT=1) endif ifeq ($(do_tools_bpftool),true) $(kmake) CROSS_COMPILE=$(CROSS_COMPILE) -C $(builddirpa)/tools/bpf/bpftool From 222e2cc9a39f736b3e518cb3c69df8e5e95d7876 Mon Sep 17 00:00:00 2001 From: Nirmoy Das Date: Thu, 5 Feb 2026 08:26:16 +0000 Subject: [PATCH 095/311] NVIDIA: SAUCE: vfio: Remove vfio_device_from_file() declaration BugLink: https://bugs.launchpad.net/bugs/2138892 Remove this declaration which is now used within the file after merging upstream "vfio/nvgrace-gpu: register device memory for poison handling". Signed-off-by: Nirmoy Das Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg (cherry picked from commit e78ec36bfb5fa7de13a49e2bb588d6ed633eb7c7 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- include/linux/vfio.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/linux/vfio.h b/include/linux/vfio.h index 591b4a148bc70..e90859956514a 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -337,7 +337,6 @@ static inline bool vfio_file_has_dev(struct file *file, struct vfio_device *devi return false; } #endif -struct vfio_device *vfio_device_from_file(struct file *file); bool vfio_file_is_valid(struct file *file); bool vfio_file_enforced_coherent(struct file *file); void vfio_file_set_kvm(struct file *file, struct kvm *kvm); From 41a52ded523321e4293ea60030bcb9cf93c090d8 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Fri, 13 Feb 2026 16:27:35 -0600 Subject: [PATCH 096/311] UBUNTU: [Packaging] Depend on 580 NVIDIA graphics driver components explicitly The virtual nvidia-kernel-source and nvidia-dkms-kernel dependencies would sometimes pull the 470 driver, which is incompatible with the nvidia-fs build. Stick to the latest LTS. This could be made to use the virtual packages again once the 470 driver transitionals are released. Ignore: yes Signed-off-by: Jacob Martin (cherry picked from commit 73a27bbf7e984cd94f37ca5f9d414f530755a78c noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- debian.nvidia/control.stub.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian.nvidia/control.stub.in b/debian.nvidia/control.stub.in index 310e68d312c8e..95082258882eb 100644 --- a/debian.nvidia/control.stub.in +++ b/debian.nvidia/control.stub.in @@ -53,8 +53,8 @@ Build-Depends: uuid-dev , zstd , bpftool:native [amd64 arm64] , - nvidia-dkms-kernel [amd64 arm64] , - nvidia-kernel-source [amd64 arm64] , + nvidia-dkms-580-open [amd64 arm64] , + nvidia-kernel-source-580-open [amd64 arm64] , Build-Depends-Indep: asciidoc , bzip2 , From 14d5ecd233b9e2678d12f60dac4378afb1e014e1 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Sat, 14 Feb 2026 07:56:30 -0600 Subject: [PATCH 097/311] UBUNTU: [Packaging] Add libopencsd-dev as a build dependency BugLink: https://bugs.launchpad.net/bugs/2093957 The patch "UBUNTU: [Packaging] Enable coresight in Perf if arm64" enables perf to be built with CORESIGHT=1 on arm64. This requires libopencsd. Signed-off-by: Jacob Martin (cherry picked from commit f95fa30baf72c7446f22c2e7c53883a9d2fc85fe noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- debian.nvidia/control.stub.in | 1 + 1 file changed, 1 insertion(+) diff --git a/debian.nvidia/control.stub.in b/debian.nvidia/control.stub.in index 95082258882eb..17859fbf26fd1 100644 --- a/debian.nvidia/control.stub.in +++ b/debian.nvidia/control.stub.in @@ -55,6 +55,7 @@ Build-Depends: bpftool:native [amd64 arm64] , nvidia-dkms-580-open [amd64 arm64] , nvidia-kernel-source-580-open [amd64 arm64] , + libopencsd-dev [arm64] , Build-Depends-Indep: asciidoc , bzip2 , From b52d12a03649355d5d77e1af3fb0b658bda8d1f4 Mon Sep 17 00:00:00 2001 From: Nirmoy Das Date: Fri, 13 Feb 2026 09:32:59 -0800 Subject: [PATCH 098/311] NVIDIA: SAUCE: r8127: fix NAPI warning on module removal BugLink: https://bugs.launchpad.net/bugs/2141780 When the r8127 module is unloaded, __netif_napi_del_locked() can trigger a WARN because NAPI is removed while still enabled. unregister_netdev() calls ndo_stop, which disables NAPI; deleting NAPI before that runs violates the netdev/NAPI teardown order. Move rtl8127_del_napi() to after unregister_netdev() so NAPI is disabled in ndo_stop before it is removed. Aligns with the upstream r8169 fix in commit 12b1bc75cd46 ("r8169: improve rtl_remove_one"). Signed-off-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Matthew R. Ochs Acked-by: Noah Wager Acked-by: Jacob Martin Signed-off-by: Brad Figg (cherry picked from commit cda2af9cd9d2b45f09e9a615e4354b4e236e3909 noble:linux-nvidia-6.17) Signed-off-by: Jacob Martin --- drivers/net/ethernet/realtek/r8127/r8127_n.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/realtek/r8127/r8127_n.c b/drivers/net/ethernet/realtek/r8127/r8127_n.c index 9e39016ea2c80..03d99ae8d3853 100755 --- a/drivers/net/ethernet/realtek/r8127/r8127_n.c +++ b/drivers/net/ethernet/realtek/r8127/r8127_n.c @@ -14334,9 +14334,6 @@ rtl8127_remove_one(struct pci_dev *pdev) rtl8127_cancel_all_schedule_work(tp); -#ifdef CONFIG_R8127_NAPI - rtl8127_del_napi(tp); -#endif if (HW_DASH_SUPPORT_DASH(tp)) rtl8127_driver_stop(tp); @@ -14347,6 +14344,9 @@ rtl8127_remove_one(struct pci_dev *pdev) #endif //ENABLE_R8127_SYSFS unregister_netdev(dev); +#ifdef CONFIG_R8127_NAPI + rtl8127_del_napi(tp); +#endif rtl8127_disable_msi(pdev, tp); #ifdef ENABLE_R8127_PROCFS rtl8127_proc_remove(dev); From 2ce11f732870985e32ba34b3235364c3f80a09fe Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Tue, 24 Mar 2026 17:50:51 -0500 Subject: [PATCH 099/311] UBUNTU: [Packaging] update dkms-versions Ignore: yes Signed-off-by: Jacob Martin --- debian.nvidia/dkms-versions | 6 ------ 1 file changed, 6 deletions(-) diff --git a/debian.nvidia/dkms-versions b/debian.nvidia/dkms-versions index 5ccbdb3d3e79c..3735399207a12 100644 --- a/debian.nvidia/dkms-versions +++ b/debian.nvidia/dkms-versions @@ -1,8 +1,2 @@ zfs-linux 2.4.0-1ubuntu3 modulename=zfs debpath=pool/universe/z/%package%/zfs-dkms_%version%_all.deb arch=amd64 arch=arm64 arch=ppc64el arch=s390x rprovides=spl-modules rprovides=spl-dkms rprovides=zfs-modules rprovides=zfs-dkms -evdi 1.14.12+dfsg-1ubuntu1 modulename=evdi debpath=pool/universe/e/%package%/evdi-dkms_%version%_all.deb rprovides=evdi-modules rprovides=evdi-dkms type=standalone -ipu6-drivers 0~git202511120800.9766e218-0ubuntu2 modulename=ipu6 debpath=pool/universe/i/%package%/intel-ipu6-dkms_%version%_amd64.deb arch=amd64 rprovides=ipu6-modules rprovides=intel-ipu6-dkms type=standalone -ipu7-drivers 0~git202511120800.fc335577-0ubuntu1 modulename=ipu7 debpath=pool/universe/i/%package%/intel-ipu7-dkms_%version%_amd64.deb arch=amd64 rprovides=ipu7-modules rprovides=intel-ipu7-dkms type=standalone -backport-iwlwifi-dkms 1:0~96.13623-gitd16e74cc-0ubuntu2 modulename=iwlwifi debpath=pool/universe/b/%package%/backport-iwlwifi-dkms_%version%_all.deb arch=amd64 rprovides=iwlwifi-modules rprovides=backport-iwlwifi-dkms type=standalone v4l2loopback 0.15.3-1ubuntu2 modulename=v4l2loopback debpath=pool/universe/v/%package%/v4l2loopback-dkms_%version%_all.deb arch=amd64 rprovides=v4l2loopback-modules rprovides=v4l2loopback-dkms -usbio-drivers 0~git202510282139.ee221eca-0ubuntu1 modulename=usbio debpath=pool/universe/u/%package%/intel-usbio-dkms_%version%_amd64.deb arch=amd64 rprovides=usbio-modules rprovides=intel-usbio-dkms type=standalone -vision-drivers 0~git202511121832.a8d772f2-0ubuntu1 modulename=vision debpath=pool/universe/v/%package%/intel-vision-dkms_%version%_amd64.deb arch=amd64 rprovides=vision-modules rprovides=intel-vision-dkms type=standalone From 7d637b8e1aebb2a7719e55c278bc6076e228eb08 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Wed, 25 Mar 2026 15:35:38 -0500 Subject: [PATCH 100/311] UBUNTU: [Packaging] nvidia: control.stub.in: add new build dependencies from master Ignore: yes Signed-off-by: Jacob Martin --- debian.nvidia/control.stub.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debian.nvidia/control.stub.in b/debian.nvidia/control.stub.in index 17859fbf26fd1..ad3894f188532 100644 --- a/debian.nvidia/control.stub.in +++ b/debian.nvidia/control.stub.in @@ -21,6 +21,7 @@ Build-Depends: java-common , kmod , libaudit-dev , + libbpf-dev , libcap-dev , libdebuginfod-dev [amd64 arm64] , libdw-dev , @@ -28,6 +29,8 @@ Build-Depends: libiberty-dev , liblzma-dev , libnewt-dev , + libnl-3-dev, + libnl-genl-3-dev, libnuma-dev [amd64 arm64] , libpci-dev , libssl-dev , From bba54370a9e8d588541c058d240a7fc81903d023 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Wed, 25 Mar 2026 15:48:21 -0500 Subject: [PATCH 101/311] UBUNTU: [Config] nvidia: updateconfigs Ignore: yes Signed-off-by: Jacob Martin --- debian.nvidia/config/annotations | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index 4d07edf8ec02a..78487314f1f9d 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -217,5 +217,8 @@ CONFIG_VFIO_IOMMU_TYPE1 note<'LP: #2095028'> # ---- Annotations without notes ---- CONFIG_BCH policy<{'amd64': 'm', 'arm64': 'y'}> -CONFIG_CC_VERSION_TEXT policy<{'amd64': '"x86_64-linux-gnu-gcc (Ubuntu 15.2.0-15ubuntu1) 15.2.0"', 'arm64': '"aarch64-linux-gnu-gcc (Ubuntu 15.2.0-15ubuntu1) 15.2.0"'}> +CONFIG_CC_VERSION_TEXT policy<{'amd64': '"x86_64-linux-gnu-gcc (Ubuntu 15.2.0-16ubuntu1) 15.2.0"', 'arm64': '"aarch64-linux-gnu-gcc (Ubuntu 15.2.0-16ubuntu1) 15.2.0"'}> CONFIG_MTD_NAND_CORE policy<{'amd64': 'm', 'arm64': 'y'}> +CONFIG_PAHOLE_VERSION policy<{'amd64': '131', 'arm64': '131'}> +CONFIG_RUSTC_VERSION policy<{'amd64': '109301', 'arm64': '109301'}> +CONFIG_RUSTC_VERSION_TEXT policy<{'amd64': '"rustc 1.93.1 (01f6ddf75 2026-02-11) (built from a source tarball)"', 'arm64': '"rustc 1.93.1 (01f6ddf75 2026-02-11) (built from a source tarball)"'}> From 5ba382d298c23df9db054c95625520b8c292c079 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Thu, 26 Mar 2026 11:14:07 -0500 Subject: [PATCH 102/311] UBUNTU: [Packaging] debian.nvidia/dkms-versions -- update from kernel-versions (adhoc/d2026.02.16) BugLink: https://bugs.launchpad.net/bugs/1786013 Signed-off-by: Jacob Martin --- debian.nvidia/dkms-versions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian.nvidia/dkms-versions b/debian.nvidia/dkms-versions index 3735399207a12..fd8e9c633b136 100644 --- a/debian.nvidia/dkms-versions +++ b/debian.nvidia/dkms-versions @@ -1,2 +1,2 @@ -zfs-linux 2.4.0-1ubuntu3 modulename=zfs debpath=pool/universe/z/%package%/zfs-dkms_%version%_all.deb arch=amd64 arch=arm64 arch=ppc64el arch=s390x rprovides=spl-modules rprovides=spl-dkms rprovides=zfs-modules rprovides=zfs-dkms -v4l2loopback 0.15.3-1ubuntu2 modulename=v4l2loopback debpath=pool/universe/v/%package%/v4l2loopback-dkms_%version%_all.deb arch=amd64 rprovides=v4l2loopback-modules rprovides=v4l2loopback-dkms +zfs-linux 2.4.1-1ubuntu1 modulename=zfs debpath=pool/universe/z/%package%/zfs-dkms_%version%_all.deb arch=amd64 arch=arm64 arch=ppc64el arch=riscv64 arch=s390x rprovides=spl-modules rprovides=spl-dkms rprovides=zfs-modules rprovides=zfs-dkms off_series=true +v4l2loopback 0.15.3-1ubuntu2 modulename=v4l2loopback debpath=pool/universe/v/%package%/v4l2loopback-dkms_%version%_all.deb arch=amd64 rprovides=v4l2loopback-modules rprovides=v4l2loopback-dkms off_series=true From 9525d1ef3d501ddee706d54d52228f1ef3f61a04 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Thu, 26 Mar 2026 11:14:51 -0500 Subject: [PATCH 103/311] UBUNTU: Ubuntu-nvidia-7.0.0-1003.3 Signed-off-by: Jacob Martin --- debian.nvidia/changelog | 436 +++++++++++++++++++++++++++++++++++++- debian.nvidia/reconstruct | 33 --- 2 files changed, 431 insertions(+), 38 deletions(-) diff --git a/debian.nvidia/changelog b/debian.nvidia/changelog index ddea5556ec148..86bdfb972ee0a 100644 --- a/debian.nvidia/changelog +++ b/debian.nvidia/changelog @@ -1,10 +1,436 @@ -linux-nvidia (7.0.0-1003.3) UNRELEASED; urgency=medium +linux-nvidia (7.0.0-1003.3) resolute; urgency=medium - CHANGELOG: Do not edit directly. Autogenerated at release. - CHANGELOG: Use the printchanges target to see the current changes. - CHANGELOG: Use the insertchanges target to create the final log. + * Packaging resync (LP: #1786013) + - [Packaging] debian.nvidia/dkms-versions -- update from kernel-versions + (adhoc/d2026.02.16) - -- Jacob Martin Tue, 10 Mar 2026 10:56:27 -0500 + * Backport NVIDIA: SAUCE: vfio/nvgrace-egm: split zapping EGM into 1GB + chunks (LP: #2142160) + - NVIDIA: SAUCE: vfio/nvgrace-egm: split zapping EGM into 1GB chunks + + * r8127 module unload triggers NAPI WARN in netif_napi_del_locked() + (LP: #2141780) + - NVIDIA: SAUCE: r8127: fix NAPI warning on module removal + + * Enable Coresight in Perf (LP: #2093957) + - [Packaging] Enable coresight in Perf if arm64 + - [Packaging] Add libopencsd-dev as a build dependency + + * Backport nvgrace-gpu hugepfnmap, ecc patches and miscellaneous cleanups + (LP: #2138892) + - NVIDIA: SAUCE: vfio/nvgrace-egm: register EGM PFNMAP range with + memory_failure + - NVIDIA: SAUCE: vfio: Remove vfio_device_from_file() declaration + + * missing prototype for vfio_device_from_file() (LP: #2138132) + - NVIDIA: SAUCE: vfio: Fix missing prototype warning + + * r8127: Downgrade GPL claim to info (LP: #2137588) + - NVIDIA: SAUCE: r8127: print GPL_CLAIM with KERN_INFO + + * mt7925: Incorrect MLO mode in firmware control (LP: #2138755) + - NVIDIA: SAUCE: wifi: mt76: mt7925: Fix incorrect MLO mode in firmware + control + + * Enable GDS in the 6.8 based linux-nvidia kernel (LP: #2059814) + - NVIDIA: [Packaging] Add nvidia-fs build dependencies + + * Add PCIe Hotplug Driver for CX7 on DGX Spark (LP: #2138269) + - NVIDIA: SAUCE: MEDIATEK: platform: Add PCIe Hotplug Driver for CX7 on + DGX Spark + + * Backport support for Grace MPAM (LP: #2122432) + - NVIDIA: SAUCE: DT: cacheinfo: Expose the code to generate a cache-id + from a device_node + - NVIDIA: SAUCE: DT: dt-bindings: arm: Add MPAM MSC binding + - NVIDIA: SAUCE: arm64: mpam: Context switch the MPAM registers + - NVIDIA: SAUCE: arm64: mpam: Re-initialise MPAM regs when CPU comes + online + - NVIDIA: SAUCE: arm64: mpam: Advertise the CPUs MPAM limits to the driver + - NVIDIA: SAUCE: arm64: mpam: Add cpu_pm notifier to restore MPAM sysregs + - NVIDIA: SAUCE: arm64: mpam: Add helpers to change a tasks and cpu mpam + partid/pmg values + - NVIDIA: SAUCE: cacheinfo: Add helper to find the cache size from + cpu+level + - NVIDIA: SAUCE: arm_mpam: resctrl: Add boilerplate cpuhp and domain + allocation + - NVIDIA: SAUCE: arm_mpam: resctrl: Pick the caches we will use as resctrl + resources + - NVIDIA: SAUCE: arm_mpam: resctrl: Implement + resctrl_arch_reset_all_ctrls() + - NVIDIA: SAUCE: arm_mpam: resctrl: Add resctrl_arch_get_config() + - NVIDIA: SAUCE: arm_mpam: resctrl: Implement helpers to update + configuration + - NVIDIA: SAUCE: arm_mpam: resctrl: Add plumbing against arm64 task and + cpu hooks + - NVIDIA: SAUCE: arm_mpam: resctrl: Add CDP emulation + - NVIDIA: SAUCE: arm_mpam: resctrl: Add rmid index helpers + - NVIDIA: SAUCE: arm_mpam: resctrl: Convert to/from MPAMs bitmaps and + fixed-point formats + - NVIDIA: SAUCE: arm_mpam: resctrl: Add support for 'MB' resource + - NVIDIA: SAUCE: arm_mpam: resctrl: Reject oversized memory bandwidth + portion bitmaps + - NVIDIA: SAUCE: arm_mpam: resctrl: Fix MB min_bandwidth value exposed to + userspace + - NVIDIA: SAUCE: arm_mpam: resctrl: Add kunit test for control format + conversions + - NVIDIA: SAUCE: arm_mpam: resctrl: Add support for csu counters + - NVIDIA: SAUCE: arm_mpam: resctrl: Pre-allocate free running monitors + - NVIDIA: SAUCE: arm_mpam: resctrl: Pre-allocate assignable monitors + - NVIDIA: SAUCE: arm_mpam: resctrl: Add kunit test for ABMC/CDP + interactions + - NVIDIA: SAUCE: arm_mpam: resctrl: Add resctrl_arch_config_cntr() for + ABMC use + - NVIDIA: SAUCE: arm_mpam: resctrl: Allow resctrl to allocate monitors + - NVIDIA: SAUCE: arm_mpam: resctrl: Add resctrl_arch_rmid_read() and + resctrl_arch_reset_rmid() + - NVIDIA: SAUCE: arm_mpam: resctrl: Add resctrl_arch_cntr_read() & + resctrl_arch_reset_cntr() + - NVIDIA: SAUCE: untested: arm_mpam: resctrl: Allow monitors to be + configured with filters + - NVIDIA: SAUCE: arm_mpam: resctrl: Add empty definitions for fine-grained + enables + - NVIDIA: SAUCE: arm64: mpam: Select ARCH_HAS_CPU_RESCTRL + - NVIDIA: SAUCE: fs/resctrl: Don't touch rmid_ptrs[] in free_rmid() when + there are no monitors + - NVIDIA: SAUCE: fs/resctrl: Avoid a race with dom_data_exit() and + closid_num_dirty_rmid[] + - NVIDIA: SAUCE: fs/resctrl: Avoid a race with dom_data_exit() and + rmid_ptrs[] + - NVIDIA: SAUCE: perf/arm-cmn: Stop claiming all the resources + - NVIDIA: SAUCE: arm_mpam: resctrl: Call resctrl_init() on platforms that + can support resctrl + - NVIDIA: SAUCE: arm_mpam: resctrl: Call resctrl_exit() in the event of + errors + - NVIDIA: SAUCE: arm_mpam: resctrl: Update the rmid reallocation limit + - NVIDIA: SAUCE: arm_mpam: resctrl: Sort the order of the domain lists + - NVIDIA: SAUCE: arm_mpam: Generate a configuration for min controls + - NVIDIA: SAUCE: arm_mpam: Add quirk framework + - NVIDIA: SAUCE: arm_mpam: Add workaround for T241-MPAM-1 + - NVIDIA: SAUCE: arm_mpam: Add workaround for T241-MPAM-4 + - NVIDIA: SAUCE: arm_mpam: Add workaround for T241-MPAM-6 + - NVIDIA: SAUCE: arm_mpam: Quirk CMN-650's CSU NRDY behaviour + - NVIDIA: SAUCE: debugfs: Add helpers for creating cpumask entries in + debugfs + - NVIDIA: SAUCE: arm_mpam: Add debugfs entries to show the MSC/RIS the + driver discovered + - NVIDIA: SAUCE: arm_mpam: Add force-disable debugfs trigger + - NVIDIA: SAUCE: arm_mpam: Expose the number of NRDY retries in debugfs + - NVIDIA: SAUCE: arm_mpam: Add resctrl_arch_round_bw() + - NVIDIA: SAUCE: fs/resctrl,x86/resctrl: Factor mba rounding to be per- + arch + - NVIDIA: SAUCE: arm_mpam: Relax num_rmids parameter advertised to + userspace + - NVIDIA: SAUCE: arm_mpam: Split the locking around the mon_sel registers + - NVIDIA: SAUCE: arm_mpam: Allow the maximum partid to be overridden from + the command line + - NVIDIA: SAUCE: arm_mpam: Allow MSC to be forced to have an unknown + location + - NVIDIA: SAUCE: fs/resctrl: Add this_is_not_abi mount option + - NVIDIA: SAUCE: iommu/arm-smmu-v3: Register SMMU capabilities with MPAM + - NVIDIA: SAUCE: iommu/arm-smmu-v3: Add mpam helpers to query and set + state + - NVIDIA: SAUCE: iommu: Add helpers to get and set the QoS state + - NVIDIA: SAUCE: iommu: Add helpers to retrieve iommu_groups by id or + kobject + - NVIDIA: SAUCE: iommu: Add helper to retrieve iommu kset + - NVIDIA: SAUCE: kobject: Add kset_get_next_obj() to allow a kset to be + walked + - NVIDIA: SAUCE: arm_mpam: resctrl: Add iommu helpers to get/set the + partid and pmg + - NVIDIA: SAUCE: fs/resctrl: Add support for assigning iommu_groups to + resctrl groups + - NVIDIA: SAUCE: firmware: arm_scmi: add MPAM-FB SCMI protocol stub + - NVIDIA: SAUCE: arm_mpam: add MPAM-FB MSC firmware access support + - NVIDIA: SAUCE: arm_mpam: Allow duplicate PCC subspace_ids + - NVIDIA: SAUCE: untested: mpam: Convert pcc_channels list to XArray and + cleanup + - NVIDIA: SAUCE: x86/resctrl: Add stub to allow other architecture to + disable monitor overflow + - NVIDIA: SAUCE: arm_mpam: resctrl: Determine if any exposed counter can + overflow + - NVIDIA: SAUCE: fs/restrl: Allow the overflow handler to be disabled + - NVIDIA: SAUCE: fs/resctrl: Uniform data type of + component_id/domid/id/cache_id + - NVIDIA: SAUCE: arm_mpam: Allow cmax/cmin to be configured + - NVIDIA: SAUCE: arm_mpam: Rename mbw conversion to 'fract16' for code re- + use + - NVIDIA: SAUCE: fs/resctrl: Group all the MBA specific properties in a + separate struct + - NVIDIA: SAUCE: fs/resctrl: Abstract duplicate domain test to a helper + - NVIDIA: SAUCE: fs/resctrl: Move MBA supported check to parse_line() + instead of parse_bw() + - NVIDIA: SAUCE: fs/resctrl: Rename resctrl_get_default_ctrl() to include + resource + - NVIDIA: SAUCE: fs/resctrl: Add a schema format to the schema, allowing + it to be different + - NVIDIA: SAUCE: fs/resctrl: Use schema format to check the resource is a + bitmap + - NVIDIA: SAUCE: fs/resctrl: Add specific schema types for 'range' + - NVIDIA: SAUCE: x86/resctrl: Move over to specifying MBA control formats + - NVIDIA: SAUCE: arm_mpam: resctrl: Convert MB resource to use percentage + - NVIDIA: SAUCE: fs/resctrl: Remove 'range' schema format + - NVIDIA: SAUCE: fs/resctrl: Add additional files for percentage and + bitmap controls + - NVIDIA: SAUCE: fs/resctrl: Add fflags_from_schema() for files based on + schema format + - NVIDIA: SAUCE: fs/resctrl: Expose the schema format to user-space + - NVIDIA: SAUCE: fs/resctrl: Add L2 and L3 'MAX' resource schema + - NVIDIA: SAUCE: arm_mpam: resctrl: Add the glue code to convert to/from + cmax + - NVIDIA: SAUCE: mm,memory_hotplug: Add lockdep assertion helper + - NVIDIA: SAUCE: fs/resctrl: Take memory hotplug lock whenever taking CPU + hotplug lock + - NVIDIA: SAUCE: fs/resctrl: Add mount option for mb_uses_numa_nid and + arch stubs + - NVIDIA: SAUCE: Fix unused variable warning + - NVIDIA: SAUCE: arm_mpam: resctrl: Pick whether MB can use NUMA nid + instead of cache-id + - NVIDIA: SAUCE: arm_mpam: resctrl: Change domain_hdr online/offline to + work with a set of CPUs + - NVIDIA: SAUCE: untested: arm_mpam: resctrl: Split + mpam_resctrl_alloc_domain() to have CPU and node + - NVIDIA: SAUCE: arm_mpam: resctrl: Add NUMA node notifier for domain + online/offline + - NVIDIA: SAUCE: untested: arm_mpam: resctrl: Allow resctrl to enable NUMA + nid as MB domain-id + - NVIDIA: SAUCE: [Config] RESCTRL configs added to annotations + - NVIDIA: SAUCE: arm_mpam: Fix missing SHIFT definitions + - NVIDIA: SAUCE: arm_mpam: resctrl: Fix MPAM kunit + - NVIDIA: SAUCE: resctrl/mpam: Align packed mpam_props to fix arm64 KUnit + alignment fault + - NVIDIA: SAUCE: resctrl/tests: mpam_devices: compare only meaningful + bytes of mpam_props + + * r8127: fix for LTS test panic (LP: #2134991) + - NVIDIA: SAUCE: r8127: Remove registers2 proc entry + + * Add two more Spark iGPU IDs for the existing iommu quirk (LP: #2132033) + - NVIDIA: SAUCE: iommu/arm-smmu-v3: Add two more DGX Spark iGPU IDs for + existing iommu quirk + + * Pull CPPC mailing list patches for Spark (LP: #2131705) + - NVIDIA: SAUCE: ACPI: CPPC: Add cppc_get_perf() API to read performance + controls + - NVIDIA: SAUCE: ACPI: CPPC: extend APIs to support auto_sel and epp + - NVIDIA: SAUCE: ACPI: CPPC: add APIs and sysfs interface for min/max_perf + - NVIDIA: SAUCE: ACPI: CPPC: add APIs and sysfs interface for perf_limited + register + - NVIDIA: SAUCE: cpufreq: CPPC: Add sysfs for min/max_perf and + perf_limited + - NVIDIA: SAUCE: cpufreq: CPPC: update policy min/max when toggling + auto_select + - NVIDIA: SAUCE: cpufreq: CPPC: add autonomous mode boot parameter support + + * r8127: fix kernel panic when dump all registers (LP: #2130445) + - NVIDIA: SAUCE: r8127: fix a kernel panic when dump all registers + - NVIDIA: SAUCE: r8127: add support for RTL8127 cable diagnostic test + + * Set CONFIG_IOMMU_DEFAULT_PASSTHROUGH as default for Nvidia CPUs + (LP: #2129776) + - NVIDIA: SAUCE: iommu/arm-smmu-v3: Set DGX Spark iGPU default domain type + to DMA + - [Config] nvidia: Update annotations to set + CONFIG_IOMMU_DEFAULT_PASSTHROUGH + + * mt7925: Introduce CSA support in non-MLO mode (LP: #2129209) + - NVIDIA: SAUCE: wifi: mt76: mt7925: introduce CSA support in non-MLO mode + + * IOMMU: Support contiguous bit in translation tables (LP: #2112600) + - NVIDIA: SAUCE: iommu/io-pgtable-arm: backport contiguous bit support + + * NVIDIA: SAUCE: MEDIATEK: usb: host: xhci-hub: fix MT89xx SoCs return + PORTLI value (LP: #2125126) + - NVIDIA: SAUCE: MEDIATEK: usb: host: xhci-hub: fix MT89xx SoCs return + PORTLI value + + * NVIDIA: SAUCE: ffa notification count initialization fix (LP: #2123861) + - NVIDIA: SAUCE: Fix FFA notification count initialization + + * Pull-request for setting CPU frequency gov to performance (LP: #2028576) + - [Config] nvidia: Use performance CPU frequency governor on amd64 + + * Set CONFIG_IOMMU_DEFAULT_DMA_LAZY as default for Nvidia CPUs + (LP: #2119661) + - [Config] nvidia: Update annotations to set CONFIG_IOMMU_DEFAULT_DMA_LAZY + + * Backport support for Grace virtualization features: vEVENTQ, HW QUEUE, and + vEGM (LP: #2119656) + - NVIDIA: SAUCE: arm64: configs: Build NVGRACE_GPU_VFIO_PCI as LKM + - NVIDIA: SAUCE: arm64: configs: Enable IOMMUFD and VFIO_DEVICE_CDEV + - NVIDIA: SAUCE: vfio/nvgrace-egm: Introduce module to manage EGM + - NVIDIA: SAUCE: vfio/nvgrace-egm: Handle pages with ECC errors on the EGM + - NVIDIA: SAUCE: arm64: configs: Build CONFIG_NVGRACE_EGM as LKM + - NVIDIA: SAUCE: vfio/nvgrace-egm: Move the egm header file to include + - NVIDIA: SAUCE: vfio/nvgrace-egm: Free region memory during + unregistration + - NVIDIA: SAUCE: vfio/nvgrace-egm: Move region hash initialization + - NVIDIA: SAUCE: vfio/nvgrace-egm: Handle and convey EGM registration + errors + - NVIDIA: SAUCE: vfio/nvgrace-gpu: Handle EGM registration failure + - NVIDIA: SAUCE: vfio/nvgrace-egm: Address sparse errors + - NVIDIA: SAUCE: vfio/nvgrace-gpu: Address smatch errors + - NVIDIA: SAUCE: vfio/nvgrace-egm: Ensure ACPI value reads are successful + - NVIDIA: SAUCE: vfio/nvgrace-egm: Avoid invalid retired pages base + - NVIDIA: SAUCE: vfio/nvgrace-egm: Update EGM unregistration API + - NVIDIA: SAUCE: vfio/nvgrace-egm: track GPUs associated with the EGM + regions + - NVIDIA: SAUCE: vfio/nvgrace-egm: list gpus through sysfs + - NVIDIA: SAUCE: vfio/nvgrace-egm: expose the egm size through sysfs + - NVIDIA: SAUCE: arm64: configs: enable NVGRACE_EGM as module + + * Backport support for arm64 BRBE and a future NVIDIA CPU ID (LP: #2118663) + - [Config] nvidia: Enable BRBE + + * nvidia-ffa-ec: Fix FFH data response length (LP: #2118357) + - NVIDIA: SAUCE: Fix FFH data response length + + * Add pincontrol driver for MT8901 chip (LP: #2117784) + - NVIDIA: SAUCE: MEDIATEK: pinctrl: mediatek: Add gpio-range record in + pinctrl driver + - NVIDIA: SAUCE: MEDIATEK: pinctrl: mediatek: Add acpi support + - NVIDIA: SAUCE: MEDIATEK: pinctrl: mt8901: Add pinctrl driver + - [Config] nvidia: Update annotations to enable CONFIG_PINCTRL_MT8901 + + * NVIDIA: SAUCE: Add FFA and EC Secure Service Driver to -nvidia kernel + (LP: #2114230) + - NVIDIA: SAUCE: Add support for custom ARM FFH offset handler + - NVIDIA: SAUCE: Add nvidia ffa driver for EC communication + - NVIDIA: SAUCE: Add ffa driver for each secure EC service + - NVIDIA: SAUCE: Add support for EC secure service communication + - NVIDIA: SAUCE: Rescan acpi devices that uses secure EC communication + - NVIDIA: SAUCE: irqchip/gic-v3: Allow unused SGIs for drivers/modules + - NVIDIA: SAUCE: Add support for notifications from secure EC services + - [Config] nvidia: Update annotations to enable NVIDIA FFA EC driver + + * Backport: TPM Service Command Response Buffer Interface Over FF-A + (LP: #2111511) + - [Config] nvidia-6.14: Update annotations to enable TPM over FFA + + * Backport: ALSA: hda - Add new driver for HDA controllers listed via ACPI + (LP: #2111447) + - NVIDIA: SAUCE: [Config] nvidia: CONFIG_SND_HDA_ACPI=m on arm64 + + * Pull request to enable GPU passthrough for CUDA (LP: #2095028) + - NVIDIA: SAUCE: WAR: iommufd/pages: Bypass PFNMAP + - NVIDIA: SAUCE: [Config] nvidia: Update annotations for Grace I/O + virtualization + - [Config] nvidia-6.14: Drop CONFIG_TEGRA241_CMDQV from annotations + + * Add Realtek r8127 ethernet driver (LP: #2109730) + - NVIDIA: SAUCE: r8127: Add Realtek r8127 ethernet driver + - NVIDIA: SAUCE: r8127: Remove Realtek r8127 non required files + - NVIDIA: SAUCE: r8127: Moved files from r8127/src to r8127 folder + - NVIDIA: SAUCE: Add r8127 in kernel build + - [Config] nvidia-6.11: Update annotations to enable realtek R8127 module + + * Pull request: Add quirk and disable SBR on Gen5 ports (LP: #2107509) + - NVIDIA: SAUCE: MEDIATEK: usb: host: xhci-plat: support usb3 bulks stream + low power + + * Apply backport of upstream commit to enable Realtek Bluetooth module + (LP: #2096882) + - NVIDIA: SAUCE: Adds MT7925 BT devices + + * Apply SAUCE patch to enable 8250 serial device (LP: #2096888) + - NVIDIA: SAUCE: serial: 8250_mtk: Add ACPI support + + * Backport: "Add support for AArch64 AMUv1-based average freq" Series + (LP: #2100032) + - NVIDIA: [Config] set CONFIG_CPUFREQ_ARCH_CUR_FREQ=y for x86 + + * MANA: include driver fixes and enable module on ARM64 (LP: #2084598) + - [Config] nvidia-6.17: Enable MANA configs on x86 and arm64 + + * Apply patch to set CONFIG_EFI_CAPSULE_LOADER=y for arm64 (LP: #2067111) + - NVIDIA: [Config] EFI: set CAPSULE_LOADER=y for arm64 + + * linux-nvidia-6.5_6.5.0-1014.14 breaks with earlier BIOS release, and + modeset/resolutions are wrong (LP: #2061930) // Blacklist coresight_etm4x + (LP: #2067106) + - [Packaging] blacklist coresight_etm4x + + * backport arm64 THP improvements from 6.9 (LP: #2059316) + - NVIDIA: [Config] arm64: ARM64_CONTPTE=y + + * Reapply the linux-nvidia kernel config options from the 5.15 and 6.5 + kernels (LP: #2060327) + - NVIDIA: [Config]: Disable the NOUVEAU driver which is not used with + -nvidia kernels + - NVIDIA: [Config]: Adding CORESIGHT and ARM64_ERRATUM configs to + annotations + + [ Ubuntu: 7.0.0-6.6 ] + + * resolute/linux: 7.0.0-6.6 -proposed tracker (LP: #2143745) + * Miscellaneous Ubuntu changes + - [Packaging] drop unstable suffix + + [ Ubuntu: 7.0.0-5.5 ] + + * resolute/linux-unstable: 7.0.0-5.5 -proposed tracker (LP: #2143700) + * Resolute real-time patchset: 7.0-rc1-rt1 (LP: #2143181) + - SAUCE: Reapply "serial: 8250: Switch to nbcon console" + - SAUCE: Reapply "serial: 8250: Revert "drop lockdep annotation from + serial8250_clear_IER()"" + - SAUCE: drm/i915: Use preempt_disable/enable_rt() where recommended + - SAUCE: drm/i915: Don't disable interrupts on PREEMPT_RT during atomic + updates + - SAUCE: drm/i915: Disable tracing points on PREEMPT_RT + - SAUCE: drm/i915/gt: Use spin_lock_irq() instead of local_irq_disable() + + spin_lock() + - SAUCE: drm/i915: Drop the irqs_disabled() check + - SAUCE: drm/i915/guc: Consider also RCU depth in busy loop. + - SAUCE: drm/i915: Consider RCU read section as atomic. + - SAUCE: Revert "drm/i915: Depend on !PREEMPT_RT." + - SAUCE: sysfs: Add /sys/kernel/realtime entry + - Real-time patchset 7.0-rc1-rt1 + * Miscellaneous Ubuntu changes + - [Config] rust toolchain version update + + [ Ubuntu-unstable: 7.0.0-4.4 ] + + * resolute/linux-unstable: 7.0.0-4.4 -proposed tracker (LP: #2143123) + * efi: Fix swapped arguments to bsearch() in efi_status_to_*() SAUCE patch + (LP: #2141276) + - SAUCE efi: Fix swapped arguments to bsearch() in efi_status_to_*() + * Plucky preinstalled server fails to boot on rb3gen2 (LP: #2106681) // + Questing preinstalled server fails to boot on sa8775p boards + (LP: #2121347) + - [Config] move more qcom interconnect/pinctrl/gcc options to builtin + * linux-tools: consider linking perf against LLVM (LP: #2138328) + - [Packaging] Add llvm-21-dev to build-depends for perf + * Miscellaneous Ubuntu changes + - [Packaging] Add intel-speed-select to linux-tools + - [Packaging] remove stale debian/dkms-versions + - [Packaging] remove stale debian/dkms-versions scripting + + [ Ubuntu-unstable: 7.0.0-3.3 ] + + * resolute/linux-unstable: 7.0.0-3.3 -proposed tracker (LP: #2143020) + * Miscellaneous Ubuntu changes + - [Config] updateconfig after rebase to v7.0-rc2 + - [Config] switch to PREEMPT_LAZY + + [ Ubuntu-unstable: 7.0.0-2.2 ] + + * resolute/linux-unstable: 7.0.0-2.2 -proposed tracker (LP: #2142764) + + [ Ubuntu-unstable: 7.0.0-1.1 ] + + * resolute/linux-unstable: 7.0.0-1.1 -proposed tracker (LP: #2142402) + * Miscellaneous Ubuntu changes + - [packaging] rename to linux-unstable + - [Config] updateconfig after rebase to v7.0-rc1 + - Update Changes.md + - [Packaging] add libbpf-dev to Build-Depends + - [Config] disable AMD_ISP4, FTBFS + - [Packaging] debian.master/dkms-versions -- temporarily remove zfs FTBFS + - [Packaging] debian.master/dkms-versions -- temporarily remove evdi FTBFS + - [Config] updateconfig after rebase to v7.0-rc1 + - [Config] update toolchain version + + -- Jacob Martin Thu, 26 Mar 2026 11:14:51 -0500 linux-nvidia (6.19.0-1001.1) resolute; urgency=medium diff --git a/debian.nvidia/reconstruct b/debian.nvidia/reconstruct index 16e52ee71b8a0..ca916d098dabf 100644 --- a/debian.nvidia/reconstruct +++ b/debian.nvidia/reconstruct @@ -1,34 +1 @@ -# Recreate any symlinks created since the orig. -[ ! -L 'ubuntu/igh-ecat/master/rtdm-ioctl.c' ] && ln -sf 'ioctl.c' 'ubuntu/igh-ecat/master/rtdm-ioctl.c' -chmod +x 'debian/cloud-tools/hv_get_dhcp_info' -chmod +x 'debian/cloud-tools/hv_get_dns_info' -chmod +x 'debian/cloud-tools/hv_set_ifconfig' -chmod +x 'debian/rules' -chmod +x 'debian/scripts/checks/final-checks' -chmod +x 'debian/scripts/checks/module-signature-check' -chmod +x 'debian/scripts/control-create' -chmod +x 'debian/scripts/dkms-build' -chmod +x 'debian/scripts/dkms-build--nvidia-N' -chmod +x 'debian/scripts/dkms-build-configure--zfs' -chmod +x 'debian/scripts/file-downloader' -chmod +x 'debian/scripts/link-headers' -chmod +x 'debian/scripts/link-lib-rust' -chmod +x 'debian/scripts/misc/annotations' -chmod +x 'debian/scripts/misc/find-missing-sauce.sh' -chmod +x 'debian/scripts/misc/gen-auto-reconstruct' -chmod +x 'debian/scripts/misc/git-ubuntu-log' -chmod +x 'debian/scripts/misc/insert-changes' -chmod +x 'debian/scripts/misc/insert-ubuntu-changes' -chmod +x 'debian/scripts/misc/kernelconfig' -chmod +x 'debian/scripts/sign-module' -chmod +x 'debian/templates/extra.postinst.in' -chmod +x 'debian/templates/extra.postrm.in' -chmod +x 'debian/templates/headers.postinst.in' -chmod +x 'debian/templates/image.postinst.in' -chmod +x 'debian/templates/image.postrm.in' -chmod +x 'debian/templates/image.preinst.in' -chmod +x 'debian/templates/image.prerm.in' -chmod +x 'debian/tests/rebuild' -chmod +x 'debian/tests/ubuntu-regression-suite' -# Remove any files deleted from the orig. exit 0 From 59162bf64d1f13a3eda2d00cd9d2c42835316615 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Mon, 30 Mar 2026 17:32:05 -0500 Subject: [PATCH 104/311] UBUNTU: Start new release Ignore: yes Signed-off-by: Jacob Martin --- debian.nvidia/changelog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debian.nvidia/changelog b/debian.nvidia/changelog index 86bdfb972ee0a..f7ca3aed42f5c 100644 --- a/debian.nvidia/changelog +++ b/debian.nvidia/changelog @@ -1,3 +1,11 @@ +linux-nvidia (7.0.0-1005.5) UNRELEASED; urgency=medium + + CHANGELOG: Do not edit directly. Autogenerated at release. + CHANGELOG: Use the printchanges target to see the current changes. + CHANGELOG: Use the insertchanges target to create the final log. + + -- Jacob Martin Mon, 30 Mar 2026 17:32:05 -0500 + linux-nvidia (7.0.0-1003.3) resolute; urgency=medium * Packaging resync (LP: #1786013) From e98e3a3c03662247e3c0ac481ebf3108b2ea5efe Mon Sep 17 00:00:00 2001 From: Paolo Pisati Date: Wed, 18 Mar 2026 11:13:35 +0100 Subject: [PATCH 105/311] UBUNTU: [Packaging] recommends dracut instead of initramfs-tools BugLink: https://bugs.launchpad.net/bugs/2142775 Signed-off-by: Paolo Pisati Signed-off-by: Jacob Martin --- debian.nvidia/control.d/flavour-signed-control.stub | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian.nvidia/control.d/flavour-signed-control.stub b/debian.nvidia/control.d/flavour-signed-control.stub index b8551a52e743f..b059c7aeceedd 100644 --- a/debian.nvidia/control.d/flavour-signed-control.stub +++ b/debian.nvidia/control.d/flavour-signed-control.stub @@ -5,7 +5,7 @@ Section: kernel Priority: optional Provides: linux-image, fuse-module, =PROVIDES=${linux:rprovides} Depends: ${misc:Depends}, ${shlibs:Depends}, kmod, linux-base (>= 4.5ubuntu1~16.04.1), linux-modules-PKGVER-ABINUM-FLAVOUR -Recommends: BOOTLOADER, initramfs-tools | linux-initramfs-tool +Recommends: BOOTLOADER, dracut | linux-initramfs-tool Breaks: flash-kernel (<< 3.90ubuntu2) [arm64 armhf], s390-tools (<< 2.3.0-0ubuntu3) [s390x] Conflicts: linux-image=SIGN-PEER-PKG=-PKGVER-ABINUM-FLAVOUR Suggests: bpftool, linux-perf, SRCPKGNAME-tools, linux-headers-PKGVER-ABINUM-FLAVOUR From a3b3899d0d54193dca6f79da272a74962ebeb01b Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Wed, 1 Apr 2026 17:15:43 -0500 Subject: [PATCH 106/311] UBUNTU: [Packaging] update variants BugLink: https://bugs.launchpad.net/bugs/1786013 Signed-off-by: Jacob Martin --- debian.nvidia/variants | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian.nvidia/variants b/debian.nvidia/variants index 6606318691bcd..881c9938362e4 100644 --- a/debian.nvidia/variants +++ b/debian.nvidia/variants @@ -1,4 +1,4 @@ --6.19 +-7.0 -- -hwe-24.04 -hwe-24.04-edge From f1c1661460d49e88f1c7da43ba94e5c1553561ff Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Thu, 2 Apr 2026 10:45:00 -0500 Subject: [PATCH 107/311] UBUNTU: link-to-tracker: update tracking bug BugLink: https://bugs.launchpad.net/bugs/2145971 Properties: no-test-build Signed-off-by: Jacob Martin --- debian.nvidia/tracking-bug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian.nvidia/tracking-bug b/debian.nvidia/tracking-bug index eaf24103d1343..0e843a8655190 100644 --- a/debian.nvidia/tracking-bug +++ b/debian.nvidia/tracking-bug @@ -1 +1 @@ -2142114 d2026.02.16-1 +2145971 d2026.03.17-1 From 92a44dc7e15b646a8de4af529ad9e4485fbc7fc2 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Thu, 2 Apr 2026 10:47:21 -0500 Subject: [PATCH 108/311] UBUNTU: [Config] nvidia: update configs Ignore: yes Signed-off-by: Jacob Martin --- debian.nvidia/config/annotations | 4 ---- 1 file changed, 4 deletions(-) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index 78487314f1f9d..a97ed74f91e3d 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -217,8 +217,4 @@ CONFIG_VFIO_IOMMU_TYPE1 note<'LP: #2095028'> # ---- Annotations without notes ---- CONFIG_BCH policy<{'amd64': 'm', 'arm64': 'y'}> -CONFIG_CC_VERSION_TEXT policy<{'amd64': '"x86_64-linux-gnu-gcc (Ubuntu 15.2.0-16ubuntu1) 15.2.0"', 'arm64': '"aarch64-linux-gnu-gcc (Ubuntu 15.2.0-16ubuntu1) 15.2.0"'}> CONFIG_MTD_NAND_CORE policy<{'amd64': 'm', 'arm64': 'y'}> -CONFIG_PAHOLE_VERSION policy<{'amd64': '131', 'arm64': '131'}> -CONFIG_RUSTC_VERSION policy<{'amd64': '109301', 'arm64': '109301'}> -CONFIG_RUSTC_VERSION_TEXT policy<{'amd64': '"rustc 1.93.1 (01f6ddf75 2026-02-11) (built from a source tarball)"', 'arm64': '"rustc 1.93.1 (01f6ddf75 2026-02-11) (built from a source tarball)"'}> From 75625403956a22db3c67f2d1bc0b9455c171dc7d Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Thu, 2 Apr 2026 10:48:01 -0500 Subject: [PATCH 109/311] UBUNTU: Ubuntu-nvidia-7.0.0-1005.5 Signed-off-by: Jacob Martin --- debian.nvidia/changelog | 364 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 359 insertions(+), 5 deletions(-) diff --git a/debian.nvidia/changelog b/debian.nvidia/changelog index f7ca3aed42f5c..fc365d8b73a9f 100644 --- a/debian.nvidia/changelog +++ b/debian.nvidia/changelog @@ -1,10 +1,364 @@ -linux-nvidia (7.0.0-1005.5) UNRELEASED; urgency=medium +linux-nvidia (7.0.0-1005.5) resolute; urgency=medium - CHANGELOG: Do not edit directly. Autogenerated at release. - CHANGELOG: Use the printchanges target to see the current changes. - CHANGELOG: Use the insertchanges target to create the final log. + * resolute/linux-nvidia: 7.0.0-1005.5 -proposed tracker (LP: #2145971) - -- Jacob Martin Mon, 30 Mar 2026 17:32:05 -0500 + * Packaging resync (LP: #1786013) + - [Packaging] update variants + + * Please make dracut the default initrd generator (LP: #2142775) + - [Packaging] recommends dracut instead of initramfs-tools + + [ Ubuntu: 7.0.0-12.12 ] + + * resolute/linux: 7.0.0-12.12 -proposed tracker (LP: #2146778) + * Packaging resync (LP: #1786013) + - [Packaging] update variants + * linux-generic does not run scripts in /usr/share/kernel/*.d (LP: #2147005) + - [Packaging] templates: Use consistent indentation + - [Packaging] templates: Run scripts in /usr/share/kernel/*.d too + * RISC-V kernel config is out of sync with other archs (LP: #1981437) + - [Config] riscv64: Enable COUNTER=m + - [Config] riscv64: Use GENDWARFKSYMS like other architectures + * unconfined profile denies userns_create for chromium based processes + (LP: #1990064) + - [Config] disable CONFIG_SECURITY_APPARMOR_RESTRICT_USERNS + * FFe: add network interface mediation to 26.04 (LP: #2144679) + - SAUCE: apparmor5.0.0 [57/57]: apparmor: add the ability to use interface + in network mediation. + * Jellyfin Desktop Flatpak doesn't work with the current AppArmor profile + (LP: #2142956) + - SAUCE: apparmor5.0.0 [29/57]: apparmor: fix fine grained inet mediation + sock_file_perm + - SAUCE: apparmor5.0.0 [30/57]: apparmor-next 7.1: aapparmor: use target + task's context in apparmor_getprocattr() + - SAUCE: apparmor5.0.0 [31/57]: apparmor-next 7.1: apparmor: return error + on namespace mismatch in verify_header + - SAUCE: apparmor5.0.0 [32/57]: apparmor-next 7.1: apparmor: enable + differential encoding + - SAUCE: apparmor5.0.0 [33/57]: apparmor-next 7.1: apparmor: propagate + -ENOMEM correctly in unpack_table + - SAUCE: apparmor5.0.0 [34/57]: apparmor-next 7.1: apparmor: Replace + memcpy + NUL termination with kmemdup_nul in do_setattr + - SAUCE: apparmor5.0.0 [35/57]: apparmor-next 7.1: apparmor: Remove + redundant if check in sk_peer_get_label + - SAUCE: apparmor5.0.0 [36/57]: apparmor-next 7.1: apparmor: use + __label_make_stale in __aa_proxy_redirect + - SAUCE: apparmor5.0.0 [37/57]: apparmor-next 7.1: apparmor: fix net.h and + policy.h circular include pattern + - SAUCE: apparmor5.0.0 [39/57]: apparmor-next 7.1: apparmor: make include + headers self-contained + - SAUCE: apparmor5.0.0 [40/57]: apparmor-next 7.1: apparmor: Use + sysfs_emit in param_get_{audit,mode} + - SAUCE: apparmor5.0.0 [41/57]: apparmor-next 7.1: apparmor: fix + rawdata_f_data implicit flex array + - SAUCE: apparmor5.0.0 [42/57]: apparmor-next 7.1: apparmor: free rawdata + as soon as possible + - SAUCE: apparmor5.0.0 [43/57]: apparmor-next 7.1: apparmor: Initial + support for compressed policies + - SAUCE: apparmor5.0.0 [44/57]: apparmor-next 7.1: apparmor: fix potential + UAF in aa_replace_profiles + - SAUCE: apparmor5.0.0 [45/57]: apparmor-next 7.1: apparmor: hide unused + get_loaddata_common_ref() function + - SAUCE: apparmor5.0.0 [46/57]: apparmor-next 7.1: apparmor: Fix string + overrun due to missing termination + - SAUCE: apparmor5.0.0 [47/57]: apparmor: fix packed tag on v5 header + struct + - SAUCE: apparmor5.0.0 [48/57]: apparmor: add temporal caching to audit + responses. + - SAUCE: apparmor5.0.0 [49/57]: apparmor: change fn_label_build() call to + not return NULL + - SAUCE: apparmor5.0.0 [50/57]: apparmor: make fn_label_build() capable of + handling not supported + - SAUCE: apparmor5.0.0 [51/57]: apparmor: move netfilter functions next to + the LSM network operations + - SAUCE: apparmor5.0.0 [52/57]: apparmor: move sock_rvc_skb() next to + inet_conn_request + - SAUCE: apparmor5.0.0 [53/57]: apparmor: fix af_unix local addr mediation + binding + - SAUCE: apparmor5.0.0 [54/57]: cleanups of apparmor af_unix mediation + - SAUCE: apparmor5.0.0 [55/57]: apparmor: fix apparmor_secmark_check() + when !inet and secmark defined. + - SAUCE: apparmor5.0.0 [56/57]: apparmor: fix auditing of non-mediation + falures + * snap service cannot change apparmor hat (LP: #2139664) // Jellyfin Desktop + Flatpak doesn't work with the current AppArmor profile (LP: #2142956) + - SAUCE: apparmor5.0.0 [38/57]: apparmor-next 7.1: apparmor: grab ns lock + and refresh when looking up changehat child profiles + * AppArmor blocks write(2) to network sockets with Linux 6.19 (LP: #2141298) + - SAUCE: apparmor5.0.0 [28/57]: apparmor: fix aa_label_sk_perm to check + for RULE_MEDIATES_NET + * update apparmor and LSM stacking patch set (LP: #2028253) + - SAUCE: apparmor5.0.0 [1/57]: Stacking: LSM: Single calls in secid hooks + - SAUCE: apparmor5.0.0 [2/57]: Stacking: LSM: Exclusive secmark usage + - SAUCE: apparmor5.0.0 [3/57]: Stacking: AppArmor: Remove the exclusive + flag + - SAUCE: apparmor5.0.0 [4/57]: Revert "apparmor: fix dbus permission + queries to v9 ABI" + - SAUCE: apparmor5.0.0 [5/57]: Revert "apparmor: gate make fine grained + unix mediation behind v9 abi" + - SAUCE: apparmor5.0.0 [6/57]: apparmor: net: patch to provide + compatibility with v2.x net rules + - SAUCE: apparmor5.0.0 [7/57]: apparmor: net: add fine grained ipv4/ipv6 + mediation + - SAUCE: apparmor5.0.0 [8/57]: apparmor: lift compatibility check out of + profile_af_perm + - SAUCE: apparmor5.0.0 [9/57]: apparmor: userns: add unprivileged user ns + mediation + - SAUCE: apparmor5.0.0 [10/57]: apparmor: userns: Add sysctls for + additional controls of unpriv userns restrictions + - SAUCE: apparmor5.0.0 [12/57]: apparmor: userns: open userns related + sysctl so lxc can check if restriction are in place + - SAUCE: apparmor5.0.0 [13/57]: apparmor: userns: allow profile to be + transitioned when a userns is created + - SAUCE: apparmor5.0.0 [14/57]: apparmor: mqueue: call + security_inode_init_security on inode creation + - SAUCE: apparmor5.0.0 [15/57]: apparmor: mqueue: add fine grained + mediation of posix mqueues + - SAUCE: apparmor5.0.0 [16/57]: apparmor: uring: add io_uring mediation + - SAUCE: apparmor5.0.0 [19/57]: apparmor: prompt: setup slab cache for + audit data + - SAUCE: apparmor5.0.0 [20/57]: apparmor: prompt: add the ability for + profiles to have a learning cache + - SAUCE: apparmor5.0.0 [21/57]: apparmor: prompt: enable userspace upcall + for mediation + - SAUCE: apparmor5.0.0 [22/57]: apparmor: prompt: pass prompt boolean + through into path_name as well + - SAUCE: apparmor5.0.0 [23/57]: apparmor: check for supported version in + notification messages. + - SAUCE: apparmor5.0.0 [24/57]: apparmor: refactor building notice so it + is easier to extend + - SAUCE: apparmor5.0.0 [25/57]: apparmor: switch from ENOTSUPP to + EPROTONOSUPPORT + - SAUCE: apparmor5.0.0 [26/57]: apparmor: add support for meta data tags + - SAUCE: apparmor5.0.0 [27/57]: apparmor: prevent profile->disconnected + double free in aa_free_profile + * update apparmor and LSM stacking patch set (LP: #2028253) // Installation + of AppArmor on a 6.14 kernel produces error message "Illegal number: yes" + (LP: #2102680) + - SAUCE: apparmor5.0.0 [17/57]: apparmor: create an + AA_SFS_TYPE_BOOLEAN_INTPRINT sysctl variant + - SAUCE: apparmor5.0.0 [18/57]: apparmor: Use AA_SFS_FILE_BOOLEAN_INTPRINT + for userns and io_uring sysctls + * update apparmor and LSM stacking patch set (LP: #2028253) // [FFe] + apparmor-4.0.0-alpha2 for unprivileged user namespace restrictions in + mantic (LP: #2032602) + - SAUCE: apparmor5.0.0 [11/57]: apparmor: userns - make it so special + unconfined profiles can mediate user namespaces + * Enable new Intel WCL soundwire support (LP: #2143301) + - ASoC: sdw_utils: Add CS42L43B codec info + - ASoC: dt-bindings: cirrus, cs42l43: Add CS42L43B variant + - mfd: cs42l43: Add support for the B variant + - ASoC: cs42l43: Add support for the B variant + * Enable audio functions on Dell Huracan/Renegade platforms w/o built-in + microphone (LP: #2143902) + - ASoC: SDCA: Add default value for mipi-sdca-function-reset-max-delay + - ASoC: SDCA: Update counting of SU/GE DAPM routes + - ASoC: SDCA: Improve mapping of Q7.8 SDCA volumes + - ASoC: SDCA: Pull the Q7.8 volume helpers out of soc-ops + - ASoC: add snd_soc_lookup_component_by_name helper + - ASoC: soc_sdw_utils: partial match the codec name + - ASoC: soc_sdw_utils: remove index from sdca codec name + * [SRU] MIPI camera is not working after upgrading to 6.17-oem + (LP: #2145171) + - SAUCE: ACPI: respect items already in honor_dep before skipping + * linux-tools: consider linking perf against LLVM (LP: #2138328) + - [Packaging] Actually enable llvm for perf + * Pull patch in qla2xxx to Resolute (LP: #2144856) + - scsi: qla2xxx: Add support to report MPI FW state + * Ubuntu Resolute Desktop image arm64 - Boot on SC8280XP stalls with gpi-dma + errors (LP: #2142403) + - Revert "arm64: dts: qcom: sc8280xp: Enable GPI DMA" + * 26.04 Snapdragon X Elite: Sync concept kernel changes (LP: #2144643) + - SAUCE: arm64: dts: add missing denali-oled.dtb to Makefile + - SAUCE: dt-bindings: phy: qcom: Add CSI2 C-PHY/DPHY schema + - SAUCE: phy: qcom-mipi-csi2: Add a CSI2 MIPI DPHY driver + - SAUCE: dt-bindings: media: qcom,x1e80100-camss: Add simple-mfd + compatible + - SAUCE: dt-bindings: media: qcom,x1e80100-camss: Add optional PHY handle + definitions + - SAUCE: dt-bindings: media: qcom,x1e80100-camss: Add support for combo- + mode endpoints + - SAUCE: dt-bindings: media: qcom,x1e80100-camss: Describe iommu entries + - SAUCE: media: qcom: camss: Add legacy_phy flag to SoC definition + structures + - SAUCE: media: qcom: camss: Add support for PHY API devices + - SAUCE: media: qcom: camss: Drop legacy PHY descriptions from x1e + - SAUCE: arm64: dts: qcom: x1e80100: Add CAMCC block definition + - SAUCE: arm64: dts: qcom: x1e80100: Add CCI definitions + - SAUCE: arm64: dts: qcom: x1e80100: Add CAMSS block definition + - SAUCE: arm64: dts: qcom: x1e80100-crd: Add pm8010 CRD pmic,id=m + regulators + - SAUCE: arm64: dts: qcom: x1e80100-crd: Add ov08x40 RGB sensor on CSIPHY4 + - SAUCE: arm64: dts: qcom: x1e80100-t14s: Add pm8010 camera PMIC with + voltage levels for IR and RGB camera + - SAUCE: arm64: dts: qcom: x1e80100-t14s: Add on ov02c10 RGB sensor on + CSIPHY4 + - SAUCE: arm64: dts: qcom: x1e80100-lenovo-yoga-slim7x: Add pm8010 camera + PMIC with voltage levels for IR and RGB camera + - SAUCE: arm64: dts: qcom: x1e80100-lenovo-yoga-slim7x: Add l7b_2p8 + voltage regulator for RGB camera + - SAUCE: arm64: dts: qcom: x1e80100-lenovo-yoga-slim7x: Add ov02c10 RGB + sensor on CSIPHY4 + - SAUCE: arm64: dts: qcom: x1e80100-dell-inspiron14-7441: Switch on CAMSS + RGB sensor + - SAUCE: arm64: dts: qcom: x1-asus-zenbook-a14: Add on OV02C10 RGB sensor + on CSIPHY4 + - SAUCE: arm64: dts: qcom: x1e80100-dell-xps13-9345: add camera support + - SAUCE: arm64: dts: qcom: x1e78100-t14s: enable camera privacy indicator + - SAUCE: arm64: dts: qcom: x1e80100-lenovo-yoga-slim7x: enable camera + privacy indicator + - SAUCE: arm64: dts: qcom: x1e80100-dell-xps13-9345: enable camera privacy + indicator + - SAUCE: dt-bindings: arm: qcom: Add ASUS Vivobook X1P42100 variant + - SAUCE: arm64: dts: qcom: x1-vivobook-s15: create a common dtsi for Hamoa + and Purwa variants + - SAUCE: arm64: dts: qcom: x1-vivobook-s15: add Purwa-compatible device + tree + - SAUCE: firmware: qcom: scm: allow QSEECOM on ASUS Vivobook X1P42100 + variant + - SAUCE: arm64: dts: qcom: hamoa: Move PCIe PERST and Wake GPIOs to port + nodes + - SAUCE: arm64: dts: qcom: x1e-acer-swift-14: Move PCIe PERST and Wake + GPIOs to port nodes + * 25.10 Snapdragon X Elite: Sync concept kernel changes (LP: #2121477) + - SAUCE: wip: arm64: dts: qcom: x1e78100-t14s: enable bluetooth + * Miscellaneous Ubuntu changes + - SAUCE: dt-bindings: arm: qcom: Document HP EliteBook 6 G1q + - SAUCE: firmware: qcom: scm: Allow QSEECOM for HP EliteBook 6 G1q + - SAUCE: arm64: dts: qcom: x1p42100-hp-elitebook-6-g1q: DT for HP + EliteBook 6 G1q + - [Config] PHY_QCOM_MIPI_CSI2=m + - SAUCE: arm64: dts: x1e80100-lenovo-yoga-slim7x: Fix RGB camera supplies + - [Config] toolchain version update + - Update Changes.md after v7.0-rc5 rebase + - [Packaging] update Ubuntu.md + - [Config] enable SECURITY_APPARMOR_PACKET_MEDIATION_ENABLED + - [Packaging] Add linux-main-modules-zfs to linux-modules depends + * Miscellaneous upstream changes + - Revert "UBUNTU: SAUCE: Add Bluetooth support for the Lenovo Yoga Slim + 7x" + + [ Ubuntu: 7.0.0-10.10 ] + + * resolute/linux: 7.0.0-10.10 -proposed tracker (LP: #2144865) + * Miscellaneous upstream changes + - Revert "powerpc: fix KUAP warning in VMX usercopy path" + + [ Ubuntu: 7.0.0-9.9 ] + + * resolute/linux: 7.0.0-9.9 -proposed tracker (LP: #2144735) + * Please make dracut the default initrd generator (LP: #2142775) + - [Packaging] recommends dracut instead of initramfs-tools + * Miscellaneous Ubuntu changes + - SAUCE: Change RISC-V target to RVA23 (riscv64a23-unknown-linux-gnu) + + [ Ubuntu: 7.0.0-8.8 ] + + * resolute/linux: 7.0.0-8.8 -proposed tracker (LP: #2144652) + * UBUNTU: SAUCE: igc: Increase Thunderbolt MAC passthrough delay to 1000ms + (LP: #2143197) + - SAUCE: igc: Increase Thunderbolt MAC passthrough delay to 1000ms + * [usrmerge] evaluate kernel owned packages for DEP17 compliance + (LP: #2139276) + - [Packaging] Install modules in /usr/lib/modules + * Miscellaneous Ubuntu changes + - [Config] hardening: enable LIST_HARDENED + - [Config] hardening: disable LDISC_AUTOLOAD + - [Config] hardening: disable LEGACY_PTYS + - [Config] updateconfigs following v7.0-rc4 rebase + + [ Ubuntu: 7.0.0-7.7 ] + + * resolute/linux: 7.0.0-7.7 -proposed tracker (LP: #2143974) + * unconfined profile denies userns_create for chromium based processes + (LP: #1990064) + - [Config] disable CONFIG_SECURITY_APPARMOR_RESTRICT_USERNS + * Jellyfin Desktop Flatpak doesn't work with the current AppArmor profile + (LP: #2142956) + - SAUCE: apparmor5.0.0 [29/29]: apparmor: fix fine grained inet mediation + sock_file_perm + * AppArmor blocks write(2) to network sockets with Linux 6.19 (LP: #2141298) + - SAUCE: apparmor5.0.0 [28/29]: apparmor: fix aa_label_sk_perm to check + for RULE_MEDIATES_NET + * update apparmor and LSM stacking patch set (LP: #2028253) + - SAUCE: apparmor5.0.0 [1/29]: Stacking: LSM: Single calls in secid hooks + - SAUCE: apparmor5.0.0 [2/29]: Stacking: LSM: Exclusive secmark usage + - SAUCE: apparmor5.0.0 [3/29]: Stacking: AppArmor: Remove the exclusive + flag + - SAUCE: apparmor5.0.0 [4/29]: Revert "apparmor: fix dbus permission + queries to v9 ABI" + - SAUCE: apparmor5.0.0 [5/29]: Revert "apparmor: gate make fine grained + unix mediation behind v9 abi" + - SAUCE: apparmor5.0.0 [6/29]: apparmor: net: patch to provide + compatibility with v2.x net rules + - SAUCE: apparmor5.0.0 [7/29]: apparmor: net: add fine grained ipv4/ipv6 + mediation + - SAUCE: apparmor5.0.0 [8/29]: apparmor: lift compatibility check out of + profile_af_perm + - SAUCE: apparmor5.0.0 [9/29]: apparmor: userns: add unprivileged user ns + mediation + - SAUCE: apparmor5.0.0 [10/29]: apparmor: userns: Add sysctls for + additional controls of unpriv userns restrictions + - SAUCE: apparmor5.0.0 [12/29]: apparmor: userns: open userns related + sysctl so lxc can check if restriction are in place + - SAUCE: apparmor5.0.0 [13/29]: apparmor: userns: allow profile to be + transitioned when a userns is created + - SAUCE: apparmor5.0.0 [14/29]: apparmor: mqueue: call + security_inode_init_security on inode creation + - SAUCE: apparmor5.0.0 [15/29]: apparmor: mqueue: add fine grained + mediation of posix mqueues + - SAUCE: apparmor5.0.0 [16/29]: apparmor: uring: add io_uring mediation + - SAUCE: apparmor5.0.0 [19/29]: apparmor: prompt: setup slab cache for + audit data + - SAUCE: apparmor5.0.0 [20/29]: apparmor: prompt: add the ability for + profiles to have a learning cache + - SAUCE: apparmor5.0.0 [21/29]: apparmor: prompt: enable userspace upcall + for mediation + - SAUCE: apparmor5.0.0 [22/29]: apparmor: prompt: pass prompt boolean + through into path_name as well + - SAUCE: apparmor5.0.0 [23/29]: apparmor: check for supported version in + notification messages. + - SAUCE: apparmor5.0.0 [24/29]: apparmor: refactor building notice so it + is easier to extend + - SAUCE: apparmor5.0.0 [25/29]: apparmor: switch from ENOTSUPP to + EPROTONOSUPPORT + - SAUCE: apparmor5.0.0 [26/29]: apparmor: add support for meta data tags + - SAUCE: apparmor5.0.0 [27/29]: apparmor: prevent profile->disconnected + double free in aa_free_profile + * update apparmor and LSM stacking patch set (LP: #2028253) // Installation + of AppArmor on a 6.14 kernel produces error message "Illegal number: yes" + (LP: #2102680) + - SAUCE: apparmor5.0.0 [17/29]: apparmor: create an + AA_SFS_TYPE_BOOLEAN_INTPRINT sysctl variant + - SAUCE: apparmor5.0.0 [18/29]: apparmor: Use AA_SFS_FILE_BOOLEAN_INTPRINT + for userns and io_uring sysctls + * update apparmor and LSM stacking patch set (LP: #2028253) // [FFe] + apparmor-4.0.0-alpha2 for unprivileged user namespace restrictions in + mantic (LP: #2032602) + - SAUCE: apparmor5.0.0 [11/29]: apparmor: userns - make it so special + unconfined profiles can mediate user namespaces + * NPU utilization on amdxdna is missing (LP: #2143243) + - SAUCE: accel/amdxdna: Add IOCTL to retrieve realtime NPU power estimate + - SAUCE: accel/amdxdna: Support sensors for column utilization + - SAUCE: accel/amdxdna: Import AMD_PMF namespace + * Adopting dark mode by default for OLED panel (LP: #2143203) + - SAUCE: drm/connector: Add a new 'panel_type' property + - SAUCE: drm/amd/display: Attach OLED property to eDP panels + * Support AMD Image Signal Processing (ISP) unit V4.0 (LP: #2110092) + - SAUCE: media: platform: amd: Introduce amd isp4 capture driver + - SAUCE: media: platform: amd: low level support for isp4 firmware + - SAUCE: media: platform: amd: Add isp4 fw and hw interface + - SAUCE: media: platform: amd: isp4 subdev and firmware loading handling + added + - SAUCE: media: platform: amd: isp4 video node and buffers handling added + - SAUCE: Documentation: add documentation of AMD isp 4 driver + - SAUCE: media: platform: amd: isp4 debug fs logging and more descriptive + errors + - [Config] Enable VIDEO_AMD_ISP4_CAPTURE + * Miscellaneous Ubuntu changes + - [Config] temporarily disable OBJTOOL_WERROR + + -- Jacob Martin Thu, 02 Apr 2026 10:48:01 -0500 linux-nvidia (7.0.0-1003.3) resolute; urgency=medium From 2b87da780630a1a5f0c450eeb810b333c61e1366 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Tue, 14 Apr 2026 11:27:52 -0500 Subject: [PATCH 110/311] UBUNTU: Start new release Ignore: yes Signed-off-by: Jacob Martin --- debian.nvidia/changelog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debian.nvidia/changelog b/debian.nvidia/changelog index fc365d8b73a9f..e89684b3d1f4f 100644 --- a/debian.nvidia/changelog +++ b/debian.nvidia/changelog @@ -1,3 +1,11 @@ +linux-nvidia (7.0.0-1006.6) UNRELEASED; urgency=medium + + CHANGELOG: Do not edit directly. Autogenerated at release. + CHANGELOG: Use the printchanges target to see the current changes. + CHANGELOG: Use the insertchanges target to create the final log. + + -- Jacob Martin Tue, 14 Apr 2026 11:27:52 -0500 + linux-nvidia (7.0.0-1005.5) resolute; urgency=medium * resolute/linux-nvidia: 7.0.0-1005.5 -proposed tracker (LP: #2145971) From b1b01bf4197d23eb641ab18dea90299dabbb3bf4 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Tue, 14 Apr 2026 16:16:17 -0500 Subject: [PATCH 111/311] UBUNTU: link-to-tracker: update tracking bug BugLink: https://bugs.launchpad.net/bugs/2148214 Properties: no-test-build Signed-off-by: Jacob Martin --- debian.nvidia/tracking-bug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian.nvidia/tracking-bug b/debian.nvidia/tracking-bug index 0e843a8655190..e1ada5cb8e86e 100644 --- a/debian.nvidia/tracking-bug +++ b/debian.nvidia/tracking-bug @@ -1 +1 @@ -2145971 d2026.03.17-1 +2148214 d2026.04.13-1 From 3cc037cf75acb0f3e937461549d79c3647722872 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Tue, 14 Apr 2026 16:19:48 -0500 Subject: [PATCH 112/311] UBUNTU: Ubuntu-nvidia-7.0.0-1006.6 Signed-off-by: Jacob Martin --- debian.nvidia/changelog | 63 +++++++++++++++++++++++++++++++++++---- debian.nvidia/reconstruct | 44 +++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 6 deletions(-) diff --git a/debian.nvidia/changelog b/debian.nvidia/changelog index e89684b3d1f4f..be5d8e21181fb 100644 --- a/debian.nvidia/changelog +++ b/debian.nvidia/changelog @@ -1,10 +1,61 @@ -linux-nvidia (7.0.0-1006.6) UNRELEASED; urgency=medium - - CHANGELOG: Do not edit directly. Autogenerated at release. - CHANGELOG: Use the printchanges target to see the current changes. - CHANGELOG: Use the insertchanges target to create the final log. +linux-nvidia (7.0.0-1006.6) resolute; urgency=medium + + * resolute/linux-nvidia: 7.0.0-1006.6 -proposed tracker (LP: #2148214) + + [ Ubuntu: 7.0.0-14.14 ] + + * resolute/linux: 7.0.0-14.14 -proposed tracker (LP: #2148159) + * support vflip/hflip for Sony IMX471 camera sensor (LP: #2138841) + - SAUCE: media: ipu-bridge: add TBE20A0 ACPI id for Sony IMX471 + * AA: disable SECURITY_APPARMOR_PACKET_MEDIATION_ENABLED (LP: #2147533) + - [Config] disable SECURITY_APPARMOR_PACKET_MEDIATION_ENABLED + * System doesn't response with mt76 call trace (LP: #2137448) + - wifi: mt76: mt792x: Fix a potential deadlock in high-load situations + * The second tbt storage plugged on the dock will not be recognized + (LP: #2139572) + - SAUCE: thunderbolt: Fix PCIe device enumeration with delayed rescan + * dma-buf filesystem flags fix (LP: #2139656) + - SAUCE: dma-buf: set SB_I_NOEXEC and SB_I_NODEV on dmabuf filesystem + * Bluetooth device (MT7925) not detected on USB bus with linux-oem-6.17 + (LP: #2145164) + - SAUCE: USB: hub: call ACPI _PRR reset during port power-cycle on + enumeration failure + * drm/i915/lnl+/tc: Fix false disconnect of active DP-alt TC port during + long HPD pulse (LP: #2143879) + - SAUCE: drm/i915/lnl+/tc: Fix false disconnect of active DP-alt TC port + during long HPD pulse + * i915 WARN_ON call trace during CB/WB on MTL/ARL platforms (LP: #2144537) + - SAUCE: drm/i915/xelpdp/tc: Convert TCSS power check WARN to a debug + message + * Miscellaneous Ubuntu changes + - [Packaging] Add support for per-flavour depends + - [Packaging] Don't hard-code lmm zfs dependency + - [Config] updateconfigs following v7.0 release + + [ Ubuntu: 7.0.0-13.13 ] + + * resolute/linux: 7.0.0-13.13 -proposed tracker (LP: #2147403) + * ubuntu_kselftests:_net/net:gre_gso.sh failing (LP: #2136820) + - SAUCE increase socat timeout in gre_gso.sh + * Canonical Kmod 2025 key rotation (LP: #2147447) + - [Packaging] ubuntu-compatible-signing -- make Ubuntu-Compatible-Signing + extensible + - [Packaging] ubuntu-compatible-signing -- allow consumption of positive + certs + - [Packaging] ubuntu-compatible-signing -- report the livepatch:2025 key + - [Config] prepare for Canonical Kmod key rotation + - [Packaging] ubuntu-compatible-signing -- report the kmod:2025 key + - [Packaging] ensure our cert rollups are always fresh + * On Dell system, the internal OLED display drops to a visibly low FPS after + suspend/resume (LP: #2144712) + - drm/i915/psr: Disable Panel Replay on Dell XPS 14 DA14260 as a quirk + - drm/i915/psr: Fixes for Dell XPS DA14260 quirk + * Realtek RTL8116AF SFP option module fails to get connected (LP: #2116144) + - SAUCE: r8169: add quirk for RTL8116af SerDes + * Miscellaneous Ubuntu changes + - [Config] updateconfigs following v7.0-rc7 rebase - -- Jacob Martin Tue, 14 Apr 2026 11:27:52 -0500 + -- Jacob Martin Tue, 14 Apr 2026 16:19:48 -0500 linux-nvidia (7.0.0-1005.5) resolute; urgency=medium diff --git a/debian.nvidia/reconstruct b/debian.nvidia/reconstruct index ca916d098dabf..d1b95906eb4b8 100644 --- a/debian.nvidia/reconstruct +++ b/debian.nvidia/reconstruct @@ -1 +1,45 @@ +# Recreate any symlinks created since the orig. +[ ! -L 'ubuntu/igh-ecat/master/rtdm-ioctl.c' ] && ln -sf 'ioctl.c' 'ubuntu/igh-ecat/master/rtdm-ioctl.c' +chmod +x 'debian/cloud-tools/hv_get_dhcp_info' +chmod +x 'debian/cloud-tools/hv_get_dns_info' +chmod +x 'debian/cloud-tools/hv_set_ifconfig' +chmod +x 'debian/rules' +chmod +x 'debian/scripts/checks/final-checks' +chmod +x 'debian/scripts/checks/module-signature-check' +chmod +x 'debian/scripts/control-create' +chmod +x 'debian/scripts/dkms-build' +chmod +x 'debian/scripts/dkms-build--nvidia-N' +chmod +x 'debian/scripts/dkms-build-configure--zfs' +chmod +x 'debian/scripts/file-downloader' +chmod +x 'debian/scripts/link-headers' +chmod +x 'debian/scripts/link-lib-rust' +chmod +x 'debian/scripts/misc/annotations' +chmod +x 'debian/scripts/misc/find-missing-sauce.sh' +chmod +x 'debian/scripts/misc/gen-auto-reconstruct' +chmod +x 'debian/scripts/misc/git-ubuntu-log' +chmod +x 'debian/scripts/misc/insert-changes' +chmod +x 'debian/scripts/misc/insert-ubuntu-changes' +chmod +x 'debian/scripts/misc/kernelconfig' +chmod +x 'debian/scripts/sign-module' +chmod +x 'debian/templates/extra.postinst.in' +chmod +x 'debian/templates/extra.postrm.in' +chmod +x 'debian/templates/headers.postinst.in' +chmod +x 'debian/templates/image.postinst.in' +chmod +x 'debian/templates/image.postrm.in' +chmod +x 'debian/templates/image.preinst.in' +chmod +x 'debian/templates/image.prerm.in' +chmod +x 'debian/tests/rebuild' +chmod +x 'debian/tests/ubuntu-regression-suite' +chmod +x 'drivers/net/ethernet/realtek/r8127/Makefile' +chmod +x 'drivers/net/ethernet/realtek/r8127/r8127.h' +chmod +x 'drivers/net/ethernet/realtek/r8127/r8127_dash.h' +chmod +x 'drivers/net/ethernet/realtek/r8127/r8127_firmware.h' +chmod +x 'drivers/net/ethernet/realtek/r8127/r8127_n.c' +chmod +x 'drivers/net/ethernet/realtek/r8127/r8127_realwow.h' +chmod +x 'drivers/net/ethernet/realtek/r8127/r8127_rss.h' +chmod +x 'drivers/net/ethernet/realtek/r8127/rtl_eeprom.c' +chmod +x 'drivers/net/ethernet/realtek/r8127/rtl_eeprom.h' +chmod +x 'drivers/net/ethernet/realtek/r8127/rtltool.c' +chmod +x 'drivers/net/ethernet/realtek/r8127/rtltool.h' +# Remove any files deleted from the orig. exit 0 From b74b990460188c9166a0272814f85fa6adc93e28 Mon Sep 17 00:00:00 2001 From: Jamie Nguyen Date: Fri, 13 Mar 2026 09:55:03 -0700 Subject: [PATCH 113/311] NVIDIA: SAUCE: r8169: remove PCI IDs claimed by r8127 driver BugLink: https://bugs.launchpad.net/bugs/2144345 Remove device IDs 0x8127 and 0x0e10 from the r8169 PCI device table so that the r8127 vendor driver binds to these devices instead. Tested-by: Keith Berger Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Nirmoy Das Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/net/ethernet/realtek/r8169_main.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index 4f27f2bbb391e..fbd0d7d903b50 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -241,10 +241,8 @@ static const struct pci_device_id rtl8169_pci_tbl[] = { { 0x0001, 0x8168, PCI_ANY_ID, 0x2410 }, { PCI_VDEVICE(REALTEK, 0x8125) }, { PCI_VDEVICE(REALTEK, 0x8126) }, - { PCI_VDEVICE(REALTEK, 0x8127) }, { PCI_VDEVICE(REALTEK, 0x3000) }, { PCI_VDEVICE(REALTEK, 0x5000) }, - { PCI_VDEVICE(REALTEK, 0x0e10) }, {} }; From 66ab4494288d1a0db20a16cc075ee80a4e244767 Mon Sep 17 00:00:00 2001 From: Nirmoy Das Date: Thu, 16 Apr 2026 10:33:57 -0700 Subject: [PATCH 114/311] Revert "NVIDIA: SAUCE: serial: 8250_mtk: Add ACPI support" BugLink: https://bugs.launchpad.net/bugs/2148607 This reverts commit 61a48d5ce4d1975c68c8fc1914ef570715738750. Signed-off-by: Nirmoy Das Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/tty/serial/8250/8250_mtk.c | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/drivers/tty/serial/8250/8250_mtk.c b/drivers/tty/serial/8250/8250_mtk.c index 39e8268cd4b9a..5875a7b9b4b10 100644 --- a/drivers/tty/serial/8250/8250_mtk.c +++ b/drivers/tty/serial/8250/8250_mtk.c @@ -19,7 +19,6 @@ #include #include #include -#include #include "8250.h" @@ -522,7 +521,6 @@ static int mtk8250_probe(struct platform_device *pdev) struct mtk8250_data *data; struct resource *regs; int irq, err; - struct fwnode_handle *fwnode = dev_fwnode(&pdev->dev); irq = platform_get_irq(pdev, 0); if (irq < 0) @@ -545,13 +543,12 @@ static int mtk8250_probe(struct platform_device *pdev) data->clk_count = 0; - if (is_of_node(fwnode)) { + if (pdev->dev.of_node) { err = mtk8250_probe_of(pdev, &uart.port, data); if (err) return err; - } else if (!fwnode) { + } else return -ENODEV; - } spin_lock_init(&uart.port.lock); uart.port.mapbase = regs->start; @@ -567,18 +564,14 @@ static int mtk8250_probe(struct platform_device *pdev) uart.port.startup = mtk8250_startup; uart.port.set_termios = mtk8250_set_termios; uart.port.uartclk = clk_get_rate(data->uart_clk); - if (!uart.port.uartclk) - uart.port.uartclk = 26 * HZ_PER_MHZ; #ifdef CONFIG_SERIAL_8250_DMA if (data->dma) uart.dma = data->dma; #endif - if (is_of_node(fwnode)) { - /* Disable Rate Fix function */ - writel(0x0, uart.port.membase + + /* Disable Rate Fix function */ + writel(0x0, uart.port.membase + (MTK_UART_RATE_FIX << uart.port.regshift)); - } platform_set_drvdata(pdev, data); @@ -656,18 +649,11 @@ static const struct of_device_id mtk8250_of_match[] = { }; MODULE_DEVICE_TABLE(of, mtk8250_of_match); -static const struct acpi_device_id mtk8250_acpi_match[] = { - { "MTKI0511" }, - {} -}; -MODULE_DEVICE_TABLE(acpi, mtk8250_acpi_match); - static struct platform_driver mtk8250_platform_driver = { .driver = { .name = "mt6577-uart", .pm = &mtk8250_pm_ops, .of_match_table = mtk8250_of_match, - .acpi_match_table = mtk8250_acpi_match, }, .probe = mtk8250_probe, .remove = mtk8250_remove, From e17d0c257425d525d718046dfd24fbe3b9b5deeb Mon Sep 17 00:00:00 2001 From: "Zhiyong.Tao" Date: Mon, 5 Jan 2026 10:39:55 +0800 Subject: [PATCH 115/311] NVIDIA: SAUCE: MEDIATEK: serial: 8250_mtk: Add ACPI support BugLink: https://bugs.launchpad.net/bugs/2148607 Add ACPI support to 8250_mtk driver. This makes it possible to use UART on ARM-based desktops with EDK2 UEFI firmware. Signed-off-by: Yenchia Chen Signed-off-by: Zhiyong.Tao (backported from https://lore.kernel.org/all/20260105024103.2027085-2-zhiyong.tao@mediatek.com/) Signed-off-by: Nirmoy Das Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/tty/serial/8250/8250_mtk.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/drivers/tty/serial/8250/8250_mtk.c b/drivers/tty/serial/8250/8250_mtk.c index 5875a7b9b4b10..e6a56cf54ae0c 100644 --- a/drivers/tty/serial/8250/8250_mtk.c +++ b/drivers/tty/serial/8250/8250_mtk.c @@ -19,6 +19,7 @@ #include #include #include +#include #include "8250.h" @@ -521,6 +522,7 @@ static int mtk8250_probe(struct platform_device *pdev) struct mtk8250_data *data; struct resource *regs; int irq, err; + struct fwnode_handle *fwnode = dev_fwnode(&pdev->dev); irq = platform_get_irq(pdev, 0); if (irq < 0) @@ -543,12 +545,13 @@ static int mtk8250_probe(struct platform_device *pdev) data->clk_count = 0; - if (pdev->dev.of_node) { + if (is_of_node(fwnode)) { err = mtk8250_probe_of(pdev, &uart.port, data); if (err) return err; - } else + } else if (!fwnode) { return -ENODEV; + } spin_lock_init(&uart.port.lock); uart.port.mapbase = regs->start; @@ -564,14 +567,18 @@ static int mtk8250_probe(struct platform_device *pdev) uart.port.startup = mtk8250_startup; uart.port.set_termios = mtk8250_set_termios; uart.port.uartclk = clk_get_rate(data->uart_clk); + if (!uart.port.uartclk) + uart.port.uartclk = 26 * HZ_PER_MHZ; #ifdef CONFIG_SERIAL_8250_DMA if (data->dma) uart.dma = data->dma; #endif - /* Disable Rate Fix function */ - writel(0x0, uart.port.membase + + if (is_of_node(fwnode)) { + /* Disable Rate Fix function */ + writel(0x0, uart.port.membase + (MTK_UART_RATE_FIX << uart.port.regshift)); + } platform_set_drvdata(pdev, data); @@ -649,11 +656,19 @@ static const struct of_device_id mtk8250_of_match[] = { }; MODULE_DEVICE_TABLE(of, mtk8250_of_match); +static const struct acpi_device_id mtk8250_acpi_match[] = { + { "MTKI0511" }, + { "NVDA0240" }, + {} +}; +MODULE_DEVICE_TABLE(acpi, mtk8250_acpi_match); + static struct platform_driver mtk8250_platform_driver = { .driver = { .name = "mt6577-uart", .pm = &mtk8250_pm_ops, .of_match_table = mtk8250_of_match, + .acpi_match_table = mtk8250_acpi_match, }, .probe = mtk8250_probe, .remove = mtk8250_remove, From cd68f4807b52975ce0b111f5af1cc260a1df7e73 Mon Sep 17 00:00:00 2001 From: Sourab Gupta Date: Thu, 20 Nov 2025 18:10:03 +0000 Subject: [PATCH 116/311] NVIDIA: SAUCE: Patch NVMe/NVMeoF driver to support GDS on Linux 7.0 Kernel BugLink: https://bugs.launchpad.net/bugs/2150289 BugLink: https://bugs.launchpad.net/bugs/2134960 With this change, the NVMe and NVMeoF driver would be enabled to support GPUDirectStorage(GDS). NVMe driver introduced a way to use the blk_rq_dma_map API to DMA map requests instead of scatter gather lists. With these changes, GDS path also adopts a similar framework where we introduce blk based APIs(nvfs_blk_rq_dma_map_iter_start and nvfs_blk_rq_dma_map_iter_next) to map a DMA request. The NVMeoF path remains the same as previous releases. Signed-off-by: Sourab Gupta Reviewed-by: Kiran Modukuri Acked-by: Matthew R. Ochs Acked-by: Nirmoy Das Acked-by: Jamie Nguyen Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/nvme/host/Makefile | 5 +- drivers/nvme/host/nvfs-dma.c | 51 +++++++++ drivers/nvme/host/nvfs-dma.h | 197 ++++++++++++++++++++++++++++++++++ drivers/nvme/host/nvfs-rdma.c | 52 +++++++++ drivers/nvme/host/nvfs-rdma.h | 86 +++++++++++++++ drivers/nvme/host/nvfs.h | 156 +++++++++++++++++++++++++++ drivers/nvme/host/pci.c | 114 +++++++++++++++++++- drivers/nvme/host/rdma.c | 22 ++++ 8 files changed, 676 insertions(+), 7 deletions(-) create mode 100644 drivers/nvme/host/nvfs-dma.c create mode 100644 drivers/nvme/host/nvfs-dma.h create mode 100644 drivers/nvme/host/nvfs-rdma.c create mode 100644 drivers/nvme/host/nvfs-rdma.h create mode 100644 drivers/nvme/host/nvfs.h diff --git a/drivers/nvme/host/Makefile b/drivers/nvme/host/Makefile index 6414ec968f99a..2fdd327bf6a88 100644 --- a/drivers/nvme/host/Makefile +++ b/drivers/nvme/host/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 ccflags-y += -I$(src) - +ccflags-y += -DCONFIG_NVFS obj-$(CONFIG_NVME_CORE) += nvme-core.o obj-$(CONFIG_BLK_DEV_NVME) += nvme.o obj-$(CONFIG_NVME_FABRICS) += nvme-fabrics.o @@ -20,10 +20,11 @@ nvme-core-$(CONFIG_NVME_HWMON) += hwmon.o nvme-core-$(CONFIG_NVME_HOST_AUTH) += auth.o nvme-y += pci.o - +nvme-y += nvfs-dma.o nvme-fabrics-y += fabrics.o nvme-rdma-y += rdma.o +nvme-rdma-y += nvfs-rdma.o nvme-fc-y += fc.o diff --git a/drivers/nvme/host/nvfs-dma.c b/drivers/nvme/host/nvfs-dma.c new file mode 100644 index 0000000000000..8d821e7b08846 --- /dev/null +++ b/drivers/nvme/host/nvfs-dma.c @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ +#ifdef CONFIG_NVFS +#define NVFS_USE_DMA_ITER_API +#define MODULE_PREFIX nvme_v2 +#include "nvfs.h" + +struct nvfs_dma_rw_blk_iter_ops *nvfs_ops = NULL; + +atomic_t nvfs_shutdown = ATOMIC_INIT(1); + +DEFINE_PER_CPU(long, nvfs_n_ops); + +#define NVIDIA_FS_COMPAT_FT(ops) \ + (NVIDIA_FS_CHECK_FT_BLK_DMA_MAP_ITER_START(ops) && NVIDIA_FS_CHECK_FT_BLK_DMA_MAP_ITER_NEXT(ops)) + +// protected via nvfs_module_mutex +int REGISTER_FUNC(struct nvfs_dma_rw_blk_iter_ops *ops) +{ + if (NVIDIA_FS_COMPAT_FT(ops)) { + nvfs_ops = ops; + atomic_set(&nvfs_shutdown, 0); + return 0; + } else + return -EOPNOTSUPP; + +} +EXPORT_SYMBOL_GPL(REGISTER_FUNC); + +// protected via nvfs_module_mutex +void UNREGISTER_FUNC(void) +{ + (void) atomic_cmpxchg(&nvfs_shutdown, 0, 1); + do { + msleep(NVFS_HOLD_TIME_MS); + } while(nvfs_count_ops()); + nvfs_ops = NULL; +} +EXPORT_SYMBOL_GPL(UNREGISTER_FUNC); +#endif diff --git a/drivers/nvme/host/nvfs-dma.h b/drivers/nvme/host/nvfs-dma.h new file mode 100644 index 0000000000000..6dc6654adb2a8 --- /dev/null +++ b/drivers/nvme/host/nvfs-dma.h @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#ifndef NVFS_DMA_H +#define NVFS_DMA_H + +/* Forward declarations for functions from pci.c that we need */ +static blk_status_t nvme_pci_setup_data_prp(struct request *req, + struct blk_dma_iter *iter); +static blk_status_t nvme_pci_setup_data_sgl(struct request *req, + struct blk_dma_iter *iter); +static inline struct dma_pool *nvme_dma_pool(struct nvme_queue *nvmeq, + struct nvme_iod *iod); +static inline dma_addr_t nvme_pci_first_desc_dma_addr(struct nvme_command *cmd); + +static inline bool nvme_nvfs_unmap_sgls(struct request *req) +{ + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; + struct device *dma_dev = nvmeq->dev->dev; + unsigned int sqe_dma_len = le32_to_cpu(iod->cmd.common.dptr.sgl.length); + struct nvme_sgl_desc *sg_list = iod->descriptors[0]; + enum dma_data_direction dir = rq_dma_dir(req); + + /* + * nr_descriptors == 0 means dma_pool_alloc failed before any SGL + * entries were recorded; the first iter mapping is handled by + * nvme_nvfs_map_data() directly, so nothing to unmap here. + */ + if (iod->nr_descriptors) { + unsigned int nr_entries = sqe_dma_len / sizeof(*sg_list), i; + + for (i = 0; i < nr_entries; i++) { + nvfs_ops->nvfs_dma_unmap_page(dma_dev, + iod->nvfs_cookie, + le64_to_cpu(sg_list[i].addr), + le32_to_cpu(sg_list[i].length), + dir); + } + } + + return true; +} + +static inline bool nvme_nvfs_unmap_prps(struct request *req) +{ + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; + struct device *dma_dev = nvmeq->dev->dev; + enum dma_data_direction dma_dir = rq_dma_dir(req); + unsigned int i; + + /* Check if dma_vecs was allocated - if setup failed early, it might be NULL */ + if (!iod->dma_vecs) + return true; + + /* Unmap all DMA vectors - pass page pointer from dma_vecs */ + for (i = 0; i < iod->nr_dma_vecs; i++) { + nvfs_ops->nvfs_dma_unmap_page(dma_dev, + iod->nvfs_cookie, + iod->dma_vecs[i].addr, + iod->dma_vecs[i].len, + dma_dir); + } + + /* Free the dma_vecs mempool allocation */ + mempool_free(iod->dma_vecs, nvmeq->dev->dmavec_mempool); + iod->dma_vecs = NULL; + iod->nr_dma_vecs = 0; + + return true; +} + +static inline void nvme_nvfs_free_descriptors(struct request *req) +{ + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; + const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1; + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + dma_addr_t dma_addr = nvme_pci_first_desc_dma_addr(&iod->cmd); + int i; + + if (iod->nr_descriptors == 1) { + dma_pool_free(nvme_dma_pool(nvmeq, iod), iod->descriptors[0], + dma_addr); + return; + } + + for (i = 0; i < iod->nr_descriptors; i++) { + __le64 *prp_list = iod->descriptors[i]; + dma_addr_t next_dma_addr = le64_to_cpu(prp_list[last_prp]); + + dma_pool_free(nvmeq->descriptor_pools.large, prp_list, + dma_addr); + dma_addr = next_dma_addr; + } +} + +static inline bool nvme_nvfs_unmap_data(struct request *req) +{ + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + bool ret; + + /* Check if this was an NVFS I/O by checking the IOD_NVFS_IO flag */ + if (!(iod->flags & IOD_NVFS_IO)) + return false; + + /* Clear the NVFS flag */ + iod->flags &= ~IOD_NVFS_IO; + + /* Call appropriate unmap function based on command type */ + if (nvme_pci_cmd_use_sgl(&iod->cmd)) + ret = nvme_nvfs_unmap_sgls(req); + else + ret = nvme_nvfs_unmap_prps(req); + + if (iod->nr_descriptors) + nvme_nvfs_free_descriptors(req); + + nvfs_put_ops(); + return ret; +} + +static inline blk_status_t nvme_nvfs_map_data(struct request *req, + bool *is_nvfs_io) +{ + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; + struct nvme_dev *dev = nvmeq->dev; + struct device *dma_dev = nvmeq->dev->dev; + enum nvme_use_sgl use_sgl = nvme_pci_use_sgls(dev, req); + struct blk_dma_iter iter; + blk_status_t ret = BLK_STS_RESOURCE; + + *is_nvfs_io = false; + + /* Check integrity and try to get nvfs_ops */ + if (blk_integrity_rq(req) || !nvfs_get_ops()) { + return ret; + } + + /* Initialize total_len for this request */ + iod->total_len = 0; + + if (!nvfs_ops->nvfs_blk_rq_dma_map_iter_start(req, dma_dev, + &iod->dma_state, &iter, &iod->nvfs_cookie)) { + nvfs_put_ops(); + if (iter.status == BLK_STS_IOERR) { + /* GPU DMA error — do not fall through to CPU path */ + *is_nvfs_io = true; + ret = iter.status; + } + /* else: CPU page, let caller fall through to CPU path */ + return ret; + } + + /* NVFS can handle this request, set the flag */ + *is_nvfs_io = true; + iod->flags |= IOD_NVFS_IO; + + if (use_sgl == SGL_FORCED || + (use_sgl == SGL_SUPPORTED && + (sgl_threshold && nvme_pci_avg_seg_size(req) >= sgl_threshold))) + ret = nvme_pci_setup_data_sgl(req, &iter); + else + ret = nvme_pci_setup_data_prp(req, &iter); + + /* If setup failed, cleanup: unmap DMA, clear flag, release ops */ + if (ret != BLK_STS_OK) { + /* + * If setup failed before any mappings were tracked (dma_vecs is + * NULL for PRP, or nr_descriptors is 0 for SGL), the first page + * mapped by nvfs_blk_rq_dma_map_iter_start() won't be covered by + * nvme_nvfs_unmap_data(). Unmap it directly using iter. + */ + bool early_fail = nvme_pci_cmd_use_sgl(&iod->cmd) ? + !iod->nr_descriptors : !iod->dma_vecs; + if (early_fail) + nvfs_ops->nvfs_dma_unmap_page(dma_dev, iod->nvfs_cookie, + iter.addr, iter.len, rq_dma_dir(req)); + nvme_nvfs_unmap_data(req); + } + + return ret; +} + +#endif /* NVFS_DMA_H */ diff --git a/drivers/nvme/host/nvfs-rdma.c b/drivers/nvme/host/nvfs-rdma.c new file mode 100644 index 0000000000000..75a269143f3a8 --- /dev/null +++ b/drivers/nvme/host/nvfs-rdma.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#ifdef CONFIG_NVFS +#define MODULE_PREFIX nvme_rdma_v1 +#include "nvfs.h" + +struct nvfs_dma_rw_ops *nvfs_ops; + +atomic_t nvfs_shutdown = ATOMIC_INIT(1); + +DEFINE_PER_CPU(long, nvfs_n_ops); + +// must have for compatability +#define NVIDIA_FS_COMPAT_FT(ops) \ + (NVIDIA_FS_CHECK_FT_SGLIST_PREP(ops) && NVIDIA_FS_CHECK_FT_SGLIST_DMA(ops)) + +// protected via nvfs_module_mutex +int REGISTER_FUNC(struct nvfs_dma_rw_ops *ops) +{ + if (NVIDIA_FS_COMPAT_FT(ops)) { + nvfs_ops = ops; + atomic_set(&nvfs_shutdown, 0); + return 0; + } else + return -EOPNOTSUPP; + +} +EXPORT_SYMBOL_GPL(REGISTER_FUNC); + +// protected via nvfs_module_mutex +void UNREGISTER_FUNC(void) +{ + (void) atomic_cmpxchg(&nvfs_shutdown, 0, 1); + do { + msleep(NVFS_HOLD_TIME_MS); + } while(nvfs_count_ops()); + nvfs_ops = NULL; +} +EXPORT_SYMBOL_GPL(UNREGISTER_FUNC); +#endif diff --git a/drivers/nvme/host/nvfs-rdma.h b/drivers/nvme/host/nvfs-rdma.h new file mode 100644 index 0000000000000..f9051e2ab22b3 --- /dev/null +++ b/drivers/nvme/host/nvfs-rdma.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#ifndef NVFS_RDMA_H +#define NVFS_RDMA_H + +static bool nvme_rdma_nvfs_unmap_data(struct ib_device *ibdev, + struct request *rq) + +{ + struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq); + enum dma_data_direction dma_dir = rq_dma_dir(rq); + int count; + + if (!blk_integrity_rq(rq) && nvfs_ops != NULL) { + count = nvfs_ops->nvfs_dma_unmap_sg(ibdev->dma_device, req->data_sgl.sg_table.sgl, req->data_sgl.nents, + dma_dir); + if (count) { + nvfs_put_ops(); + sg_free_table_chained(&req->data_sgl.sg_table, NVME_INLINE_SG_CNT); + return true; + } + } + return false; +} + +static int nvme_rdma_nvfs_map_data(struct ib_device *ibdev, struct request *rq, bool *is_nvfs_io, int* count) +{ + struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq); + enum dma_data_direction dma_dir = rq_dma_dir(rq); + int ret = 0; + + *is_nvfs_io = false; + *count = 0; + if (!blk_integrity_rq(rq) && nvfs_get_ops()) { + + // associates bio pages to scatterlist + *count = nvfs_ops->nvfs_blk_rq_map_sg(rq->q, rq , req->data_sgl.sg_table.sgl); + if (!*count) { + nvfs_put_ops(); + return 0; // fall to cpu path + } + + *is_nvfs_io = true; + if (unlikely((*count == NVFS_IO_ERR))) { + nvfs_put_ops(); + pr_err("%s: failed to map sg_nents=:%d\n", __func__, req->data_sgl.nents); + return -EIO; + } + req->data_sgl.nents = *count; + + *count = nvfs_ops->nvfs_dma_map_sg_attrs(ibdev->dma_device, + req->data_sgl.sg_table.sgl, + req->data_sgl.nents, + dma_dir, + DMA_ATTR_NO_WARN); + + if (unlikely((*count == NVFS_IO_ERR))) { + nvfs_put_ops(); + return -EIO; + } + + if (unlikely(*count == NVFS_CPU_REQ)) { + nvfs_put_ops(); + return -EIO; + } + + return ret; + } + + // Fall to CPU path + return 0; +} + +#endif diff --git a/drivers/nvme/host/nvfs.h b/drivers/nvme/host/nvfs.h new file mode 100644 index 0000000000000..0101a88dcc6c7 --- /dev/null +++ b/drivers/nvme/host/nvfs.h @@ -0,0 +1,156 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ +#ifndef NVFS_H +#define NVFS_H + +#include +#include +#include +#include +#include +#include +#include + +/* Forward declarations */ +struct blk_dma_iter; +struct dma_iova_state; + +#define REGSTR2(x) x##_register_nvfs_dma_ops +#define REGSTR(x) REGSTR2(x) + +#define UNREGSTR2(x) x##_unregister_nvfs_dma_ops +#define UNREGSTR(x) UNREGSTR2(x) + +#define REGISTER_FUNC REGSTR(MODULE_PREFIX) +#define UNREGISTER_FUNC UNREGSTR(MODULE_PREFIX) + +#define NVFS_IO_ERR -1 +#define NVFS_CPU_REQ -2 + +#define NVFS_HOLD_TIME_MS 1000 + +#ifdef NVFS_USE_DMA_ITER_API +extern struct nvfs_dma_rw_blk_iter_ops *nvfs_ops; +#else +extern struct nvfs_dma_rw_ops *nvfs_ops; +#endif + +extern atomic_t nvfs_shutdown; + +DECLARE_PER_CPU(long, nvfs_n_ops); + +static inline long nvfs_count_ops(void) +{ + int i; + long sum = 0; + for_each_possible_cpu(i) + sum += per_cpu(nvfs_n_ops, i); + return sum; +} + +static inline bool nvfs_get_ops(void) +{ + if (nvfs_ops && !atomic_read(&nvfs_shutdown)) { + this_cpu_inc(nvfs_n_ops); + return true; + } + return false; +} + +static inline void nvfs_put_ops(void) +{ + this_cpu_dec(nvfs_n_ops); +} + + +struct nvfs_dma_rw_blk_iter_ops { + unsigned long long ft_bmap; // feature bitmap + + int (*nvfs_blk_rq_dma_map_iter_start) (struct request *req, + struct device *dma_dev, + struct dma_iova_state *state, + struct blk_dma_iter *iter, + void **cookie); + + int (*nvfs_blk_rq_dma_map_iter_next) (struct request *req, + struct device *dma_dev, + struct dma_iova_state *state, + struct blk_dma_iter *iter); + + int (*nvfs_dma_unmap_page) (struct device *device, + void* cookie, + dma_addr_t addr, + size_t size, + enum dma_data_direction dir); + + bool (*nvfs_is_gpu_page) (struct page *page); + + unsigned int (*nvfs_gpu_index) (struct page *page); + + unsigned int (*nvfs_device_priority) (struct device *dev, unsigned int gpu_index); + +}; + +struct nvfs_dma_rw_ops { + unsigned long long ft_bmap; // feature bitmap + + int (*nvfs_blk_rq_map_sg) (struct request_queue *q, + struct request *req, + struct scatterlist *sglist); + + int (*nvfs_dma_map_sg_attrs) (struct device *device, + struct scatterlist *sglist, + int nents, + enum dma_data_direction dma_dir, + unsigned long attrs); + + int (*nvfs_dma_unmap_sg) (struct device *device, + struct scatterlist *sglist, + int nents, + enum dma_data_direction dma_dir); + + bool (*nvfs_is_gpu_page) (struct page *page); + + unsigned int (*nvfs_gpu_index) (struct page *page); + + unsigned int (*nvfs_device_priority) (struct device *dev, unsigned int gpu_index); +}; + +// feature list for dma_ops, values indicate bit pos +enum ft_bits { + nvfs_ft_prep_sglist = 1ULL << 0, + nvfs_ft_map_sglist = 1ULL << 1, + nvfs_ft_is_gpu_page = 1ULL << 2, + nvfs_ft_device_priority = 1ULL << 3, + nvfs_ft_blk_dma_map_iter_start = 1ULL << 5, + nvfs_ft_blk_dma_map_iter_next = 1ULL << 6, +}; + +// check features for use in registration with vendor drivers +#define NVIDIA_FS_CHECK_FT_SGLIST_PREP(ops) ((ops)->ft_bmap & nvfs_ft_prep_sglist) +#define NVIDIA_FS_CHECK_FT_SGLIST_DMA(ops) ((ops)->ft_bmap & nvfs_ft_map_sglist) +#define NVIDIA_FS_CHECK_FT_GPU_PAGE(ops) ((ops)->ft_bmap & nvfs_ft_is_gpu_page) +#define NVIDIA_FS_CHECK_FT_DEVICE_PRIORITY(ops) ((ops)->ft_bmap & nvfs_ft_device_priority) +#define NVIDIA_FS_CHECK_FT_BLK_DMA_MAP_ITER_START(ops) ((ops)->ft_bmap & nvfs_ft_blk_dma_map_iter_start) +#define NVIDIA_FS_CHECK_FT_BLK_DMA_MAP_ITER_NEXT(ops) ((ops)->ft_bmap & nvfs_ft_blk_dma_map_iter_next) + +#ifdef NVFS_USE_DMA_ITER_API +int REGISTER_FUNC(struct nvfs_dma_rw_blk_iter_ops *ops); +#else +int REGISTER_FUNC(struct nvfs_dma_rw_ops *ops); +#endif + +void UNREGISTER_FUNC(void); + +#endif /* NVFS_H */ diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index db5fc9bf66272..2a56a97937c3c 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -30,6 +30,10 @@ #include "trace.h" #include "nvme.h" +#ifdef CONFIG_NVFS +#define NVFS_USE_DMA_ITER_API +#include "nvfs.h" +#endif #define SQ_SIZE(q) ((q)->q_depth << (q)->sqes) #define CQ_SIZE(q) ((q)->q_depth * sizeof(struct nvme_completion)) @@ -37,6 +41,15 @@ /* Optimisation for I/Os between 4k and 128k */ #define NVME_SMALL_POOL_SIZE 256 +#ifdef CONFIG_NVFS +/* GPU physical pages are minimum 64K. Worst-case SGL entries with misalignment: + * ceil(payload/64K) + 1. Safe payload for small pool = (entries - 1) * 64K, + * where entries = NVME_SMALL_POOL_SIZE / sizeof(struct nvme_sgl_desc). */ +#define NVFS_GPU_PAGE_SIZE (64UL * 1024) +#define NVFS_SMALL_POOL_PAYLOAD \ + ((NVME_SMALL_POOL_SIZE / sizeof(struct nvme_sgl_desc) - 1) * NVFS_GPU_PAGE_SIZE) +#endif + /* * Arbitrary upper bound. */ @@ -418,6 +431,11 @@ enum nvme_iod_flags { /* Metadata using non-coalesced MPTR */ IOD_SINGLE_META_SEGMENT = 1U << 7, + +#ifdef CONFIG_NVFS + /* NVFS GPU Direct Storage I/O */ + IOD_NVFS_IO = 1U << 31, +#endif }; struct nvme_dma_vec { @@ -431,7 +449,11 @@ struct nvme_dma_vec { struct nvme_iod { struct nvme_request req; struct nvme_command cmd; +#ifdef CONFIG_NVFS + u32 flags; +#else u8 flags; +#endif u8 nr_descriptors; size_t total_len; @@ -444,6 +466,9 @@ struct nvme_iod { size_t meta_total_len; struct dma_iova_state meta_dma_state; struct nvme_sgl_desc *meta_descriptor; +#ifdef CONFIG_NVFS + void *nvfs_cookie; +#endif }; static inline unsigned int nvme_dbbuf_size(struct nvme_dev *dev) @@ -924,6 +949,10 @@ static void nvme_unmap_metadata(struct request *req) iod->meta_descriptor, iod->meta_dma); } +#ifdef CONFIG_NVFS +#include "nvfs-dma.h" +#endif + static void nvme_unmap_data(struct request *req) { enum pci_p2pdma_map_type map = PCI_P2PDMA_MAP_NONE; @@ -932,6 +961,12 @@ static void nvme_unmap_data(struct request *req) struct device *dma_dev = nvmeq->dev->dev; unsigned int attrs = 0; +#ifdef CONFIG_NVFS + /* Check if this was an NVFS I/O and handle unmapping */ + if (nvme_nvfs_unmap_data(req)) + return; +#endif + if (iod->flags & IOD_SINGLE_SEGMENT) { static_assert(offsetof(union nvme_data_ptr, prp1) == offsetof(union nvme_data_ptr, sgl.addr)); @@ -991,6 +1026,20 @@ static bool nvme_pci_prp_iter_next(struct request *req, struct device *dma_dev, { if (iter->len) return true; +#ifdef CONFIG_NVFS + { + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + if (iod->flags & IOD_NVFS_IO) { + if (!nvfs_ops->nvfs_blk_rq_dma_map_iter_next(req, dma_dev, + &iod->dma_state, iter)) + return false; + iod->dma_vecs[iod->nr_dma_vecs].addr = iter->addr; + iod->dma_vecs[iod->nr_dma_vecs].len = iter->len; + iod->nr_dma_vecs++; + return true; + } + } +#endif if (!blk_rq_dma_map_iter_next(req, dma_dev, iter)) return false; return nvme_pci_prp_save_mapping(req, dma_dev, iter); @@ -1006,6 +1055,16 @@ static blk_status_t nvme_pci_setup_data_prp(struct request *req, unsigned int prp_len, i; __le64 *prp_list; +#ifdef CONFIG_NVFS + if (iod->flags & IOD_NVFS_IO) { + iod->dma_vecs = mempool_alloc(nvmeq->dev->dmavec_mempool, GFP_ATOMIC); + if (!iod->dma_vecs) + return BLK_STS_RESOURCE; + iod->dma_vecs[0].addr = iter->addr; + iod->dma_vecs[0].len = iter->len; + iod->nr_dma_vecs = 1; + } else +#endif if (!nvme_pci_prp_save_mapping(req, nvmeq->dev->dev, iter)) return iter->status; @@ -1104,6 +1163,11 @@ static blk_status_t nvme_pci_setup_data_prp(struct request *req, */ iod->cmd.common.dptr.prp1 = cpu_to_le64(prp1_dma); iod->cmd.common.dptr.prp2 = cpu_to_le64(prp2_dma); +#ifdef CONFIG_NVFS + /* For NVFS, don't call nvme_unmap_data - cleanup happens in nvme_nvfs_unmap_data */ + if (iod->flags & IOD_NVFS_IO) + return iter->status; +#endif if (unlikely(iter->status)) nvme_unmap_data(req); return iter->status; @@ -1144,12 +1208,34 @@ static blk_status_t nvme_pci_setup_data_sgl(struct request *req, /* set the transfer type as SGL */ iod->cmd.common.flags = NVME_CMD_SGL_METABUF; - if (entries == 1 || blk_rq_dma_map_coalesce(&iod->dma_state)) { - nvme_pci_sgl_set_data(&iod->cmd.common.dptr.sgl, iter); - iod->total_len += iter->len; - return BLK_STS_OK; +#ifdef CONFIG_NVFS + if (!(iod->flags & IOD_NVFS_IO)) +#endif + { + if (entries == 1 || blk_rq_dma_map_coalesce(&iod->dma_state)) { + nvme_pci_sgl_set_data(&iod->cmd.common.dptr.sgl, iter); + iod->total_len += iter->len; + return BLK_STS_OK; + } } +#ifdef CONFIG_NVFS + if (iod->flags & IOD_NVFS_IO) { + /* + * blk_rq_nr_phys_segments() reflects shadow buffer contiguity, + * not GPU physical segments. GPU pages are 64K minimum; worst-case + * entries with misalignment = ceil(payload/64K) + 1. + * Small pool (16 entries) is safe for payload < NVFS_SMALL_POOL_PAYLOAD. + * Large pool capacity = NVME_CTRL_PAGE_SIZE / sizeof(*sg_list) = 256. + */ + if (blk_rq_payload_bytes(req) < NVFS_SMALL_POOL_PAYLOAD) { + entries = NVME_SMALL_POOL_SIZE / sizeof(*sg_list); + iod->flags |= IOD_SMALL_DESCRIPTOR; + } else { + entries = NVME_CTRL_PAGE_SIZE / sizeof(*sg_list); + } + } else +#endif if (entries <= NVME_SMALL_POOL_SIZE / sizeof(*sg_list)) iod->flags |= IOD_SMALL_DESCRIPTOR; @@ -1166,9 +1252,21 @@ static blk_status_t nvme_pci_setup_data_sgl(struct request *req, } nvme_pci_sgl_set_data(&sg_list[mapped++], iter); iod->total_len += iter->len; - } while (blk_rq_dma_map_iter_next(req, nvmeq->dev->dev, iter)); + } while ( +#ifdef CONFIG_NVFS + (iod->flags & IOD_NVFS_IO) ? + (mapped < entries && + nvfs_ops->nvfs_blk_rq_dma_map_iter_next(req, nvmeq->dev->dev, + &iod->dma_state, iter)) : +#endif + blk_rq_dma_map_iter_next(req, nvmeq->dev->dev, iter)); nvme_pci_sgl_set_seg(&iod->cmd.common.dptr.sgl, sgl_dma, mapped); +#ifdef CONFIG_NVFS + /* For NVFS, don't call nvme_unmap_data - cleanup happens in nvme_nvfs_unmap_data */ + if (iod->flags & IOD_NVFS_IO) + return iter->status; +#endif if (unlikely(iter->status)) nvme_unmap_data(req); return iter->status; @@ -1222,6 +1320,12 @@ static blk_status_t nvme_map_data(struct request *req) struct blk_dma_iter iter; blk_status_t ret; +#ifdef CONFIG_NVFS + bool is_nvfs_io = false; + ret = nvme_nvfs_map_data(req, &is_nvfs_io); + if (is_nvfs_io) + return ret; +#endif /* * Try to skip the DMA iterator for single segment requests, as that * significantly improves performances for small I/O sizes. diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 57111139e84fa..53b4823d57c5f 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -27,6 +27,9 @@ #include "nvme.h" #include "fabrics.h" +#ifdef CONFIG_NVFS +#include "nvfs.h" +#endif #define NVME_RDMA_CM_TIMEOUT_MS 3000 /* 3 second */ @@ -1212,6 +1215,9 @@ static int nvme_rdma_inv_rkey(struct nvme_rdma_queue *queue, return ib_post_send(queue->qp, &wr, NULL); } +#ifdef CONFIG_NVFS +#include "nvfs-rdma.h" +#endif static void nvme_rdma_dma_unmap_req(struct ib_device *ibdev, struct request *rq) { struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq); @@ -1223,6 +1229,11 @@ static void nvme_rdma_dma_unmap_req(struct ib_device *ibdev, struct request *rq) NVME_INLINE_METADATA_SG_CNT); } +#ifdef CONFIG_NVFS + if (nvme_rdma_nvfs_unmap_data(ibdev, rq)) + return; +#endif + ib_dma_unmap_sg(ibdev, req->data_sgl.sg_table.sgl, req->data_sgl.nents, rq_dma_dir(rq)); sg_free_table_chained(&req->data_sgl.sg_table, NVME_INLINE_SG_CNT); @@ -1476,6 +1487,17 @@ static int nvme_rdma_dma_map_req(struct ib_device *ibdev, struct request *rq, if (ret) return -ENOMEM; +#ifdef CONFIG_NVFS + { + bool is_nvfs_io = false; + ret = nvme_rdma_nvfs_map_data(ibdev, rq, &is_nvfs_io, count); + if (is_nvfs_io) { + if (ret) + goto out_free_table; + return 0; + } + } +#endif req->data_sgl.nents = blk_rq_map_sg(rq, req->data_sgl.sg_table.sgl); *count = ib_dma_map_sg(ibdev, req->data_sgl.sg_table.sgl, From 20c89ac6e9415c270159a958f53d0c2f328f06e3 Mon Sep 17 00:00:00 2001 From: Koba Ko Date: Wed, 15 Apr 2026 11:03:03 +0800 Subject: [PATCH 117/311] NVIDIA: SAUCE: iommu/arm-smmu-v3: Use identity domain for ASPEED BMC devices BugLink: https://bugs.launchpad.net/bugs/2150470 ASPEED BMC devices behind an AST1150 PCIe-to-PCI bridge receive DMA from BMC firmware using host physical addresses that bypass the kernel's DMA API entirely. When these devices are assigned a DMA translated domain, the SMMU generates F_TRANSLATION faults because the BMC's physical addresses have no corresponding IOVA mappings in the SMMU page tables. Fix this by returning IOMMU_DOMAIN_IDENTITY for PCI devices whose parent bridge has both the PCI_BRIDGE_NO_ALIASES flag and an ASPEED vendor ID, so the SMMU passes BMC DMA transactions through untranslated. Signed-off-by: Koba Ko (backported from commit 738fff0e2060b6b383c21afdf1366330d9c79698 linux-nvidia-6.17) [koba: rename PCI_DEV_FLAGS_PCI_BRIDGE_NO_ALIASES -> PCI_DEV_FLAGS_PCI_BRIDGE_NO_ALIAS; Nirmoy's upstream AST1150 NO_ALIAS quirk in 7.0 uses the singular form (bit 14 in include/linux/pci.h) vs the plural form (bit 15) in 6.17-next] Signed-off-by: Koba Ko Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 821e7d3da07bb..c8595733d2fc8 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3731,6 +3731,19 @@ static int arm_smmu_def_domain_type(struct device *dev) if (IS_HISI_PTT_DEVICE(pdev)) return IOMMU_DOMAIN_IDENTITY; + /* + * ASPEED BMC devices behind an AST1150 PCIe-to-PCI bridge + * receive DMA from BMC firmware using host physical addresses + * that bypass the kernel DMA API. Use identity mapping so + * the SMMU passes these transactions through untranslated. + */ + if (pdev->bus->self && + (pdev->bus->self->dev_flags & + PCI_DEV_FLAGS_PCI_BRIDGE_NO_ALIAS) && + pdev->bus->self->vendor == PCI_VENDOR_ID_ASPEED && + pdev->bus->self->device == 0x1150) + return IOMMU_DOMAIN_IDENTITY; + if (pdev->vendor == PCI_VENDOR_ID_NVIDIA && (pdev->device == 0x2E12 || pdev->device == 0x2E2A || pdev->device == 0x2E2B)) From 4700d520f6a392dff804e0769d9103f1ece56b9a Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Thu, 12 Mar 2026 09:12:02 -0700 Subject: [PATCH 118/311] workqueue: fix parse_affn_scope() prefix matching bug BugLink: https://bugs.launchpad.net/bugs/2150467 parse_affn_scope() uses strncasecmp() with the length of the candidate name, which means it only checks if the input *starts with* a known scope name. Given that the upcoming diff will create "cache_shard" affinity scope, writing "cache_shard" to a workqueue's affinity_scope sysfs attribute always matches "cache" first, making it impossible to select "cache_shard" via sysfs, so, this fix enable it to distinguish "cache" and "cache_shard" Fix by replacing the hand-rolled prefix matching loop with sysfs_match_string(), which uses sysfs_streq() for exact matching (modulo trailing newlines). Also add the missing const qualifier to the wq_affn_names[] array declaration. Note that sysfs_streq() is case-sensitive, unlike the previous strncasecmp() approach. This is intentional and consistent with how other sysfs attributes handle string matching in the kernel. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo (cherry picked from commit 1abaae9b38a85c9dabff67a22d8c99f7254c423a) Signed-off-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Matthew R. Ochs Signed-off-by: Brad Figg --- kernel/workqueue.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index c6ea96d5b7167..54e1bf873f0ae 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -404,7 +404,7 @@ struct work_offq_data { u32 flags; }; -static const char *wq_affn_names[WQ_AFFN_NR_TYPES] = { +static const char * const wq_affn_names[WQ_AFFN_NR_TYPES] = { [WQ_AFFN_DFL] = "default", [WQ_AFFN_CPU] = "cpu", [WQ_AFFN_SMT] = "smt", @@ -7078,13 +7078,7 @@ int workqueue_unbound_housekeeping_update(const struct cpumask *hk) static int parse_affn_scope(const char *val) { - int i; - - for (i = 0; i < ARRAY_SIZE(wq_affn_names); i++) { - if (!strncasecmp(val, wq_affn_names[i], strlen(wq_affn_names[i]))) - return i; - } - return -EINVAL; + return sysfs_match_string(wq_affn_names, val); } static int wq_affn_dfl_set(const char *val, const struct kernel_param *kp) From ec7fa748f32a4cac6ef2bb7c5ce112b784f65f3c Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 1 Apr 2026 06:03:52 -0700 Subject: [PATCH 119/311] workqueue: fix typo in WQ_AFFN_SMT comment BugLink: https://bugs.launchpad.net/bugs/2150467 Fix "poer" -> "per" in the WQ_AFFN_SMT enum comment. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo (cherry picked from commit 9dc42c9070282c81058a875fea5acae057610980) Signed-off-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Matthew R. Ochs Signed-off-by: Brad Figg --- include/linux/workqueue.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h index a4749f56398fd..17543aec2a6e1 100644 --- a/include/linux/workqueue.h +++ b/include/linux/workqueue.h @@ -131,7 +131,7 @@ struct rcu_work { enum wq_affn_scope { WQ_AFFN_DFL, /* use system default */ WQ_AFFN_CPU, /* one pod per CPU */ - WQ_AFFN_SMT, /* one pod poer SMT */ + WQ_AFFN_SMT, /* one pod per SMT */ WQ_AFFN_CACHE, /* one pod per LLC */ WQ_AFFN_NUMA, /* one pod per NUMA node */ WQ_AFFN_SYSTEM, /* one pod across the whole system */ From 940a111abb145fd2d454d60ed34b7b171dddf065 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 1 Apr 2026 06:03:53 -0700 Subject: [PATCH 120/311] workqueue: add WQ_AFFN_CACHE_SHARD affinity scope BugLink: https://bugs.launchpad.net/bugs/2150467 On systems where many CPUs share one LLC, unbound workqueues using WQ_AFFN_CACHE collapse to a single worker pool, causing heavy spinlock contention on pool->lock. For example, Chuck Lever measured 39% of cycles lost to native_queued_spin_lock_slowpath on a 12-core shared-L3 NFS-over-RDMA system. The existing affinity hierarchy (cpu, smt, cache, numa, system) offers no intermediate option between per-LLC and per-SMT-core granularity. Add WQ_AFFN_CACHE_SHARD, which subdivides each LLC into groups of at most wq_cache_shard_size cores (default 8, tunable via boot parameter). Shards are always split on core (SMT group) boundaries so that Hyper-Threading siblings are never placed in different pods. Cores are distributed across shards as evenly as possible -- for example, 36 cores in a single LLC with max shard size 8 produces 5 shards of 8+7+7+7+7 cores. The implementation follows the same comparator pattern as other affinity scopes: precompute_cache_shard_ids() pre-fills the cpu_shard_id[] array from the already-initialized WQ_AFFN_CACHE and WQ_AFFN_SMT topology, and cpus_share_cache_shard() is passed to init_pod_type(). Benchmark on NVIDIA Grace (72 CPUs, single LLC, 50k items/thread), show cache_shard delivers ~5x the throughput and ~6.5x lower p50 latency compared to cache scope on this 72-core single-LLC system. Suggested-by: Tejun Heo Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo (cherry picked from commit 5920d046f7ae3bf9cf51b9d915c1fff13d299d84) Signed-off-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Matthew R. Ochs Signed-off-by: Brad Figg --- include/linux/workqueue.h | 1 + kernel/workqueue.c | 183 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+) diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h index 17543aec2a6e1..50bdb7e30d35f 100644 --- a/include/linux/workqueue.h +++ b/include/linux/workqueue.h @@ -133,6 +133,7 @@ enum wq_affn_scope { WQ_AFFN_CPU, /* one pod per CPU */ WQ_AFFN_SMT, /* one pod per SMT */ WQ_AFFN_CACHE, /* one pod per LLC */ + WQ_AFFN_CACHE_SHARD, /* synthetic sub-LLC shards */ WQ_AFFN_NUMA, /* one pod per NUMA node */ WQ_AFFN_SYSTEM, /* one pod across the whole system */ diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 54e1bf873f0ae..cdee3abcce97b 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -130,6 +130,14 @@ enum wq_internal_consts { WORKER_ID_LEN = 10 + WQ_NAME_LEN, /* "kworker/R-" + WQ_NAME_LEN */ }; +/* Layout of shards within one LLC pod */ +struct llc_shard_layout { + int nr_large_shards; /* number of large shards (cores_per_shard + 1) */ + int cores_per_shard; /* base number of cores per default shard */ + int nr_shards; /* total number of shards */ + /* nr_default shards = (nr_shards - nr_large_shards) */ +}; + /* * We don't want to trap softirq for too long. See MAX_SOFTIRQ_TIME and * MAX_SOFTIRQ_RESTART in kernel/softirq.c. These are macros because @@ -409,6 +417,7 @@ static const char * const wq_affn_names[WQ_AFFN_NR_TYPES] = { [WQ_AFFN_CPU] = "cpu", [WQ_AFFN_SMT] = "smt", [WQ_AFFN_CACHE] = "cache", + [WQ_AFFN_CACHE_SHARD] = "cache_shard", [WQ_AFFN_NUMA] = "numa", [WQ_AFFN_SYSTEM] = "system", }; @@ -431,6 +440,9 @@ module_param_named(cpu_intensive_warning_thresh, wq_cpu_intensive_warning_thresh static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT); module_param_named(power_efficient, wq_power_efficient, bool, 0444); +static unsigned int wq_cache_shard_size = 8; +module_param_named(cache_shard_size, wq_cache_shard_size, uint, 0444); + static bool wq_online; /* can kworkers be created yet? */ static bool wq_topo_initialized __read_mostly = false; @@ -8138,6 +8150,175 @@ static bool __init cpus_share_numa(int cpu0, int cpu1) return cpu_to_node(cpu0) == cpu_to_node(cpu1); } +/* Maps each CPU to its shard index within the LLC pod it belongs to */ +static int cpu_shard_id[NR_CPUS] __initdata; + +/** + * llc_count_cores - count distinct cores (SMT groups) within an LLC pod + * @pod_cpus: the cpumask of CPUs in the LLC pod + * @smt_pods: the SMT pod type, used to identify sibling groups + * + * A core is represented by the lowest-numbered CPU in its SMT group. Returns + * the number of distinct cores found in @pod_cpus. + */ +static int __init llc_count_cores(const struct cpumask *pod_cpus, + struct wq_pod_type *smt_pods) +{ + const struct cpumask *sibling_cpus; + int nr_cores = 0, c; + + /* + * Count distinct cores by only counting the first CPU in each + * SMT sibling group. + */ + for_each_cpu(c, pod_cpus) { + sibling_cpus = smt_pods->pod_cpus[smt_pods->cpu_pod[c]]; + if (cpumask_first(sibling_cpus) == c) + nr_cores++; + } + + return nr_cores; +} + +/* + * llc_shard_size - number of cores in a given shard + * + * Cores are spread as evenly as possible. The first @nr_large_shards shards are + * "large shards" with (cores_per_shard + 1) cores; the rest are "default + * shards" with cores_per_shard cores. + */ +static int __init llc_shard_size(int shard_id, int cores_per_shard, int nr_large_shards) +{ + /* The first @nr_large_shards shards are large shards */ + if (shard_id < nr_large_shards) + return cores_per_shard + 1; + + /* The remaining shards are default shards */ + return cores_per_shard; +} + +/* + * llc_calc_shard_layout - compute the shard layout for an LLC pod + * @nr_cores: number of distinct cores in the LLC pod + * + * Chooses the number of shards that keeps average shard size closest to + * wq_cache_shard_size. Returns a struct describing the total number of shards, + * the base size of each, and how many are large shards. + */ +static struct llc_shard_layout __init llc_calc_shard_layout(int nr_cores) +{ + struct llc_shard_layout layout; + + /* Ensure at least one shard; pick the count closest to the target size */ + layout.nr_shards = max(1, DIV_ROUND_CLOSEST(nr_cores, wq_cache_shard_size)); + layout.cores_per_shard = nr_cores / layout.nr_shards; + layout.nr_large_shards = nr_cores % layout.nr_shards; + + return layout; +} + +/* + * llc_shard_is_full - check whether a shard has reached its core capacity + * @cores_in_shard: number of cores already assigned to this shard + * @shard_id: index of the shard being checked + * @layout: the shard layout computed by llc_calc_shard_layout() + * + * Returns true if @cores_in_shard equals the expected size for @shard_id. + */ +static bool __init llc_shard_is_full(int cores_in_shard, int shard_id, + const struct llc_shard_layout *layout) +{ + return cores_in_shard == llc_shard_size(shard_id, layout->cores_per_shard, + layout->nr_large_shards); +} + +/** + * llc_populate_cpu_shard_id - populate cpu_shard_id[] for each CPU in an LLC pod + * @pod_cpus: the cpumask of CPUs in the LLC pod + * @smt_pods: the SMT pod type, used to identify sibling groups + * @nr_cores: number of distinct cores in @pod_cpus (from llc_count_cores()) + * + * Walks @pod_cpus in order. At each SMT group leader, advances to the next + * shard once the current shard is full. Results are written to cpu_shard_id[]. + */ +static void __init llc_populate_cpu_shard_id(const struct cpumask *pod_cpus, + struct wq_pod_type *smt_pods, + int nr_cores) +{ + struct llc_shard_layout layout = llc_calc_shard_layout(nr_cores); + const struct cpumask *sibling_cpus; + /* Count the number of cores in the current shard_id */ + int cores_in_shard = 0; + /* This is a cursor for the shards. Go from zero to nr_shards - 1*/ + int shard_id = 0; + int c; + + /* Iterate at every CPU for a given LLC pod, and assign it a shard */ + for_each_cpu(c, pod_cpus) { + sibling_cpus = smt_pods->pod_cpus[smt_pods->cpu_pod[c]]; + if (cpumask_first(sibling_cpus) == c) { + /* This is the CPU leader for the siblings */ + if (llc_shard_is_full(cores_in_shard, shard_id, &layout)) { + shard_id++; + cores_in_shard = 0; + } + cores_in_shard++; + cpu_shard_id[c] = shard_id; + } else { + /* + * The siblings' shard MUST be the same as the leader. + * never split threads in the same core. + */ + cpu_shard_id[c] = cpu_shard_id[cpumask_first(sibling_cpus)]; + } + } + + WARN_ON_ONCE(shard_id != (layout.nr_shards - 1)); +} + +/** + * precompute_cache_shard_ids - assign each CPU its shard index within its LLC + * + * Iterates over all LLC pods. For each pod, counts distinct cores then assigns + * shard indices to all CPUs in the pod. Must be called after WQ_AFFN_CACHE and + * WQ_AFFN_SMT have been initialized. + */ +static void __init precompute_cache_shard_ids(void) +{ + struct wq_pod_type *llc_pods = &wq_pod_types[WQ_AFFN_CACHE]; + struct wq_pod_type *smt_pods = &wq_pod_types[WQ_AFFN_SMT]; + const struct cpumask *cpus_sharing_llc; + int nr_cores; + int pod; + + if (!wq_cache_shard_size) { + pr_warn("workqueue: cache_shard_size must be > 0, setting to 1\n"); + wq_cache_shard_size = 1; + } + + for (pod = 0; pod < llc_pods->nr_pods; pod++) { + cpus_sharing_llc = llc_pods->pod_cpus[pod]; + + /* Number of cores in this given LLC */ + nr_cores = llc_count_cores(cpus_sharing_llc, smt_pods); + llc_populate_cpu_shard_id(cpus_sharing_llc, smt_pods, nr_cores); + } +} + +/* + * cpus_share_cache_shard - test whether two CPUs belong to the same cache shard + * + * Two CPUs share a cache shard if they are in the same LLC and have the same + * shard index. Used as the pod affinity callback for WQ_AFFN_CACHE_SHARD. + */ +static bool __init cpus_share_cache_shard(int cpu0, int cpu1) +{ + if (!cpus_share_cache(cpu0, cpu1)) + return false; + + return cpu_shard_id[cpu0] == cpu_shard_id[cpu1]; +} + /** * workqueue_init_topology - initialize CPU pods for unbound workqueues * @@ -8153,6 +8334,8 @@ void __init workqueue_init_topology(void) init_pod_type(&wq_pod_types[WQ_AFFN_CPU], cpus_dont_share); init_pod_type(&wq_pod_types[WQ_AFFN_SMT], cpus_share_smt); init_pod_type(&wq_pod_types[WQ_AFFN_CACHE], cpus_share_cache); + precompute_cache_shard_ids(); + init_pod_type(&wq_pod_types[WQ_AFFN_CACHE_SHARD], cpus_share_cache_shard); init_pod_type(&wq_pod_types[WQ_AFFN_NUMA], cpus_share_numa); wq_topo_initialized = true; From 91cd645b7f065f66adb1e57bd00c034918d76edf Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 1 Apr 2026 06:03:54 -0700 Subject: [PATCH 121/311] workqueue: set WQ_AFFN_CACHE_SHARD as the default affinity scope BugLink: https://bugs.launchpad.net/bugs/2150467 Set WQ_AFFN_CACHE_SHARD as the default affinity scope for unbound workqueues. On systems where many CPUs share one LLC, the previous default (WQ_AFFN_CACHE) collapses all CPUs to a single worker pool, causing heavy spinlock contention on pool->lock. WQ_AFFN_CACHE_SHARD subdivides each LLC into smaller groups, providing a better balance between locality and contention. Users can revert to the previous behavior with workqueue.default_affinity_scope=cache. On systems with 8 or fewer cores per LLC, CACHE_SHARD produces a single shard covering the entire LLC, making it functionally identical to the previous CACHE default. The sharding only activates when an LLC has more than 8 cores. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo (cherry picked from commit 4cdc8a7389d5025051f6c4a60fb5b7cb9b7960bb) Signed-off-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Matthew R. Ochs Signed-off-by: Brad Figg --- kernel/workqueue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index cdee3abcce97b..469e7628f0581 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -449,7 +449,7 @@ static bool wq_topo_initialized __read_mostly = false; static struct kmem_cache *pwq_cache; static struct wq_pod_type wq_pod_types[WQ_AFFN_NR_TYPES]; -static enum wq_affn_scope wq_affn_dfl = WQ_AFFN_CACHE; +static enum wq_affn_scope wq_affn_dfl = WQ_AFFN_CACHE_SHARD; /* buf for wq_update_unbound_pod_attrs(), protected by CPU hotplug exclusion */ static struct workqueue_attrs *unbound_wq_update_pwq_attrs_buf; From 710c8af89ce1770c3e3e7f2fe4285a0bf8c7b230 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 1 Apr 2026 06:03:55 -0700 Subject: [PATCH 122/311] tools/workqueue: add CACHE_SHARD support to wq_dump.py BugLink: https://bugs.launchpad.net/bugs/2150467 The WQ_AFFN_CACHE_SHARD affinity scope was added to the kernel but wq_dump.py was not updated to enumerate it. Add the missing constant lookup and include it in the affinity scopes iteration so that drgn output shows the CACHE_SHARD pod topology alongside the other scopes. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo (cherry picked from commit 738390a5321c7d34f468bc69f7232db711210bc0) Signed-off-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Matthew R. Ochs Signed-off-by: Brad Figg --- tools/workqueue/wq_dump.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/workqueue/wq_dump.py b/tools/workqueue/wq_dump.py index d29b918306b48..06948ffcfc4b6 100644 --- a/tools/workqueue/wq_dump.py +++ b/tools/workqueue/wq_dump.py @@ -107,6 +107,7 @@ def wq_type_str(wq): WQ_AFFN_CPU = prog['WQ_AFFN_CPU'] WQ_AFFN_SMT = prog['WQ_AFFN_SMT'] WQ_AFFN_CACHE = prog['WQ_AFFN_CACHE'] +WQ_AFFN_CACHE_SHARD = prog['WQ_AFFN_CACHE_SHARD'] WQ_AFFN_NUMA = prog['WQ_AFFN_NUMA'] WQ_AFFN_SYSTEM = prog['WQ_AFFN_SYSTEM'] @@ -138,7 +139,7 @@ def print_pod_type(pt): print(f' [{cpu}]={pt.cpu_pod[cpu].value_()}', end='') print('') -for affn in [WQ_AFFN_CPU, WQ_AFFN_SMT, WQ_AFFN_CACHE, WQ_AFFN_NUMA, WQ_AFFN_SYSTEM]: +for affn in [WQ_AFFN_CPU, WQ_AFFN_SMT, WQ_AFFN_CACHE, WQ_AFFN_CACHE_SHARD, WQ_AFFN_NUMA, WQ_AFFN_SYSTEM]: print('') print(f'{wq_affn_names[affn].string_().decode().upper()}{" (default)" if affn == wq_affn_dfl else ""}') print_pod_type(wq_pod_types[affn]) From 933150044cd3196c9d6c039e5aa7a40ee92c1f60 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 1 Apr 2026 06:03:56 -0700 Subject: [PATCH 123/311] workqueue: add test_workqueue benchmark module BugLink: https://bugs.launchpad.net/bugs/2150467 Add a kernel module that benchmarks queue_work() throughput on an unbound workqueue to measure pool->lock contention under different affinity scope configurations (cache vs cache_shard). The module spawns N kthreads (default: num_online_cpus()), each bound to a different CPU. All threads start simultaneously and queue work items, measuring the latency of each queue_work() call. Results are reported as p50/p90/p95 latencies for each affinity scope. The affinity scope is switched between runs via the workqueue's sysfs affinity_scope attribute (WQ_SYSFS), avoiding the need for any new exported symbols. The module runs as __init-only, returning -EAGAIN to auto-unload, and can be re-run via insmod. Example of the output: running 50 threads, 50000 items/thread cpu 6806017 items/sec p50=2574 p90=5068 p95=5818 ns smt 6821040 items/sec p50=2624 p90=5168 p95=5949 ns cache_shard 1633653 items/sec p50=5337 p90=9694 p95=11207 ns cache 286069 items/sec p50=72509 p90=82304 p95=85009 ns numa 319403 items/sec p50=63745 p90=73480 p95=76505 ns system 308461 items/sec p50=66561 p90=75714 p95=78048 ns Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo (cherry picked from commit 24b2e73f9700e0682575feb34556b756e59d4548) Signed-off-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Matthew R. Ochs Signed-off-by: Brad Figg --- lib/Kconfig.debug | 10 ++ lib/Makefile | 1 + lib/test_workqueue.c | 294 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 305 insertions(+) create mode 100644 lib/test_workqueue.c diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 93f356d2b3d95..38bee649697f3 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2628,6 +2628,16 @@ config TEST_VMALLOC If unsure, say N. +config TEST_WORKQUEUE + tristate "Test module for stress/performance analysis of workqueue" + default n + help + This builds the "test_workqueue" module for benchmarking + workqueue throughput under contention. Useful for evaluating + affinity scope changes (e.g., cache_shard vs cache). + + If unsure, say N. + config TEST_BPF tristate "Test BPF filter functionality" depends on m && NET diff --git a/lib/Makefile b/lib/Makefile index 1b9ee167517f3..ea660cca04f40 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -79,6 +79,7 @@ UBSAN_SANITIZE_test_ubsan.o := y obj-$(CONFIG_TEST_KSTRTOX) += test-kstrtox.o obj-$(CONFIG_TEST_LKM) += test_module.o obj-$(CONFIG_TEST_VMALLOC) += test_vmalloc.o +obj-$(CONFIG_TEST_WORKQUEUE) += test_workqueue.o obj-$(CONFIG_TEST_RHASHTABLE) += test_rhashtable.o obj-$(CONFIG_TEST_STATIC_KEYS) += test_static_keys.o obj-$(CONFIG_TEST_STATIC_KEYS) += test_static_key_base.o diff --git a/lib/test_workqueue.c b/lib/test_workqueue.c new file mode 100644 index 0000000000000..f2ae1ac4bd937 --- /dev/null +++ b/lib/test_workqueue.c @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Test module for stress and performance analysis of workqueue. + * + * Benchmarks queue_work() throughput on an unbound workqueue to measure + * pool->lock contention under different affinity scope configurations + * (e.g., cache vs cache_shard). + * + * The affinity scope is changed between runs via the workqueue's sysfs + * affinity_scope attribute (WQ_SYSFS). + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates + * Copyright (c) 2026 Breno Leitao + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define WQ_NAME "bench_wq" +#define SCOPE_PATH "/sys/bus/workqueue/devices/" WQ_NAME "/affinity_scope" + +static int nr_threads; +module_param(nr_threads, int, 0444); +MODULE_PARM_DESC(nr_threads, + "Number of threads to spawn (default: 0 = num_online_cpus())"); + +static int wq_items = 50000; +module_param(wq_items, int, 0444); +MODULE_PARM_DESC(wq_items, + "Number of work items each thread queues (default: 50000)"); + +static struct workqueue_struct *bench_wq; +static atomic_t threads_done; +static DECLARE_COMPLETION(start_comp); +static DECLARE_COMPLETION(all_done_comp); + +struct thread_ctx { + struct completion work_done; + struct work_struct work; + u64 *latencies; + int cpu; + int items; +}; + +static void bench_work_fn(struct work_struct *work) +{ + struct thread_ctx *ctx = container_of(work, struct thread_ctx, work); + + complete(&ctx->work_done); +} + +static int bench_kthread_fn(void *data) +{ + struct thread_ctx *ctx = data; + ktime_t t_start, t_end; + int i; + + /* Wait for all threads to be ready */ + wait_for_completion(&start_comp); + + if (kthread_should_stop()) + return 0; + + for (i = 0; i < ctx->items; i++) { + reinit_completion(&ctx->work_done); + INIT_WORK(&ctx->work, bench_work_fn); + + t_start = ktime_get(); + queue_work(bench_wq, &ctx->work); + t_end = ktime_get(); + + ctx->latencies[i] = ktime_to_ns(ktime_sub(t_end, t_start)); + wait_for_completion(&ctx->work_done); + } + + if (atomic_dec_and_test(&threads_done)) + complete(&all_done_comp); + + /* + * Wait for kthread_stop() so the module text isn't freed + * while we're still executing. + */ + while (!kthread_should_stop()) + schedule(); + + return 0; +} + +static int cmp_u64(const void *a, const void *b) +{ + u64 va = *(const u64 *)a; + u64 vb = *(const u64 *)b; + + if (va < vb) + return -1; + if (va > vb) + return 1; + return 0; +} + +static int __init set_affn_scope(const char *scope) +{ + struct file *f; + loff_t pos = 0; + ssize_t ret; + + f = filp_open(SCOPE_PATH, O_WRONLY, 0); + if (IS_ERR(f)) { + pr_err("test_workqueue: open %s failed: %ld\n", + SCOPE_PATH, PTR_ERR(f)); + return PTR_ERR(f); + } + + ret = kernel_write(f, scope, strlen(scope), &pos); + filp_close(f, NULL); + + if (ret < 0) { + pr_err("test_workqueue: write '%s' failed: %zd\n", scope, ret); + return ret; + } + + return 0; +} + +static int __init run_bench(int n_threads, const char *scope, const char *label) +{ + struct task_struct **tasks; + unsigned long total_items; + struct thread_ctx *ctxs; + u64 *all_latencies; + ktime_t start, end; + int cpu, i, j, ret; + s64 elapsed_us; + + ret = set_affn_scope(scope); + if (ret) + return ret; + + ctxs = kcalloc(n_threads, sizeof(*ctxs), GFP_KERNEL); + if (!ctxs) + return -ENOMEM; + + tasks = kcalloc(n_threads, sizeof(*tasks), GFP_KERNEL); + if (!tasks) { + kfree(ctxs); + return -ENOMEM; + } + + total_items = (unsigned long)n_threads * wq_items; + all_latencies = kvmalloc_array(total_items, sizeof(u64), GFP_KERNEL); + if (!all_latencies) { + kfree(tasks); + kfree(ctxs); + return -ENOMEM; + } + + /* Allocate per-thread latency arrays */ + for (i = 0; i < n_threads; i++) { + ctxs[i].latencies = kvmalloc_array(wq_items, sizeof(u64), + GFP_KERNEL); + if (!ctxs[i].latencies) { + while (--i >= 0) + kvfree(ctxs[i].latencies); + kvfree(all_latencies); + kfree(tasks); + kfree(ctxs); + return -ENOMEM; + } + } + + atomic_set(&threads_done, n_threads); + reinit_completion(&all_done_comp); + reinit_completion(&start_comp); + + /* Create kthreads, each bound to a different online CPU */ + i = 0; + for_each_online_cpu(cpu) { + if (i >= n_threads) + break; + + ctxs[i].cpu = cpu; + ctxs[i].items = wq_items; + init_completion(&ctxs[i].work_done); + + tasks[i] = kthread_create(bench_kthread_fn, &ctxs[i], + "wq_bench/%d", cpu); + if (IS_ERR(tasks[i])) { + ret = PTR_ERR(tasks[i]); + pr_err("test_workqueue: failed to create kthread %d: %d\n", + i, ret); + /* Unblock threads waiting on start_comp before stopping them */ + complete_all(&start_comp); + while (--i >= 0) + kthread_stop(tasks[i]); + goto out_free; + } + + kthread_bind(tasks[i], cpu); + wake_up_process(tasks[i]); + i++; + } + + /* Start timing and release all threads */ + start = ktime_get(); + complete_all(&start_comp); + + /* Wait for all threads to finish the benchmark */ + wait_for_completion(&all_done_comp); + + /* Drain any remaining work */ + flush_workqueue(bench_wq); + + /* Ensure all kthreads have fully exited before module memory is freed */ + for (i = 0; i < n_threads; i++) + kthread_stop(tasks[i]); + + end = ktime_get(); + elapsed_us = ktime_us_delta(end, start); + + /* Merge all per-thread latencies and sort for percentile calculation */ + j = 0; + for (i = 0; i < n_threads; i++) { + memcpy(&all_latencies[j], ctxs[i].latencies, + wq_items * sizeof(u64)); + j += wq_items; + } + + sort(all_latencies, total_items, sizeof(u64), cmp_u64, NULL); + + pr_info("test_workqueue: %-16s %llu items/sec\tp50=%llu\tp90=%llu\tp95=%llu ns\n", + label, + elapsed_us ? total_items * 1000000ULL / elapsed_us : 0, + all_latencies[total_items * 50 / 100], + all_latencies[total_items * 90 / 100], + all_latencies[total_items * 95 / 100]); + + ret = 0; +out_free: + for (i = 0; i < n_threads; i++) + kvfree(ctxs[i].latencies); + kvfree(all_latencies); + kfree(tasks); + kfree(ctxs); + + return ret; +} + +static const char * const bench_scopes[] = { + "cpu", "smt", "cache_shard", "cache", "numa", "system", +}; + +static int __init test_workqueue_init(void) +{ + int n_threads = min(nr_threads ?: num_online_cpus(), num_online_cpus()); + int i; + + if (wq_items <= 0) { + pr_err("test_workqueue: wq_items must be > 0\n"); + return -EINVAL; + } + + bench_wq = alloc_workqueue(WQ_NAME, WQ_UNBOUND | WQ_SYSFS, 0); + if (!bench_wq) + return -ENOMEM; + + pr_info("test_workqueue: running %d threads, %d items/thread\n", + n_threads, wq_items); + + for (i = 0; i < ARRAY_SIZE(bench_scopes); i++) + run_bench(n_threads, bench_scopes[i], bench_scopes[i]); + + destroy_workqueue(bench_wq); + + /* Return -EAGAIN so the module doesn't stay loaded after the benchmark */ + return -EAGAIN; +} + +module_init(test_workqueue_init); +MODULE_AUTHOR("Breno Leitao "); +MODULE_DESCRIPTION("Stress/performance benchmark for workqueue subsystem"); +MODULE_LICENSE("GPL"); From 68aca42394d4de475f46a4f18986cf994cf981c9 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 1 Apr 2026 06:03:57 -0700 Subject: [PATCH 124/311] docs: workqueue: document WQ_AFFN_CACHE_SHARD affinity scope BugLink: https://bugs.launchpad.net/bugs/2150467 Update kernel-parameters.txt and workqueue.rst to reflect the new cache_shard affinity scope and the default change from cache to cache_shard. Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo (cherry picked from commit 41e3ccca00b374b7f39cf68e818b59a921cd7069) Signed-off-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Matthew R. Ochs Signed-off-by: Brad Figg --- Documentation/admin-guide/kernel-parameters.txt | 3 ++- Documentation/core-api/workqueue.rst | 14 ++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index d3e7034bece52..a030ce253b4b7 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -8541,7 +8541,8 @@ Kernel parameters workqueue.default_affinity_scope= Select the default affinity scope to use for unbound workqueues. Can be one of "cpu", "smt", "cache", - "numa" and "system". Default is "cache". For more + "cache_shard", "numa" and "system". Default is + "cache_shard". For more information, see the Affinity Scopes section in Documentation/core-api/workqueue.rst. diff --git a/Documentation/core-api/workqueue.rst b/Documentation/core-api/workqueue.rst index 165ca73e83514..411e1b28b8dec 100644 --- a/Documentation/core-api/workqueue.rst +++ b/Documentation/core-api/workqueue.rst @@ -378,9 +378,9 @@ Affinity Scopes An unbound workqueue groups CPUs according to its affinity scope to improve cache locality. For example, if a workqueue is using the default affinity -scope of "cache", it will group CPUs according to last level cache -boundaries. A work item queued on the workqueue will be assigned to a worker -on one of the CPUs which share the last level cache with the issuing CPU. +scope of "cache_shard", it will group CPUs into sub-LLC shards. A work item +queued on the workqueue will be assigned to a worker on one of the CPUs +within the same shard as the issuing CPU. Once started, the worker may or may not be allowed to move outside the scope depending on the ``affinity_strict`` setting of the scope. @@ -402,7 +402,13 @@ Workqueue currently supports the following affinity scopes. ``cache`` CPUs are grouped according to cache boundaries. Which specific cache boundary is used is determined by the arch code. L3 is used in a lot of - cases. This is the default affinity scope. + cases. + +``cache_shard`` + CPUs are grouped into sub-LLC shards of at most ``wq_cache_shard_size`` + cores (default 8, tunable via the ``workqueue.cache_shard_size`` boot + parameter). Shards are always split on core (SMT group) boundaries. + This is the default affinity scope. ``numa`` CPUs are grouped according to NUMA boundaries. From 501aca5fb07bdb34d48309247c73b5839f58bbb7 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Thu, 2 Apr 2026 22:59:03 +0200 Subject: [PATCH 125/311] workqueue: avoid unguarded 64-bit division BugLink: https://bugs.launchpad.net/bugs/2150467 The printk() requires a division that is not allowed on 32-bit architectures: x86_64-linux-ld: lib/test_workqueue.o: in function `test_workqueue_init': test_workqueue.c:(.init.text+0x36f): undefined reference to `__udivdi3' Use div_u64() to print the resulting elapsed microseconds. Fixes: 24b2e73f9700 ("workqueue: add test_workqueue benchmark module") Signed-off-by: Arnd Bergmann Signed-off-by: Tejun Heo (cherry picked from commit c6890f36fc49848c61d2113a3442eb1b59e0bc4b) Signed-off-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Matthew R. Ochs Signed-off-by: Brad Figg --- lib/test_workqueue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/test_workqueue.c b/lib/test_workqueue.c index f2ae1ac4bd937..99e160bd5ad17 100644 --- a/lib/test_workqueue.c +++ b/lib/test_workqueue.c @@ -242,7 +242,7 @@ static int __init run_bench(int n_threads, const char *scope, const char *label) pr_info("test_workqueue: %-16s %llu items/sec\tp50=%llu\tp90=%llu\tp95=%llu ns\n", label, - elapsed_us ? total_items * 1000000ULL / elapsed_us : 0, + elapsed_us ? div_u64(total_items * 1000000ULL, elapsed_us) : 0, all_latencies[total_items * 50 / 100], all_latencies[total_items * 90 / 100], all_latencies[total_items * 95 / 100]); From 080eb2c499af543e4de17a0bdf85ed2175d7d6b5 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Mon, 13 Apr 2026 07:26:47 -0700 Subject: [PATCH 126/311] workqueue: validate cpumask_first() result in llc_populate_cpu_shard_id() BugLink: https://bugs.launchpad.net/bugs/2150467 On uniprocessor (UP) configs such as nios2, NR_CPUS is 1, so cpu_shard_id[] is a single-element array (int[1]). In llc_populate_cpu_shard_id(), cpumask_first(sibling_cpus) returns an unsigned int that the compiler cannot prove is always 0, triggering a -Warray-bounds warning when the result is used to index cpu_shard_id[]: kernel/workqueue.c:8321:55: warning: array subscript 1 is above array bounds of 'int[1]' [-Warray-bounds] 8321 | cpu_shard_id[c] = cpu_shard_id[cpumask_first(sibling_cpus)]; | ~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This is a false positive: sibling_cpus can never be empty here because 'c' itself is always set in it, so cpumask_first() will always return a valid CPU. However, the compiler cannot prove this statically, and the warning only manifests on UP configs where the array size is 1. Add a bounds check with WARN_ON_ONCE to silence the warning, and store the result in a local variable to make the code clearer and avoid calling cpumask_first() twice. Fixes: 5920d046f7ae ("workqueue: add WQ_AFFN_CACHE_SHARD affinity scope") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202604022343.GQtkF2vO-lkp@intel.com/ Signed-off-by: Breno Leitao Signed-off-by: Tejun Heo (cherry picked from commit 76af54648899abbd6b449c035583e47fd407078a) Signed-off-by: Carol L Soto Acked-by: Jamie Nguyen Acked-by: Matthew R. Ochs Signed-off-by: Brad Figg --- kernel/workqueue.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 469e7628f0581..a3309f41e4106 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -8249,6 +8249,7 @@ static void __init llc_populate_cpu_shard_id(const struct cpumask *pod_cpus, const struct cpumask *sibling_cpus; /* Count the number of cores in the current shard_id */ int cores_in_shard = 0; + unsigned int leader; /* This is a cursor for the shards. Go from zero to nr_shards - 1*/ int shard_id = 0; int c; @@ -8269,7 +8270,17 @@ static void __init llc_populate_cpu_shard_id(const struct cpumask *pod_cpus, * The siblings' shard MUST be the same as the leader. * never split threads in the same core. */ - cpu_shard_id[c] = cpu_shard_id[cpumask_first(sibling_cpus)]; + leader = cpumask_first(sibling_cpus); + + /* + * This check silences a Warray-bounds warning on UP + * configs where NR_CPUS=1 makes cpu_shard_id[] + * a single-element array, and the compiler can't + * prove the index is always 0. + */ + if (WARN_ON_ONCE(leader >= nr_cpu_ids)) + continue; + cpu_shard_id[c] = cpu_shard_id[leader]; } } From 0e8d7fb3d32bf8d2daf3152d6b7c277e37be0cee Mon Sep 17 00:00:00 2001 From: Abhishek Sahu Date: Mon, 27 Apr 2026 05:04:56 +0000 Subject: [PATCH 127/311] NVIDIA: SAUCE: iommu/arm-smmu-v3: Use device ID range for DGX Spark iGPU iommu quirk BugLink: https://bugs.launchpad.net/bugs/2150487 Replace the explicit DGX Spark iGPU device ID list with a range check covering 0x2E00-0x2E3F to accommodate all possible DGX Spark iGPU PCI device IDs without requiring individual additions. The original quirk was introduced in commit ab858638d96a ("NVIDIA: SAUCE: iommu/arm-smmu-v3: Set DGX Spark iGPU default domain type to DMA") and extended with two more IDs in commit 8dc61abaa2eb ("NVIDIA: SAUCE: iommu/arm-smmu-v3: Add two more DGX Spark iGPU IDs for existing iommu quirk"). Using a range avoids further per-ID additions as new DGX Spark variants are introduced. Signed-off-by: Abhishek Sahu Acked-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Nirmoy Das Signed-off-by: Brad Figg --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index c8595733d2fc8..d86e888d300b9 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3745,8 +3745,7 @@ static int arm_smmu_def_domain_type(struct device *dev) return IOMMU_DOMAIN_IDENTITY; if (pdev->vendor == PCI_VENDOR_ID_NVIDIA && - (pdev->device == 0x2E12 || pdev->device == 0x2E2A || - pdev->device == 0x2E2B)) + pdev->device >= 0x2E00 && pdev->device <= 0x2E3F) return IOMMU_DOMAIN_DMA; } From d6c06af50ef2844844721d2a6e3ea6a59e4fa508 Mon Sep 17 00:00:00 2001 From: "Matthew R. Ochs" Date: Fri, 1 May 2026 13:41:52 -0700 Subject: [PATCH 128/311] UBUNTU: [Config] nvidia: Disable default CMA reservation BugLink: https://bugs.launchpad.net/bugs/2150898 Set CONFIG_CMA_SIZE_MBYTES=0 for arm64 linux-nvidia kernels. Signed-off-by: Matthew R. Ochs Acked-by: Jamie Nguyen Acked-by: Carol L Soto Signed-off-by: Brad Figg --- debian.nvidia/config/annotations | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index a97ed74f91e3d..3368520a2a6b9 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -39,8 +39,8 @@ CONFIG_ARM_FFA_TRANSPORT note<'LP: #2111511'> CONFIG_ARM_SMMU_V3_IOMMUFD policy<{'arm64': 'y'}> CONFIG_ARM_SMMU_V3_IOMMUFD note<'LP: #2095028'> -CONFIG_CMA_SIZE_MBYTES policy<{'amd64': '0', 'arm64': '32', 'arm64-nvidia': '128', 'arm64-nvidia-64k': '1024'}> -CONFIG_CMA_SIZE_MBYTES note<'LP: #2095028'> +CONFIG_CMA_SIZE_MBYTES policy<{'amd64': '0', 'arm64': '0'}> +CONFIG_CMA_SIZE_MBYTES note<'LP: #2150898'> CONFIG_CORESIGHT policy<{'arm64': 'm'}> CONFIG_CORESIGHT note<'Required for Grace enablement'> From 17c683ab2c18df4758a2cbcedada31a0c3fe68d1 Mon Sep 17 00:00:00 2001 From: Carol L Soto Date: Wed, 6 May 2026 08:00:19 -0700 Subject: [PATCH 129/311] UBUNTU: [Config] nvidia: Defaults for CONFIG_TEST_WORKQUEUE BugLink: https://bugs.launchpad.net/bugs/2150467 Set defaults for CONFIG_TEST_WORKQUEUE. Signed-off-by: Carol L Soto Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Acked-by: Nirmoy Das Signed-off-by: Brad Figg --- debian.nvidia/config/annotations | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index 3368520a2a6b9..b70730cab4cf6 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -201,6 +201,9 @@ CONFIG_TCG_ARM_CRB_FFA note<'LP: #2111511'> CONFIG_TCG_TIS_SPI policy<{'amd64': 'm', 'arm64': 'y'}> CONFIG_TCG_TIS_SPI note<'Ensures the TPM is available before the IMA driver initializes'> +CONFIG_TEST_WORKQUEUE policy<{'amd64': 'n', 'arm64': 'n'}> +CONFIG_TEST_WORKQUEUE note<'LP: #2150467'> + CONFIG_UBUNTU_ODM_DRIVERS policy<{'amd64': 'n', 'arm64': 'n'}> CONFIG_UBUNTU_ODM_DRIVERS note<'Disable all Ubuntu ODM drivers'> From 84b6a6af590120ced3702529bbe9f62b5b037092 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Mon, 20 Apr 2026 09:24:01 +0200 Subject: [PATCH 130/311] NVIDIA: VR: SAUCE: sched/fair: Attach sched_domain_shared to sd_asym_cpucapacity BugLink: https://bugs.launchpad.net/bugs/2150671 On asymmetric CPU capacity systems, the wakeup path uses select_idle_capacity(), which scans the span of sd_asym_cpucapacity rather than sd_llc. The has_idle_cores hint however lives on sd_llc->shared, so the wakeup-time read of has_idle_cores operates on an LLC-scoped blob while the actual scan/decision spans the wider asym domain; nr_busy_cpus also lives in the same shared sched_domain data, but it's never used in the asym CPU capacity scenario. Therefore, move the sched_domain_shared object to sd_asym_cpucapacity whenever the CPU has a SD_ASYM_CPUCAPACITY_FULL ancestor and that ancestor is non-overlapping (i.e., not built from SD_NUMA). In that case the scope of has_idle_cores matches the scope of the wakeup scan. Fall back to attaching the shared object to sd_llc in three cases: 1) plain symmetric systems (no SD_ASYM_CPUCAPACITY_FULL anywhere); 2) CPUs in an exclusive cpuset that carves out a symmetric capacity island: has_asym is system-wide but those CPUs have no SD_ASYM_CPUCAPACITY_FULL ancestor in their hierarchy and follow the symmetric LLC path in select_idle_sibling(); 3) exotic topologies where SD_ASYM_CPUCAPACITY_FULL lands on an SD_NUMA-built domain. init_sched_domain_shared() keys the shared blob off cpumask_first(span), which on overlapping NUMA domains would alias unrelated spans onto the same blob. Keep the shared object on the LLC there; select_idle_capacity() gracefully skips the has_idle_cores preference when sd->shared is NULL. While at it, also rename the per-CPU sd_llc_shared to sd_balance_shared, as it is no longer strictly tied to the LLC. Co-developed-by: Andrea Righi Signed-off-by: Andrea Righi Signed-off-by: K Prateek Nayak (backported from https://lore.kernel.org/all/20260428051720.3180182-1-arighi@nvidia.com) [ arighi: - backport full logic to attach sd->shared in build_sched_domains() - do not rename sd_llc_shared to reduce the risk of conflicts ] Signed-off-by: Andrea Righi Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Acked-by: Carol L Soto Signed-off-by: Brad Figg --- kernel/sched/fair.c | 6 ++- kernel/sched/topology.c | 101 +++++++++++++++++++++++++++++++++------- 2 files changed, 89 insertions(+), 18 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ab4114712be74..bd2c6ebc85e7b 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -12617,7 +12617,8 @@ static void set_cpu_sd_state_busy(int cpu) goto unlock; sd->nohz_idle = 0; - atomic_inc(&sd->shared->nr_busy_cpus); + if (sd->shared) + atomic_inc(&sd->shared->nr_busy_cpus); unlock: rcu_read_unlock(); } @@ -12646,7 +12647,8 @@ static void set_cpu_sd_state_idle(int cpu) goto unlock; sd->nohz_idle = 1; - atomic_dec(&sd->shared->nr_busy_cpus); + if (sd->shared) + atomic_dec(&sd->shared->nr_busy_cpus); unlock: rcu_read_unlock(); } diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 32dcddaead82d..7bc2d13b3bf57 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -680,16 +680,38 @@ static void update_top_cache_domain(int cpu) int id = cpu; int size = 1; + sd = lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY_FULL); + /* + * The shared object is attached to sd_asym_cpucapacity only when the + * asym domain is non-overlapping (i.e., not built from SD_NUMA). + * On overlapping (NUMA) asym domains we fall back to letting the + * SD_SHARE_LLC path own the shared object, so sd->shared may be NULL + * here. + */ + if (sd && sd->shared) + sds = sd->shared; + + rcu_assign_pointer(per_cpu(sd_asym_cpucapacity, cpu), sd); + sd = highest_flag_domain(cpu, SD_SHARE_LLC); if (sd) { id = cpumask_first(sched_domain_span(sd)); size = cpumask_weight(sched_domain_span(sd)); - sds = sd->shared; + + /* + * If sd_asym_cpucapacity didn't claim the shared object, + * sd_llc must have one linked. + */ + if (!sds) { + WARN_ON_ONCE(!sd->shared); + sds = sd->shared; + } } rcu_assign_pointer(per_cpu(sd_llc, cpu), sd); per_cpu(sd_llc_size, cpu) = size; per_cpu(sd_llc_id, cpu) = id; + rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds); sd = lowest_flag_domain(cpu, SD_CLUSTER); @@ -708,9 +730,6 @@ static void update_top_cache_domain(int cpu) sd = highest_flag_domain(cpu, SD_ASYM_PACKING); rcu_assign_pointer(per_cpu(sd_asym_packing, cpu), sd); - - sd = lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY_FULL); - rcu_assign_pointer(per_cpu(sd_asym_cpucapacity, cpu), sd); } /* @@ -1640,7 +1659,7 @@ sd_init(struct sched_domain_topology_level *tl, { struct sd_data *sdd = &tl->data; struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu); - int sd_id, sd_weight, sd_flags = 0; + int sd_weight, sd_flags = 0; struct cpumask *sd_span; sd_weight = cpumask_weight(tl->mask(tl, cpu)); @@ -1688,7 +1707,6 @@ sd_init(struct sched_domain_topology_level *tl, sd_span = sched_domain_span(sd); cpumask_and(sd_span, cpu_map, tl->mask(tl, cpu)); - sd_id = cpumask_first(sd_span); sd->flags |= asym_cpu_capacity_classify(sd_span, cpu_map); @@ -1727,16 +1745,6 @@ sd_init(struct sched_domain_topology_level *tl, sd->cache_nice_tries = 1; } - /* - * For all levels sharing cache; connect a sched_domain_shared - * instance. - */ - if (sd->flags & SD_SHARE_LLC) { - sd->shared = *per_cpu_ptr(sdd->sds, sd_id); - atomic_inc(&sd->shared->ref); - atomic_set(&sd->shared->nr_busy_cpus, sd_weight); - } - sd->private = sdd; return sd; @@ -2548,6 +2556,16 @@ static bool topology_span_sane(const struct cpumask *cpu_map) return true; } +static void init_sched_domain_shared(struct sched_domain *sd) +{ + struct sd_data *sdd = sd->private; + int sd_id = cpumask_first(sched_domain_span(sd)); + + sd->shared = *per_cpu_ptr(sdd->sds, sd_id); + atomic_set(&sd->shared->nr_busy_cpus, sd->span_weight); + atomic_inc(&sd->shared->ref); +} + /* * Build sched domains for a given set of CPUs and attach the sched domains * to the individual CPUs @@ -2605,6 +2623,57 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att } } + for_each_cpu(i, cpu_map) { + struct sched_domain *sd_asym = NULL; + bool asym_claimed = false; + + sd = *per_cpu_ptr(d.sd, i); + if (!sd) + continue; + + /* + * In case of ASYM_CPUCAPACITY, attach sd->shared to + * sd_asym_cpucapacity for wakeup stat tracking. + * + * Caveats: + * + * 1) has_asym is system-wide, but a given CPU may still + * lack an SD_ASYM_CPUCAPACITY_FULL ancestor (e.g., an + * exclusive cpuset carving out a symmetric capacity island). + * Such CPUs must fall through to the LLC seeding path below. + * + * 2) Skip the asym attach if the asym ancestor is an + * overlapping domain (SD_NUMA). On those topologies let the + * LLC path own the shared object instead. + * + * XXX: This assumes SD_ASYM_CPUCAPACITY_FULL domain + * always has more than one group else it is prone to + * degeneration. + */ + sd_asym = sd; + while (sd_asym && !(sd_asym->flags & SD_ASYM_CPUCAPACITY_FULL)) + sd_asym = sd_asym->parent; + + if (sd_asym && !(sd_asym->flags & SD_NUMA)) { + init_sched_domain_shared(sd_asym); + asym_claimed = true; + } + + /* First, find the topmost SD_SHARE_LLC domain */ + sd = *per_cpu_ptr(d.sd, i); + while (sd->parent && (sd->parent->flags & SD_SHARE_LLC)) + sd = sd->parent; + + if (sd->flags & SD_SHARE_LLC) { + /* + * Initialize the sd->shared for SD_SHARE_LLC unless + * the asym path above already claimed it. + */ + if (!asym_claimed) + init_sched_domain_shared(sd); + } + } + /* * Calculate an allowed NUMA imbalance such that LLCs do not get * imbalanced. From c2f6baf5d80262dbe2a5e1455671ff17a875f3e0 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Wed, 11 Mar 2026 18:43:19 +0100 Subject: [PATCH 131/311] NVIDIA: VR: SAUCE: sched/fair: Prefer fully-idle SMT cores in asym-capacity idle selection BugLink: https://bugs.launchpad.net/bugs/2150671 On systems with asymmetric CPU capacity (e.g., ACPI/CPPC reporting different per-core frequencies), the wakeup path uses select_idle_capacity() and prioritizes idle CPUs with higher capacity for better task placement. However, when those CPUs belong to SMT cores, their effective capacity can be much lower than the nominal capacity when the sibling thread is busy: SMT siblings compete for shared resources, so a "high capacity" CPU that is idle but whose sibling is busy does not deliver its full capacity. This effective capacity reduction cannot be modeled by the static capacity value alone. Introduce SMT awareness in the asym-capacity idle selection policy: when SMT is active, always prefer fully-idle SMT cores over partially-idle ones. Prioritizing fully-idle SMT cores yields better task placement because the effective capacity of partially-idle SMT cores is reduced; always preferring them when available leads to more accurate capacity usage on task wakeup. On an SMT system with asymmetric CPU capacities, SMT-aware idle selection has been shown to improve throughput by around 15-18% for CPU-bound workloads, running an amount of tasks equal to the amount of SMT cores. Cc: Vincent Guittot Cc: Dietmar Eggemann Cc: Christian Loehle Cc: Koba Ko Reviewed-by: K Prateek Nayak Reported-by: Felix Abecassis Signed-off-by: Andrea Righi (cherry picked from https://lore.kernel.org/all/20260428051720.3180182-1-arighi@nvidia.com) Signed-off-by: Andrea Righi Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Acked-by: Carol L Soto Signed-off-by: Brad Figg --- kernel/sched/fair.c | 70 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 5 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index bd2c6ebc85e7b..e4393d393f70d 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -7762,6 +7762,22 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool return idle_cpu; } +/* + * Idle-capacity scan ranks transformed util_fits_cpu() outcomes; lower values + * are more preferred (see select_idle_capacity()). + */ +enum asym_fits_state { + /* In descending order of preference */ + ASYM_IDLE_CORE_UCLAMP_MISFIT = -4, + ASYM_IDLE_CORE_COMPLETE_MISFIT, + ASYM_IDLE_THREAD_FITS, + ASYM_IDLE_THREAD_UCLAMP_MISFIT, + ASYM_IDLE_COMPLETE_MISFIT, + + /* asym_fits_cpu() bias for an idle core. */ + ASYM_IDLE_CORE_BIAS = -3, +}; + /* * Scan the asym_capacity domain for idle CPUs; pick the first idle one on which * the task fits. If no CPU is big enough, but there are idle ones, try to @@ -7770,8 +7786,9 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool static int select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) { + bool prefers_idle_core = sched_smt_active() && test_idle_cores(target); unsigned long task_util, util_min, util_max, best_cap = 0; - int fits, best_fits = 0; + int fits, best_fits = ASYM_IDLE_COMPLETE_MISFIT; int cpu, best_cpu = -1; struct cpumask *cpus; @@ -7783,6 +7800,7 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) util_max = uclamp_eff_value(p, UCLAMP_MAX); for_each_cpu_wrap(cpu, cpus, target) { + bool preferred_core = !prefers_idle_core || is_core_idle(cpu); unsigned long cpu_cap = capacity_of(cpu); if (!available_idle_cpu(cpu) && !sched_idle_cpu(cpu)) @@ -7791,7 +7809,7 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) fits = util_fits_cpu(task_util, util_min, util_max, cpu); /* This CPU fits with all requirements */ - if (fits > 0) + if (fits > 0 && preferred_core) return cpu; /* * Only the min performance hint (i.e. uclamp_min) doesn't fit. @@ -7799,9 +7817,33 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) */ else if (fits < 0) cpu_cap = get_actual_cpu_capacity(cpu); + /* + * fits > 0 implies we are not on a preferred core + * but the util fits CPU capacity. Set fits to ASYM_IDLE_THREAD_FITS + * so the effective range becomes + * [ASYM_IDLE_THREAD_FITS, ASYM_IDLE_COMPLETE_MISFIT] where: + * ASYM_IDLE_COMPLETE_MISFIT - does not fit + * ASYM_IDLE_THREAD_UCLAMP_MISFIT - fits with the exception of UCLAMP_MIN + * ASYM_IDLE_THREAD_FITS - fits with the exception of preferred_core + */ + else if (fits > 0) + fits = ASYM_IDLE_THREAD_FITS; + + /* + * If we are on a preferred core, translate the range of fits + * of [ASYM_IDLE_THREAD_UCLAMP_MISFIT, ASYM_IDLE_COMPLETE_MISFIT] to + * [ASYM_IDLE_CORE_UCLAMP_MISFIT, ASYM_IDLE_CORE_COMPLETE_MISFIT]. + * This ensures that an idle core is always given priority over + * (partially) busy core. + * + * A fully fitting idle core would have returned early and hence + * fits > 0 for preferred_core need not be dealt with. + */ + if (preferred_core) + fits += ASYM_IDLE_CORE_BIAS; /* - * First, select CPU which fits better (-1 being better than 0). + * First, select CPU which fits better (lower is more preferred). * Then, select the one with best capacity at same level. */ if ((fits < best_fits) || @@ -7812,6 +7854,19 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) } } + /* + * A value in the [ASYM_IDLE_CORE_UCLAMP_MISFIT, ASYM_IDLE_CORE_BIAS] + * range means the chosen CPU is in a fully idle SMT core. Values above + * ASYM_IDLE_CORE_BIAS mean we never ranked such a CPU best. + * + * The asym-capacity wakeup path returns from select_idle_sibling() + * after this function and never runs select_idle_cpu(), so the usual + * select_idle_cpu() tail that clears idle cores must live here when the + * idle-core preference did not win. + */ + if (prefers_idle_core && best_fits > ASYM_IDLE_CORE_BIAS) + set_idle_cores(target, false); + return best_cpu; } @@ -7820,12 +7875,17 @@ static inline bool asym_fits_cpu(unsigned long util, unsigned long util_max, int cpu) { - if (sched_asym_cpucap_active()) + if (sched_asym_cpucap_active()) { /* * Return true only if the cpu fully fits the task requirements * which include the utilization and the performance hints. + * + * When SMT is active, also require that the core has no busy + * siblings. */ - return (util_fits_cpu(util, util_min, util_max, cpu) > 0); + return (!sched_smt_active() || is_core_idle(cpu)) && + (util_fits_cpu(util, util_min, util_max, cpu) > 0); + } return true; } From 468da0828edc2c1dd92ac3941e132489ed0908fe Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Wed, 25 Mar 2026 16:39:32 +0100 Subject: [PATCH 132/311] NVIDIA: VR: SAUCE: sched/fair: Reject misfit pulls onto busy SMT siblings on asym-capacity BugLink: https://bugs.launchpad.net/bugs/2150671 When SD_ASYM_CPUCAPACITY load balancing considers pulling a misfit task, capacity_of(dst_cpu) can overstate available compute if the SMT sibling is busy: the core does not deliver its full nominal capacity. If SMT is active and dst_cpu is not on a fully idle core, skip this destination so we do not migrate a misfit expecting a capacity upgrade we cannot actually provide. Cc: Vincent Guittot Cc: Dietmar Eggemann Cc: Christian Loehle Cc: Koba Ko Cc: K Prateek Nayak Reported-by: Felix Abecassis Signed-off-by: Andrea Righi (cherry picked from https://lore.kernel.org/all/20260428051720.3180182-1-arighi@nvidia.com) Signed-off-by: Andrea Righi Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Acked-by: Carol L Soto Signed-off-by: Brad Figg --- kernel/sched/fair.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index e4393d393f70d..3513c47f94c8f 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -9390,6 +9390,7 @@ struct lb_env { int dst_cpu; struct rq *dst_rq; + bool dst_core_idle; struct cpumask *dst_grpmask; int new_dst_cpu; @@ -10635,10 +10636,16 @@ static bool update_sd_pick_busiest(struct lb_env *env, * We can use max_capacity here as reduction in capacity on some * CPUs in the group should either be possible to resolve * internally or be covered by avg_load imbalance (eventually). + * + * When SMT is active, only pull a misfit to dst_cpu if it is on a + * fully idle core; otherwise the effective capacity of the core is + * reduced and we may not actually provide more capacity than the + * source. */ if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && (sgs->group_type == group_misfit_task) && - (!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) || + (!env->dst_core_idle || + !capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) || sds->local_stat.group_type != group_has_spare)) return false; @@ -11204,6 +11211,8 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd unsigned long sum_util = 0; bool sg_overloaded = 0, sg_overutilized = 0; + env->dst_core_idle = !sched_smt_active() || is_core_idle(env->dst_cpu); + do { struct sg_lb_stats *sgs = &tmp_sgs; int local_group; From dc8871d83e23210487a26f7b003c64b72e02eb83 Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Tue, 21 Apr 2026 16:52:46 +0530 Subject: [PATCH 133/311] NVIDIA: VR: SAUCE: sched/fair: Add SIS_UTIL support to select_idle_capacity() BugLink: https://bugs.launchpad.net/bugs/2150671 Add to select_idle_capacity() the same SIS_UTIL-controlled idle-scan mechanism, already used by select_idle_cpu(): when sched_feat(SIS_UTIL) is enabled and the LLC domain has sched_domain_shared data, derive the per-attempt scan limit from sd->shared->nr_idle_scan. That bounds the walk on large LLCs and allows an early return once the scan limit is reached, if we already picked a sufficiently strong idle-core candidate (best_fits == ASYM_IDLE_CORE_UCLAMP_MISFIT). Co-developed-by: Andrea Righi Signed-off-by: Andrea Righi Signed-off-by: K Prateek Nayak (cherry picked from https://lore.kernel.org/all/20260428051720.3180182-1-arighi@nvidia.com) Signed-off-by: Andrea Righi Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Acked-by: Carol L Soto Signed-off-by: Brad Figg --- kernel/sched/fair.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 3513c47f94c8f..82714027a6564 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -7791,6 +7791,7 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) int fits, best_fits = ASYM_IDLE_COMPLETE_MISFIT; int cpu, best_cpu = -1; struct cpumask *cpus; + int nr = INT_MAX; cpus = this_cpu_cpumask_var_ptr(select_rq_mask); cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); @@ -7799,10 +7800,28 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) util_min = uclamp_eff_value(p, UCLAMP_MIN); util_max = uclamp_eff_value(p, UCLAMP_MAX); + if (sched_feat(SIS_UTIL) && sd->shared) { + /* + * Same nr_idle_scan hint as select_idle_cpu(), nr only limits + * the scan when not preferring an idle core. + */ + nr = READ_ONCE(sd->shared->nr_idle_scan) + 1; + /* overloaded domain is unlikely to have idle cpu/core */ + if (nr == 1) + return -1; + } + for_each_cpu_wrap(cpu, cpus, target) { bool preferred_core = !prefers_idle_core || is_core_idle(cpu); unsigned long cpu_cap = capacity_of(cpu); + /* + * Good-enough early exit (mirrors select_idle_cpu() logic). + */ + if (!prefers_idle_core && + --nr <= 0 && best_fits == ASYM_IDLE_CORE_UCLAMP_MISFIT) + return best_cpu; + if (!available_idle_cpu(cpu) && !sched_idle_cpu(cpu)) continue; From b3aea101c0e3bfa6a3bd64966e4b6559870de5ea Mon Sep 17 00:00:00 2001 From: Nicolai Buchwitz Date: Mon, 23 Feb 2026 09:54:42 +0100 Subject: [PATCH 134/311] net: microchip: lan743x: add ethtool nway_reset support BugLink: https://bugs.launchpad.net/bugs/2152064 Wire phylink_ethtool_nway_reset() as the .nway_reset ethtool operation, allowing userspace to restart PHY autonegotiation via 'ethtool -r'. Signed-off-by: Nicolai Buchwitz Reviewed-by: Russel King (Oracle) Link: https://patch.msgid.link/20260223085442.42852-1-nb@tipi-net.de Signed-off-by: Paolo Abeni (cherry picked from commit 8636385b9f0175220318bc23f6926b3fd013b131) Signed-off-by: David Thompson Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/net/ethernet/microchip/lan743x_ethtool.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/net/ethernet/microchip/lan743x_ethtool.c b/drivers/net/ethernet/microchip/lan743x_ethtool.c index 8a3c1ecc7866c..9195419ecee02 100644 --- a/drivers/net/ethernet/microchip/lan743x_ethtool.c +++ b/drivers/net/ethernet/microchip/lan743x_ethtool.c @@ -1079,6 +1079,13 @@ static int lan743x_ethtool_set_eee(struct net_device *netdev, return phylink_ethtool_set_eee(adapter->phylink, eee); } +static int lan743x_ethtool_nway_reset(struct net_device *netdev) +{ + struct lan743x_adapter *adapter = netdev_priv(netdev); + + return phylink_ethtool_nway_reset(adapter->phylink); +} + static int lan743x_ethtool_set_link_ksettings(struct net_device *netdev, const struct ethtool_link_ksettings *cmd) @@ -1369,6 +1376,7 @@ const struct ethtool_ops lan743x_ethtool_ops = { .set_rxfh = lan743x_ethtool_set_rxfh, .get_rxfh_fields = lan743x_ethtool_get_rxfh_fields, .get_ts_info = lan743x_ethtool_get_ts_info, + .nway_reset = lan743x_ethtool_nway_reset, .get_eee = lan743x_ethtool_get_eee, .set_eee = lan743x_ethtool_set_eee, .get_link_ksettings = lan743x_ethtool_get_link_ksettings, From 2099e2c530e7a950535dcc043f4f9abc617b1f69 Mon Sep 17 00:00:00 2001 From: Thangaraj Samynathan Date: Wed, 18 Mar 2026 12:02:28 +0530 Subject: [PATCH 135/311] net: lan743x: fix SGMII detection on PCI1xxxx B0+ during warm reset BugLink: https://bugs.launchpad.net/bugs/2152064 A warm reset on boards using an EEPROM-only strap configuration (where no MAC address is set in the image) can cause the driver to incorrectly revert to RGMII mode. This occurs because the ENET_CONFIG_LOAD_STARTED bit may not persist or behave as expected. Update pci11x1x_strap_get_status() to use revision-specific validation: - For PCI11x1x A0: Continue using the legacy check (config load started or reset protection) to validate the SGMII strap. - For PCI11x1x B0 and later: Use the newly available STRAP_READ_USE_SGMII_EN_ bit in the upper strap register to validate the lower SGMII_EN bit. This ensures the SGMII interface is correctly identified even after a warm reboot. Signed-off-by: Thangaraj Samynathan Link: https://patch.msgid.link/20260318063228.17110-1-thangaraj.s@microchip.com Signed-off-by: Jakub Kicinski (cherry picked from commit e783e40fb689381caca31e03d28c39e10c82e722) Signed-off-by: David Thompson Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/net/ethernet/microchip/lan743x_main.c | 15 +++++++++++---- drivers/net/ethernet/microchip/lan743x_main.h | 1 + 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c index f0b5dd752f084..b4cabde6625a2 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.c +++ b/drivers/net/ethernet/microchip/lan743x_main.c @@ -28,6 +28,12 @@ #define RFE_RD_FIFO_TH_3_DWORDS 0x3 +static bool pci11x1x_is_a0(struct lan743x_adapter *adapter) +{ + u32 dev_rev = adapter->csr.id_rev & ID_REV_CHIP_REV_MASK_; + return dev_rev == ID_REV_CHIP_REV_PCI11X1X_A0_; +} + static void pci11x1x_strap_get_status(struct lan743x_adapter *adapter) { u32 chip_rev; @@ -47,10 +53,11 @@ static void pci11x1x_strap_get_status(struct lan743x_adapter *adapter) cfg_load = lan743x_csr_read(adapter, ETH_SYS_CONFIG_LOAD_STARTED_REG); lan743x_hs_syslock_release(adapter); hw_cfg = lan743x_csr_read(adapter, HW_CFG); - - if (cfg_load & GEN_SYS_LOAD_STARTED_REG_ETH_ || - hw_cfg & HW_CFG_RST_PROTECT_) { - strap = lan743x_csr_read(adapter, STRAP_READ); + strap = lan743x_csr_read(adapter, STRAP_READ); + if ((pci11x1x_is_a0(adapter) && + (cfg_load & GEN_SYS_LOAD_STARTED_REG_ETH_ || + hw_cfg & HW_CFG_RST_PROTECT_)) || + (strap & STRAP_READ_USE_SGMII_EN_)) { if (strap & STRAP_READ_SGMII_EN_) adapter->is_sgmii_en = true; else diff --git a/drivers/net/ethernet/microchip/lan743x_main.h b/drivers/net/ethernet/microchip/lan743x_main.h index 02a28b7091630..160d94a7cee66 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.h +++ b/drivers/net/ethernet/microchip/lan743x_main.h @@ -27,6 +27,7 @@ #define ID_REV_CHIP_REV_MASK_ (0x0000FFFF) #define ID_REV_CHIP_REV_A0_ (0x00000000) #define ID_REV_CHIP_REV_B0_ (0x00000010) +#define ID_REV_CHIP_REV_PCI11X1X_A0_ (0x000000A0) #define ID_REV_CHIP_REV_PCI11X1X_B0_ (0x000000B0) #define FPGA_REV (0x04) From bf173b212a4b2b42d2dcb57dacdf7bd2b22c48c9 Mon Sep 17 00:00:00 2001 From: Thangaraj Samynathan Date: Fri, 10 Apr 2026 14:27:10 +0530 Subject: [PATCH 136/311] net: lan743x: rename chip_rev to fpga_rev BugLink: https://bugs.launchpad.net/bugs/2152064 The variable chip_rev stores the value read from the FPGA_REV register and represents the FPGA revision. Rename it to fpga_rev to better reflect its meaning. No functional change intended. Signed-off-by: Thangaraj Samynathan Link: https://patch.msgid.link/20260410085710.9246-1-thangaraj.s@microchip.com Signed-off-by: Jakub Kicinski (cherry picked from commit 469faa546e7a82be85114e322cec6438790870ff) Signed-off-by: David Thompson Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/net/ethernet/microchip/lan743x_main.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c index b4cabde6625a2..f3332417162e6 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.c +++ b/drivers/net/ethernet/microchip/lan743x_main.c @@ -36,7 +36,7 @@ static bool pci11x1x_is_a0(struct lan743x_adapter *adapter) static void pci11x1x_strap_get_status(struct lan743x_adapter *adapter) { - u32 chip_rev; + u32 fpga_rev; u32 cfg_load; u32 hw_cfg; u32 strap; @@ -63,9 +63,9 @@ static void pci11x1x_strap_get_status(struct lan743x_adapter *adapter) else adapter->is_sgmii_en = false; } else { - chip_rev = lan743x_csr_read(adapter, FPGA_REV); - if (chip_rev) { - if (chip_rev & FPGA_SGMII_OP) + fpga_rev = lan743x_csr_read(adapter, FPGA_REV); + if (fpga_rev) { + if (fpga_rev & FPGA_SGMII_OP) adapter->is_sgmii_en = true; else adapter->is_sgmii_en = false; From 462b6f0bb2d9bb9d92585190437eb6e7aa0d89cf Mon Sep 17 00:00:00 2001 From: Sumit Gupta Date: Fri, 6 Feb 2026 19:56:52 +0530 Subject: [PATCH 137/311] ACPI: CPPC: Add cppc_get_perf() API to read performance controls BugLink: https://bugs.launchpad.net/bugs/2131705 Add cppc_get_perf() function to read values of performance control registers including desired_perf, min_perf, max_perf, energy_perf, and auto_sel. This provides a read interface to complement the existing cppc_set_perf() write interface for performance control registers. Note that auto_sel is read by cppc_get_perf() but not written by cppc_set_perf() to avoid unintended mode changes during performance updates. It can be updated with existing dedicated cppc_set_auto_sel() API. Use cppc_get_perf() in cppc_cpufreq_get_cpu_data() to initialize perf_ctrls with current hardware register values during cpufreq policy initialization. Signed-off-by: Sumit Gupta Reviewed-by: Pierre Gondois Reviewed-by: Lifeng Zheng Link: https://patch.msgid.link/20260206142658.72583-2-sumitg@nvidia.com Signed-off-by: Rafael J. Wysocki (cherry picked from commit 658fa7b1c47a857af484c5c5dff8d0164b7c7bfb) Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/acpi/cppc_acpi.c | 80 ++++++++++++++++++++++++++++++++++ drivers/cpufreq/cppc_cpufreq.c | 6 +++ include/acpi/cppc_acpi.h | 5 +++ 3 files changed, 91 insertions(+) diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index f0e513e9ed5d3..5122e99bddb09 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -1738,6 +1738,86 @@ int cppc_set_enable(int cpu, bool enable) } EXPORT_SYMBOL_GPL(cppc_set_enable); +/** + * cppc_get_perf - Get a CPU's performance controls. + * @cpu: CPU for which to get performance controls. + * @perf_ctrls: ptr to cppc_perf_ctrls. See cppc_acpi.h + * + * Return: 0 for success with perf_ctrls, -ERRNO otherwise. + */ +int cppc_get_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) +{ + struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu); + struct cpc_register_resource *desired_perf_reg, + *min_perf_reg, *max_perf_reg, + *energy_perf_reg, *auto_sel_reg; + u64 desired_perf = 0, min = 0, max = 0, energy_perf = 0, auto_sel = 0; + int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu); + struct cppc_pcc_data *pcc_ss_data = NULL; + int ret = 0, regs_in_pcc = 0; + + if (!cpc_desc) { + pr_debug("No CPC descriptor for CPU:%d\n", cpu); + return -ENODEV; + } + + if (!perf_ctrls) { + pr_debug("Invalid perf_ctrls pointer\n"); + return -EINVAL; + } + + desired_perf_reg = &cpc_desc->cpc_regs[DESIRED_PERF]; + min_perf_reg = &cpc_desc->cpc_regs[MIN_PERF]; + max_perf_reg = &cpc_desc->cpc_regs[MAX_PERF]; + energy_perf_reg = &cpc_desc->cpc_regs[ENERGY_PERF]; + auto_sel_reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE]; + + /* Are any of the regs PCC ?*/ + if (CPC_IN_PCC(desired_perf_reg) || CPC_IN_PCC(min_perf_reg) || + CPC_IN_PCC(max_perf_reg) || CPC_IN_PCC(energy_perf_reg) || + CPC_IN_PCC(auto_sel_reg)) { + if (pcc_ss_id < 0) { + pr_debug("Invalid pcc_ss_id for CPU:%d\n", cpu); + return -ENODEV; + } + pcc_ss_data = pcc_data[pcc_ss_id]; + regs_in_pcc = 1; + down_write(&pcc_ss_data->pcc_lock); + /* Ring doorbell once to update PCC subspace */ + if (send_pcc_cmd(pcc_ss_id, CMD_READ) < 0) { + ret = -EIO; + goto out_err; + } + } + + /* Read optional elements if present */ + if (CPC_SUPPORTED(max_perf_reg)) + cpc_read(cpu, max_perf_reg, &max); + perf_ctrls->max_perf = max; + + if (CPC_SUPPORTED(min_perf_reg)) + cpc_read(cpu, min_perf_reg, &min); + perf_ctrls->min_perf = min; + + if (CPC_SUPPORTED(desired_perf_reg)) + cpc_read(cpu, desired_perf_reg, &desired_perf); + perf_ctrls->desired_perf = desired_perf; + + if (CPC_SUPPORTED(energy_perf_reg)) + cpc_read(cpu, energy_perf_reg, &energy_perf); + perf_ctrls->energy_perf = energy_perf; + + if (CPC_SUPPORTED(auto_sel_reg)) + cpc_read(cpu, auto_sel_reg, &auto_sel); + perf_ctrls->auto_sel = (bool)auto_sel; + +out_err: + if (regs_in_pcc) + up_write(&pcc_ss_data->pcc_lock); + return ret; +} +EXPORT_SYMBOL_GPL(cppc_get_perf); + /** * cppc_set_perf - Set a CPU's performance controls. * @cpu: CPU for which to set performance controls. diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index 011f35cb47b94..a61a24e0dcaed 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -594,6 +594,12 @@ static struct cppc_cpudata *cppc_cpufreq_get_cpu_data(unsigned int cpu) goto free_mask; } + ret = cppc_get_perf(cpu, &cpu_data->perf_ctrls); + if (ret) { + pr_debug("Err reading CPU%d perf ctrls: ret:%d\n", cpu, ret); + goto free_mask; + } + return cpu_data; free_mask: diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h index 4d644f03098e3..3fc796c0d9022 100644 --- a/include/acpi/cppc_acpi.h +++ b/include/acpi/cppc_acpi.h @@ -151,6 +151,7 @@ extern int cppc_get_desired_perf(int cpunum, u64 *desired_perf); extern int cppc_get_nominal_perf(int cpunum, u64 *nominal_perf); extern int cppc_get_highest_perf(int cpunum, u64 *highest_perf); extern int cppc_get_perf_ctrs(int cpu, struct cppc_perf_fb_ctrs *perf_fb_ctrs); +extern int cppc_get_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls); extern int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls); extern int cppc_set_enable(int cpu, bool enable); extern int cppc_get_perf_caps(int cpu, struct cppc_perf_caps *caps); @@ -193,6 +194,10 @@ static inline int cppc_get_perf_ctrs(int cpu, struct cppc_perf_fb_ctrs *perf_fb_ { return -EOPNOTSUPP; } +static inline int cppc_get_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) +{ + return -EOPNOTSUPP; +} static inline int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) { return -EOPNOTSUPP; From 17327b9e142183003798142db59c5733b5aa5bb8 Mon Sep 17 00:00:00 2001 From: Sumit Gupta Date: Fri, 6 Feb 2026 19:56:53 +0530 Subject: [PATCH 138/311] ACPI: CPPC: Warn on missing mandatory DESIRED_PERF register BugLink: https://bugs.launchpad.net/bugs/2131705 Add a warning during CPPC processor probe if the Desired Performance register is not supported when it should be. As per 8.4.6.1.2.3 section of ACPI 6.6 specification, "The Desired Performance Register is optional only when OSPM indicates support for CPPC2 in the platform-wide _OSC capabilities and the Autonomous Selection Enable field is encoded as an Integer with a value of 1." In other words: - In CPPC v1, DESIRED_PERF is mandatory - In CPPC v2, it becomes optional only when AUTO_SEL_ENABLE is supported This helps detect firmware configuration issues early during boot. Link: https://lore.kernel.org/lkml/9fa21599-004a-4af8-acc2-190fd0404e35@nvidia.com/ Suggested-by: Pierre Gondois Signed-off-by: Sumit Gupta Reviewed-by: Pierre Gondois Reviewed-by: Lifeng Zheng Link: https://patch.msgid.link/20260206142658.72583-3-sumitg@nvidia.com Signed-off-by: Rafael J. Wysocki (cherry picked from commit b3e45fb2db9d8a733e94b315f1272e2c4468ed4b) Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/acpi/cppc_acpi.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 5122e99bddb09..3f758d2944e23 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -853,6 +853,16 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) } per_cpu(cpu_pcc_subspace_idx, pr->id) = pcc_subspace_id; + /* + * In CPPC v1, DESIRED_PERF is mandatory. In CPPC v2, it is optional + * only when AUTO_SEL_ENABLE is supported. + */ + if (!CPC_SUPPORTED(&cpc_ptr->cpc_regs[DESIRED_PERF]) && + (!osc_sb_cppc2_support_acked || + !CPC_SUPPORTED(&cpc_ptr->cpc_regs[AUTO_SEL_ENABLE]))) + pr_warn("Desired perf. register is mandatory if CPPC v2 is not supported " + "or autonomous selection is disabled\n"); + /* * Initialize the remaining cpc_regs as unsupported. * Example: In case FW exposes CPPC v2, the below loop will initialize From 218d1828a48ca7d7239894fa1969d96a8005a3d0 Mon Sep 17 00:00:00 2001 From: Sumit Gupta Date: Fri, 6 Feb 2026 19:56:54 +0530 Subject: [PATCH 139/311] ACPI: CPPC: Extend cppc_set_epp_perf() for FFH/SystemMemory BugLink: https://bugs.launchpad.net/bugs/2131705 Extend cppc_set_epp_perf() to write both auto_sel and energy_perf registers when they are in FFH or SystemMemory address space. This keeps the behavior consistent with PCC case where both registers are already updated together, but was missing for FFH/SystemMemory. Signed-off-by: Sumit Gupta Reviewed-by: Pierre Gondois Reviewed-by: Lifeng Zheng Link: https://patch.msgid.link/20260206142658.72583-4-sumitg@nvidia.com Signed-off-by: Rafael J. Wysocki (cherry picked from commit 38428a680026c52a1fc64212325d161974c3e4cf) Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/acpi/cppc_acpi.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 3f758d2944e23..94a7ffa8be3c3 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -1571,6 +1571,8 @@ int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable) struct cpc_register_resource *auto_sel_reg; struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu); struct cppc_pcc_data *pcc_ss_data = NULL; + bool autosel_ffh_sysmem; + bool epp_ffh_sysmem; int ret; if (!cpc_desc) { @@ -1581,6 +1583,11 @@ int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable) auto_sel_reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE]; epp_set_reg = &cpc_desc->cpc_regs[ENERGY_PERF]; + epp_ffh_sysmem = CPC_SUPPORTED(epp_set_reg) && + (CPC_IN_FFH(epp_set_reg) || CPC_IN_SYSTEM_MEMORY(epp_set_reg)); + autosel_ffh_sysmem = CPC_SUPPORTED(auto_sel_reg) && + (CPC_IN_FFH(auto_sel_reg) || CPC_IN_SYSTEM_MEMORY(auto_sel_reg)); + if (CPC_IN_PCC(epp_set_reg) || CPC_IN_PCC(auto_sel_reg)) { if (pcc_ss_id < 0) { pr_debug("Invalid pcc_ss_id for CPU:%d\n", cpu); @@ -1606,11 +1613,22 @@ int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable) ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE); up_write(&pcc_ss_data->pcc_lock); } else if (osc_cpc_flexible_adr_space_confirmed && - CPC_SUPPORTED(epp_set_reg) && CPC_IN_FFH(epp_set_reg)) { - ret = cpc_write(cpu, epp_set_reg, perf_ctrls->energy_perf); + (epp_ffh_sysmem || autosel_ffh_sysmem)) { + if (autosel_ffh_sysmem) { + ret = cpc_write(cpu, auto_sel_reg, enable); + if (ret) + return ret; + } + + if (epp_ffh_sysmem) { + ret = cpc_write(cpu, epp_set_reg, + perf_ctrls->energy_perf); + if (ret) + return ret; + } } else { ret = -ENOTSUPP; - pr_debug("_CPC in PCC and _CPC in FFH are not supported\n"); + pr_debug("_CPC in PCC/FFH/SystemMemory are not supported\n"); } return ret; From 74470f5ff53300365cd5b9b37367b53fd0e4782c Mon Sep 17 00:00:00 2001 From: Sumit Gupta Date: Fri, 6 Feb 2026 19:56:55 +0530 Subject: [PATCH 140/311] cpufreq: CPPC: Update cached perf_ctrls on sysfs write BugLink: https://bugs.launchpad.net/bugs/2131705 Update the cached perf_ctrls values when writing via sysfs to keep them in sync with hardware registers: - store_auto_select(): update perf_ctrls.auto_sel - store_energy_performance_preference_val(): update perf_ctrls.energy_perf This ensures consistent cached values after sysfs writes, which complements the cppc_get_perf() initialization during policy setup. Signed-off-by: Sumit Gupta Reviewed-by: Pierre Gondois Reviewed-by: Lifeng Zheng Link: https://patch.msgid.link/20260206142658.72583-5-sumitg@nvidia.com Signed-off-by: Rafael J. Wysocki (cherry picked from commit 24ad4c6c136bdaa4c92c5c5948856752ce3e9f76) Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/cpufreq/cppc_cpufreq.c | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index a61a24e0dcaed..ebb5746df220e 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -855,6 +855,7 @@ static ssize_t show_auto_select(struct cpufreq_policy *policy, char *buf) static ssize_t store_auto_select(struct cpufreq_policy *policy, const char *buf, size_t count) { + struct cppc_cpudata *cpu_data = policy->driver_data; bool val; int ret; @@ -866,6 +867,8 @@ static ssize_t store_auto_select(struct cpufreq_policy *policy, if (ret) return ret; + cpu_data->perf_ctrls.auto_sel = val; + return count; } @@ -916,8 +919,32 @@ static ssize_t store_##_name(struct cpufreq_policy *policy, \ CPPC_CPUFREQ_ATTR_RW_U64(auto_act_window, cppc_get_auto_act_window, cppc_set_auto_act_window) -CPPC_CPUFREQ_ATTR_RW_U64(energy_performance_preference_val, - cppc_get_epp_perf, cppc_set_epp) +static ssize_t +show_energy_performance_preference_val(struct cpufreq_policy *policy, char *buf) +{ + return cppc_cpufreq_sysfs_show_u64(policy->cpu, cppc_get_epp_perf, buf); +} + +static ssize_t +store_energy_performance_preference_val(struct cpufreq_policy *policy, + const char *buf, size_t count) +{ + struct cppc_cpudata *cpu_data = policy->driver_data; + u64 val; + int ret; + + ret = kstrtou64(buf, 0, &val); + if (ret) + return ret; + + ret = cppc_set_epp(policy->cpu, val); + if (ret) + return ret; + + cpu_data->perf_ctrls.energy_perf = val; + + return count; +} cpufreq_freq_attr_ro(freqdomain_cpus); cpufreq_freq_attr_rw(auto_select); From 4e5d3bce818d126a5aab9113f0aa7437505e8623 Mon Sep 17 00:00:00 2001 From: Sumit Gupta Date: Fri, 6 Feb 2026 19:56:56 +0530 Subject: [PATCH 141/311] cpufreq: cppc: Update MIN_PERF/MAX_PERF in target callbacks BugLink: https://bugs.launchpad.net/bugs/2131705 Update MIN_PERF and MAX_PERF registers from policy->min and policy->max in the .target() and .fast_switch() callbacks. This allows controlling performance bounds via standard scaling_min_freq and scaling_max_freq sysfs interfaces. Similar to intel_cpufreq which updates HWP min/max limits in .target(), cppc_cpufreq now programs MIN_PERF/MAX_PERF along with DESIRED_PERF. Since MIN_PERF/MAX_PERF can be updated even when auto_sel is disabled, they are updated unconditionally. Also program MIN_PERF/MAX_PERF in store_auto_select() when enabling autonomous selection so the platform uses correct bounds immediately. Suggested-by: Rafael J. Wysocki Signed-off-by: Sumit Gupta Link: https://patch.msgid.link/20260206142658.72583-6-sumitg@nvidia.com Signed-off-by: Rafael J. Wysocki (cherry picked from commit ea3db45ae476889a1ba0ab3617e6afdeeefbda3d) Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/cpufreq/cppc_cpufreq.c | 41 +++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index ebb5746df220e..8a8cf76828ee2 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -287,6 +287,21 @@ static inline void cppc_freq_invariance_exit(void) } #endif /* CONFIG_ACPI_CPPC_CPUFREQ_FIE */ +static void cppc_cpufreq_update_perf_limits(struct cppc_cpudata *cpu_data, + struct cpufreq_policy *policy) +{ + struct cppc_perf_caps *caps = &cpu_data->perf_caps; + u32 min_perf, max_perf; + + min_perf = cppc_khz_to_perf(caps, policy->min); + max_perf = cppc_khz_to_perf(caps, policy->max); + + cpu_data->perf_ctrls.min_perf = + clamp_t(u32, min_perf, caps->lowest_perf, caps->highest_perf); + cpu_data->perf_ctrls.max_perf = + clamp_t(u32, max_perf, caps->lowest_perf, caps->highest_perf); +} + static int cppc_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int target_freq, unsigned int relation) @@ -298,6 +313,8 @@ static int cppc_cpufreq_set_target(struct cpufreq_policy *policy, cpu_data->perf_ctrls.desired_perf = cppc_khz_to_perf(&cpu_data->perf_caps, target_freq); + cppc_cpufreq_update_perf_limits(cpu_data, policy); + freqs.old = policy->cur; freqs.new = target_freq; @@ -322,8 +339,9 @@ static unsigned int cppc_cpufreq_fast_switch(struct cpufreq_policy *policy, desired_perf = cppc_khz_to_perf(&cpu_data->perf_caps, target_freq); cpu_data->perf_ctrls.desired_perf = desired_perf; - ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); + cppc_cpufreq_update_perf_limits(cpu_data, policy); + ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); if (ret) { pr_debug("Failed to set target on CPU:%d. ret:%d\n", cpu, ret); @@ -869,6 +887,27 @@ static ssize_t store_auto_select(struct cpufreq_policy *policy, cpu_data->perf_ctrls.auto_sel = val; + if (val) { + u32 old_min_perf = cpu_data->perf_ctrls.min_perf; + u32 old_max_perf = cpu_data->perf_ctrls.max_perf; + + /* + * When enabling autonomous selection, program MIN_PERF and + * MAX_PERF from current policy limits so that the platform + * uses the correct performance bounds immediately. + */ + cppc_cpufreq_update_perf_limits(cpu_data, policy); + + ret = cppc_set_perf(policy->cpu, &cpu_data->perf_ctrls); + if (ret) { + cpu_data->perf_ctrls.min_perf = old_min_perf; + cpu_data->perf_ctrls.max_perf = old_max_perf; + cppc_set_auto_sel(policy->cpu, false); + cpu_data->perf_ctrls.auto_sel = false; + return ret; + } + } + return count; } From 33bf7e15b9ba95f0c5bee5ffd93a371b54677890 Mon Sep 17 00:00:00 2001 From: Sumit Gupta Date: Fri, 6 Feb 2026 19:56:57 +0530 Subject: [PATCH 142/311] ACPI: CPPC: add APIs and sysfs interface for perf_limited BugLink: https://bugs.launchpad.net/bugs/2131705 Add sysfs interface to read/write the Performance Limited register. The Performance Limited register indicates to the OS that an unpredictable event (like thermal throttling) has limited processor performance. It contains two sticky bits set by the platform: - Bit 0 (Desired_Excursion): Set when delivered performance is constrained below desired performance. Not used when Autonomous Selection is enabled. - Bit 1 (Minimum_Excursion): Set when delivered performance is constrained below minimum performance. These bits remain set until OSPM explicitly clears them. The write operation accepts a bitmask of bits to clear: - Write 0x1 to clear bit 0 - Write 0x2 to clear bit 1 - Write 0x3 to clear both bits This enables users to detect if platform throttling impacted a workload. Users clear the register before execution, run the workload, then check afterward - if set, hardware throttling occurred during that time window. The interface is exposed as: /sys/devices/system/cpu/cpuX/cpufreq/perf_limited Signed-off-by: Sumit Gupta Reviewed-by: Pierre Gondois Reviewed-by: Lifeng Zheng Link: https://patch.msgid.link/20260206142658.72583-7-sumitg@nvidia.com Signed-off-by: Rafael J. Wysocki (cherry picked from commit 13c45a26635fa51a68911aa57e6778bdad18b103) Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/acpi/cppc_acpi.c | 56 ++++++++++++++++++++++++++++++++++ drivers/cpufreq/cppc_cpufreq.c | 5 +++ include/acpi/cppc_acpi.h | 15 +++++++++ 3 files changed, 76 insertions(+) diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 94a7ffa8be3c3..53a6ffd995a1a 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -1978,6 +1978,62 @@ int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) } EXPORT_SYMBOL_GPL(cppc_set_perf); +/** + * cppc_get_perf_limited - Get the Performance Limited register value. + * @cpu: CPU from which to get Performance Limited register. + * @perf_limited: Pointer to store the Performance Limited value. + * + * The returned value contains sticky status bits indicating platform-imposed + * performance limitations. + * + * Return: 0 for success, -EIO on failure, -EOPNOTSUPP if not supported. + */ +int cppc_get_perf_limited(int cpu, u64 *perf_limited) +{ + return cppc_get_reg_val(cpu, PERF_LIMITED, perf_limited); +} +EXPORT_SYMBOL_GPL(cppc_get_perf_limited); + +/** + * cppc_set_perf_limited() - Clear bits in the Performance Limited register. + * @cpu: CPU on which to write register. + * @bits_to_clear: Bitmask of bits to clear in the perf_limited register. + * + * The Performance Limited register contains two sticky bits set by platform: + * - Bit 0 (Desired_Excursion): Set when delivered performance is constrained + * below desired performance. Not used when Autonomous Selection is enabled. + * - Bit 1 (Minimum_Excursion): Set when delivered performance is constrained + * below minimum performance. + * + * These bits are sticky and remain set until OSPM explicitly clears them. + * This function only allows clearing bits (the platform sets them). + * + * Return: 0 for success, -EINVAL for invalid bits, -EIO on register + * access failure, -EOPNOTSUPP if not supported. + */ +int cppc_set_perf_limited(int cpu, u64 bits_to_clear) +{ + u64 current_val, new_val; + int ret; + + /* Only bits 0 and 1 are valid */ + if (bits_to_clear & ~CPPC_PERF_LIMITED_MASK) + return -EINVAL; + + if (!bits_to_clear) + return 0; + + ret = cppc_get_perf_limited(cpu, ¤t_val); + if (ret) + return ret; + + /* Clear the specified bits */ + new_val = current_val & ~bits_to_clear; + + return cppc_set_reg_val(cpu, PERF_LIMITED, new_val); +} +EXPORT_SYMBOL_GPL(cppc_set_perf_limited); + /** * cppc_get_transition_latency - returns frequency transition latency in ns * @cpu_num: CPU number for per_cpu(). diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index 8a8cf76828ee2..94d489a4c90d1 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -985,16 +985,21 @@ store_energy_performance_preference_val(struct cpufreq_policy *policy, return count; } +CPPC_CPUFREQ_ATTR_RW_U64(perf_limited, cppc_get_perf_limited, + cppc_set_perf_limited) + cpufreq_freq_attr_ro(freqdomain_cpus); cpufreq_freq_attr_rw(auto_select); cpufreq_freq_attr_rw(auto_act_window); cpufreq_freq_attr_rw(energy_performance_preference_val); +cpufreq_freq_attr_rw(perf_limited); static struct freq_attr *cppc_cpufreq_attr[] = { &freqdomain_cpus, &auto_select, &auto_act_window, &energy_performance_preference_val, + &perf_limited, NULL, }; diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h index 3fc796c0d9022..f7afa20b8ad9d 100644 --- a/include/acpi/cppc_acpi.h +++ b/include/acpi/cppc_acpi.h @@ -42,6 +42,11 @@ #define CPPC_EPP_PERFORMANCE_PREF 0x00 #define CPPC_EPP_ENERGY_EFFICIENCY_PREF 0xFF +#define CPPC_PERF_LIMITED_DESIRED_EXCURSION BIT(0) +#define CPPC_PERF_LIMITED_MINIMUM_EXCURSION BIT(1) +#define CPPC_PERF_LIMITED_MASK (CPPC_PERF_LIMITED_DESIRED_EXCURSION | \ + CPPC_PERF_LIMITED_MINIMUM_EXCURSION) + /* Each register has the folowing format. */ struct cpc_reg { u8 descriptor; @@ -174,6 +179,8 @@ extern int cppc_get_auto_act_window(int cpu, u64 *auto_act_window); extern int cppc_set_auto_act_window(int cpu, u64 auto_act_window); extern int cppc_get_auto_sel(int cpu, bool *enable); extern int cppc_set_auto_sel(int cpu, bool enable); +extern int cppc_get_perf_limited(int cpu, u64 *perf_limited); +extern int cppc_set_perf_limited(int cpu, u64 bits_to_clear); extern int amd_get_highest_perf(unsigned int cpu, u32 *highest_perf); extern int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator); extern int amd_detect_prefcore(bool *detected); @@ -270,6 +277,14 @@ static inline int cppc_set_auto_sel(int cpu, bool enable) { return -EOPNOTSUPP; } +static inline int cppc_get_perf_limited(int cpu, u64 *perf_limited) +{ + return -EOPNOTSUPP; +} +static inline int cppc_set_perf_limited(int cpu, u64 bits_to_clear) +{ + return -EOPNOTSUPP; +} static inline int amd_get_highest_perf(unsigned int cpu, u32 *highest_perf) { return -ENODEV; From 8e37567df27b8393be8d4eecadd6bbfab0d92ca3 Mon Sep 17 00:00:00 2001 From: Sumit Gupta Date: Fri, 6 Feb 2026 19:56:58 +0530 Subject: [PATCH 143/311] cpufreq: CPPC: Add sysfs documentation for perf_limited BugLink: https://bugs.launchpad.net/bugs/2131705 Add ABI documentation for the Performance Limited Register sysfs interface in the cppc_cpufreq driver. Signed-off-by: Sumit Gupta Reviewed-by: Randy Dunlap Reviewed-by: Pierre Gondois Reviewed-by: Lifeng Zheng Link: https://patch.msgid.link/20260206142658.72583-8-sumitg@nvidia.com Signed-off-by: Rafael J. Wysocki (cherry picked from commit 856250ba2e810e772dc95b3234ebf0d6393a51d9) Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- .../ABI/testing/sysfs-devices-system-cpu | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu index 3a05604c21bf8..82d10d556cc89 100644 --- a/Documentation/ABI/testing/sysfs-devices-system-cpu +++ b/Documentation/ABI/testing/sysfs-devices-system-cpu @@ -327,6 +327,24 @@ Description: Energy performance preference This file is only present if the cppc-cpufreq driver is in use. +What: /sys/devices/system/cpu/cpuX/cpufreq/perf_limited +Date: February 2026 +Contact: linux-pm@vger.kernel.org +Description: Performance Limited + + Read to check if platform throttling (thermal/power/current + limits) caused delivered performance to fall below the + requested level. A non-zero value indicates throttling occurred. + + Write the bitmask of bits to clear: + + - 0x1 = clear bit 0 (desired performance excursion) + - 0x2 = clear bit 1 (minimum performance excursion) + - 0x3 = clear both bits + + The platform sets these bits; OSPM can only clear them. + + This file is only present if the cppc-cpufreq driver is in use. What: /sys/devices/system/cpu/cpu*/cache/index3/cache_disable_{0,1} Date: August 2008 From 1e44b7a834653fca22c0c20f62fc8ac27124977c Mon Sep 17 00:00:00 2001 From: Pengjie Zhang Date: Fri, 13 Feb 2026 18:09:35 +0800 Subject: [PATCH 144/311] ACPI: CPPC: Move reference performance to capabilities BugLink: https://bugs.launchpad.net/bugs/2131705 Currently, the `Reference Performance` register is read every time the CPU frequency is sampled in `cppc_get_perf_ctrs()`. This function is on the hot path of the cppc_cpufreq driver. Reference Performance indicates the performance level that corresponds to the Reference Counter incrementing and is not expected to change dynamically during runtime (unlike the Delivered and Reference counters). Reading this register in the hot path incurs unnecessary overhead, particularly on platforms where CPC registers are located in the PCC (Platform Communication Channel) subspace. This patch moves `reference_perf` from the dynamic feedback counters structure (`cppc_perf_fb_ctrs`) to the static capabilities structure (`cppc_perf_caps`). Signed-off-by: Pengjie Zhang [ rjw: Changelog adjustment ] Link: https://patch.msgid.link/20260213100935.19111-1-zhangpengjie2@huawei.com Signed-off-by: Rafael J. Wysocki (cherry picked from commit 8505bfb4e4eca28ef1b20d3369435ec2d6a125c6) Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/acpi/cppc_acpi.c | 55 +++++++++++++++------------------- drivers/cpufreq/cppc_cpufreq.c | 21 +++++++------ include/acpi/cppc_acpi.h | 2 +- 3 files changed, 37 insertions(+), 41 deletions(-) diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 53a6ffd995a1a..07bbf5b366a42 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -177,12 +177,12 @@ __ATTR(_name, 0444, show_##_name, NULL) show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, highest_perf); show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, lowest_perf); show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, nominal_perf); +show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, reference_perf); show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, lowest_nonlinear_perf); show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, guaranteed_perf); show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, lowest_freq); show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, nominal_freq); -show_cppc_data(cppc_get_perf_ctrs, cppc_perf_fb_ctrs, reference_perf); show_cppc_data(cppc_get_perf_ctrs, cppc_perf_fb_ctrs, wraparound_time); /* Check for valid access_width, otherwise, fallback to using bit_width */ @@ -1352,9 +1352,10 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps) { struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum); struct cpc_register_resource *highest_reg, *lowest_reg, - *lowest_non_linear_reg, *nominal_reg, *guaranteed_reg, - *low_freq_reg = NULL, *nom_freq_reg = NULL; - u64 high, low, guaranteed, nom, min_nonlinear, low_f = 0, nom_f = 0; + *lowest_non_linear_reg, *nominal_reg, *reference_reg, + *guaranteed_reg, *low_freq_reg = NULL, *nom_freq_reg = NULL; + u64 high, low, guaranteed, nom, ref, min_nonlinear, + low_f = 0, nom_f = 0; int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpunum); struct cppc_pcc_data *pcc_ss_data = NULL; int ret = 0, regs_in_pcc = 0; @@ -1368,6 +1369,7 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps) lowest_reg = &cpc_desc->cpc_regs[LOWEST_PERF]; lowest_non_linear_reg = &cpc_desc->cpc_regs[LOW_NON_LINEAR_PERF]; nominal_reg = &cpc_desc->cpc_regs[NOMINAL_PERF]; + reference_reg = &cpc_desc->cpc_regs[REFERENCE_PERF]; low_freq_reg = &cpc_desc->cpc_regs[LOWEST_FREQ]; nom_freq_reg = &cpc_desc->cpc_regs[NOMINAL_FREQ]; guaranteed_reg = &cpc_desc->cpc_regs[GUARANTEED_PERF]; @@ -1375,6 +1377,7 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps) /* Are any of the regs PCC ?*/ if (CPC_IN_PCC(highest_reg) || CPC_IN_PCC(lowest_reg) || CPC_IN_PCC(lowest_non_linear_reg) || CPC_IN_PCC(nominal_reg) || + (CPC_SUPPORTED(reference_reg) && CPC_IN_PCC(reference_reg)) || CPC_IN_PCC(low_freq_reg) || CPC_IN_PCC(nom_freq_reg) || CPC_IN_PCC(guaranteed_reg)) { if (pcc_ss_id < 0) { @@ -1400,6 +1403,17 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps) cpc_read(cpunum, nominal_reg, &nom); perf_caps->nominal_perf = nom; + /* + * If reference perf register is not supported then we should + * use the nominal perf value + */ + if (CPC_SUPPORTED(reference_reg)) { + cpc_read(cpunum, reference_reg, &ref); + perf_caps->reference_perf = ref; + } else { + perf_caps->reference_perf = nom; + } + if (guaranteed_reg->type != ACPI_TYPE_BUFFER || IS_NULL_REG(&guaranteed_reg->cpc_entry.reg)) { perf_caps->guaranteed_perf = 0; @@ -1411,7 +1425,7 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps) cpc_read(cpunum, lowest_non_linear_reg, &min_nonlinear); perf_caps->lowest_nonlinear_perf = min_nonlinear; - if (!high || !low || !nom || !min_nonlinear) + if (!high || !low || !nom || !ref || !min_nonlinear) ret = -EFAULT; /* Read optional lowest and nominal frequencies if present */ @@ -1441,20 +1455,10 @@ EXPORT_SYMBOL_GPL(cppc_get_perf_caps); bool cppc_perf_ctrs_in_pcc_cpu(unsigned int cpu) { struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu); - struct cpc_register_resource *ref_perf_reg; - - /* - * If reference perf register is not supported then we should use the - * nominal perf value - */ - ref_perf_reg = &cpc_desc->cpc_regs[REFERENCE_PERF]; - if (!CPC_SUPPORTED(ref_perf_reg)) - ref_perf_reg = &cpc_desc->cpc_regs[NOMINAL_PERF]; return CPC_IN_PCC(&cpc_desc->cpc_regs[DELIVERED_CTR]) || CPC_IN_PCC(&cpc_desc->cpc_regs[REFERENCE_CTR]) || - CPC_IN_PCC(&cpc_desc->cpc_regs[CTR_WRAP_TIME]) || - CPC_IN_PCC(ref_perf_reg); + CPC_IN_PCC(&cpc_desc->cpc_regs[CTR_WRAP_TIME]); } EXPORT_SYMBOL_GPL(cppc_perf_ctrs_in_pcc_cpu); @@ -1491,10 +1495,10 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs) { struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum); struct cpc_register_resource *delivered_reg, *reference_reg, - *ref_perf_reg, *ctr_wrap_reg; + *ctr_wrap_reg; int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpunum); struct cppc_pcc_data *pcc_ss_data = NULL; - u64 delivered, reference, ref_perf, ctr_wrap_time; + u64 delivered, reference, ctr_wrap_time; int ret = 0, regs_in_pcc = 0; if (!cpc_desc) { @@ -1504,19 +1508,11 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs) delivered_reg = &cpc_desc->cpc_regs[DELIVERED_CTR]; reference_reg = &cpc_desc->cpc_regs[REFERENCE_CTR]; - ref_perf_reg = &cpc_desc->cpc_regs[REFERENCE_PERF]; ctr_wrap_reg = &cpc_desc->cpc_regs[CTR_WRAP_TIME]; - /* - * If reference perf register is not supported then we should - * use the nominal perf value - */ - if (!CPC_SUPPORTED(ref_perf_reg)) - ref_perf_reg = &cpc_desc->cpc_regs[NOMINAL_PERF]; - /* Are any of the regs PCC ?*/ if (CPC_IN_PCC(delivered_reg) || CPC_IN_PCC(reference_reg) || - CPC_IN_PCC(ctr_wrap_reg) || CPC_IN_PCC(ref_perf_reg)) { + CPC_IN_PCC(ctr_wrap_reg)) { if (pcc_ss_id < 0) { pr_debug("Invalid pcc_ss_id\n"); return -ENODEV; @@ -1533,8 +1529,6 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs) cpc_read(cpunum, delivered_reg, &delivered); cpc_read(cpunum, reference_reg, &reference); - cpc_read(cpunum, ref_perf_reg, &ref_perf); - /* * Per spec, if ctr_wrap_time optional register is unsupported, then the * performance counters are assumed to never wrap during the lifetime of @@ -1544,14 +1538,13 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs) if (CPC_SUPPORTED(ctr_wrap_reg)) cpc_read(cpunum, ctr_wrap_reg, &ctr_wrap_time); - if (!delivered || !reference || !ref_perf) { + if (!delivered || !reference) { ret = -EFAULT; goto out_err; } perf_fb_ctrs->delivered = delivered; perf_fb_ctrs->reference = reference; - perf_fb_ctrs->reference_perf = ref_perf; perf_fb_ctrs->wraparound_time = ctr_wrap_time; out_err: if (regs_in_pcc) diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index 94d489a4c90d1..5dfb109cf1f4e 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -50,7 +50,8 @@ struct cppc_freq_invariance { static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv); static struct kthread_worker *kworker_fie; -static int cppc_perf_from_fbctrs(struct cppc_perf_fb_ctrs *fb_ctrs_t0, +static int cppc_perf_from_fbctrs(u64 reference_perf, + struct cppc_perf_fb_ctrs *fb_ctrs_t0, struct cppc_perf_fb_ctrs *fb_ctrs_t1); /** @@ -70,7 +71,7 @@ static void __cppc_scale_freq_tick(struct cppc_freq_invariance *cppc_fi) struct cppc_perf_fb_ctrs fb_ctrs = {0}; struct cppc_cpudata *cpu_data; unsigned long local_freq_scale; - u64 perf; + u64 perf, ref_perf; cpu_data = cppc_fi->cpu_data; @@ -79,7 +80,9 @@ static void __cppc_scale_freq_tick(struct cppc_freq_invariance *cppc_fi) return; } - perf = cppc_perf_from_fbctrs(&cppc_fi->prev_perf_fb_ctrs, &fb_ctrs); + ref_perf = cpu_data->perf_caps.reference_perf; + perf = cppc_perf_from_fbctrs(ref_perf, + &cppc_fi->prev_perf_fb_ctrs, &fb_ctrs); if (!perf) return; @@ -747,13 +750,11 @@ static inline u64 get_delta(u64 t1, u64 t0) return (u32)t1 - (u32)t0; } -static int cppc_perf_from_fbctrs(struct cppc_perf_fb_ctrs *fb_ctrs_t0, +static int cppc_perf_from_fbctrs(u64 reference_perf, + struct cppc_perf_fb_ctrs *fb_ctrs_t0, struct cppc_perf_fb_ctrs *fb_ctrs_t1) { u64 delta_reference, delta_delivered; - u64 reference_perf; - - reference_perf = fb_ctrs_t0->reference_perf; delta_reference = get_delta(fb_ctrs_t1->reference, fb_ctrs_t0->reference); @@ -790,7 +791,7 @@ static unsigned int cppc_cpufreq_get_rate(unsigned int cpu) struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); struct cppc_perf_fb_ctrs fb_ctrs_t0 = {0}, fb_ctrs_t1 = {0}; struct cppc_cpudata *cpu_data; - u64 delivered_perf; + u64 delivered_perf, reference_perf; int ret; if (!policy) @@ -807,7 +808,9 @@ static unsigned int cppc_cpufreq_get_rate(unsigned int cpu) return 0; } - delivered_perf = cppc_perf_from_fbctrs(&fb_ctrs_t0, &fb_ctrs_t1); + reference_perf = cpu_data->perf_caps.reference_perf; + delivered_perf = cppc_perf_from_fbctrs(reference_perf, + &fb_ctrs_t0, &fb_ctrs_t1); if (!delivered_perf) goto out_invalid_counters; diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h index f7afa20b8ad9d..d8e405becdc31 100644 --- a/include/acpi/cppc_acpi.h +++ b/include/acpi/cppc_acpi.h @@ -121,6 +121,7 @@ struct cppc_perf_caps { u32 guaranteed_perf; u32 highest_perf; u32 nominal_perf; + u32 reference_perf; u32 lowest_perf; u32 lowest_nonlinear_perf; u32 lowest_freq; @@ -138,7 +139,6 @@ struct cppc_perf_ctrls { struct cppc_perf_fb_ctrs { u64 reference; u64 delivered; - u64 reference_perf; u64 wraparound_time; }; From 565ba769d01f70e911d7289561572ae228cd6594 Mon Sep 17 00:00:00 2001 From: Pengjie Zhang Date: Wed, 11 Mar 2026 15:13:34 +0800 Subject: [PATCH 145/311] ACPI: CPPC: Fix uninitialized ref variable in cppc_get_perf_caps() BugLink: https://bugs.launchpad.net/bugs/2131705 Commit 8505bfb4e4ec ("ACPI: CPPC: Move reference performance to capabilities") introduced a logical error when retrieving the reference performance. On platforms lacking the reference performance register, the fallback logic leaves the local 'ref' variable uninitialized (0). This causes the subsequent sanity check to incorrectly return -EFAULT, breaking amd_pstate initialization. Fix this by assigning 'ref = nom' in the fallback path. Fixes: 8505bfb4e4ec ("ACPI: CPPC: Move reference performance to capabilities") Reported-by: Nathan Chancellor Closes: https://lore.kernel.org/all/20260310003026.GA2639793@ax162/ Tested-by: Nathan Chancellor Signed-off-by: Pengjie Zhang [ rjw: Subject tweak ] Link: https://patch.msgid.link/20260311071334.1494960-1-zhangpengjie2@huawei.com Signed-off-by: Rafael J. Wysocki (cherry picked from commit be473f0591f183990a998edee02161b319047eaa) Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/acpi/cppc_acpi.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 07bbf5b366a42..5ad922eb937a9 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -1407,12 +1407,11 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps) * If reference perf register is not supported then we should * use the nominal perf value */ - if (CPC_SUPPORTED(reference_reg)) { + if (CPC_SUPPORTED(reference_reg)) cpc_read(cpunum, reference_reg, &ref); - perf_caps->reference_perf = ref; - } else { - perf_caps->reference_perf = nom; - } + else + ref = nom; + perf_caps->reference_perf = ref; if (guaranteed_reg->type != ACPI_TYPE_BUFFER || IS_NULL_REG(&guaranteed_reg->cpc_entry.reg)) { From 71b690539c9c63d9e6c595716003b7eb5eae0b01 Mon Sep 17 00:00:00 2001 From: Sumit Gupta Date: Wed, 18 Mar 2026 15:20:05 +0530 Subject: [PATCH 146/311] ACPI: CPPC: Check cpc_read() return values consistently BugLink: https://bugs.launchpad.net/bugs/2131705 Callers of cpc_read() ignore its return value, which can lead to using uninitialized or stale values when the read fails. Fix this by consistently checking cpc_read() return values in cppc_get_perf_caps(), cppc_get_perf_ctrs(), and cppc_get_perf(). Link: https://lore.kernel.org/lkml/48bdf87e-39f1-402f-a7dc-1a0e1e7a819d@nvidia.com/ Suggested-by: Rafael J. Wysocki Signed-off-by: Sumit Gupta Link: https://patch.msgid.link/20260318095005.2437960-1-sumitg@nvidia.com Signed-off-by: Rafael J. Wysocki (cherry picked from commit 0cc24977224a6c7d470860265a4990109f0a32ee) Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/acpi/cppc_acpi.c | 99 +++++++++++++++++++++++++++++----------- 1 file changed, 72 insertions(+), 27 deletions(-) diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 5ad922eb937a9..053fc6765a592 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -1394,45 +1394,66 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps) } } - cpc_read(cpunum, highest_reg, &high); + ret = cpc_read(cpunum, highest_reg, &high); + if (ret) + goto out_err; perf_caps->highest_perf = high; - cpc_read(cpunum, lowest_reg, &low); + ret = cpc_read(cpunum, lowest_reg, &low); + if (ret) + goto out_err; perf_caps->lowest_perf = low; - cpc_read(cpunum, nominal_reg, &nom); + ret = cpc_read(cpunum, nominal_reg, &nom); + if (ret) + goto out_err; perf_caps->nominal_perf = nom; /* * If reference perf register is not supported then we should * use the nominal perf value */ - if (CPC_SUPPORTED(reference_reg)) - cpc_read(cpunum, reference_reg, &ref); - else + if (CPC_SUPPORTED(reference_reg)) { + ret = cpc_read(cpunum, reference_reg, &ref); + if (ret) + goto out_err; + } else { ref = nom; + } perf_caps->reference_perf = ref; if (guaranteed_reg->type != ACPI_TYPE_BUFFER || IS_NULL_REG(&guaranteed_reg->cpc_entry.reg)) { perf_caps->guaranteed_perf = 0; } else { - cpc_read(cpunum, guaranteed_reg, &guaranteed); + ret = cpc_read(cpunum, guaranteed_reg, &guaranteed); + if (ret) + goto out_err; perf_caps->guaranteed_perf = guaranteed; } - cpc_read(cpunum, lowest_non_linear_reg, &min_nonlinear); + ret = cpc_read(cpunum, lowest_non_linear_reg, &min_nonlinear); + if (ret) + goto out_err; perf_caps->lowest_nonlinear_perf = min_nonlinear; - if (!high || !low || !nom || !ref || !min_nonlinear) + if (!high || !low || !nom || !ref || !min_nonlinear) { ret = -EFAULT; + goto out_err; + } /* Read optional lowest and nominal frequencies if present */ - if (CPC_SUPPORTED(low_freq_reg)) - cpc_read(cpunum, low_freq_reg, &low_f); + if (CPC_SUPPORTED(low_freq_reg)) { + ret = cpc_read(cpunum, low_freq_reg, &low_f); + if (ret) + goto out_err; + } - if (CPC_SUPPORTED(nom_freq_reg)) - cpc_read(cpunum, nom_freq_reg, &nom_f); + if (CPC_SUPPORTED(nom_freq_reg)) { + ret = cpc_read(cpunum, nom_freq_reg, &nom_f); + if (ret) + goto out_err; + } perf_caps->lowest_freq = low_f; perf_caps->nominal_freq = nom_f; @@ -1526,16 +1547,25 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs) } } - cpc_read(cpunum, delivered_reg, &delivered); - cpc_read(cpunum, reference_reg, &reference); + ret = cpc_read(cpunum, delivered_reg, &delivered); + if (ret) + goto out_err; + + ret = cpc_read(cpunum, reference_reg, &reference); + if (ret) + goto out_err; + /* * Per spec, if ctr_wrap_time optional register is unsupported, then the * performance counters are assumed to never wrap during the lifetime of * platform */ ctr_wrap_time = (u64)(~((u64)0)); - if (CPC_SUPPORTED(ctr_wrap_reg)) - cpc_read(cpunum, ctr_wrap_reg, &ctr_wrap_time); + if (CPC_SUPPORTED(ctr_wrap_reg)) { + ret = cpc_read(cpunum, ctr_wrap_reg, &ctr_wrap_time); + if (ret) + goto out_err; + } if (!delivered || !reference) { ret = -EFAULT; @@ -1811,24 +1841,39 @@ int cppc_get_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) } /* Read optional elements if present */ - if (CPC_SUPPORTED(max_perf_reg)) - cpc_read(cpu, max_perf_reg, &max); + if (CPC_SUPPORTED(max_perf_reg)) { + ret = cpc_read(cpu, max_perf_reg, &max); + if (ret) + goto out_err; + } perf_ctrls->max_perf = max; - if (CPC_SUPPORTED(min_perf_reg)) - cpc_read(cpu, min_perf_reg, &min); + if (CPC_SUPPORTED(min_perf_reg)) { + ret = cpc_read(cpu, min_perf_reg, &min); + if (ret) + goto out_err; + } perf_ctrls->min_perf = min; - if (CPC_SUPPORTED(desired_perf_reg)) - cpc_read(cpu, desired_perf_reg, &desired_perf); + if (CPC_SUPPORTED(desired_perf_reg)) { + ret = cpc_read(cpu, desired_perf_reg, &desired_perf); + if (ret) + goto out_err; + } perf_ctrls->desired_perf = desired_perf; - if (CPC_SUPPORTED(energy_perf_reg)) - cpc_read(cpu, energy_perf_reg, &energy_perf); + if (CPC_SUPPORTED(energy_perf_reg)) { + ret = cpc_read(cpu, energy_perf_reg, &energy_perf); + if (ret) + goto out_err; + } perf_ctrls->energy_perf = energy_perf; - if (CPC_SUPPORTED(auto_sel_reg)) - cpc_read(cpu, auto_sel_reg, &auto_sel); + if (CPC_SUPPORTED(auto_sel_reg)) { + ret = cpc_read(cpu, auto_sel_reg, &auto_sel); + if (ret) + goto out_err; + } perf_ctrls->auto_sel = (bool)auto_sel; out_err: From 5974566e381460ea9a325b06dd6b5dec05853cad Mon Sep 17 00:00:00 2001 From: Pierre Gondois Date: Thu, 26 Mar 2026 21:44:00 +0100 Subject: [PATCH 147/311] cpufreq: Remove max_freq_req update for pre-existing policy BugLink: https://bugs.launchpad.net/bugs/2131705 policy->max_freq_req QoS constraint represents the maximal allowed frequency than can be requested. It is set by: - writing to policyX/scaling_max sysfs file - toggling the cpufreq/boost sysfs file Upon calling freq_qos_update_request(), a successful update of the max_freq_req value triggers cpufreq_notifier_max(), followed by cpufreq_set_policy() which update the requested frequency for the policy. If the new max_freq_req value is not different from the original value, no frequency update is triggered. In a specific sequence of toggling: - cpufreq/boost sysfs file - CPU hot-plugging a CPU could end up with boost enabled but running at the maximal non-boost frequency, cpufreq_notifier_max() not being triggered. The following fixed that: commit 1608f0230510 ("cpufreq: Fix re-boost issue after hotplugging a CPU") The following: commit dd016f379ebc ("cpufreq: Introduce a more generic way to set default per-policy boost flag") also fixed the issue by correctly setting the max_freq_req constraint of a policy that is re-activated. This makes the first fix unnecessary. As the original issue is fixed by another method, this patch reverts: commit 1608f0230510 ("cpufreq: Fix re-boost issue after hotplugging a CPU") Reviewed-by: Lifeng Zheng Signed-off-by: Pierre Gondois Acked-by: Viresh Kumar Link: https://patch.msgid.link/20260326204404.1401849-2-pierre.gondois@arm.com Signed-off-by: Rafael J. Wysocki (cherry picked from commit 04aa9d0726cc6a23b348498815a9722b42d27c91) Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/cpufreq/cpufreq.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 1f794524a1d92..fbc04bff06e35 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1484,10 +1484,6 @@ static int cpufreq_policy_online(struct cpufreq_policy *policy, blocking_notifier_call_chain(&cpufreq_policy_notifier_list, CPUFREQ_CREATE_POLICY, policy); - } else { - ret = freq_qos_update_request(policy->max_freq_req, policy->max); - if (ret < 0) - goto out_destroy_policy; } if (cpufreq_driver->get && has_target()) { From ab99983f0f5ea4621a98f97c54d55edabf39dd6e Mon Sep 17 00:00:00 2001 From: Pierre Gondois Date: Thu, 26 Mar 2026 21:44:01 +0100 Subject: [PATCH 148/311] cpufreq: Add boost_freq_req QoS request BugLink: https://bugs.launchpad.net/bugs/2131705 The Power Management Quality of Service (PM QoS) allows to aggregate constraints from multiple entities. It is currently used to manage the min/max frequency of a given policy. Frequency constraints can come for instance from: - Thermal framework: acpi_thermal_cpufreq_init() - Firmware: _PPC objects: acpi_processor_ppc_init() - User: by setting policyX/scaling_[min|max]_freq The minimum of the max frequency constraints is used to compute the resulting maximum allowed frequency. When enabling boost frequencies, the same frequency request object (policy->max_freq_req) as to handle requests from users is used. As a result, when setting: - scaling_max_freq - boost The last sysfs file used overwrites the request from the other sysfs file. To avoid this, create a per-policy boost_freq_req to save the boost constraints instead of overwriting the last scaling_max_freq constraint. policy_set_boost() calls the cpufreq set_boost callback. Update the newly added boost_freq_req request from there: - whenever boost is toggled - to cover all possible paths In the existing .set_boost() callbacks: - Don't update policy->max as this is done through the qos notifier cpufreq_notifier_max() which calls cpufreq_set_policy(). - Remove freq_qos_update_request() calls as the qos request is now done in policy_set_boost() and updates the new boost_freq_req $ ## Init state scaling_max_freq:1000000 cpuinfo_max_freq:1000000 $ echo 700000 > scaling_max_freq scaling_max_freq:700000 cpuinfo_max_freq:1000000 $ echo 1 > ../boost scaling_max_freq:1200000 cpuinfo_max_freq:1200000 $ echo 800000 > scaling_max_freq scaling_max_freq:800000 cpuinfo_max_freq:1200000 $ ## Final step: $ ## Without the patches: $ echo 0 > ../boost scaling_max_freq:1000000 cpuinfo_max_freq:1000000 $ ## With the patches: $ echo 0 > ../boost scaling_max_freq:800000 cpuinfo_max_freq:1000000 Note: cpufreq_frequency_table_cpuinfo() updates policy->min and max from: A. cpufreq_boost_set_sw() \-cpufreq_frequency_table_cpuinfo() B. cpufreq_policy_online() \-cpufreq_table_validate_and_sort() \-cpufreq_frequency_table_cpuinfo() Keep these updates as some drivers expect policy->min and max to be set through B. Reviewed-by: Lifeng Zheng Signed-off-by: Pierre Gondois Acked-by: Viresh Kumar Link: https://patch.msgid.link/20260326204404.1401849-3-pierre.gondois@arm.com Signed-off-by: Rafael J. Wysocki (cherry picked from commit 6e39ba4e5a82aa5469b2ac517b74a71accb0540f) Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/cpufreq/amd-pstate.c | 2 -- drivers/cpufreq/cppc_cpufreq.c | 10 ++------ drivers/cpufreq/cpufreq.c | 46 +++++++++++++++++++++++----------- include/linux/cpufreq.h | 1 + 4 files changed, 34 insertions(+), 25 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 5aa9fcd80cf51..d0675d6a19fe1 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -769,8 +769,6 @@ static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) else if (policy->cpuinfo.max_freq > nominal_freq) policy->cpuinfo.max_freq = nominal_freq; - policy->max = policy->cpuinfo.max_freq; - if (cppc_state == AMD_PSTATE_PASSIVE) { ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq); if (ret < 0) diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index 5dfb109cf1f4e..7e7f9dfb7a24c 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -834,17 +834,11 @@ static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state) { struct cppc_cpudata *cpu_data = policy->driver_data; struct cppc_perf_caps *caps = &cpu_data->perf_caps; - int ret; if (state) - policy->max = cppc_perf_to_khz(caps, caps->highest_perf); + policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, caps->highest_perf); else - policy->max = cppc_perf_to_khz(caps, caps->nominal_perf); - policy->cpuinfo.max_freq = policy->max; - - ret = freq_qos_update_request(policy->max_freq_req, policy->max); - if (ret < 0) - return ret; + policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, caps->nominal_perf); return 0; } diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index fbc04bff06e35..0e4440bfff54e 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -609,10 +609,19 @@ static int policy_set_boost(struct cpufreq_policy *policy, bool enable) policy->boost_enabled = enable; ret = cpufreq_driver->set_boost(policy, enable); - if (ret) + if (ret) { policy->boost_enabled = !policy->boost_enabled; + return ret; + } - return ret; + ret = freq_qos_update_request(policy->boost_freq_req, policy->cpuinfo.max_freq); + if (ret < 0) { + policy->boost_enabled = !policy->boost_enabled; + cpufreq_driver->set_boost(policy, policy->boost_enabled); + return ret; + } + + return 0; } static ssize_t store_local_boost(struct cpufreq_policy *policy, @@ -1377,6 +1386,7 @@ static void cpufreq_policy_free(struct cpufreq_policy *policy) } freq_qos_remove_request(policy->min_freq_req); + freq_qos_remove_request(policy->boost_freq_req); kfree(policy->min_freq_req); cpufreq_policy_put_kobj(policy); @@ -1442,26 +1452,38 @@ static int cpufreq_policy_online(struct cpufreq_policy *policy, cpumask_and(policy->cpus, policy->cpus, cpu_online_mask); if (new_policy) { + unsigned int count; + for_each_cpu(j, policy->related_cpus) { per_cpu(cpufreq_cpu_data, j) = policy; add_cpu_dev_symlink(policy, j, get_cpu_device(j)); } - policy->min_freq_req = kzalloc(2 * sizeof(*policy->min_freq_req), + count = policy->boost_supported ? 3 : 2; + policy->min_freq_req = kzalloc(count * sizeof(*policy->min_freq_req), GFP_KERNEL); if (!policy->min_freq_req) { ret = -ENOMEM; goto out_destroy_policy; } + if (policy->boost_supported) { + policy->boost_freq_req = policy->min_freq_req + 2; + + ret = freq_qos_add_request(&policy->constraints, + policy->boost_freq_req, + FREQ_QOS_MAX, + policy->cpuinfo.max_freq); + if (ret < 0) { + policy->boost_freq_req = NULL; + goto out_destroy_policy; + } + } + ret = freq_qos_add_request(&policy->constraints, policy->min_freq_req, FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); if (ret < 0) { - /* - * So we don't call freq_qos_remove_request() for an - * uninitialized request. - */ kfree(policy->min_freq_req); policy->min_freq_req = NULL; goto out_destroy_policy; @@ -2785,16 +2807,10 @@ int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state) return -ENXIO; ret = cpufreq_frequency_table_cpuinfo(policy); - if (ret) { + if (ret) pr_err("%s: Policy frequency update failed\n", __func__); - return ret; - } - - ret = freq_qos_update_request(policy->max_freq_req, policy->max); - if (ret < 0) - return ret; - return 0; + return ret; } EXPORT_SYMBOL_GPL(cpufreq_boost_set_sw); diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index cc894fc389710..89157e367eefa 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -81,6 +81,7 @@ struct cpufreq_policy { struct freq_constraints constraints; struct freq_qos_request *min_freq_req; struct freq_qos_request *max_freq_req; + struct freq_qos_request *boost_freq_req; struct cpufreq_frequency_table *freq_table; enum cpufreq_table_sorting freq_table_sorted; From cae1d25fd8b9c6f83a7b2744214b7052fc02c58c Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Tue, 31 Mar 2026 10:33:46 +0530 Subject: [PATCH 149/311] cpufreq: Allocate QoS freq_req objects with policy BugLink: https://bugs.launchpad.net/bugs/2131705 A recent change exposed a bug in the error path: if freq_qos_add_request(boost_freq_req) fails, min_freq_req may remain a valid pointer even though it was never successfully added. During policy teardown, this leads to an unconditional call to freq_qos_remove_request(), triggering a WARN. The current design allocates all three freq_req objects together, making the lifetime rules unclear and error handling fragile. Simplify this by allocating the QoS freq_req objects at policy allocation time. The policy itself is dynamically allocated, and two of the three requests are always needed anyway. This ensures consistent lifetime management and eliminates the inconsistent state in failure paths. Reported-by: Zhongqiu Han Fixes: 6e39ba4e5a82 ("cpufreq: Add boost_freq_req QoS request") Signed-off-by: Viresh Kumar Reviewed-by: Lifeng Zheng Tested-by: Pierre Gondois Reviewed-by: Zhongqiu Han Link: https://patch.msgid.link/a293f29d841b86c51f34699c6e717e01858d8ada.1774933424.git.viresh.kumar@linaro.org Signed-off-by: Rafael J. Wysocki (cherry picked from commit 9266b4da051a410d9e6c5c0b0ef0c877855aa1b8) Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/cpufreq/cpufreq.c | 53 +++++++++++---------------------------- include/linux/cpufreq.h | 6 ++--- 2 files changed, 17 insertions(+), 42 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 0e4440bfff54e..5a5cb04ee12da 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -614,7 +614,7 @@ static int policy_set_boost(struct cpufreq_policy *policy, bool enable) return ret; } - ret = freq_qos_update_request(policy->boost_freq_req, policy->cpuinfo.max_freq); + ret = freq_qos_update_request(&policy->boost_freq_req, policy->cpuinfo.max_freq); if (ret < 0) { policy->boost_enabled = !policy->boost_enabled; cpufreq_driver->set_boost(policy, policy->boost_enabled); @@ -769,7 +769,7 @@ static ssize_t store_##file_name \ if (ret) \ return ret; \ \ - ret = freq_qos_update_request(policy->object##_freq_req, val);\ + ret = freq_qos_update_request(&policy->object##_freq_req, val); \ return ret >= 0 ? count : ret; \ } @@ -1374,7 +1374,7 @@ static void cpufreq_policy_free(struct cpufreq_policy *policy) /* Cancel any pending policy->update work before freeing the policy. */ cancel_work_sync(&policy->update); - if (policy->max_freq_req) { + if (freq_qos_request_active(&policy->max_freq_req)) { /* * Remove max_freq_req after sending CPUFREQ_REMOVE_POLICY * notification, since CPUFREQ_CREATE_POLICY notification was @@ -1382,12 +1382,13 @@ static void cpufreq_policy_free(struct cpufreq_policy *policy) */ blocking_notifier_call_chain(&cpufreq_policy_notifier_list, CPUFREQ_REMOVE_POLICY, policy); - freq_qos_remove_request(policy->max_freq_req); + freq_qos_remove_request(&policy->max_freq_req); } - freq_qos_remove_request(policy->min_freq_req); - freq_qos_remove_request(policy->boost_freq_req); - kfree(policy->min_freq_req); + if (freq_qos_request_active(&policy->min_freq_req)) + freq_qos_remove_request(&policy->min_freq_req); + if (freq_qos_request_active(&policy->boost_freq_req)) + freq_qos_remove_request(&policy->boost_freq_req); cpufreq_policy_put_kobj(policy); free_cpumask_var(policy->real_cpus); @@ -1452,57 +1453,31 @@ static int cpufreq_policy_online(struct cpufreq_policy *policy, cpumask_and(policy->cpus, policy->cpus, cpu_online_mask); if (new_policy) { - unsigned int count; - for_each_cpu(j, policy->related_cpus) { per_cpu(cpufreq_cpu_data, j) = policy; add_cpu_dev_symlink(policy, j, get_cpu_device(j)); } - count = policy->boost_supported ? 3 : 2; - policy->min_freq_req = kzalloc(count * sizeof(*policy->min_freq_req), - GFP_KERNEL); - if (!policy->min_freq_req) { - ret = -ENOMEM; - goto out_destroy_policy; - } - if (policy->boost_supported) { - policy->boost_freq_req = policy->min_freq_req + 2; - ret = freq_qos_add_request(&policy->constraints, - policy->boost_freq_req, + &policy->boost_freq_req, FREQ_QOS_MAX, policy->cpuinfo.max_freq); - if (ret < 0) { - policy->boost_freq_req = NULL; + if (ret < 0) goto out_destroy_policy; - } } ret = freq_qos_add_request(&policy->constraints, - policy->min_freq_req, FREQ_QOS_MIN, + &policy->min_freq_req, FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); - if (ret < 0) { - kfree(policy->min_freq_req); - policy->min_freq_req = NULL; + if (ret < 0) goto out_destroy_policy; - } - - /* - * This must be initialized right here to avoid calling - * freq_qos_remove_request() on uninitialized request in case - * of errors. - */ - policy->max_freq_req = policy->min_freq_req + 1; ret = freq_qos_add_request(&policy->constraints, - policy->max_freq_req, FREQ_QOS_MAX, + &policy->max_freq_req, FREQ_QOS_MAX, FREQ_QOS_MAX_DEFAULT_VALUE); - if (ret < 0) { - policy->max_freq_req = NULL; + if (ret < 0) goto out_destroy_policy; - } blocking_notifier_call_chain(&cpufreq_policy_notifier_list, CPUFREQ_CREATE_POLICY, policy); diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 89157e367eefa..ae3f15fd4e6e2 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -79,9 +79,9 @@ struct cpufreq_policy { * called, but you're in IRQ context */ struct freq_constraints constraints; - struct freq_qos_request *min_freq_req; - struct freq_qos_request *max_freq_req; - struct freq_qos_request *boost_freq_req; + struct freq_qos_request min_freq_req; + struct freq_qos_request max_freq_req; + struct freq_qos_request boost_freq_req; struct cpufreq_frequency_table *freq_table; enum cpufreq_table_sorting freq_table_sorted; From 8b622e1fd7c55faf6e5771410b07303cc7582104 Mon Sep 17 00:00:00 2001 From: "Mario Limonciello (AMD)" Date: Thu, 26 Mar 2026 14:36:20 -0500 Subject: [PATCH 150/311] cpufreq/amd-pstate: Cache the max frequency in cpudata BugLink: https://bugs.launchpad.net/bugs/2131705 The value of maximum frequency is fixed and never changes. Doing calculations every time based off of perf is unnecessary. Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20260326193620.649441-1-mario.limonciello@amd.com Signed-off-by: Mario Limonciello (AMD) (backported from commit 8cdc494013dfcd48f31eafe19b18fd67c224dd8a) [jamien: minor context-line drift in amd-pstate.c hunks from 3-way auto-merge; +/- content is byte-identical to upstream.] Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/cpufreq/amd-pstate.c | 27 +++++++++------------------ drivers/cpufreq/amd-pstate.h | 2 ++ 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index d0675d6a19fe1..dce694d13c4e9 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -757,15 +757,13 @@ static void amd_pstate_adjust_perf(unsigned int cpu, static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) { struct amd_cpudata *cpudata = policy->driver_data; - union perf_cached perf = READ_ONCE(cpudata->perf); - u32 nominal_freq, max_freq; + u32 nominal_freq; int ret = 0; nominal_freq = READ_ONCE(cpudata->nominal_freq); - max_freq = perf_to_freq(perf, cpudata->nominal_freq, perf.highest_perf); if (on) - policy->cpuinfo.max_freq = max_freq; + policy->cpuinfo.max_freq = cpudata->max_freq; else if (policy->cpuinfo.max_freq > nominal_freq) policy->cpuinfo.max_freq = nominal_freq; @@ -950,13 +948,15 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) WRITE_ONCE(cpudata->nominal_freq, nominal_freq); + /* max_freq is calculated according to (nominal_freq * highest_perf)/nominal_perf */ max_freq = perf_to_freq(perf, nominal_freq, perf.highest_perf); + WRITE_ONCE(cpudata->max_freq, max_freq); + lowest_nonlinear_freq = perf_to_freq(perf, nominal_freq, perf.lowest_nonlinear_perf); WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); /** * Below values need to be initialized correctly, otherwise driver will fail to load - * max_freq is calculated according to (nominal_freq * highest_perf)/nominal_perf * lowest_nonlinear_freq is a value between [min_freq, nominal_freq] * Check _CPC in ACPI table objects if any values are incorrect */ @@ -1019,9 +1019,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, cpudata->nominal_freq, perf.lowest_perf); - policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf, - cpudata->nominal_freq, - perf.highest_perf); + policy->cpuinfo.max_freq = policy->max = cpudata->max_freq; ret = amd_pstate_cppc_enable(policy); if (ret) @@ -1088,14 +1086,9 @@ static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, char *buf) { - struct amd_cpudata *cpudata; - union perf_cached perf; - - cpudata = policy->driver_data; - perf = READ_ONCE(cpudata->perf); + struct amd_cpudata *cpudata = policy->driver_data; - return sysfs_emit(buf, "%u\n", - perf_to_freq(perf, cpudata->nominal_freq, perf.highest_perf)); + return sysfs_emit(buf, "%u\n", cpudata->max_freq); } static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, @@ -1501,9 +1494,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, cpudata->nominal_freq, perf.lowest_perf); - policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf, - cpudata->nominal_freq, - perf.highest_perf); + policy->cpuinfo.max_freq = policy->max = cpudata->max_freq; policy->driver_data = cpudata; ret = amd_pstate_cppc_enable(policy); diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index cb45fdca27a6c..0d26e56b7938b 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -68,6 +68,7 @@ struct amd_aperf_mperf { * @min_limit_freq: Cached value of policy->min (in khz) * @max_limit_freq: Cached value of policy->max (in khz) * @nominal_freq: the frequency (in khz) that mapped to nominal_perf + * @max_freq: in ideal conditions the maximum frequency (in khz) possible frequency * @lowest_nonlinear_freq: the frequency (in khz) that mapped to lowest_nonlinear_perf * @cur: Difference of Aperf/Mperf/tsc count between last and current sample * @prev: Last Aperf/Mperf/tsc count value read from register @@ -94,6 +95,7 @@ struct amd_cpudata { u32 min_limit_freq; u32 max_limit_freq; u32 nominal_freq; + u32 max_freq; u32 lowest_nonlinear_freq; struct amd_aperf_mperf cur; From e8c99b5c81bffe5290748f058f70c4b0ceadfdee Mon Sep 17 00:00:00 2001 From: Pierre Gondois Date: Mon, 11 May 2026 15:55:28 +0200 Subject: [PATCH 151/311] NVIDIA: SAUCE: cpufreq: Extract cpufreq_policy_init_qos() function BugLink: https://bugs.launchpad.net/bugs/2131705 Extract the QoS related logic from cpufreq_policy_online() to make the function shorter/simpler. The logic is placed in cpufreq_policy_init_qos() and is now executed right after the following calls: - cpufreq_driver->init() - cpufreq_table_validate_and_sort() This helps preparing following patches that will, in cpufreq_policy_init_qos(): - treat the policy->min/max values set by drivers as QoS requests. - set a default policy->min/max value to all policies. No functional change. Signed-off-by: Pierre Gondois (backported from https://lore.kernel.org/lkml/20260511135538.522653-1-pierre.gondois@arm.com/) Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/cpufreq/cpufreq.c | 53 +++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 5a5cb04ee12da..44eb68c80ea13 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1397,6 +1397,32 @@ static void cpufreq_policy_free(struct cpufreq_policy *policy) kfree(policy); } +static int cpufreq_policy_init_qos(struct cpufreq_policy *policy) +{ + int ret; + + if (policy->boost_supported) { + ret = freq_qos_add_request(&policy->constraints, + &policy->boost_freq_req, + FREQ_QOS_MAX, + policy->cpuinfo.max_freq); + if (ret < 0) + return ret; + } + + ret = freq_qos_add_request(&policy->constraints, &policy->min_freq_req, + FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); + if (ret < 0) + return ret; + + ret = freq_qos_add_request(&policy->constraints, &policy->max_freq_req, + FREQ_QOS_MAX, FREQ_QOS_MAX_DEFAULT_VALUE); + if (ret < 0) + return ret; + + return ret; +} + static int cpufreq_policy_online(struct cpufreq_policy *policy, unsigned int cpu, bool new_policy) { @@ -1442,6 +1468,12 @@ static int cpufreq_policy_online(struct cpufreq_policy *policy, if (ret) goto out_offline_policy; + if (new_policy) { + ret = cpufreq_policy_init_qos(policy); + if (ret < 0) + goto out_offline_policy; + } + /* related_cpus should at least include policy->cpus. */ cpumask_copy(policy->related_cpus, policy->cpus); } @@ -1458,27 +1490,6 @@ static int cpufreq_policy_online(struct cpufreq_policy *policy, add_cpu_dev_symlink(policy, j, get_cpu_device(j)); } - if (policy->boost_supported) { - ret = freq_qos_add_request(&policy->constraints, - &policy->boost_freq_req, - FREQ_QOS_MAX, - policy->cpuinfo.max_freq); - if (ret < 0) - goto out_destroy_policy; - } - - ret = freq_qos_add_request(&policy->constraints, - &policy->min_freq_req, FREQ_QOS_MIN, - FREQ_QOS_MIN_DEFAULT_VALUE); - if (ret < 0) - goto out_destroy_policy; - - ret = freq_qos_add_request(&policy->constraints, - &policy->max_freq_req, FREQ_QOS_MAX, - FREQ_QOS_MAX_DEFAULT_VALUE); - if (ret < 0) - goto out_destroy_policy; - blocking_notifier_call_chain(&cpufreq_policy_notifier_list, CPUFREQ_CREATE_POLICY, policy); } From 5a1d60dcb0b696c1fe3578546dd1dd425aa17ad5 Mon Sep 17 00:00:00 2001 From: Pierre Gondois Date: Mon, 11 May 2026 15:55:29 +0200 Subject: [PATCH 152/311] NVIDIA: SAUCE: cpufreq: Set default policy->min/max values for all drivers BugLink: https://bugs.launchpad.net/bugs/2131705 Some drivers set policy->min/max in their .init() callback. cpufreq_set_policy() will ultimately override them through: cpufreq_policy_online() \-cpufreq_init_policy() \-cpufreq_set_policy() \-/* Set policy->min/max */ Thus the policy min/max values provided are only temporary. There is an exception if CPUFREQ_NEED_INITIAL_FREQ_CHECK is set and: cpufreq_policy_online() \-__cpufreq_driver_target() \-cpufreq_driver->target() To prepare for a following patch that will remove all policy->min/max initialization in the driver .init() callback if the min/max value is equal to the cpuinfo.min/max_freq, set a default policy->min/max value for all drivers. Signed-off-by: Pierre Gondois (backported from https://lore.kernel.org/lkml/20260511135538.522653-1-pierre.gondois@arm.com/) Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/cpufreq/cpufreq.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 44eb68c80ea13..d193f6e446008 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1401,6 +1401,13 @@ static int cpufreq_policy_init_qos(struct cpufreq_policy *policy) { int ret; + /* + * If the driver didn't set policy->min/max, set them as + * they are used to clamp frequency requests. + */ + policy->min = policy->min ? policy->min : policy->cpuinfo.min_freq; + policy->max = policy->max ? policy->max : policy->cpuinfo.max_freq; + if (policy->boost_supported) { ret = freq_qos_add_request(&policy->constraints, &policy->boost_freq_req, From 7e7a996831b189e96946ed54734afbd533015a21 Mon Sep 17 00:00:00 2001 From: Pierre Gondois Date: Mon, 11 May 2026 15:55:30 +0200 Subject: [PATCH 153/311] NVIDIA: SAUCE: cpufreq: Remove driver default policy->min/max init BugLink: https://bugs.launchpad.net/bugs/2131705 Prior to [1], drivers were setting policy->min/max and the value was used as a QoS constraint. After that change, the values were only temporarily used: cpufreq_set_policy() ultimately overriding them through: cpufreq_policy_online() \-cpufreq_init_policy() \-cpufreq_set_policy() \-/* Set policy->min/max */ This patch reinstate the initial behaviour. This will allow drivers to request min/max QoS frequencies if desired. For instance, the cppc driver advertises a lowest non-linear frequency, which should be used as a min QoS value. To avoid having drivers setting policy->min/max to default values which are considered as QoS values (i.e. the reason why [1] was introduced), remove the initialization of policy->min/max in .init() callbacks wherever the policy->min/max values are identical to the policy->cpuinfo.min/max_freq. Indeed, the previous patch ("cpufreq: Set default policy->min/max values for all drivers") makes this initialization redundant. The only drivers where these values are different are: - gx-suspmod.c (min) - cppc-cpufreq.c (min) - longrun.c [1] commit 521223d8b3ec ("cpufreq: Fix initialization of min and max frequency QoS requests") Signed-off-by: Pierre Gondois (backported from https://lore.kernel.org/lkml/20260511135538.522653-1-pierre.gondois@arm.com/) [jamien: 3-way auto-merge resolved context drift in amd-pstate.c and intel_pstate.c against this tree; +/- content is byte-identical to v2 3/4.] Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/cpufreq/amd-pstate.c | 14 ++++++-------- drivers/cpufreq/cppc_cpufreq.c | 5 ++--- drivers/cpufreq/cpufreq-nforce2.c | 4 ++-- drivers/cpufreq/freq_table.c | 7 +++---- drivers/cpufreq/gx-suspmod.c | 2 +- drivers/cpufreq/intel_pstate.c | 3 --- drivers/cpufreq/pcc-cpufreq.c | 10 ++++------ drivers/cpufreq/pxa3xx-cpufreq.c | 5 ++--- drivers/cpufreq/sh-cpufreq.c | 6 ++---- drivers/cpufreq/virtual-cpufreq.c | 5 +---- 10 files changed, 23 insertions(+), 38 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index dce694d13c4e9..c2d9d409867db 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1016,10 +1016,9 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) perf = READ_ONCE(cpudata->perf); - policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, - cpudata->nominal_freq, - perf.lowest_perf); - policy->cpuinfo.max_freq = policy->max = cpudata->max_freq; + policy->cpuinfo.min_freq = perf_to_freq(perf, cpudata->nominal_freq, + perf.lowest_perf); + policy->cpuinfo.max_freq = cpudata->max_freq; ret = amd_pstate_cppc_enable(policy); if (ret) @@ -1491,10 +1490,9 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) perf = READ_ONCE(cpudata->perf); - policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, - cpudata->nominal_freq, - perf.lowest_perf); - policy->cpuinfo.max_freq = policy->max = cpudata->max_freq; + policy->cpuinfo.min_freq = perf_to_freq(perf, cpudata->nominal_freq, + perf.lowest_perf); + policy->cpuinfo.max_freq = cpudata->max_freq; policy->driver_data = cpudata; ret = amd_pstate_cppc_enable(policy); diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index 7e7f9dfb7a24c..5abac50df7508 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -660,8 +660,6 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy) * Section 8.4.7.1.1.5 of ACPI 6.1 spec) */ policy->min = cppc_perf_to_khz(caps, caps->lowest_nonlinear_perf); - policy->max = cppc_perf_to_khz(caps, policy->boost_enabled ? - caps->highest_perf : caps->nominal_perf); /* * Set cpuinfo.min_freq to Lowest to make the full range of performance @@ -669,7 +667,8 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy) * nonlinear perf */ policy->cpuinfo.min_freq = cppc_perf_to_khz(caps, caps->lowest_perf); - policy->cpuinfo.max_freq = policy->max; + policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, policy->boost_enabled ? + caps->highest_perf : caps->nominal_perf); policy->transition_delay_us = cppc_cpufreq_get_transition_delay_us(cpu); policy->shared_type = cpu_data->shared_type; diff --git a/drivers/cpufreq/cpufreq-nforce2.c b/drivers/cpufreq/cpufreq-nforce2.c index fbbbe501cf2dc..831102522ad64 100644 --- a/drivers/cpufreq/cpufreq-nforce2.c +++ b/drivers/cpufreq/cpufreq-nforce2.c @@ -355,8 +355,8 @@ static int nforce2_cpu_init(struct cpufreq_policy *policy) min_fsb = NFORCE2_MIN_FSB; /* cpuinfo and default policy values */ - policy->min = policy->cpuinfo.min_freq = min_fsb * fid * 100; - policy->max = policy->cpuinfo.max_freq = max_fsb * fid * 100; + policy->cpuinfo.min_freq = min_fsb * fid * 100; + policy->cpuinfo.max_freq = max_fsb * fid * 100; return 0; } diff --git a/drivers/cpufreq/freq_table.c b/drivers/cpufreq/freq_table.c index 5b364d8da4f92..ea994647abc88 100644 --- a/drivers/cpufreq/freq_table.c +++ b/drivers/cpufreq/freq_table.c @@ -49,16 +49,15 @@ int cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy) max_freq = freq; } - policy->min = policy->cpuinfo.min_freq = min_freq; - policy->max = max_freq; + policy->cpuinfo.min_freq = min_freq; /* * If the driver has set its own cpuinfo.max_freq above max_freq, leave * it as is. */ if (policy->cpuinfo.max_freq < max_freq) - policy->max = policy->cpuinfo.max_freq = max_freq; + policy->cpuinfo.max_freq = max_freq; - if (policy->min == ~0) + if (min_freq == ~0) return -EINVAL; else return 0; diff --git a/drivers/cpufreq/gx-suspmod.c b/drivers/cpufreq/gx-suspmod.c index d269a4f26f98e..d40c9e0bbb740 100644 --- a/drivers/cpufreq/gx-suspmod.c +++ b/drivers/cpufreq/gx-suspmod.c @@ -421,7 +421,7 @@ static int cpufreq_gx_cpu_init(struct cpufreq_policy *policy) policy->min = maxfreq / max_duration; else policy->min = maxfreq / POLICY_MIN_DIV; - policy->max = maxfreq; + policy->cpuinfo.min_freq = maxfreq / max_duration; policy->cpuinfo.max_freq = maxfreq; diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 11c58af419006..ed3fd134fb8b7 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -3049,9 +3049,6 @@ static int __intel_pstate_cpu_init(struct cpufreq_policy *policy) policy->cpuinfo.max_freq = READ_ONCE(global.no_turbo) ? cpu->pstate.max_freq : cpu->pstate.turbo_freq; - policy->min = policy->cpuinfo.min_freq; - policy->max = policy->cpuinfo.max_freq; - intel_pstate_init_acpi_perf_limits(policy); policy->fast_switch_possible = true; diff --git a/drivers/cpufreq/pcc-cpufreq.c b/drivers/cpufreq/pcc-cpufreq.c index ac2e90a65f0c4..0f185a13577f8 100644 --- a/drivers/cpufreq/pcc-cpufreq.c +++ b/drivers/cpufreq/pcc-cpufreq.c @@ -551,13 +551,11 @@ static int pcc_cpufreq_cpu_init(struct cpufreq_policy *policy) goto out; } - policy->max = policy->cpuinfo.max_freq = - ioread32(&pcch_hdr->nominal) * 1000; - policy->min = policy->cpuinfo.min_freq = - ioread32(&pcch_hdr->minimum_frequency) * 1000; + policy->cpuinfo.max_freq = ioread32(&pcch_hdr->nominal) * 1000; + policy->cpuinfo.min_freq = ioread32(&pcch_hdr->minimum_frequency) * 1000; - pr_debug("init: policy->max is %d, policy->min is %d\n", - policy->max, policy->min); + pr_debug("init: max_freq is %d, min_freq is %d\n", + policy->cpuinfo.max_freq, policy->cpuinfo.min_freq); out: return result; } diff --git a/drivers/cpufreq/pxa3xx-cpufreq.c b/drivers/cpufreq/pxa3xx-cpufreq.c index 50ff3b6a69000..06b27cbc59d6a 100644 --- a/drivers/cpufreq/pxa3xx-cpufreq.c +++ b/drivers/cpufreq/pxa3xx-cpufreq.c @@ -185,9 +185,8 @@ static int pxa3xx_cpufreq_init(struct cpufreq_policy *policy) int ret = -EINVAL; /* set default policy and cpuinfo */ - policy->min = policy->cpuinfo.min_freq = 104000; - policy->max = policy->cpuinfo.max_freq = - (cpu_is_pxa320()) ? 806000 : 624000; + policy->cpuinfo.min_freq = 104000; + policy->cpuinfo.max_freq = (cpu_is_pxa320()) ? 806000 : 624000; policy->cpuinfo.transition_latency = 1000; /* FIXME: 1 ms, assumed */ if (cpu_is_pxa300() || cpu_is_pxa310()) diff --git a/drivers/cpufreq/sh-cpufreq.c b/drivers/cpufreq/sh-cpufreq.c index 642ddb9ea217e..3c99d7009cbe2 100644 --- a/drivers/cpufreq/sh-cpufreq.c +++ b/drivers/cpufreq/sh-cpufreq.c @@ -124,10 +124,8 @@ static int sh_cpufreq_cpu_init(struct cpufreq_policy *policy) dev_notice(dev, "no frequency table found, falling back " "to rate rounding.\n"); - policy->min = policy->cpuinfo.min_freq = - (clk_round_rate(cpuclk, 1) + 500) / 1000; - policy->max = policy->cpuinfo.max_freq = - (clk_round_rate(cpuclk, ~0UL) + 500) / 1000; + policy->cpuinfo.min_freq = (clk_round_rate(cpuclk, 1) + 500) / 1000; + policy->cpuinfo.max_freq = (clk_round_rate(cpuclk, ~0UL) + 500) / 1000; } return 0; diff --git a/drivers/cpufreq/virtual-cpufreq.c b/drivers/cpufreq/virtual-cpufreq.c index 4159f31349b16..dc78b74409af4 100644 --- a/drivers/cpufreq/virtual-cpufreq.c +++ b/drivers/cpufreq/virtual-cpufreq.c @@ -164,10 +164,7 @@ static int virt_cpufreq_get_freq_info(struct cpufreq_policy *policy) policy->cpuinfo.min_freq = 1; policy->cpuinfo.max_freq = virt_cpufreq_get_perftbl_entry(policy->cpu, 0); - policy->min = policy->cpuinfo.min_freq; - policy->max = policy->cpuinfo.max_freq; - - policy->cur = policy->max; + policy->cur = policy->cpuinfo.max_freq; return 0; } From aed8b74510c5a20f1069161727bdeefff6425699 Mon Sep 17 00:00:00 2001 From: Pierre Gondois Date: Mon, 11 May 2026 15:55:31 +0200 Subject: [PATCH 154/311] NVIDIA: SAUCE: cpufreq: Use policy->min/max init as QoS request BugLink: https://bugs.launchpad.net/bugs/2131705 Consider policy->min/max being set in the driver .init() callback as a QoS request. Impacted driver are: - gx-suspmod.c (min) - cppc-cpufreq.c (min) - longrun.c (min/max) Update the documentation accordingly. Signed-off-by: Pierre Gondois (backported from https://lore.kernel.org/lkml/20260511135538.522653-1-pierre.gondois@arm.com/) Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- Documentation/cpu-freq/cpu-drivers.rst | 10 ++++++++-- drivers/cpufreq/cpufreq.c | 12 ++++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Documentation/cpu-freq/cpu-drivers.rst b/Documentation/cpu-freq/cpu-drivers.rst index c5635ac3de547..ab4f3c0f3a89b 100644 --- a/Documentation/cpu-freq/cpu-drivers.rst +++ b/Documentation/cpu-freq/cpu-drivers.rst @@ -114,8 +114,14 @@ Then, the driver must fill in the following values: |policy->cur | The current operating frequency of | | | this CPU (if appropriate) | +-----------------------------------+--------------------------------------+ -|policy->min, | | -|policy->max, | | +|policy->min | If set by the driver in ->init(), | +| | used as initial minimum frequency | +| | QoS request. | ++-----------------------------------+--------------------------------------+ +|policy->max | If set by the driver in ->init(), | +| | used as initial maximum frequency | +| | QoS request. | ++-----------------------------------+--------------------------------------+ |policy->policy and, if necessary, | | |policy->governor | must contain the "default policy" for| | | this CPU. A few moments later, | diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index d193f6e446008..a2fba4d3e5070 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1399,8 +1399,16 @@ static void cpufreq_policy_free(struct cpufreq_policy *policy) static int cpufreq_policy_init_qos(struct cpufreq_policy *policy) { + unsigned int min_freq, max_freq; int ret; + /* Use policy->min/max set by the driver as QoS requests. */ + min_freq = max(FREQ_QOS_MIN_DEFAULT_VALUE, policy->min); + if (policy->max) + max_freq = min(FREQ_QOS_MAX_DEFAULT_VALUE, policy->max); + else + max_freq = FREQ_QOS_MAX_DEFAULT_VALUE; + /* * If the driver didn't set policy->min/max, set them as * they are used to clamp frequency requests. @@ -1418,12 +1426,12 @@ static int cpufreq_policy_init_qos(struct cpufreq_policy *policy) } ret = freq_qos_add_request(&policy->constraints, &policy->min_freq_req, - FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); + FREQ_QOS_MIN, min_freq); if (ret < 0) return ret; ret = freq_qos_add_request(&policy->constraints, &policy->max_freq_req, - FREQ_QOS_MAX, FREQ_QOS_MAX_DEFAULT_VALUE); + FREQ_QOS_MAX, max_freq); if (ret < 0) return ret; From 6709607b60ecb3031b26d86682b241df2e2f845b Mon Sep 17 00:00:00 2001 From: Sumit Gupta Date: Sat, 25 Apr 2026 01:48:14 +0530 Subject: [PATCH 155/311] NVIDIA: SAUCE: cpufreq: CPPC: add autonomous mode boot parameter support BugLink: https://bugs.launchpad.net/bugs/2131705 Add a kernel boot parameter 'cppc_cpufreq.auto_sel_mode' to enable CPPC autonomous performance selection on all CPUs at system startup. When autonomous mode is enabled, the hardware automatically adjusts CPU performance based on workload demands using Energy Performance Preference (EPP) hints. When auto_sel_mode=1: - Configure all CPUs for autonomous operation on first init - Set EPP to performance preference (0x0) - Use HW min/max_perf when available; otherwise initialize from caps - Clamp desired_perf to bounds before enabling autonomous mode - Hardware controls frequency instead of the OS governor The boot parameter is applied only during first policy initialization. Skip applying it on CPU hotplug to preserve runtime sysfs configuration. This patch depends on patch [2] ("cpufreq: Set policy->min and max as real QoS constraints") so that the policy->min/max set in cppc_cpufreq_cpu_init() are not overridden by cpufreq_set_policy() during init. Reviewed-by: Randy Dunlap Signed-off-by: Sumit Gupta (backported from https://lore.kernel.org/lkml/20260424201814.230071-1-sumitg@nvidia.com/) [jamien: hunk #2 (cppc_set_enable() insertion in cppc_cpufreq_cpu_init) rebased onto Pierre's v2 series, which replaced the local min/max vars with direct policy->min assignment; insertion point and code are unchanged.] Signed-off-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- .../admin-guide/kernel-parameters.txt | 13 +++ drivers/cpufreq/cppc_cpufreq.c | 89 +++++++++++++++++-- 2 files changed, 97 insertions(+), 5 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index a030ce253b4b7..2915345847fde 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1052,6 +1052,19 @@ Kernel parameters policy to use. This governor must be registered in the kernel before the cpufreq driver probes. + cppc_cpufreq.auto_sel_mode= + [CPU_FREQ] Enable ACPI CPPC autonomous performance + selection. When enabled, hardware automatically adjusts + CPU frequency on all CPUs based on workload demands. + In Autonomous mode, Energy Performance Preference (EPP) + hints guide hardware toward performance (0x0) or energy + efficiency (0xff). + Requires ACPI CPPC autonomous selection register support. + Format: + Default: 0 (disabled) + 0: use cpufreq governors + 1: enable if supported by hardware + cpu_init_udelay=N [X86,EARLY] Delay for N microsec between assert and de-assert of APIC INIT to start processors. This delay occurs diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index 5abac50df7508..be28d7e6cf63e 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -28,6 +28,9 @@ static struct cpufreq_driver cppc_cpufreq_driver; +/* Autonomous Selection boot parameter */ +static bool auto_sel_mode; + #ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE static enum { FIE_UNSET = -1, @@ -655,6 +658,14 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy) caps = &cpu_data->perf_caps; policy->driver_data = cpu_data; + /* + * Enable CPPC for both OS-driven and autonomous modes. + * The Enable register is optional - some platforms may not support it + */ + ret = cppc_set_enable(cpu, true); + if (ret && ret != -EOPNOTSUPP) + pr_warn("Failed to enable CPPC for CPU%d (%d)\n", cpu, ret); + /* * Set min to lowest nonlinear perf to avoid any efficiency penalty (see * Section 8.4.7.1.1.5 of ACPI 6.1 spec) @@ -707,11 +718,71 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy) policy->cur = cppc_perf_to_khz(caps, caps->highest_perf); cpu_data->perf_ctrls.desired_perf = caps->highest_perf; - ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); - if (ret) { - pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n", - caps->highest_perf, cpu, ret); - goto out; + /* + * Enable autonomous mode on first init if boot param is set. + * Check last_governor to detect first init and skip if auto_sel + * is already enabled. + */ + if (auto_sel_mode && policy->last_governor[0] == '\0' && + !cpu_data->perf_ctrls.auto_sel) { + /* Init min/max_perf from caps if not already set by HW. */ + if (!cpu_data->perf_ctrls.min_perf) + cpu_data->perf_ctrls.min_perf = caps->lowest_nonlinear_perf; + if (!cpu_data->perf_ctrls.max_perf) + cpu_data->perf_ctrls.max_perf = policy->boost_enabled ? + caps->highest_perf : caps->nominal_perf; + + cpu_data->perf_ctrls.desired_perf = + clamp_t(u32, cpu_data->perf_ctrls.desired_perf, + cpu_data->perf_ctrls.min_perf, + cpu_data->perf_ctrls.max_perf); + + policy->cur = cppc_perf_to_khz(caps, + cpu_data->perf_ctrls.desired_perf); + + /* EPP is optional - some platforms may not support it */ + ret = cppc_set_epp(cpu, CPPC_EPP_PERFORMANCE_PREF); + if (ret && ret != -EOPNOTSUPP) + pr_warn("Failed to set EPP for CPU%d (%d)\n", cpu, ret); + else if (!ret) + cpu_data->perf_ctrls.energy_perf = CPPC_EPP_PERFORMANCE_PREF; + + /* Program min/max/desired into CPPC regs before enabling auto_sel. */ + ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); + if (ret) { + pr_debug("Err setting perf for autonomous mode CPU:%d ret:%d\n", + cpu, ret); + goto out; + } + + ret = cppc_set_auto_sel(cpu, true); + if (ret && ret != -EOPNOTSUPP) { + pr_warn("Failed autonomous config for CPU%d (%d)\n", + cpu, ret); + goto out; + } + if (!ret) + cpu_data->perf_ctrls.auto_sel = true; + } + + if (cpu_data->perf_ctrls.auto_sel) { + /* Sync policy limits from HW when autonomous mode is active */ + policy->min = cppc_perf_to_khz(caps, + cpu_data->perf_ctrls.min_perf ?: + caps->lowest_nonlinear_perf); + policy->max = cppc_perf_to_khz(caps, + cpu_data->perf_ctrls.max_perf ?: + (policy->boost_enabled ? + caps->highest_perf : + caps->nominal_perf)); + } else { + /* Normal mode: governors control frequency */ + ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); + if (ret) { + pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n", + caps->highest_perf, cpu, ret); + goto out; + } } cppc_cpufreq_cpu_fie_init(policy); @@ -1031,10 +1102,18 @@ static int __init cppc_cpufreq_init(void) static void __exit cppc_cpufreq_exit(void) { + unsigned int cpu; + + for_each_present_cpu(cpu) + cppc_set_auto_sel(cpu, false); + cpufreq_unregister_driver(&cppc_cpufreq_driver); cppc_freq_invariance_exit(); } +module_param(auto_sel_mode, bool, 0444); +MODULE_PARM_DESC(auto_sel_mode, "Enable CPPC autonomous performance selection at boot"); + module_exit(cppc_cpufreq_exit); MODULE_AUTHOR("Ashwin Chaugule"); MODULE_DESCRIPTION("CPUFreq driver based on the ACPI CPPC v5.0+ spec"); From 7eb6dfee8ab509e0d51a974115267e39c844ee9d Mon Sep 17 00:00:00 2001 From: Nirmoy Das Date: Fri, 15 May 2026 09:35:38 -0700 Subject: [PATCH 156/311] NVIDIA: SAUCE: ovl: keep err zero after successful ovl_cache_get() BugLink: https://bugs.launchpad.net/bugs/2150640 ovl_iterate_merged() stores PTR_ERR(cache) in err before checking IS_ERR(cache). On success err holds the truncated cache pointer and can be returned as a bogus non-zero error. The syzbot reproducer reaches this through overlay-on-overlay readdir: getdents64 iterate_dir(outer overlay file) ovl_iterate_merged() ovl_cache_get() ovl_dir_read_merged() ovl_dir_read() iterate_dir(inner overlay file) ovl_iterate_merged() Only compute PTR_ERR(cache) on the error path. Fixes: d25e4b739f83 ("ovl: refactor ovl_iterate() and port to cred guard") Reported-by: syzbot+a16fb0cce329a320661c@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=a16fb0cce329a320661c Cc: stable@vger.kernel.org Signed-off-by: Nirmoy Das (backported from https://lore.kernel.org/r/20260514144258.3068715-1-nirmoyd@nvidia.com) Signed-off-by: Nirmoy Das Acked-by: Jamie Nguyen Acked-by: Carol L Soto Signed-off-by: Brad Figg --- fs/overlayfs/readdir.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fs/overlayfs/readdir.c b/fs/overlayfs/readdir.c index 1dcc75b3a90f9..e7fe29cb6028b 100644 --- a/fs/overlayfs/readdir.c +++ b/fs/overlayfs/readdir.c @@ -838,15 +838,14 @@ static int ovl_iterate_merged(struct file *file, struct dir_context *ctx) struct ovl_dir_file *od = file->private_data; struct dentry *dentry = file->f_path.dentry; struct ovl_cache_entry *p; - int err = 0; + int err; if (!od->cache) { struct ovl_dir_cache *cache; cache = ovl_cache_get(dentry); - err = PTR_ERR(cache); if (IS_ERR(cache)) - return err; + return PTR_ERR(cache); od->cache = cache; ovl_seek_cursor(od, ctx->pos); @@ -869,7 +868,7 @@ static int ovl_iterate_merged(struct file *file, struct dir_context *ctx) od->cursor = p->l_node.next; ctx->pos++; } - return err; + return 0; } static bool ovl_need_adjust_d_ino(struct file *file) From 0f6abca012f5beb84b2e99bff9f552ed11f2aef5 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Wed, 20 May 2026 16:57:43 -0500 Subject: [PATCH 157/311] UBUNTU: [Packaging] update variants BugLink: https://bugs.launchpad.net/bugs/1786013 Signed-off-by: Jacob Martin --- debian.nvidia/variants | 2 -- 1 file changed, 2 deletions(-) diff --git a/debian.nvidia/variants b/debian.nvidia/variants index 881c9938362e4..cab2f62a9df90 100644 --- a/debian.nvidia/variants +++ b/debian.nvidia/variants @@ -1,4 +1,2 @@ -7.0 -- --hwe-24.04 --hwe-24.04-edge From 35bbc60314ddc30805d4579a8c73e148e9052db3 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Wed, 20 May 2026 17:00:07 -0500 Subject: [PATCH 158/311] UBUNTU: Start new release Ignore: yes Signed-off-by: Jacob Martin --- debian.nvidia/changelog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debian.nvidia/changelog b/debian.nvidia/changelog index be5d8e21181fb..cb77935736a79 100644 --- a/debian.nvidia/changelog +++ b/debian.nvidia/changelog @@ -1,3 +1,11 @@ +linux-nvidia (7.0.0-1014.14) UNRELEASED; urgency=medium + + CHANGELOG: Do not edit directly. Autogenerated at release. + CHANGELOG: Use the printchanges target to see the current changes. + CHANGELOG: Use the insertchanges target to create the final log. + + -- Jacob Martin Wed, 20 May 2026 17:00:07 -0500 + linux-nvidia (7.0.0-1006.6) resolute; urgency=medium * resolute/linux-nvidia: 7.0.0-1006.6 -proposed tracker (LP: #2148214) From 1ec5c9f9b5023f89db3016b656766ab804a30f2a Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Wed, 20 May 2026 17:04:01 -0500 Subject: [PATCH 159/311] UBUNTU: link-to-tracker: update tracking bug BugLink: https://bugs.launchpad.net/bugs/2153496 Properties: no-test-build Signed-off-by: Jacob Martin --- debian.nvidia/tracking-bug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian.nvidia/tracking-bug b/debian.nvidia/tracking-bug index e1ada5cb8e86e..164377f318e42 100644 --- a/debian.nvidia/tracking-bug +++ b/debian.nvidia/tracking-bug @@ -1 +1 @@ -2148214 d2026.04.13-1 +2153496 d2026.05.20-1 From e32f4dbeccb151708ebd17266d508af5785560fd Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Fri, 22 May 2026 08:38:19 -0500 Subject: [PATCH 160/311] UBUNTU: [Packaging] dkms-build: Pass --force to `dkms build` This is necessary to bypass dependencies declared by the nvidia-fs dkms.conf that are present on the system, detected by the nvidia-fs build, but not in the source directory used by dkms and so not detected by dkms. Ignore: yes Signed-off-by: Jacob Martin --- debian/scripts/dkms-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/scripts/dkms-build b/debian/scripts/dkms-build index 8e9adc56f03de..d64946a808e9b 100755 --- a/debian/scripts/dkms-build +++ b/debian/scripts/dkms-build @@ -171,7 +171,7 @@ echo "II: dkms-build building $package" fakeroot="" [ $(id -u) -ne 0 ] && fakeroot="/usr/bin/fakeroot" rc=0 -$fakeroot /usr/sbin/dkms build --no-prepare-kernel --no-clean-kernel \ +$fakeroot /usr/sbin/dkms build --force --no-prepare-kernel --no-clean-kernel \ -k "$abi_flavour" ${ARCH:+-a $ARCH} \ --sourcetree "$dkms_dir/source" \ --dkmstree "$dkms_dir/build" \ From 094189485e8e7410b938ac16cce3e34cf560ca64 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Fri, 22 May 2026 11:18:36 -0500 Subject: [PATCH 161/311] UBUNTU: [Packaging] debian.nvidia/dkms-versions -- update from kernel-versions (adhoc/d2026.05.20) BugLink: https://bugs.launchpad.net/bugs/1786013 Signed-off-by: Jacob Martin --- debian.nvidia/dkms-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian.nvidia/dkms-versions b/debian.nvidia/dkms-versions index fd8e9c633b136..ba40c69369282 100644 --- a/debian.nvidia/dkms-versions +++ b/debian.nvidia/dkms-versions @@ -1,2 +1,2 @@ -zfs-linux 2.4.1-1ubuntu1 modulename=zfs debpath=pool/universe/z/%package%/zfs-dkms_%version%_all.deb arch=amd64 arch=arm64 arch=ppc64el arch=riscv64 arch=s390x rprovides=spl-modules rprovides=spl-dkms rprovides=zfs-modules rprovides=zfs-dkms off_series=true +zfs-linux 2.4.1-1ubuntu5 modulename=zfs debpath=pool/universe/z/%package%/zfs-dkms_%version%_all.deb arch=amd64 arch=arm64 arch=ppc64el arch=riscv64 arch=s390x rprovides=spl-modules rprovides=spl-dkms rprovides=zfs-modules rprovides=zfs-dkms off_series=true v4l2loopback 0.15.3-1ubuntu2 modulename=v4l2loopback debpath=pool/universe/v/%package%/v4l2loopback-dkms_%version%_all.deb arch=amd64 rprovides=v4l2loopback-modules rprovides=v4l2loopback-dkms off_series=true From fa782aedb83ff730c498812cf30fbe6b19367761 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Thu, 18 Jun 2026 18:23:46 -0500 Subject: [PATCH 162/311] UBUNTU: Ubuntu-nvidia-7.0.0-1014.14 Signed-off-by: Jacob Martin --- debian.nvidia/changelog | 688 +++++++++++++++++++++++++++++++++++++- debian.nvidia/reconstruct | 2 + 2 files changed, 685 insertions(+), 5 deletions(-) diff --git a/debian.nvidia/changelog b/debian.nvidia/changelog index cb77935736a79..3bf2cb4618b8d 100644 --- a/debian.nvidia/changelog +++ b/debian.nvidia/changelog @@ -1,10 +1,688 @@ -linux-nvidia (7.0.0-1014.14) UNRELEASED; urgency=medium +linux-nvidia (7.0.0-1014.14) resolute; urgency=medium - CHANGELOG: Do not edit directly. Autogenerated at release. - CHANGELOG: Use the printchanges target to see the current changes. - CHANGELOG: Use the insertchanges target to create the final log. + * resolute/linux-nvidia: 7.0.0-1014.14 -proposed tracker (LP: #2153496) - -- Jacob Martin Wed, 20 May 2026 17:00:07 -0500 + * Packaging resync (LP: #1786013) + - [Packaging] update variants + - [Packaging] debian.nvidia/dkms-versions -- update from kernel-versions + (adhoc/d2026.05.20) + + * Installer fails internally with a RSync error due to page fault + (LP: #2150640) + - NVIDIA: SAUCE: ovl: keep err zero after successful ovl_cache_get() + + * Pull CPPC mailing list patches for Spark (LP: #2131705) + - ACPI: CPPC: Add cppc_get_perf() API to read performance controls + - ACPI: CPPC: Warn on missing mandatory DESIRED_PERF register + - ACPI: CPPC: Extend cppc_set_epp_perf() for FFH/SystemMemory + - cpufreq: CPPC: Update cached perf_ctrls on sysfs write + - cpufreq: cppc: Update MIN_PERF/MAX_PERF in target callbacks + - ACPI: CPPC: add APIs and sysfs interface for perf_limited + - cpufreq: CPPC: Add sysfs documentation for perf_limited + - ACPI: CPPC: Move reference performance to capabilities + - ACPI: CPPC: Fix uninitialized ref variable in cppc_get_perf_caps() + - ACPI: CPPC: Check cpc_read() return values consistently + - cpufreq: Remove max_freq_req update for pre-existing policy + - cpufreq: Add boost_freq_req QoS request + - cpufreq: Allocate QoS freq_req objects with policy + - cpufreq/amd-pstate: Cache the max frequency in cpudata + - NVIDIA: SAUCE: cpufreq: Extract cpufreq_policy_init_qos() function + - NVIDIA: SAUCE: cpufreq: Set default policy->min/max values for all + drivers + - NVIDIA: SAUCE: cpufreq: Remove driver default policy->min/max init + - NVIDIA: SAUCE: cpufreq: Use policy->min/max init as QoS request + - NVIDIA: SAUCE: cpufreq: CPPC: add autonomous mode boot parameter support + + * Backport lan743x driver patches (LP: #2152064) + - net: microchip: lan743x: add ethtool nway_reset support + - net: lan743x: fix SGMII detection on PCI1xxxx B0+ during warm reset + - net: lan743x: rename chip_rev to fpga_rev + + * Backport SMT-aware asymmetric CPU capacity idle selection (LP: #2150671) + - NVIDIA: VR: SAUCE: sched/fair: Attach sched_domain_shared to + sd_asym_cpucapacity + - NVIDIA: VR: SAUCE: sched/fair: Prefer fully-idle SMT cores in asym- + capacity idle selection + - NVIDIA: VR: SAUCE: sched/fair: Reject misfit pulls onto busy SMT + siblings on asym-capacity + - NVIDIA: VR: SAUCE: sched/fair: Add SIS_UTIL support to + select_idle_capacity() + + * Introduce a sharded cache affinity scope (LP: #2150467) + - workqueue: fix parse_affn_scope() prefix matching bug + - workqueue: fix typo in WQ_AFFN_SMT comment + - workqueue: add WQ_AFFN_CACHE_SHARD affinity scope + - workqueue: set WQ_AFFN_CACHE_SHARD as the default affinity scope + - tools/workqueue: add CACHE_SHARD support to wq_dump.py + - workqueue: add test_workqueue benchmark module + - docs: workqueue: document WQ_AFFN_CACHE_SHARD affinity scope + - workqueue: avoid unguarded 64-bit division + - workqueue: validate cpumask_first() result in + llc_populate_cpu_shard_id() + - [Config] nvidia: Defaults for CONFIG_TEST_WORKQUEUE + + * UBUNTU: [Config] nvidia: Disable default CMA reservation (LP: #2150898) + - [Config] nvidia: Disable default CMA reservation + + * Backport Use device ID range for DGX Spark iGPU (LP: #2150487) + - NVIDIA: SAUCE: iommu/arm-smmu-v3: Use device ID range for DGX Spark iGPU + iommu quirk + + * Backport NVIDIA: SAUCE: iommu/arm-smmu-v3: Use identity domain for ASPEED + BMC devices (LP: #2150470) + - NVIDIA: SAUCE: iommu/arm-smmu-v3: Use identity domain for ASPEED BMC + devices + + * Update GDS/NVMe SAUCE for v6.17 (LP: #2134960) // [linux-nvidia-7.0]: + Forward-port GDS/NVFS content (LP: #2150289) + - NVIDIA: SAUCE: Patch NVMe/NVMeoF driver to support GDS on Linux 7.0 + Kernel + + * serial: 8250_mtk: Update ACPI support to add NVDA0240 device ID + (LP: #2148607) + - Revert "NVIDIA: SAUCE: serial: 8250_mtk: Add ACPI support" + - NVIDIA: SAUCE: MEDIATEK: serial: 8250_mtk: Add ACPI support + + * fix r8169 vs r8127 contention for Spark (LP: #2144345) + - NVIDIA: SAUCE: r8169: remove PCI IDs claimed by r8127 driver + + [ Ubuntu: 7.0.0-27.27 ] + + * resolute/linux: 7.0.0-27.27 -proposed tracker (LP: #2157114) + * Packaging resync (LP: #1786013) + - [Packaging] update annotations scripts + * Ubuntu 26.04 linux kernel has non-functional nova-core GPU driver enabled, + conflicting with nouveau (LP: #2150845) + - [Config] Disable NOVA_CORE + * CVE-2026-46316 + - KVM: arm64: vgic-its: Drop the translation cache reference only for the + erased entry + * CVE-2026-46244 + - netfilter: nft_inner: Fix IPv6 inner_thoff desync + * CVE-2026-46137 + - mptcp: pm: ADD_ADDR rtx: allow ID 0 + - mptcp: pm: ADD_ADDR rtx: fix potential data-race + * CVE-2026-46185 + - smb/client: fix out-of-bounds read in symlink_data() + * CVE-2026-46195 + - smb: client: validate dacloffset before building DACL pointers + * CVE-2026-46289 + - lib/scatterlist: fix length calculations in extract_kvec_to_sg + * CVE-2026-46119 + - libceph: Fix slab-out-of-bounds access in auth message processing + * CVE-2026-46135 + - nvmet-tcp: fix race between ICReq handling and queue teardown + * CVE-2026-46155 + - smb/client: fix out-of-bounds read in smb2_compound_op() + * CVE-2026-46115 + - block: add pgmap check to biovec_phys_mergeable + * CVE-2026-46243 + - smb: client: reject userspace cifs.spnego descriptions + + [ Ubuntu: 7.0.0-26.26 ] + + * resolute/linux: 7.0.0-26.26 -proposed tracker (LP: #2154530) + * Packaging resync (LP: #1786013) + - Revert "UBUNTU: SAUCE: import Huawei ES3000_V2 (2.1.0.23)" + - [Packaging] debian.master/dkms-versions -- remove dkms-versions + (main/2026.05.18) + * Fix mic mute led on a HP EliteBook 6 G2a platform (LP: #2150065) + - ALSA: hda/realtek: Add LED fixup for HP EliteBook 6 G2a Laptops + * ov08x40 module mounted upside down on a certain DELL platforms + (LP: #2146517) + - SAUCE: media: ipu-bridge: Add DMI quirk for new Dell XPS laptops with + upside down sensors + - SAUCE: media: ipu-bridge: Add DMI quirk for Dell 14 laptops with upside + down sensors + * Support additional 2888x1808@30fps 900MHz for OVTI05C1 camera sensor + (LP: #2147409) + - SAUCE: media: ipu-bridge: Add 900MHz for OV05C10 + - SAUCE: platform/x86: int3472: increase handshake delay to 50ms for + OV05C10 + * Support Samsung S5K3J1 sensor for Intel MIPI camera (LP: #2121852) + - SAUCE: media: ipu-bridge: Support s5k3j1 sensor + * [SRU] ASoC: enable rt1320 speaker amp and DMIC on PTL SoundWire platforms + (LP: #2150196) + - ASoC: Intel: soc-acpi-intel-ptl-match: drop rt722 monolithic match + tables + - ASoC: SOF: Intel: Add a is_amp flag to fix the wrong name prefix + - ASoC: sdw_utils: add rt1320 and rt1321 dmic dai in codec_info_list + * powerpc-build in ubuntu_kernel_selftests fails to build due to + uninitialized value (LP: #2129844) + - selftests/powerpc: Suppress -Wmaybe-uninitialized with GCC 15 + * Ubuntu 26.04 linux kernel has non-functional nova-core GPU driver enabled, + conflicting with nouveau (LP: #2150845) + - [Config] Disable DRM_NOVA + * Resolute update: v7.0.6 upstream stable release (LP: #2152558) + - Linux 7.0.6 + - Upstream stable to v7.0.6 + * Resolute update: v7.0.5 upstream stable release (LP: #2152556) + - Linux 7.0.5 + - Upstream stable to v7.0.5 + * Resolute update: v7.0.4 upstream stable release (LP: #2152552) + - ALSA: usb-audio: stop parsing UAC2 rates at MAX_NR_RATES + - ALSA: usb-audio: Avoid false E-MU sample-rate notifications + - ALSA: usb-audio: Fix Audio Advantage Micro II SPDIF switch + - usb: xhci: Make usb_host_endpoint.hcpriv survive endpoint_disable() + - usb: chipidea: otg: not wait vbus drop if use role_switch + - usb: chipidea: core: allow ci_irq_handler() handle both ID and VBUS + change + - ALSA: usb-audio: Evaluate packsize caps at the right place + - LoongArch: Add spectre boundry for syscall dispatch table + - drm/nouveau: fix u32 overflow in pushbuf reloc bounds check + - leds: qcom-lpg: Check for array overflow when selecting the high + resolution + - greybus: gb-beagleplay: bound bootloader receive buffering + - greybus: gb-beagleplay: fix sleep in atomic context in hdlc_tx_frames() + - misc: ibmasm: fix OOB MMIO read in ibmasm_handle_mouse_interrupt() + - ibmasm: fix OOB reads in command_file_write due to missing size checks + - ibmasm: fix heap over-read in ibmasm_send_i2o_message() + - sysfs: attribute_group: Respect is_visible_const() when changing owner + - driver core: Don't let a device probe until it's ready + - device property: Make modifications of fwnode "flags" thread safe + - drm/nouveau: fix nvkm_device leak on aperture removal failure + - rust: dma: remove DMA_ATTR_NO_KERNEL_MAPPING from public attrs + - kbuild: rust: allow `clippy::uninlined_format_args` + - fs: afs: revert mmap_prepare() change + - firmware: google: framebuffer: Do not mark framebuffer as busy + - lib: test_hmm: evict device pages on file close to avoid use-after-free + - arm64/mm: Enable batched TLB flush in unmap_hotplug_range() + - arm64: mm: Fix rodata=full block mapping support for realm guests + - mm: migrate: requeue destination folio on deferred split queue + - mm: prevent droppable mappings from being locked + - mm: fix deferred split queue races during migration + - ocfs2: split transactions in dio completion to avoid credit exhaustion + - Input: edt-ft5x06 - fix use-after-free in debugfs teardown + - zram: do not forget to endio for partial discard requests + - wifi: rtw88: check for PCI upstream bridge existence + - wifi: mwifiex: fix use-after-free in mwifiex_adapter_cleanup() + - vfio: selftests: Fix VLA initialisation in vfio_pci_irq_set() + - vfio/xe: Add a missing vfio_pci_core_release_dev() + - vfio/virtio: Convert list_lock from spinlock to mutex + - vfio/cdx: Serialize VFIO_DEVICE_SET_IRQS with a per-device mutex + - vfio/cdx: Fix NULL pointer dereference in interrupt trigger path + - um: drivers: call kernel_strrchr() explicitly in cow_user.c + - thermal: core: Fix thermal zone governor cleanup issues + - spi: imx: fix use-after-free on unbind + - spi: ch341: fix memory leaks on probe failures + - crypto: algif_aead - snapshot IV for async AEAD requests + - crypto: pcrypt - Fix handling of MAY_BACKLOG requests + - dt-bindings: display: ti, am65x-dss: Fix AM62L DSS reg and clock + constraints + - of: unittest: fix use-after-free in of_unittest_changeset() + - of: unittest: fix use-after-free in testdrv_probe() + - hwmon: (powerz) Fix missing usb_kill_urb() on signal interrupt + - EDAC/versalnet: Fix device_node leak in mc_probe() + - PCI: imx6: Skip waiting for L2/L3 Ready on i.MX6SX + - media: amphion: Fix race between m2m job_abort and device_run + - ALSA: control: Validate buf_len before strnlen() in + snd_ctl_elem_init_enum_names() + - net: caif: clear client service pointer on teardown + - net: strparser: fix skb_head leak in strp_abort_strp() + - media: mtk-jpeg: fix use-after-free in release path due to uncancelled + work + - crypto: atmel-sha204a - Fix OTP sysfs read and error handling + - PCI: endpoint: pci-epf-ntb: Remove duplicate resource teardown + - Revert "ALSA: usb: Increase volume range that triggers a warning" + - phy: qcom: m31-eusb2: clear PLL_EN during init + - PCI: epf-mhi: Return 0, not remaining timeout, when eDMA ops complete + - lib/ts_kmp: fix integer overflow in pattern length calculation + - media: i2c: imx219: Check return value of devm_gpiod_get_optional() in + imx219_probe() + - net: qrtr: ns: Fix use-after-free in driver remove() + - ext2: reject inodes with zero i_nlink and valid mode in ext2_iget() + - mm/zsmalloc: copy KMSAN metadata in zs_page_migrate() + - ALSA: aoa: i2sbus: clear stale prepared state + - ALSA: aoa: i2sbus: fix OF node lifetime handling + - ALSA: aoa: Skip devices with no codecs in i2sbus_resume() + - ALSA: ctxfi: Add fallback to default RSR for S/PDIF + - ALSA: seq_oss: return full count for successful SEQ_FULLSIZE writes + - erofs: fix the out-of-bounds nameoff handling for trailing dirents + - ipmi:ssif: Clean up kthread on errors + - jbd2: fix deadlock in jbd2_journal_cancel_revoke() + - KVM: selftests: Fix reserved value WRMSR testcase for multi-feature MSRs + - md/raid10: fix deadlock with check operation and nowait requests + - media: rc: igorplugusb: heed coherency rules + - media: rockchip: rkcif: fix off by one bugs + - media: rockchip: rkcif: comply with minimum number of buffers + requirement + - mfd: stpmic1: Attempt system shutdown twice in case PMIC is confused + - mm/alloc_tag: clear codetag for pages allocated before page_ext + initialization + - mm/damon/core: fix damon_call() vs kdamond_fn() exit race + - mm/damon/core: fix damos_walk() vs kdamond_fn() exit race + - mm/hugetlb: fix early boot crash on parameters without '=' separator + - mtd: docg3: fix use-after-free in docg3_release() + - nvme-pci: add NVME_QUIRK_DISABLE_WRITE_ZEROES for Kingston OM3SGP4 + - nvme: respect NVME_QUIRK_DISABLE_WRITE_ZEROES when wzsl is set + - parisc: _llseek syscall is only available for 32-bit userspace + - parisc: Drop ip_fast_csum() inline assembly implementation + - PCI: cadence: Use cdns_pcie_read_sz() for byte or word read access + - PCI: imx6: Fix reference clock source selection for i.MX95 + - perf annotate: Use jump__delete when freeing LoongArch jumps + - RDMA/mana_ib: Disable RX steering on RSS QP destroy + - remoteproc: xlnx: Only access buffer information if IPI is buffered + - reset: rzv2h-usb2phy: Keep PHY clock enabled for entire device lifetime + - sched: Use u64 for bandwidth ratio calculations + - selftests/mqueue: Fix incorrectly named file + - landlock: Fix LOG_SUBDOMAINS_OFF inheritance across fork() + - landlock: Allow TSYNC with LOG_SUBDOMAINS_OFF and fd=-1 + - selftests/landlock: Drain stale audit records on init + - selftests/landlock: Fix format warning for __u64 in net_test + - selftests/landlock: Fix snprintf truncation checks in audit helpers + - selftests/landlock: Skip stale records in audit_match_record() + - rbd: fix null-ptr-deref when device_add_disk() fails + - mm/zone_device: do not touch device folio after calling ->folio_free() + - block: fix zone write plugs refcount handling in + disk_zone_wplug_schedule_bio_work() + - io_uring/zcrx: return back two step unregistration + - io_uring/timeout: check unused sqe fields + - block: relax pgmap check in bio_add_page for compatible zone device + pages + - iio: adc: ti-ads7950: use iio_push_to_buffers_with_ts_unaligned() + - io_uring/register: fix ring resizing with mixed/large SQEs/CQEs + - io_uring/zcrx: fix user_struct uaf + - io_uring/poll: fix signed comparison in io_poll_get_ownership() + - io_uring/poll: ensure EPOLL_ONESHOT is propagated for EPOLL_URING_WAKE + - module.lds,codetag: force 0 sh_addr for sections + - module.lds.S: Fix modules on 32-bit parisc architecture + - ALSA: core: Fix potential data race at fasync handling + - ALSA: caiaq: Fix control_put() result and cache rollback + - ALSA: caiaq: Handle probe errors properly + - ALSA: 6fire: Fix input volume change detection + - ALSA: hda/realtek - Add mute LED support for HP Victus 15-fa2xxx + - ALSA: pcmtest: fix reference leak on failed device registration + - ALSA: pcmtest: Fix resource leaks in module init error paths + - iio: adc: ad7768-1: fix one-shot mode data acquisition + - iio: adc: ad7768-1: remove switch to one-shot mode + - rxrpc: Fix memory leaks in rxkad_verify_response() + - rxrpc: Fix rxkad crypto unalignment handling + - rxrpc: Fix error handling in rxgk_extract_token() + - rxrpc: Fix re-decryption of RESPONSE packets + - EDAC/versalnet: Fix memory leak in remove and probe error paths + - tools/accounting: handle truncated taskstats netlink messages + - net: txgbe: fix RTNL assertion warning when remove module + - arm64: dts: marvell: uDPU: add ethernet aliases + - net: qrtr: ns: Limit the maximum server registration per node + - net: qrtr: ns: Limit the maximum number of lookups + - net: qrtr: ns: Free the node during ctrl_cmd_bye() + - net: qrtr: ns: Limit the total number of nodes + - net: rds: fix MR cleanup on copy error + - net: txgbe: fix firmware version check + - net/smc: avoid early lgr access in smc_clc_wait_msg + - net: ks8851: Reinstate disabling of BHs around IRQ handler + - net: bridge: use a stable FDB dst snapshot in RCU readers + - netconsole: avoid out-of-bounds access on empty string in trim_newline() + - net: mctp: fix don't require received header reserved bits to be zero + - net: ks8851: Avoid excess softirq scheduling + - drm/arcpgu: fix device node leak + - slub: fix data loss and overflow in krealloc() + - tracing/fprobe: Reject registration of a registered fprobe before init + - RDMA/rxe: Validate pad and ICRC before payload_size() in rxe_rcv + - printf: Compile the kunit test with DISABLE_BRANCH_PROFILING + DISABLE_BRANCH_PROFILING + - ipv4: icmp: validate reply type before using icmp_pointers + - libceph: Prevent potential null-ptr-deref in ceph_handle_auth_reply() + - spi: fix resource leaks on device setup failure + - extract-cert: Wrap key_pass with '#ifdef USE_PKCS11_ENGINE' + - tpm: avoid -Wunused-but-set-variable + - LoongArch: Make arch_irq_work_has_interrupt() true only if IPI HW exist + - LoongArch: Show CPU vulnerabilites correctly + - fbdev: defio: Disconnect deferred I/O from the lifetime of struct + fb_info + - power: supply: axp288_charger: Do not cancel work before initializing it + - hwmon: (isl28022) Fix integer overflow in power calculation on 32-bit + - hwmon: (powerz) Avoid cacheline sharing for DMA buffer + - media: rzv2h-ivc: Revise default VBLANK formula + - media: rzv2h-ivc: Fix AXIRX_VBLANK register write + - fs: prepare for adding LSM blob to backing_file + - lsm: add backing_file LSM hooks + - selinux: fix overlayfs mmap() and mprotect() access checks + - hwmon: (pt5161l) Fix bugs in pt5161l_read_block_data() + - randomize_kstack: Maintain kstack_offset per task + - mmc: block: use single block write in retry + - mmc: sdhci-of-dwcmshc: Disable clock before DLL configuration + - arm64: dts: ti: am62-verdin: Enable pullup for eMMC data pins + - crypto: qat - fix IRQ cleanup on 6xxx probe failure + - xfs: start gc on zonegc_low_space attribute updates + - xfs: fix a resource leak in xfs_alloc_buftarg() + - firmware: google: framebuffer: Do not unregister platform device + - firmware: exynos-acpm: Drop fake 'const' on handle pointer + - crypto: talitos - fix SEC1 32k ahash request limitation + - crypto: talitos - rename first/last to first_desc/last_desc + - pwm: imx-tpm: Count the number of enabled channels in probe + - tpm2-sessions: Fix missing tpm_buf_destroy() in tpm2_read_public() + - tpm: Fix auth session leak in tpm2_get_random() error path + - tpm: Use kfree_sensitive() to free auth session in tpm_dev_release() + - tpm: tpm_tis: add error logging for data transfer + - tpm: tpm_tis: stop transmit if retries are exhausted + - rtc: ntxec: fix OF node reference imbalance + - mm/vmalloc: take vmap_purge_lock in shrinker + - mm/memfd_luo: fix physical address conversion in put_folios cleanup + - mm/mempolicy: fix memory leaks in weighted_interleave_auto_store() + - mm/damon/stat: fix memory leak on damon_start() failure in + damon_stat_start() + - mm/damon/core: validate damos_quota_goal->nid for + node_mem_{used,free}_bp + - mm/damon/core: validate damos_quota_goal->nid for + node_memcg_{used,free}_bp + - mm/damon/core: use time_in_range_open() for damos quota window start + - mm/damon/core: disallow time-quota setting zero esz + - mm/damon/core: disallow non-power of two min_region_sz on damon_start() + - userfaultfd: allow registration of ranges below mmap_min_addr + - LoongArch: KVM: Use CSR_CRMD_PLV in kvm_arch_vcpu_in_kernel() + - KVM: x86: Defer non-architectural deliver of exception payload to + userspace read + - KVM: nSVM: Mark all of vmcb02 dirty when restoring nested state + - KVM: nSVM: Sync NextRIP to cached vmcb12 after VMRUN of L2 + - KVM: nSVM: Sync interrupt shadow to cached vmcb12 after VMRUN of L2 + - KVM: SVM: Inject #UD for INVLPGA if EFER.SVME=0 + - KVM: SVM: Explicitly mark vmcb01 dirty after modifying VMCB intercepts + - KVM: nSVM: Ensure AVIC is inhibited when restoring a vCPU to guest mode + - KVM: nSVM: Always use NextRIP as vmcb02's NextRIP after first L2 VMRUN + - KVM: nSVM: Delay stuffing L2's current RIP into NextRIP until vCPU run + - KVM: nSVM: Use vcpu->arch.cr2 when updating vmcb12 on nested #VMEXIT + - KVM: arm64: Account for RESx bits in __compute_fgt() + - KVM: nSVM: Avoid clearing VMCB_LBR in vmcb12 + - KVM: nSVM: Delay setting soft IRQ RIP tracking fields until vCPU run + - KVM: SVM: Switch svm_copy_lbrs() to a macro + - KVM: SVM: Add missing save/restore handling of LBR MSRs + - KVM: nSVM: Always inject a #GP if mapping VMCB12 fails on nested VMRUN + - KVM: nSVM: Refactor checking LBRV enablement in vmcb12 into a helper + - KVM: nSVM: Refactor writing vmcb12 on nested #VMEXIT as a helper + - KVM: nSVM: Triple fault if restore host CR3 fails on nested #VMEXIT + - KVM: nSVM: Triple fault if mapping VMCB12 fails on nested #VMEXIT + - KVM: nSVM: Clear GIF on nested #VMEXIT(INVALID) + - KVM: nSVM: Clear EVENTINJ fields in vmcb12 on nested #VMEXIT + - KVM: nSVM: Clear tracking of L1->L2 NMI and soft IRQ on nested #VMEXIT + - KVM: nSVM: Add missing consistency check for EFER, CR0, CR4, and CS + - KVM: nSVM: Drop the non-architectural consistency check for NP_ENABLE + - KVM: nSVM: Add missing consistency check for nCR3 validity + - KVM: nSVM: Raise #UD if unhandled VMMCALL isn't intercepted by L1 + - KVM: nSVM: Always intercept VMMCALL when L2 is active + - ARM: 9472/1: fix race condition on PG_dcache_clean in + __sync_icache_dcache() + - ring-buffer: Do not double count the reader_page + - ext4: fix bounds check in check_xattrs() to prevent out-of-bounds access + - ext4: fix missing brelse() in ext4_xattr_inode_dec_ref_all() + - udf: fix partition descriptor append bookkeeping + - mtd: spi-nor: sst: Fix write enable before AAI sequence + - mtd: spinand: winbond: Declare the QE bit on W25NxxJW + - amdgpu/jpeg: fix deepsleep register for jpeg 5_0_0 and 5_0_2 + - md/md-llbitmap: skip reading rdevs that are not in_sync + - md/md-llbitmap: raise barrier before state machine transition + - md/raid5: fix soft lockup in retry_aligned_read() + - md/raid5: validate payload size before accessing journal metadata + - check-uapi: link into shared objects + - mm, swap: speed up hibernation allocation and writeout + - HID: apple: ensure the keyboard backlight is off if suspending + - inotify: fix watch count leak when fsnotify_add_inode_mark_locked() + fails + - x86/cpu: Disable FRED when PTI is forced on + - x86/shstk: Prevent deadlock during shstk sigreturn + - wifi: rtl8xxxu: fix potential use of uninitialized value + - tcp: call sk_data_ready() after listener migration + - taskstats: set version in TGID exit notifications + - mptcp: sync the msk->sndbuf at accept() time + - mfd: core: Preserve OF node when ACPI handle is present + - 9p: fix access mode flags being ORed instead of replaced + - Bluetooth: hci_event: fix potential UAF in SSP passkey handlers + - bus: mhi: host: pci_generic: Switch to async power up to avoid boot + delays + - can: ucan: fix devres lifetime + - crypto: acomp - fix wrong pointer stored by acomp_save_req() + - crypto: arm64/aes - Fix 32-bit aes_mac_update() arg treated as 64-bit + - crypto: atmel-aes - Fix 3-page memory leak in atmel_aes_buff_cleanup + - crypto: atmel-ecc - Release client on allocation failure + - crypto: hisilicon - Fix dma_unmap_single() direction + - crypto: ccree - fix a memory leak in cc_mac_digest() + - crypto: atmel-tdes - fix DMA sync direction + - crypto: atmel-sha204a - Fix error codes in OTP reads + - crypto: atmel-sha204a - Fix potential UAF and memory leak in remove path + - crypto: atmel-sha204a - Fix uninitialized data access on OTP read error + - crypto: nx - fix bounce buffer leaks in nx842_crypto_{alloc,free}_ctx + - crypto: nx - fix context leak in nx842_crypto_free_ctx + - crypto: nx - Fix packed layout in struct nx842_crypto_header + - dm mirror: fix integer overflow in create_dirty_log() + - erofs: fix unsigned underflow in z_erofs_lz4_handle_overlap() + - ceph: fix num_ops off-by-one when crypto allocation fails + - ceph: only d_add() negative dentries when they are unhashed + - gtp: disable BH before calling udp_tunnel_xmit_skb() + - IB/core: Fix zero dmac race in neighbor resolution + - ktest: Fix the month in the name of the failure directory + - NFSv4.1: Apply session size limits on clone path + - ntfs3: add buffer boundary checks to run_unpack() + - ntfs3: fix integer overflow in run_unpack() volume boundary check + - rtmutex: Use waiter::task instead of current in remove_waiter() + - rxgk: Fix potential integer overflow in length check + - sched_ext: Documentation: Clarify ops.dispatch() role in task lifecycle + - scsi: sd: fix missing put_disk() when device_add(&disk_dev) fails + - seg6: fix seg6 lwtunnel output redirect for L2 reduced encap mode + - perf loongarch: Fix build failure with CONFIG_LIBDW_DWARF_UNWIND + - iio: frequency: admv1013: add dev variable + - iio: frequency: admv1013: fix NULL pointer dereference on str + - wifi: mt76: mt792x: describe USB WFSYS reset with a descriptor + - wifi: mt76: mt792x: fix mt7925u USB WFSYS reset handling + - mm: various small mmap_prepare cleanups + - mm: avoid deadlock when holding rmap on mmap_prepare error + - mei: me: use PCI_DEVICE_DATA macro + - mei: me: add nova lake point H DID + - crypto: authencesn - reject short ahash digests during instance creation + - driver core: Add kernel-doc for DEV_FLAG_COUNT enum value + - ALSA: caiaq: Fix potentially leftover ep1_in_urb at error path + - ALSA: caiaq: Don't abort when no input device is available + - ipv6: rpl: reserve mac_len headroom when recompressed SRH grows + - drm/amdgpu: fix zero-size GDS range init on RDNA4 + - drm/imagination: Fix segfault when updating ftrace mask + - ALSA: caiaq: fix usb_dev refcount leak on probe failure + - ALSA: aloop: Fix peer runtime UAF during format-change stop + - vmalloc: fix buffer overflow in vrealloc_node_align() + - mm/page_alloc: return NULL early from alloc_frozen_pages_nolock() in NMI + on UP + - mm/slab: return NULL early from kmalloc_nolock() in NMI on UP + - net: ipv6: fix NOREF dst use in seg6 and rpl lwtunnels + - netfilter: reject zero shift in nft_bitwise + - ipmi:ssif: Remove unnecessary indention + - ipmi:ssif: NULL thread on error + - Linux 7.0.4 + - Upstream stable to v7.0.4 + * Resolute update: v7.0.3 upstream stable release (LP: #2152550) + - Buffer overflow in drivers/xen/sys-hypervisor.c + - xen/privcmd: fix double free via VMA splitting + - Linux 7.0.3 + - Upstream stable to v7.0.3 + * Resolute update: v7.0.2 upstream stable release (LP: #2150553) + - crypto: authencesn - Fix src offset when decrypting in-place + - pwm: th1520: fix `CLIPPY=1` warning + - drm/amdgpu: replace PASID IDR with XArray + - crypto: krb5enc - fix sleepable flag handling in encrypt dispatch + - crypto: krb5enc - fix async decrypt skipping hash verification + - ksmbd: fix use-after-free in __ksmbd_close_fd() via durable scavenger + - ksmbd: validate owner of durable handle on reconnect + - scripts: generate_rust_analyzer.py: define scripts + - scripts/dtc: Remove unused dts_version in dtc-lexer.l + - fs/ntfs3: validate rec->used in journal-replay file record check + - f2fs: fix to do sanity check on dcc->discard_cmd_cnt conditionally + - f2fs: fix UAF caused by decrementing sbi->nr_pages[] in + f2fs_write_end_io() + - f2fs: fix to avoid memory leak in f2fs_rename() + - f2fs: fix to avoid uninit-value access in f2fs_sanity_check_node_footer + - fuse: reject oversized dirents in page cache + - fuse: abort on fatal signal during sync init + - fuse: Check for large folio with SPLICE_F_MOVE + - fuse: quiet down complaints in fuse_conn_limit_write + - fuse: fuse_dev_ioctl_clone() should wait for device file to be + initialized + - ksmbd: require minimum ACE size in smb_check_perm_dacl() + - smb: server: fix active_num_conn leak on transport allocation failure + - smb: client: fix dir separator in SMB1 UNIX mounts + - smb: server: fix max_connections off-by-one in tcp accept path + - smb: client: require a full NFS mode SID before reading mode bits + - smb: client: validate the whole DACL before rewriting it in cifsacl + - smb: client: fix OOB read in smb2_ioctl_query_info QUERY_INFO path + - ksmbd: validate response sizes in ipc_validate_msg() + - ksmbd: validate num_aces and harden ACE walk in smb_inherit_dacl() + - ksmbd: fix out-of-bounds write in smb2_get_ea() EA alignment + - ksmbd: use check_add_overflow() to prevent u16 DACL size overflow + - ksmbd: reset rcount per connection in ksmbd_conn_wait_idle_sess_id() + - writeback: Fix use after free in inode_switch_wbs_work_fn() + - f2fs: fix use-after-free of sbi in f2fs_compress_write_end_io() + - ALSA: usb-audio: apply quirk for MOONDROP JU Jiu + - ALSA: hda/realtek: Add quirk for Legion S7 15IMH + - ALSA: caiaq: take a reference on the USB device in create_card() + - net/packet: fix TOCTOU race on mmap'd vnet_hdr in tpacket_snd() + - crypto: ccp: Don't attempt to copy CSR to userspace if PSP command + failed + - crypto: ccp: Don't attempt to copy PDH cert to userspace if PSP command + failed + - crypto: ccp: Don't attempt to copy ID to userspace if PSP command failed + - rxrpc: Fix missing validation of ticket length in non-XDR key preparsing + - mshv_vtl: Fix vmemmap_shift exceeding MAX_FOLIO_ORDER + - Linux 7.0.2 + * Resolute update: v7.0.1 upstream stable release (LP: #2150547) + - Revert "UBUNTU: SAUCE: cdc-acm: Exclude Exar USB serial ports" + - nfc: llcp: add missing return after LLCP_CLOSED checks + - x86/CPU: Fix FPDSS on Zen1 + - can: raw: fix ro->uniq use-after-free in raw_rcv() + - i2c: s3c24xx: check the size of the SMBUS message before using it + - staging: rtl8723bs: initialize le_tmp64 in rtw_BIP_verify() + - HID: alps: fix NULL pointer dereference in alps_raw_event() + - HID: core: clamp report_size in s32ton() to avoid undefined shift + - net: usb: cdc-phonet: fix skb frags[] overflow in rx_complete() + - NFC: digital: Bounds check NFC-A cascade depth in SDD response handler + - drm/vc4: platform_get_irq_byname() returns an int + - bnge: return after auxiliary_device_uninit() in error path + - ALSA: usx2y: us144mkii: fix NULL deref on missing interface 0 + - ALSA: fireworks: bound device-supplied status before string array lookup + - fbdev: tdfxfb: avoid divide-by-zero on FBIOPUT_VSCREENINFO + - usb: gadget: f_ncm: validate minimum block_len in ncm_unwrap_ntb() + - usb: gadget: f_phonet: fix skb frags[] overflow in pn_rx_complete() + - usb: gadget: renesas_usb3: validate endpoint index in standard request + handlers + - smb: client: fix off-by-8 bounds check in check_wsl_eas() + - smb: client: fix OOB reads parsing symlink error response + - ksmbd: validate EaNameLength in smb2_get_ea() + - ksmbd: require 3 sub-authorities before reading sub_auth[2] + - ksmbd: fix mechToken leak when SPNEGO decode fails after token alloc + - smb: client: avoid double-free in smbd_free_send_io() after + smbd_send_batch_flush() + - smb: server: avoid double-free in smb_direct_free_sendmsg after + smb_direct_flush_send_list() + - usbip: validate number_of_packets in usbip_pack_ret_submit() + - usb: typec: fusb302: Switch to threaded IRQ handler + - usb: storage: Expand range of matched versions for VL817 quirks entry + - USB: cdc-acm: Add quirks for Yoga Book 9 14IAH10 INGENIC touchscreen + - usb: gadget: f_hid: don't call cdev_init while cdev in use + - usb: port: add delay after usb_hub_set_port_power() + - fbdev: udlfb: avoid divide-by-zero on FBIOPUT_VSCREENINFO + - scripts/gdb/symbols: handle module path parameters + - scripts: generate_rust_analyzer.py: avoid FD leak + - wifi: rtw88: fix device leak on probe failure + - staging: sm750fb: fix division by zero in ps_to_hz() + - selftests/mm: hmm-tests: don't hardcode THP size to 2MB + - USB: serial: option: add Telit Cinterion FN990A MBIM composition + - Docs/admin-guide/mm/damon/reclaim: warn commit_inputs vs param updates + race + - Docs/admin-guide/mm/damon/lru_sort: warn commit_inputs vs param updates + race + - ALSA: ctxfi: Limit PTP to a single page + - dcache: Limit the minimal number of bucket to two + - vfio/xe: Reorganize the init to decouple migration from reset + - arm64: mm: Handle invalid large leaf mappings correctly + - media: vidtv: fix NULL pointer dereference in + vidtv_channel_pmt_match_sections + - ocfs2: fix possible deadlock between unlink and dio_end_io_write + - ocfs2: fix use-after-free in ocfs2_fault() when VM_FAULT_RETRY + - ocfs2: handle invalid dinode in ocfs2_group_extend + - PCI: endpoint: pci-epf-vntb: Stop cmd_handler work in + epf_ntb_epc_cleanup + - PCI: endpoint: pci-epf-vntb: Remove duplicate resource teardown + - KVM: selftests: Remove duplicate LAUNCH_UPDATE_VMSA call in SEV-ES + migrate test + - KVM: SEV: Reject attempts to sync VMSA of an already-launched/encrypted + vCPU + - KVM: SEV: Protect *all* of sev_mem_enc_register_region() with kvm->lock + - KVM: SEV: Disallow LAUNCH_FINISH if vCPUs are actively being created + - KVM: SEV: Lock all vCPUs when synchronzing VMSAs for SNP launch finish + - KVM: SEV: Drop WARN on large size for KVM_MEMORY_ENCRYPT_REG_REGION + - mm: call ->free_folio() directly in folio_unmap_invalidate() + - checkpatch: add support for Assisted-by tag + - x86-64: rename misleadingly named '__copy_user_nocache()' function + - x86: rename and clean up __copy_from_user_inatomic_nocache() + - x86-64/arm64/powerpc: clean up and rename __copy_from_user_flushcache + - KVM: x86: Use scratch field in MMIO fragment to hold small write values + - ASoC: qcom: q6apm: move component registration to unmanaged version + - mm/kasan: fix double free for kasan pXds + - mm: blk-cgroup: fix use-after-free in cgwb_release_workfn() + - media: vidtv: fix nfeeds state corruption on start_streaming failure + - media: mediatek: vcodec: fix use-after-free in encoder release path + - media: em28xx: fix use-after-free in em28xx_v4l2_open() + - hwmon: (powerz) Fix use-after-free on USB disconnect + - ALSA: 6fire: fix use-after-free on disconnect + - bcache: fix cached_dev.sb_bio use-after-free and crash + - wireguard: device: use exit_rtnl callback instead of manual rtnl_lock in + pre_exit + - media: as102: fix to not free memory after the device is registered in + as102_usb_probe() + - nilfs2: fix NULL i_assoc_inode dereference in + nilfs_mdt_save_to_shadow_map + - media: vidtv: fix pass-by-value structs causing MSAN warnings + - media: hackrf: fix to not free memory after the device is registered in + hackrf_probe() + - mm/userfaultfd: fix hugetlb fault mutex hash calculation + - clockevents: Add missing resets of the next_event_forced flag + - Linux 7.0.1 + * GRO managed-frag use-after-free leading to local privilege escalation + (LP: #2154172) + - net: gro: don't merge zcopy skbs + * AppArmor Vulnerabilities (LP: #2151747) + - SAUCE: apparmor: pass big_resp to handler + - SAUCE: apparmor: remove redundant kref_init for listener->count + - SAUCE: apparmor: fix NULL pointer dereference in unpack_pdb + * AppArmor Vulnerabilities (LP: #2151747) // CVE-2026-47337 + - SAUCE: apparmor: fix NULL pointer dereference in bind_map_addr + * AppArmor Vulnerabilities (LP: #2151747) // CVE-2026-47334 + - SAUCE: apparmor: fix sleep prone memory allocation under a spin_lock + * AppArmor Vulnerabilities (LP: #2151747) // CVE-2026-47333 + - SAUCE: apparmor: fix dfa unpacking size of the notification filter + * AppArmor Vulnerabilities (LP: #2151747) // CVE-2026-47332 + - SAUCE: apparmor: fix size check against type instead of pointer + * apparmor: LLVM/clang build failure due to uninitialized variable in + notify.c (LP: #2148809) // CVE-2026-47330 + - SAUCE: apparmor: initialize variable used in uninitialized context + * AppArmor Vulnerabilities (LP: #2151747) // CVE-2026-47329 + - SAUCE: apparmor: fix name validation bypass on notification + * AppArmor Vulnerabilities (LP: #2151747) // CVE-2026-47327 // + CVE-2026-47328 + - SAUCE: apparmor: fix glob memory leak after kstrdup + * AppArmor Vulnerabilities (LP: #2151747) // CVE-2026-47326 + - SAUCE: apparmor: fix inverted NULL check after aa_get_buffer + * CVE-2026-46300 + - net: skbuff: preserve shared-frag marker during coalescing + - net: skbuff: propagate shared-frag marker through frag-transfer helpers + * net/rds: reset op_nents when zerocopy page pin fails (LP: #2153962) + - net/rds: reset op_nents when zerocopy page pin fails + * CVE-2026-46333 + - ptrace: slightly saner 'get_dumpable()' logic + * CVE-2026-43500 + - rxrpc: Fix conn-level packet handling to unshare RESPONSE packets + - rxrpc: Fix potential UAF after skb_unshare() failure + - rxrpc: Fix rxrpc_input_call_event() to only unshare DATA packets + - rxrpc: Also unshare DATA/RESPONSE packets when paged frags are present + * CVE-2026-43284 + - xfrm: esp: avoid in-place decrypt on shared skb frags + + [ Ubuntu: 7.0.0-15.15 ] + + * resolute/linux: 7.0.0-15.15 -proposed tracker (LP: #2148866) + * Qualcomm X1E: Speaker overdrive causes hardware protection shutdown + (LP: #2149808) + - SAUCE: ASoC: qcom: x1e80100: limit speaker volumes + * intel-ipu7 / intel-ipu7-isys modules are shipped unsigned in latest + Resolute kernels, breaking Secure Boot systems (LP: #2148718) + - [packaging] add intel-ipu7 to signature inclusion list + + -- Jacob Martin Thu, 18 Jun 2026 18:23:46 -0500 linux-nvidia (7.0.0-1006.6) resolute; urgency=medium diff --git a/debian.nvidia/reconstruct b/debian.nvidia/reconstruct index d1b95906eb4b8..c74b364c08a46 100644 --- a/debian.nvidia/reconstruct +++ b/debian.nvidia/reconstruct @@ -42,4 +42,6 @@ chmod +x 'drivers/net/ethernet/realtek/r8127/rtl_eeprom.h' chmod +x 'drivers/net/ethernet/realtek/r8127/rtltool.c' chmod +x 'drivers/net/ethernet/realtek/r8127/rtltool.h' # Remove any files deleted from the orig. +rm -f 'arch/parisc/lib/checksum.c' +rm -f 'tools/testing/selftests/mqueue/setting' exit 0 From e4c7bed34902eef8bc8594c885c1e1198fa509f3 Mon Sep 17 00:00:00 2001 From: Sudeep Holla Date: Tue, 28 Apr 2026 19:33:30 +0100 Subject: [PATCH 163/311] firmware: arm_ffa: Bound PARTITION_INFO_GET_REGS copies BugLink: https://bugs.launchpad.net/bugs/2154045 The register-based PARTITION_INFO_GET path trusted the firmware-provided indices when copying partition descriptors into the caller buffer. Reject inconsistent counts or index progressions so the copy loop cannot write past the allocated array. Fixes: ba85c644ac8d ("firmware: arm_ffa: Add support for FFA_PARTITION_INFO_GET_REGS") Link: https://patch.msgid.link/20260428-ffa_fixes-v2-6-8595ae450034@kernel.org (fixed cur_idx when exactly one descriptor in the first fragment) Signed-off-by: Sudeep Holla (cherry picked from commit 3974ea1938406f9bfa7c1f48d4e43533f447bb08) Signed-off-by: Jamie Nguyen Acked-by: Nirmoy Das Acked-by: Seth Forshee Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/firmware/arm_ffa/driver.c | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c index f2f94d4d533e8..51ce791405eaa 100644 --- a/drivers/firmware/arm_ffa/driver.c +++ b/drivers/firmware/arm_ffa/driver.c @@ -322,6 +322,12 @@ __ffa_partition_info_get(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, #define PART_INFO_ID_MASK GENMASK(15, 0) #define PART_INFO_EXEC_CXT_MASK GENMASK(31, 16) #define PART_INFO_PROPS_MASK GENMASK(63, 32) +#define FFA_PART_INFO_GET_REGS_FIRST_REG 3 +#define FFA_PART_INFO_GET_REGS_REGS_PER_DESC 3 +#define FFA_PART_INFO_GET_REGS_MAX_DESC \ + (((sizeof(ffa_value_t) / sizeof_field(ffa_value_t, a0)) - \ + FFA_PART_INFO_GET_REGS_FIRST_REG) / \ + FFA_PART_INFO_GET_REGS_REGS_PER_DESC) #define PART_INFO_ID(x) ((u16)(FIELD_GET(PART_INFO_ID_MASK, (x)))) #define PART_INFO_EXEC_CXT(x) ((u16)(FIELD_GET(PART_INFO_EXEC_CXT_MASK, (x)))) #define PART_INFO_PROPERTIES(x) ((u32)(FIELD_GET(PART_INFO_PROPS_MASK, (x)))) @@ -329,15 +335,13 @@ static int __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, struct ffa_partition_info *buffer, int num_parts) { - u16 buf_sz, start_idx, cur_idx, count = 0, prev_idx = 0, tag = 0; + u16 buf_sz, start_idx = 0, cur_idx, count = 0, tag = 0; struct ffa_partition_info *buf = buffer; ffa_value_t partition_info; do { __le64 *regs; - int idx; - - start_idx = prev_idx ? prev_idx + 1 : 0; + int idx, nr_desc, buf_idx; invoke_ffa_fn((ffa_value_t){ .a0 = FFA_PARTITION_INFO_GET_REGS, @@ -353,15 +357,28 @@ __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, count = PARTITION_COUNT(partition_info.a2); if (!buffer || !num_parts) /* count only */ return count; + if (count > num_parts) + return -EINVAL; cur_idx = CURRENT_INDEX(partition_info.a2); + if (cur_idx < start_idx || cur_idx >= count) + return -EINVAL; + + nr_desc = cur_idx - start_idx + 1; + if (nr_desc > FFA_PART_INFO_GET_REGS_MAX_DESC) + return -EINVAL; + + buf_idx = buf - buffer; + if (buf_idx + nr_desc > num_parts) + return -EINVAL; + tag = UUID_INFO_TAG(partition_info.a2); buf_sz = PARTITION_INFO_SZ(partition_info.a2); if (buf_sz > sizeof(*buffer)) buf_sz = sizeof(*buffer); regs = (void *)&partition_info.a3; - for (idx = 0; idx < cur_idx - start_idx + 1; idx++, buf++) { + for (idx = 0; idx < nr_desc; idx++, buf++) { union { uuid_t uuid; u64 regs[2]; @@ -379,7 +396,7 @@ __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, uuid_copy(&buf->uuid, &uuid_regs.uuid); regs += 3; } - prev_idx = cur_idx; + start_idx = cur_idx + 1; } while (cur_idx < (count - 1)); From a63096fc2db76a10e3d27b539ae77405c0742cb1 Mon Sep 17 00:00:00 2001 From: Jamie Nguyen Date: Mon, 18 May 2026 10:06:30 -0700 Subject: [PATCH 164/311] firmware: arm_ffa: Honor partition info descriptor size BugLink: https://bugs.launchpad.net/bugs/2154045 FFA_PARTITION_INFO_GET_REGS reports the size of each partition information descriptor in x2[63:48]. However, __ffa_partition_info_get_regs() walks the returned register payload with a hardcoded 24-byte stride (regs += 3), even though the size is already read into buf_sz. That works for the FF-A v1.1/v1.2 24-byte descriptor layout, where each descriptor consumes three registers. Newer FF-A revisions can extend the descriptor while keeping the existing fields at the front. For example, a 48-byte descriptor consumes six registers, so advancing by only three registers desynchronises the parser and can make it read subsequent entries from the middle of a descriptor. Use the advertised descriptor size to derive the register stride. Validate that the size is register-aligned, large enough for the fields parsed by the driver, and that the requested number of descriptors fits in the returned x3..x17 register window. The driver still copies only the fields it understands, but now skips over any trailing descriptor fields correctly. Fixes: ba85c644ac8d ("firmware: arm_ffa: Add support for FFA_PARTITION_INFO_GET_REGS") Suggested-by: Sudeep Holla Signed-off-by: Jamie Nguyen Link: https://patch.msgid.link/20260518203116.42624-1-jamien@nvidia.com (sudeep.holla: Minor rewordng of the commit message and subject) Signed-off-by: Sudeep Holla (backported from commit 01b9cae706161a39452a2cce0f281d4369344c51 linux-next) Signed-off-by: Jamie Nguyen Acked-by: Nirmoy Das Acked-by: Seth Forshee Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/firmware/arm_ffa/driver.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c index 51ce791405eaa..874b686743044 100644 --- a/drivers/firmware/arm_ffa/driver.c +++ b/drivers/firmware/arm_ffa/driver.c @@ -323,11 +323,9 @@ __ffa_partition_info_get(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, #define PART_INFO_EXEC_CXT_MASK GENMASK(31, 16) #define PART_INFO_PROPS_MASK GENMASK(63, 32) #define FFA_PART_INFO_GET_REGS_FIRST_REG 3 -#define FFA_PART_INFO_GET_REGS_REGS_PER_DESC 3 -#define FFA_PART_INFO_GET_REGS_MAX_DESC \ - (((sizeof(ffa_value_t) / sizeof_field(ffa_value_t, a0)) - \ - FFA_PART_INFO_GET_REGS_FIRST_REG) / \ - FFA_PART_INFO_GET_REGS_REGS_PER_DESC) +#define FFA_PART_INFO_GET_REGS_MIN_REGS_PER_DESC 3 +#define FFA_PART_INFO_GET_REGS_NUM_REGS \ + (sizeof(ffa_value_t) / sizeof_field(ffa_value_t, a0)) #define PART_INFO_ID(x) ((u16)(FIELD_GET(PART_INFO_ID_MASK, (x)))) #define PART_INFO_EXEC_CXT(x) ((u16)(FIELD_GET(PART_INFO_EXEC_CXT_MASK, (x)))) #define PART_INFO_PROPERTIES(x) ((u32)(FIELD_GET(PART_INFO_PROPS_MASK, (x)))) @@ -341,7 +339,7 @@ __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, do { __le64 *regs; - int idx, nr_desc, buf_idx; + int idx, nr_desc, buf_idx, regs_per_desc, max_desc; invoke_ffa_fn((ffa_value_t){ .a0 = FFA_PARTITION_INFO_GET_REGS, @@ -364,8 +362,18 @@ __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, if (cur_idx < start_idx || cur_idx >= count) return -EINVAL; + buf_sz = PARTITION_INFO_SZ(partition_info.a2); + if (buf_sz % sizeof(*regs)) + return -EINVAL; + + regs_per_desc = buf_sz / sizeof(*regs); + if (regs_per_desc < FFA_PART_INFO_GET_REGS_MIN_REGS_PER_DESC) + return -EINVAL; + nr_desc = cur_idx - start_idx + 1; - if (nr_desc > FFA_PART_INFO_GET_REGS_MAX_DESC) + max_desc = (FFA_PART_INFO_GET_REGS_NUM_REGS - + FFA_PART_INFO_GET_REGS_FIRST_REG) / regs_per_desc; + if (nr_desc > max_desc) return -EINVAL; buf_idx = buf - buffer; @@ -373,9 +381,6 @@ __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, return -EINVAL; tag = UUID_INFO_TAG(partition_info.a2); - buf_sz = PARTITION_INFO_SZ(partition_info.a2); - if (buf_sz > sizeof(*buffer)) - buf_sz = sizeof(*buffer); regs = (void *)&partition_info.a3; for (idx = 0; idx < nr_desc; idx++, buf++) { @@ -394,7 +399,7 @@ __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, buf->exec_ctxt = PART_INFO_EXEC_CXT(val); buf->properties = PART_INFO_PROPERTIES(val); uuid_copy(&buf->uuid, &uuid_regs.uuid); - regs += 3; + regs += regs_per_desc; } start_idx = cur_idx + 1; From a645386b79f39e4f4fd510eea7ea2ba8e63b7ca6 Mon Sep 17 00:00:00 2001 From: Jamie Nguyen Date: Tue, 19 May 2026 12:42:20 -0700 Subject: [PATCH 165/311] fs/ntfs3: fix mount failure on 64K page-size kernels BugLink: https://bugs.launchpad.net/bugs/2155467 On 64K page-size kernels, mounting NTFS volumes smaller than ~650 MB fails with EINVAL. The issue is in log_replay(): the initial log page size probe uses PAGE_SIZE (65536) instead of DefaultLogPageSize (4096) when PAGE_SIZE exceeds DefaultLogPageSize * 2. This makes norm_file_page() require the $LogFile to be at least 50 * 65536 = 3.2 MB, but mkfs.ntfs creates a $LogFile of only ~1.5 MB for a typical 300 MB volume. norm_file_page() returns 0 and the mount is rejected with EINVAL. On 4K kernels the #if guard evaluates to true, so use_default=true is passed and DefaultLogPageSize (4096) is used, requiring only ~200 KB. This path works fine. Fix this by always passing use_default=true, which forces the initial probe to use DefaultLogPageSize regardless of the kernel's PAGE_SIZE. This is safe because, after reading the on-disk restart area, log_replay() already re-adjusts log->page_size to match the volume's actual sys_page_size. Also fix read_log_page() to pass log->page_size instead of PAGE_SIZE to ntfs_fix_post_read(), matching the actual buffer size. Fixes: b46acd6a6a62 ("fs/ntfs3: Add NTFS journal") Tested-by: Matthew R. Ochs Signed-off-by: Jamie Nguyen Signed-off-by: Konstantin Komarov (cherry picked from commit b7a9125cac8645245d2473c6c0a50e338280ad23 linux-next) Signed-off-by: Jamie Nguyen Acked-by: Seth Forshee Acked-by: Carol L Soto Acked-by: Matthew R. Ochs Signed-off-by: Brad Figg --- fs/ntfs3/fslog.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c index 037df47fa9f3a..272ea39e3c999 100644 --- a/fs/ntfs3/fslog.c +++ b/fs/ntfs3/fslog.c @@ -1172,7 +1172,7 @@ static int read_log_page(struct ntfs_log *log, u32 vbo, goto out; if (page_buf->rhdr.sign != NTFS_FFFF_SIGNATURE) - ntfs_fix_post_read(&page_buf->rhdr, PAGE_SIZE, false); + ntfs_fix_post_read(&page_buf->rhdr, log->page_size, false); if (page_buf != *buffer) memcpy(*buffer, Add2Ptr(page_buf, page_off), bytes); @@ -3796,11 +3796,7 @@ int log_replay(struct ntfs_inode *ni, bool *initialized) log->l_size = log->orig_file_size = ni->vfs_inode.i_size; /* Get the size of page. NOTE: To replay we can use default page. */ -#if PAGE_SIZE >= DefaultLogPageSize && PAGE_SIZE <= DefaultLogPageSize * 2 log->page_size = norm_file_page(PAGE_SIZE, &log->l_size, true); -#else - log->page_size = norm_file_page(PAGE_SIZE, &log->l_size, false); -#endif if (!log->page_size) { err = -EINVAL; goto out; From c74c5c52db4176dbbdab4307c7b2637a418239b8 Mon Sep 17 00:00:00 2001 From: Zeng Heng Date: Fri, 13 Mar 2026 14:45:38 +0000 Subject: [PATCH 166/311] arm_mpam: Ensure in_reset_state is false after applying configuration BugLink: https://bugs.launchpad.net/bugs/2154527 The per-RIS flag, in_reset_state, indicates whether or not the MSC registers are in reset state, and allows avoiding resetting when they are already in reset state. However, when mpam_apply_config() updates the configuration it doesn't update the in_reset_state flag and so even after the configuration update in_reset_state can be true and mpam_reset_ris() will skip the actual register restoration on subsequent resets. Once resctrl has a MPAM backend it will use resctrl_arch_reset_all_ctrls() to reset the MSC configuration on unmount and, if the in_reset_state flag is bogusly true, fail to reset the MSC configuration. The resulting non-reset MSC configuration can lead to persistent performance restrictions even after resctrl is unmounted. Fix by clearing in_reset_state to false immediately after successful configuration application, ensuring that the next reset operation properly restores MSC register defaults. Fixes: 09b89d2a72f3 ("arm_mpam: Allow configuration to be applied and restored during cpu online") Signed-off-by: Zeng Heng Acked-by: Ben Horgan [Horgan: rewrite commit message to not be specific to resctrl unmount] Signed-off-by: Ben Horgan Reviewed-by: Gavin Shan Reviewed-by: Jonathan Cameron Reviewed-by: James Morse Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Jesse Chick Signed-off-by: James Morse (cherry picked from commit f91e913355f49c878fc77f995fd71b7800352bd2) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_devices.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 0666be6b0e88d..3c7e69de753ef 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -2694,6 +2694,7 @@ int mpam_apply_config(struct mpam_component *comp, u16 partid, srcu_read_lock_held(&mpam_srcu)) { arg.ris = ris; mpam_touch_msc(msc, __write_config, &arg); + ris->in_reset_state = false; } mutex_unlock(&msc->cfg_lock); } From 048bc39cbbd833e8cb8535eb8caa63cda849a9c5 Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Fri, 13 Mar 2026 14:45:39 +0000 Subject: [PATCH 167/311] arm_mpam: Reset when feature configuration bit unset BugLink: https://bugs.launchpad.net/bugs/2154527 To indicate that the configuration, of the controls used by resctrl, in a RIS need resetting to driver defaults the reset flags in mpam_config are set. However, these flags are only ever set temporarily at RIS scope in mpam_reset_ris() and hence mpam_cpu_online() will never reset these controls to default. As the hardware reset is unknown this leads to unknown configuration when the control values haven't been configured away from the defaults. Use the policy that an unset feature configuration bit means reset. In this way the mpam_config in the component can encode that it should be in reset state and mpam_reprogram_msc() will reset controls as needed. Fixes: 09b89d2a72f3 ("arm_mpam: Allow configuration to be applied and restored during cpu online") Signed-off-by: Ben Horgan Reviewed-by: Gavin Shan Reviewed-by: James Morse Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Jesse Chick [ morse: Removed unused reset flags from config structure ] Signed-off-by: James Morse (cherry picked from commit a1cb6577f575ba5ec2583caf4f791a86754dbf69) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_devices.c | 40 ++++++++++----------------------- drivers/resctrl/mpam_internal.h | 4 ---- 2 files changed, 12 insertions(+), 32 deletions(-) diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 3c7e69de753ef..740d99dc847eb 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -1364,17 +1364,15 @@ static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid, __mpam_intpart_sel(ris->ris_idx, partid, msc); } - if (mpam_has_feature(mpam_feat_cpor_part, rprops) && - mpam_has_feature(mpam_feat_cpor_part, cfg)) { - if (cfg->reset_cpbm) - mpam_reset_msc_bitmap(msc, MPAMCFG_CPBM, rprops->cpbm_wd); - else + if (mpam_has_feature(mpam_feat_cpor_part, rprops)) { + if (mpam_has_feature(mpam_feat_cpor_part, cfg)) mpam_write_partsel_reg(msc, CPBM, cfg->cpbm); + else + mpam_reset_msc_bitmap(msc, MPAMCFG_CPBM, rprops->cpbm_wd); } - if (mpam_has_feature(mpam_feat_mbw_part, rprops) && - mpam_has_feature(mpam_feat_mbw_part, cfg)) { - if (cfg->reset_mbw_pbm) + if (mpam_has_feature(mpam_feat_mbw_part, rprops)) { + if (mpam_has_feature(mpam_feat_mbw_part, cfg)) mpam_reset_msc_bitmap(msc, MPAMCFG_MBW_PBM, rprops->mbw_pbm_bits); else mpam_write_partsel_reg(msc, MBW_PBM, cfg->mbw_pbm); @@ -1384,16 +1382,14 @@ static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid, mpam_has_feature(mpam_feat_mbw_min, cfg)) mpam_write_partsel_reg(msc, MBW_MIN, 0); - if (mpam_has_feature(mpam_feat_mbw_max, rprops) && - mpam_has_feature(mpam_feat_mbw_max, cfg)) { - if (cfg->reset_mbw_max) - mpam_write_partsel_reg(msc, MBW_MAX, MPAMCFG_MBW_MAX_MAX); - else + if (mpam_has_feature(mpam_feat_mbw_max, rprops)) { + if (mpam_has_feature(mpam_feat_mbw_max, cfg)) mpam_write_partsel_reg(msc, MBW_MAX, cfg->mbw_max); + else + mpam_write_partsel_reg(msc, MBW_MAX, MPAMCFG_MBW_MAX_MAX); } - if (mpam_has_feature(mpam_feat_mbw_prop, rprops) && - mpam_has_feature(mpam_feat_mbw_prop, cfg)) + if (mpam_has_feature(mpam_feat_mbw_prop, rprops)) mpam_write_partsel_reg(msc, MBW_PROP, 0); if (mpam_has_feature(mpam_feat_cmax_cmax, rprops)) @@ -1493,16 +1489,6 @@ static int mpam_save_mbwu_state(void *arg) return 0; } -static void mpam_init_reset_cfg(struct mpam_config *reset_cfg) -{ - *reset_cfg = (struct mpam_config) { - .reset_cpbm = true, - .reset_mbw_pbm = true, - .reset_mbw_max = true, - }; - bitmap_fill(reset_cfg->features, MPAM_FEATURE_LAST); -} - /* * Called via smp_call_on_cpu() to prevent migration, while still being * pre-emptible. Caller must hold mpam_srcu. @@ -1510,14 +1496,12 @@ static void mpam_init_reset_cfg(struct mpam_config *reset_cfg) static int mpam_reset_ris(void *arg) { u16 partid, partid_max; - struct mpam_config reset_cfg; + struct mpam_config reset_cfg = {}; struct mpam_msc_ris *ris = arg; if (ris->in_reset_state) return 0; - mpam_init_reset_cfg(&reset_cfg); - spin_lock(&partid_max_lock); partid_max = mpam_partid_max; spin_unlock(&partid_max_lock); diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index e8971842b124f..7af762c98efc4 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -266,10 +266,6 @@ struct mpam_config { u32 mbw_pbm; u16 mbw_max; - bool reset_cpbm; - bool reset_mbw_pbm; - bool reset_mbw_max; - struct mpam_garbage garbage; }; From 78fb538a5abf6bfdacc71ac0588d9252aa384cb7 Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Fri, 13 Mar 2026 14:45:40 +0000 Subject: [PATCH 168/311] arm64/sysreg: Add MPAMSM_EL1 register BugLink: https://bugs.launchpad.net/bugs/2154527 The MPAMSM_EL1 register determines the MPAM configuration for an SMCU. Add the register definition. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Acked-by: Catalin Marinas Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 29fa1be82b83f87e603ed4c21fe86c6e05fd0282) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/arm64/tools/sysreg | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 9d1c211080571..1287cb1de6f3c 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -5172,6 +5172,14 @@ Field 31:16 PARTID_D Field 15:0 PARTID_I EndSysreg +Sysreg MPAMSM_EL1 3 0 10 5 3 +Res0 63:48 +Field 47:40 PMG_D +Res0 39:32 +Field 31:16 PARTID_D +Res0 15:0 +EndSysreg + Sysreg ISR_EL1 3 0 12 1 0 Res0 63:11 Field 10 IS From 3a1ccd9e273f77d59e766c1811afa4ff71fbf8fc Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Fri, 13 Mar 2026 14:45:41 +0000 Subject: [PATCH 169/311] KVM: arm64: Preserve host MPAM configuration when changing traps BugLink: https://bugs.launchpad.net/bugs/2154527 When KVM enables or disables MPAM traps to EL2 it clears all other bits in MPAM2_EL2. Notably, it clears the partition ids (PARTIDs) and performance monitoring groups (PMGs). Avoid changing these bits in anticipation of adding support for MPAM in the kernel. Otherwise, on a VHE system with the host running at EL2 where MPAM2_EL2 and MPAM1_EL1 access the same register, any attempt to use MPAM to monitor or partition resources for kernel space would be foiled by running a KVM guest. Additionally, MPAM2_EL2.EnMPAMSM is always set to 0 which causes MPAMSM_EL1 to always trap. Keep EnMPAMSM set to 1 when not in a guest so that the kernel can use MPAMSM_EL1. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Acked-by: Marc Zyngier Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit eda1cd1f9d29b382a07d757cf8b29f9ee636355f) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/arm64/kvm/hyp/include/hyp/switch.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 2597e8bda8672..0b50ddd530f3e 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -267,7 +267,8 @@ static inline void __deactivate_traps_hfgxtr(struct kvm_vcpu *vcpu) static inline void __activate_traps_mpam(struct kvm_vcpu *vcpu) { - u64 r = MPAM2_EL2_TRAPMPAM0EL1 | MPAM2_EL2_TRAPMPAM1EL1; + u64 clr = MPAM2_EL2_EnMPAMSM; + u64 set = MPAM2_EL2_TRAPMPAM0EL1 | MPAM2_EL2_TRAPMPAM1EL1; if (!system_supports_mpam()) return; @@ -277,18 +278,21 @@ static inline void __activate_traps_mpam(struct kvm_vcpu *vcpu) write_sysreg_s(MPAMHCR_EL2_TRAP_MPAMIDR_EL1, SYS_MPAMHCR_EL2); } else { /* From v1.1 TIDR can trap MPAMIDR, set it unconditionally */ - r |= MPAM2_EL2_TIDR; + set |= MPAM2_EL2_TIDR; } - write_sysreg_s(r, SYS_MPAM2_EL2); + sysreg_clear_set_s(SYS_MPAM2_EL2, clr, set); } static inline void __deactivate_traps_mpam(void) { + u64 clr = MPAM2_EL2_TRAPMPAM0EL1 | MPAM2_EL2_TRAPMPAM1EL1 | MPAM2_EL2_TIDR; + u64 set = MPAM2_EL2_EnMPAMSM; + if (!system_supports_mpam()) return; - write_sysreg_s(0, SYS_MPAM2_EL2); + sysreg_clear_set_s(SYS_MPAM2_EL2, clr, set); if (system_supports_mpam_hcr()) write_sysreg_s(MPAMHCR_HOST_FLAGS, SYS_MPAMHCR_EL2); From 1b347fcf0b627ec6ef2101ca33c49dd9350783d6 Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Fri, 13 Mar 2026 14:45:42 +0000 Subject: [PATCH 170/311] KVM: arm64: Make MPAMSM_EL1 accesses UNDEF BugLink: https://bugs.launchpad.net/bugs/2154527 The MPAMSM_EL1 register controls the MPAM labeling for an SMCU, Streaming Mode Compute Unit. As there is no MPAM support in KVM, make sure MPAMSM_EL1 accesses trigger an UNDEF. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Acked-by: Marc Zyngier Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 2e7c684bdb50cfaf98da80ebaab4a961fdcd1aa2) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/arm64/kvm/sys_regs.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 1b4cacb6e918a..0edd655934a97 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -3376,6 +3376,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { { SYS_DESC(SYS_MPAM1_EL1), undef_access }, { SYS_DESC(SYS_MPAM0_EL1), undef_access }, + { SYS_DESC(SYS_MPAMSM_EL1), undef_access }, + { SYS_DESC(SYS_VBAR_EL1), access_rw, reset_val, VBAR_EL1, 0 }, { SYS_DESC(SYS_DISR_EL1), NULL, reset_val, DISR_EL1, 0 }, From e235165a351bcc46998f880c653bf37909c5d539 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:45:43 +0000 Subject: [PATCH 171/311] arm64: mpam: Context switch the MPAM registers BugLink: https://bugs.launchpad.net/bugs/2154527 MPAM allows traffic in the SoC to be labeled by the OS, these labels are used to apply policy in caches and bandwidth regulators, and to monitor traffic in the SoC. The label is made up of a PARTID and PMG value. The x86 equivalent calls these CLOSID and RMID, but they don't map precisely. MPAM has two CPU system registers that is used to hold the PARTID and PMG values that traffic generated at each exception level will use. These can be set per-task by the resctrl file system. (resctrl is the defacto interface for controlling this stuff). Add a helper to switch this. struct task_struct's separate CLOSID and RMID fields are insufficient to implement resctrl using MPAM, as resctrl can change the PARTID (CLOSID) and PMG (sort of like the RMID) separately. On x86, the rmid is an independent number, so a race that writes a mismatched closid and rmid into hardware is benign. On arm64, the pmg bits extend the partid. (i.e. partid-5 has a pmg-0 that is not the same as partid-6's pmg-0). In this case, mismatching the values will 'dirty' a pmg value that resctrl believes is clean, and is not tracking with its 'limbo' code. To avoid this, the partid and pmg are always read and written as a pair. This requires a new u64 field. In struct task_struct there are two u32, rmid and closid for the x86 case, but as we can't use them here do something else. Add this new field, mpam_partid_pmg, to struct thread_info to avoid adding more architecture specific code to struct task_struct. Always use READ_ONCE()/WRITE_ONCE() when accessing this field. Resctrl allows a per-cpu 'default' value to be set, this overrides the values when scheduling a task in the default control-group, which has PARTID 0. The way 'code data prioritisation' gets emulated means the register value for the default group needs to be a variable. The current system register value is kept in a per-cpu variable to avoid writing to the system register if the value isn't going to change. Writes to this register may reset the hardware state for regulating bandwidth. Finally, there is no reason to context switch these registers unless there is a driver changing the values in struct task_struct. Hide the whole thing behind a static key. This also allows the driver to disable MPAM in response to errors reported by hardware. Move the existing static key to belong to the arch code, as in the future the MPAM driver may become a loadable module. All this should depend on whether there is an MPAM driver, hide it behind CONFIG_ARM64_MPAM. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick CC: Amit Singh Tomar Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Reviewed-by: Catalin Marinas Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 8e06d04ff1cf764066c62e5677bfb0b0c1d1fbbc) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/arm64/Kconfig | 2 + arch/arm64/include/asm/mpam.h | 67 ++++++++++++++++++++++++++++ arch/arm64/include/asm/thread_info.h | 3 ++ arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/mpam.c | 13 ++++++ arch/arm64/kernel/process.c | 7 +++ drivers/resctrl/mpam_devices.c | 2 - drivers/resctrl/mpam_internal.h | 4 +- 8 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 arch/arm64/include/asm/mpam.h create mode 100644 arch/arm64/kernel/mpam.c diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index d4b97eebf9965..f04c272757993 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2040,6 +2040,8 @@ config ARM64_MPAM MPAM is exposed to user-space via the resctrl pseudo filesystem. + This option enables the extra context switch code. + endmenu # "ARMv8.4 architectural features" menu "ARMv8.5 architectural features" diff --git a/arch/arm64/include/asm/mpam.h b/arch/arm64/include/asm/mpam.h new file mode 100644 index 0000000000000..0747e0526927d --- /dev/null +++ b/arch/arm64/include/asm/mpam.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2025 Arm Ltd. */ + +#ifndef __ASM__MPAM_H +#define __ASM__MPAM_H + +#include +#include +#include + +#include + +DECLARE_STATIC_KEY_FALSE(mpam_enabled); +DECLARE_PER_CPU(u64, arm64_mpam_default); +DECLARE_PER_CPU(u64, arm64_mpam_current); + +/* + * The value of the MPAM0_EL1 sysreg when a task is in resctrl's default group. + * This is used by the context switch code to use the resctrl CPU property + * instead. The value is modified when CDP is enabled/disabled by mounting + * the resctrl filesystem. + */ +extern u64 arm64_mpam_global_default; + +/* + * The resctrl filesystem writes to the partid/pmg values for threads and CPUs, + * which may race with reads in mpam_thread_switch(). Ensure only one of the old + * or new values are used. Particular care should be taken with the pmg field as + * mpam_thread_switch() may read a partid and pmg that don't match, causing this + * value to be stored with cache allocations, despite being considered 'free' by + * resctrl. + */ +#ifdef CONFIG_ARM64_MPAM +static inline u64 mpam_get_regval(struct task_struct *tsk) +{ + return READ_ONCE(task_thread_info(tsk)->mpam_partid_pmg); +} + +static inline void mpam_thread_switch(struct task_struct *tsk) +{ + u64 oldregval; + int cpu = smp_processor_id(); + u64 regval = mpam_get_regval(tsk); + + if (!static_branch_likely(&mpam_enabled)) + return; + + if (regval == READ_ONCE(arm64_mpam_global_default)) + regval = READ_ONCE(per_cpu(arm64_mpam_default, cpu)); + + oldregval = READ_ONCE(per_cpu(arm64_mpam_current, cpu)); + if (oldregval == regval) + return; + + write_sysreg_s(regval | MPAM1_EL1_MPAMEN, SYS_MPAM1_EL1); + isb(); + + /* Synchronising the EL0 write is left until the ERET to EL0 */ + write_sysreg_s(regval, SYS_MPAM0_EL1); + + WRITE_ONCE(per_cpu(arm64_mpam_current, cpu), regval); +} +#else +static inline void mpam_thread_switch(struct task_struct *tsk) {} +#endif /* CONFIG_ARM64_MPAM */ + +#endif /* __ASM__MPAM_H */ diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index 7942478e40658..5d7fe3e153c85 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -41,6 +41,9 @@ struct thread_info { #ifdef CONFIG_SHADOW_CALL_STACK void *scs_base; void *scs_sp; +#endif +#ifdef CONFIG_ARM64_MPAM + u64 mpam_partid_pmg; #endif u32 cpu; }; diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index fe627100d1990..74b76bb704523 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -68,6 +68,7 @@ obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_VMCORE_INFO) += vmcore_info.o obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o +obj-$(CONFIG_ARM64_MPAM) += mpam.o obj-$(CONFIG_ARM64_MTE) += mte.o obj-y += vdso-wrap.o obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o diff --git a/arch/arm64/kernel/mpam.c b/arch/arm64/kernel/mpam.c new file mode 100644 index 0000000000000..9866d2ca0faa9 --- /dev/null +++ b/arch/arm64/kernel/mpam.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2025 Arm Ltd. */ + +#include + +#include +#include + +DEFINE_STATIC_KEY_FALSE(mpam_enabled); +DEFINE_PER_CPU(u64, arm64_mpam_default); +DEFINE_PER_CPU(u64, arm64_mpam_current); + +u64 arm64_mpam_global_default; diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 489554931231e..47698955fa1e4 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -738,6 +739,12 @@ struct task_struct *__switch_to(struct task_struct *prev, if (prev->thread.sctlr_user != next->thread.sctlr_user) update_sctlr_el1(next->thread.sctlr_user); + /* + * MPAM thread switch happens after the DSB to ensure prev's accesses + * use prev's MPAM settings. + */ + mpam_thread_switch(next); + /* the actual thread switch */ last = cpu_switch_to(prev, next); diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 740d99dc847eb..ae0562a7ce218 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -29,8 +29,6 @@ #include "mpam_internal.h" -DEFINE_STATIC_KEY_FALSE(mpam_enabled); /* This moves to arch code */ - /* * mpam_list_lock protects the SRCU lists when writing. Once the * mpam_enabled key is enabled these lists are read-only, diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index 7af762c98efc4..a13fb9880cede 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -16,12 +16,12 @@ #include #include +#include + #define MPAM_MSC_MAX_NUM_RIS 16 struct platform_device; -DECLARE_STATIC_KEY_FALSE(mpam_enabled); - #ifdef CONFIG_MPAM_KUNIT_TEST #define PACKED_FOR_KUNIT __packed #else From 7f63066b275de37fc4fe65be959ffe2072dab0bd Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:45:44 +0000 Subject: [PATCH 172/311] arm64: mpam: Re-initialise MPAM regs when CPU comes online BugLink: https://bugs.launchpad.net/bugs/2154527 Now that the MPAM system registers are expected to have values that change, reprogram them based on the previous value when a CPU is brought online. Previously MPAM's 'default PARTID' of 0 was always used for MPAM in kernel-space as this is the PARTID that hardware guarantees to reset. Because there are a limited number of PARTID, this value is exposed to user-space, meaning resctrl changes to the resctrl default group would also affect kernel threads. Instead, use the task's PARTID value for kernel work on behalf of user-space too. The default of 0 is kept for both user-space and kernel-space when MPAM is not enabled. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Reviewed-by: Catalin Marinas Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 87b78a5d70e83d4dbe31e1afda2be736a3330b31) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/arm64/kernel/cpufeature.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 32c2dbcc0c641..18d7555ea98bc 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -86,6 +86,7 @@ #include #include #include +#include #include #include #include @@ -2501,13 +2502,17 @@ test_has_mpam(const struct arm64_cpu_capabilities *entry, int scope) static void cpu_enable_mpam(const struct arm64_cpu_capabilities *entry) { - /* - * Access by the kernel (at EL1) should use the reserved PARTID - * which is configured unrestricted. This avoids priority-inversion - * where latency sensitive tasks have to wait for a task that has - * been throttled to release the lock. - */ - write_sysreg_s(0, SYS_MPAM1_EL1); + int cpu = smp_processor_id(); + u64 regval = 0; + + if (IS_ENABLED(CONFIG_ARM64_MPAM) && static_branch_likely(&mpam_enabled)) + regval = READ_ONCE(per_cpu(arm64_mpam_current, cpu)); + + write_sysreg_s(regval | MPAM1_EL1_MPAMEN, SYS_MPAM1_EL1); + isb(); + + /* Synchronising the EL0 write is left until the ERET to EL0 */ + write_sysreg_s(regval, SYS_MPAM0_EL1); } static bool From 505b118aa3a42f373ec702a9cdaf528b334a05b2 Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Fri, 13 Mar 2026 14:45:45 +0000 Subject: [PATCH 173/311] arm64: mpam: Drop the CONFIG_EXPERT restriction BugLink: https://bugs.launchpad.net/bugs/2154527 In anticipation of MPAM being useful remove the CONFIG_EXPERT restriction. This was done to prevent the driver being enabled before the user-space interface was wired up. Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Reviewed-by: James Morse Acked-by: Catalin Marinas Signed-off-by: Ben Horgan [ morse: Added second paragraph ] Signed-off-by: James Morse (cherry picked from commit c544f00a473239835d22e7109b403314d8b85974) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/arm64/Kconfig | 2 +- drivers/resctrl/Kconfig | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index f04c272757993..00d79552a3c11 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2017,7 +2017,7 @@ config ARM64_TLB_RANGE config ARM64_MPAM bool "Enable support for MPAM" - select ARM64_MPAM_DRIVER if EXPERT # does nothing yet + select ARM64_MPAM_DRIVER select ACPI_MPAM if ACPI help Memory System Resource Partitioning and Monitoring (MPAM) is an diff --git a/drivers/resctrl/Kconfig b/drivers/resctrl/Kconfig index c808e04703946..c34e059c6e41f 100644 --- a/drivers/resctrl/Kconfig +++ b/drivers/resctrl/Kconfig @@ -1,6 +1,6 @@ menuconfig ARM64_MPAM_DRIVER bool "MPAM driver" - depends on ARM64 && ARM64_MPAM && EXPERT + depends on ARM64 && ARM64_MPAM help Memory System Resource Partitioning and Monitoring (MPAM) driver for System IP, e.g. caches and memory controllers. From 1b3cda5ecd31cc19ae9e6d10c155826efa1285de Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:45:46 +0000 Subject: [PATCH 174/311] arm64: mpam: Advertise the CPUs MPAM limits to the driver BugLink: https://bugs.launchpad.net/bugs/2154527 Requesters need to populate the MPAM fields for any traffic they send on the interconnect. For the CPUs these values are taken from the corresponding MPAMy_ELx register. Each requester may have a limit on the largest PARTID or PMG value that can be used. The MPAM driver has to determine the system-wide minimum supported PARTID and PMG values. To do this, the driver needs to be told what each requestor's limit is. CPUs are special, but this infrastructure is also needed for the SMMU and GIC ITS. Call the helper to tell the MPAM driver what the CPUs can do. The return value can be ignored by the arch code as it runs well before the MPAM driver starts probing. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Catalin Marinas Reviewed-by: Gavin Shan Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan [ morse: requestor->requester as argued by ispell ] Signed-off-by: James Morse (cherry picked from commit 831a7f16728c5ceef04ab99a699c3d9e519dc4b8) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/arm64/kernel/mpam.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/arm64/kernel/mpam.c b/arch/arm64/kernel/mpam.c index 9866d2ca0faa9..e6feff2324acb 100644 --- a/arch/arm64/kernel/mpam.c +++ b/arch/arm64/kernel/mpam.c @@ -3,6 +3,7 @@ #include +#include #include #include @@ -11,3 +12,14 @@ DEFINE_PER_CPU(u64, arm64_mpam_default); DEFINE_PER_CPU(u64, arm64_mpam_current); u64 arm64_mpam_global_default; + +static int __init arm64_mpam_register_cpus(void) +{ + u64 mpamidr = read_sanitised_ftr_reg(SYS_MPAMIDR_EL1); + u16 partid_max = FIELD_GET(MPAMIDR_EL1_PARTID_MAX, mpamidr); + u8 pmg_max = FIELD_GET(MPAMIDR_EL1_PMG_MAX, mpamidr); + + return mpam_register_requestor(partid_max, pmg_max); +} +/* Must occur before mpam_msc_driver_init() from subsys_initcall() */ +arch_initcall(arm64_mpam_register_cpus) From 02eec931b7f5bff0b5d4a3cc8b347b0880a20843 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:45:47 +0000 Subject: [PATCH 175/311] arm64: mpam: Add cpu_pm notifier to restore MPAM sysregs BugLink: https://bugs.launchpad.net/bugs/2154527 The MPAM system registers will be lost if the CPU is reset during PSCI's CPU_SUSPEND. Add a PM notifier to restore them. mpam_thread_switch(current) can't be used as this won't make any changes if the in-memory copy says the register already has the correct value. In reality the system register is UNKNOWN out of reset. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Reviewed-by: Catalin Marinas Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 735dad999905dfd246be1994bb8d203063aeb0d6) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/arm64/kernel/mpam.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/arch/arm64/kernel/mpam.c b/arch/arm64/kernel/mpam.c index e6feff2324acb..48ec0ffd59997 100644 --- a/arch/arm64/kernel/mpam.c +++ b/arch/arm64/kernel/mpam.c @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -13,12 +14,44 @@ DEFINE_PER_CPU(u64, arm64_mpam_current); u64 arm64_mpam_global_default; +static int mpam_pm_notifier(struct notifier_block *self, + unsigned long cmd, void *v) +{ + u64 regval; + int cpu = smp_processor_id(); + + switch (cmd) { + case CPU_PM_EXIT: + /* + * Don't use mpam_thread_switch() as the system register + * value has changed under our feet. + */ + regval = READ_ONCE(per_cpu(arm64_mpam_current, cpu)); + write_sysreg_s(regval | MPAM1_EL1_MPAMEN, SYS_MPAM1_EL1); + isb(); + + write_sysreg_s(regval, SYS_MPAM0_EL1); + + return NOTIFY_OK; + default: + return NOTIFY_DONE; + } +} + +static struct notifier_block mpam_pm_nb = { + .notifier_call = mpam_pm_notifier, +}; + static int __init arm64_mpam_register_cpus(void) { u64 mpamidr = read_sanitised_ftr_reg(SYS_MPAMIDR_EL1); u16 partid_max = FIELD_GET(MPAMIDR_EL1_PARTID_MAX, mpamidr); u8 pmg_max = FIELD_GET(MPAMIDR_EL1_PMG_MAX, mpamidr); + if (!system_supports_mpam()) + return 0; + + cpu_pm_register_notifier(&mpam_pm_nb); return mpam_register_requestor(partid_max, pmg_max); } /* Must occur before mpam_msc_driver_init() from subsys_initcall() */ From 4c86a936435171dcf81a7a5eb232ae45bba755bf Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Fri, 13 Mar 2026 14:45:48 +0000 Subject: [PATCH 176/311] arm64: mpam: Initialise and context switch the MPAMSM_EL1 register BugLink: https://bugs.launchpad.net/bugs/2154527 The MPAMSM_EL1 sets the MPAM labels, PMG and PARTID, for loads and stores generated by a shared SMCU. Disable the traps so the kernel can use it and set it to the same configuration as the per-EL cpu MPAM configuration. If an SMCU is not shared with other cpus then it is implementation defined whether the configuration from MPAMSM_EL1 is used or that from the appropriate MPAMy_ELx. As we set the same, PMG_D and PARTID_D, configuration for MPAM0_EL1, MPAM1_EL1 and MPAMSM_EL1 the resulting configuration is the same regardless. The range of valid configurations for the PARTID and PMG in MPAMSM_EL1 is not currently specified in Arm Architectural Reference Manual but the architect has confirmed that it is intended to be the same as that for the cpu configuration in the MPAMy_ELx registers. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Reviewed-by: Catalin Marinas Reviewed-by: James Morse Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 37fe0f984d9ca60e8d95fc9a85d37f4300159625) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/arm64/include/asm/el2_setup.h | 3 ++- arch/arm64/include/asm/mpam.h | 2 ++ arch/arm64/kernel/cpufeature.c | 2 ++ arch/arm64/kernel/mpam.c | 4 ++++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index 85f4c1615472d..4d15071a4f3fc 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -513,7 +513,8 @@ check_override id_aa64pfr0, ID_AA64PFR0_EL1_MPAM_SHIFT, .Linit_mpam_\@, .Lskip_mpam_\@, x1, x2 .Linit_mpam_\@: - msr_s SYS_MPAM2_EL2, xzr // use the default partition + mov x0, #MPAM2_EL2_EnMPAMSM_MASK + msr_s SYS_MPAM2_EL2, x0 // use the default partition, // and disable lower traps mrs_s x0, SYS_MPAMIDR_EL1 tbz x0, #MPAMIDR_EL1_HAS_HCR_SHIFT, .Lskip_mpam_\@ // skip if no MPAMHCR reg diff --git a/arch/arm64/include/asm/mpam.h b/arch/arm64/include/asm/mpam.h index 0747e0526927d..6bccbfdccb87e 100644 --- a/arch/arm64/include/asm/mpam.h +++ b/arch/arm64/include/asm/mpam.h @@ -53,6 +53,8 @@ static inline void mpam_thread_switch(struct task_struct *tsk) return; write_sysreg_s(regval | MPAM1_EL1_MPAMEN, SYS_MPAM1_EL1); + if (system_supports_sme()) + write_sysreg_s(regval & (MPAMSM_EL1_PARTID_D | MPAMSM_EL1_PMG_D), SYS_MPAMSM_EL1); isb(); /* Synchronising the EL0 write is left until the ERET to EL0 */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 18d7555ea98bc..f57c2ff98326b 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2509,6 +2509,8 @@ cpu_enable_mpam(const struct arm64_cpu_capabilities *entry) regval = READ_ONCE(per_cpu(arm64_mpam_current, cpu)); write_sysreg_s(regval | MPAM1_EL1_MPAMEN, SYS_MPAM1_EL1); + if (cpus_have_cap(ARM64_SME)) + write_sysreg_s(regval & (MPAMSM_EL1_PARTID_D | MPAMSM_EL1_PMG_D), SYS_MPAMSM_EL1); isb(); /* Synchronising the EL0 write is left until the ERET to EL0 */ diff --git a/arch/arm64/kernel/mpam.c b/arch/arm64/kernel/mpam.c index 48ec0ffd59997..3a490de4fa125 100644 --- a/arch/arm64/kernel/mpam.c +++ b/arch/arm64/kernel/mpam.c @@ -28,6 +28,10 @@ static int mpam_pm_notifier(struct notifier_block *self, */ regval = READ_ONCE(per_cpu(arm64_mpam_current, cpu)); write_sysreg_s(regval | MPAM1_EL1_MPAMEN, SYS_MPAM1_EL1); + if (system_supports_sme()) { + write_sysreg_s(regval & (MPAMSM_EL1_PARTID_D | MPAMSM_EL1_PMG_D), + SYS_MPAMSM_EL1); + } isb(); write_sysreg_s(regval, SYS_MPAM0_EL1); From 33d7ab1fea8520d73790dc5fc4029d14bc7aa03e Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:45:49 +0000 Subject: [PATCH 177/311] arm64: mpam: Add helpers to change a task or cpu's MPAM PARTID/PMG values BugLink: https://bugs.launchpad.net/bugs/2154527 Care must be taken when modifying the PARTID and PMG of a task in any per-task structure as writing these values may race with the task being scheduled in, and reading the modified values. Add helpers to set the task properties, and the CPU default value. These use WRITE_ONCE() that pairs with the READ_ONCE() in mpam_get_regval() to avoid causing torn values. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Cc: Dave Martin Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Catalin Marinas Reviewed-by: Gavin Shan Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 2cf9ca3fae38b7894e7f1435cec92f9a679b42f9) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/arm64/include/asm/mpam.h | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/mpam.h b/arch/arm64/include/asm/mpam.h index 6bccbfdccb87e..05aa71200f61a 100644 --- a/arch/arm64/include/asm/mpam.h +++ b/arch/arm64/include/asm/mpam.h @@ -4,6 +4,7 @@ #ifndef __ASM__MPAM_H #define __ASM__MPAM_H +#include #include #include #include @@ -22,6 +23,23 @@ DECLARE_PER_CPU(u64, arm64_mpam_current); */ extern u64 arm64_mpam_global_default; +#ifdef CONFIG_ARM64_MPAM +static inline u64 __mpam_regval(u16 partid_d, u16 partid_i, u8 pmg_d, u8 pmg_i) +{ + return FIELD_PREP(MPAM0_EL1_PARTID_D, partid_d) | + FIELD_PREP(MPAM0_EL1_PARTID_I, partid_i) | + FIELD_PREP(MPAM0_EL1_PMG_D, pmg_d) | + FIELD_PREP(MPAM0_EL1_PMG_I, pmg_i); +} + +static inline void mpam_set_cpu_defaults(int cpu, u16 partid_d, u16 partid_i, + u8 pmg_d, u8 pmg_i) +{ + u64 default_val = __mpam_regval(partid_d, partid_i, pmg_d, pmg_i); + + WRITE_ONCE(per_cpu(arm64_mpam_default, cpu), default_val); +} + /* * The resctrl filesystem writes to the partid/pmg values for threads and CPUs, * which may race with reads in mpam_thread_switch(). Ensure only one of the old @@ -30,12 +48,20 @@ extern u64 arm64_mpam_global_default; * value to be stored with cache allocations, despite being considered 'free' by * resctrl. */ -#ifdef CONFIG_ARM64_MPAM static inline u64 mpam_get_regval(struct task_struct *tsk) { return READ_ONCE(task_thread_info(tsk)->mpam_partid_pmg); } +static inline void mpam_set_task_partid_pmg(struct task_struct *tsk, + u16 partid_d, u16 partid_i, + u8 pmg_d, u8 pmg_i) +{ + u64 regval = __mpam_regval(partid_d, partid_i, pmg_d, pmg_i); + + WRITE_ONCE(task_thread_info(tsk)->mpam_partid_pmg, regval); +} + static inline void mpam_thread_switch(struct task_struct *tsk) { u64 oldregval; From 48fe003b01d6c64e20c3e274ee826c3fd69da283 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:45:51 +0000 Subject: [PATCH 178/311] arm_mpam: resctrl: Add boilerplate cpuhp and domain allocation BugLink: https://bugs.launchpad.net/bugs/2154527 resctrl has its own data structures to describe its resources. We can't use these directly as we play tricks with the 'MBA' resource, picking the MPAM controls or monitors that best apply. We may export the same component as both L3 and MBA. Add mpam_resctrl_res[] as the array of class->resctrl mappings we are exporting, and add the cpuhp hooks that allocated and free the resctrl domain structures. Only the mpam control feature are considered here and monitor support will be added later. While we're here, plumb in a few other obvious things. CONFIG_ARM_CPU_RESCTRL is used to allow this code to be built even though it can't yet be linked against resctrl. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 09e61daf8e96b9bdb04dd112bdecf9382fd3f919) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/Makefile | 1 + drivers/resctrl/mpam_devices.c | 12 ++ drivers/resctrl/mpam_internal.h | 21 +++ drivers/resctrl/mpam_resctrl.c | 324 ++++++++++++++++++++++++++++++++ include/linux/arm_mpam.h | 3 + 5 files changed, 361 insertions(+) create mode 100644 drivers/resctrl/mpam_resctrl.c diff --git a/drivers/resctrl/Makefile b/drivers/resctrl/Makefile index 898199dcf80d5..40beaf999582c 100644 --- a/drivers/resctrl/Makefile +++ b/drivers/resctrl/Makefile @@ -1,4 +1,5 @@ obj-$(CONFIG_ARM64_MPAM_DRIVER) += mpam.o mpam-y += mpam_devices.o +mpam-$(CONFIG_ARM_CPU_RESCTRL) += mpam_resctrl.o ccflags-$(CONFIG_ARM64_MPAM_DRIVER_DEBUG) += -DDEBUG diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index ae0562a7ce218..e35acf8c25d93 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -1614,6 +1614,9 @@ static int mpam_cpu_online(unsigned int cpu) mpam_reprogram_msc(msc); } + if (mpam_is_enabled()) + return mpam_resctrl_online_cpu(cpu); + return 0; } @@ -1657,6 +1660,9 @@ static int mpam_cpu_offline(unsigned int cpu) { struct mpam_msc *msc; + if (mpam_is_enabled()) + mpam_resctrl_offline_cpu(cpu); + guard(srcu)(&mpam_srcu); list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list, srcu_read_lock_held(&mpam_srcu)) { @@ -2502,6 +2508,12 @@ static void mpam_enable_once(void) mutex_unlock(&mpam_list_lock); cpus_read_unlock(); + if (!err) { + err = mpam_resctrl_setup(); + if (err) + pr_err("Failed to initialise resctrl: %d\n", err); + } + if (err) { mpam_disable_reason = "Failed to enable."; schedule_work(&mpam_broken_work); diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index a13fb9880cede..43c8e0f5f7ac5 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -333,6 +334,16 @@ struct mpam_msc_ris { struct mpam_garbage garbage; }; +struct mpam_resctrl_dom { + struct mpam_component *ctrl_comp; + struct rdt_ctrl_domain resctrl_ctrl_dom; +}; + +struct mpam_resctrl_res { + struct mpam_class *class; + struct rdt_resource resctrl_res; +}; + static inline int mpam_alloc_csu_mon(struct mpam_class *class) { struct mpam_props *cprops = &class->props; @@ -387,6 +398,16 @@ void mpam_msmon_reset_mbwu(struct mpam_component *comp, struct mon_cfg *ctx); int mpam_get_cpumask_from_cache_id(unsigned long cache_id, u32 cache_level, cpumask_t *affinity); +#ifdef CONFIG_RESCTRL_FS +int mpam_resctrl_setup(void); +int mpam_resctrl_online_cpu(unsigned int cpu); +void mpam_resctrl_offline_cpu(unsigned int cpu); +#else +static inline int mpam_resctrl_setup(void) { return 0; } +static inline int mpam_resctrl_online_cpu(unsigned int cpu) { return 0; } +static inline void mpam_resctrl_offline_cpu(unsigned int cpu) { } +#endif /* CONFIG_RESCTRL_FS */ + /* * MPAM MSCs have the following register layout. See: * Arm Memory System Resource Partitioning and Monitoring (MPAM) System diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c new file mode 100644 index 0000000000000..9a30709704142 --- /dev/null +++ b/drivers/resctrl/mpam_resctrl.c @@ -0,0 +1,324 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Arm Ltd. + +#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "mpam_internal.h" + +/* + * The classes we've picked to map to resctrl resources, wrapped + * in with their resctrl structure. + * Class pointer may be NULL. + */ +static struct mpam_resctrl_res mpam_resctrl_controls[RDT_NUM_RESOURCES]; + +#define for_each_mpam_resctrl_control(res, rid) \ + for (rid = 0, res = &mpam_resctrl_controls[rid]; \ + rid < RDT_NUM_RESOURCES; \ + rid++, res = &mpam_resctrl_controls[rid]) + +/* The lock for modifying resctrl's domain lists from cpuhp callbacks. */ +static DEFINE_MUTEX(domain_list_lock); + +bool resctrl_arch_alloc_capable(void) +{ + struct mpam_resctrl_res *res; + enum resctrl_res_level rid; + + for_each_mpam_resctrl_control(res, rid) { + if (res->resctrl_res.alloc_capable) + return true; + } + + return false; +} + +/* + * MSC may raise an error interrupt if it sees an out or range partid/pmg, + * and go on to truncate the value. Regardless of what the hardware supports, + * only the system wide safe value is safe to use. + */ +u32 resctrl_arch_get_num_closid(struct rdt_resource *ignored) +{ + return mpam_partid_max + 1; +} + +struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l) +{ + if (l >= RDT_NUM_RESOURCES) + return NULL; + + return &mpam_resctrl_controls[l].resctrl_res; +} + +static int mpam_resctrl_control_init(struct mpam_resctrl_res *res) +{ + /* TODO: initialise the resctrl resources */ + + return 0; +} + +static int mpam_resctrl_pick_domain_id(int cpu, struct mpam_component *comp) +{ + struct mpam_class *class = comp->class; + + if (class->type == MPAM_CLASS_CACHE) + return comp->comp_id; + + /* TODO: repaint domain ids to match the L3 domain ids */ + /* Otherwise, expose the ID used by the firmware table code. */ + return comp->comp_id; +} + +static void mpam_resctrl_domain_hdr_init(int cpu, struct mpam_component *comp, + enum resctrl_res_level rid, + struct rdt_domain_hdr *hdr) +{ + lockdep_assert_cpus_held(); + + INIT_LIST_HEAD(&hdr->list); + hdr->id = mpam_resctrl_pick_domain_id(cpu, comp); + hdr->rid = rid; + cpumask_set_cpu(cpu, &hdr->cpu_mask); +} + +static void mpam_resctrl_online_domain_hdr(unsigned int cpu, + struct rdt_domain_hdr *hdr) +{ + lockdep_assert_cpus_held(); + + cpumask_set_cpu(cpu, &hdr->cpu_mask); +} + +/** + * mpam_resctrl_offline_domain_hdr() - Update the domain header to remove a CPU. + * @cpu: The CPU to remove from the domain. + * @hdr: The domain's header. + * + * Removes @cpu from the header mask. If this was the last CPU in the domain, + * the domain header is removed from its parent list and true is returned, + * indicating the parent structure can be freed. + * If there are other CPUs in the domain, returns false. + */ +static bool mpam_resctrl_offline_domain_hdr(unsigned int cpu, + struct rdt_domain_hdr *hdr) +{ + lockdep_assert_held(&domain_list_lock); + + cpumask_clear_cpu(cpu, &hdr->cpu_mask); + if (cpumask_empty(&hdr->cpu_mask)) { + list_del_rcu(&hdr->list); + synchronize_rcu(); + return true; + } + + return false; +} + +static void mpam_resctrl_domain_insert(struct list_head *list, + struct rdt_domain_hdr *new) +{ + struct rdt_domain_hdr *err; + struct list_head *pos = NULL; + + lockdep_assert_held(&domain_list_lock); + + err = resctrl_find_domain(list, new->id, &pos); + if (WARN_ON_ONCE(err)) + return; + + list_add_tail_rcu(&new->list, pos); +} + +static struct mpam_resctrl_dom * +mpam_resctrl_alloc_domain(unsigned int cpu, struct mpam_resctrl_res *res) +{ + int err; + struct mpam_resctrl_dom *dom; + struct rdt_ctrl_domain *ctrl_d; + struct mpam_class *class = res->class; + struct mpam_component *comp_iter, *ctrl_comp; + struct rdt_resource *r = &res->resctrl_res; + + lockdep_assert_held(&domain_list_lock); + + ctrl_comp = NULL; + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(comp_iter, &class->components, class_list, + srcu_read_lock_held(&mpam_srcu)) { + if (cpumask_test_cpu(cpu, &comp_iter->affinity)) { + ctrl_comp = comp_iter; + break; + } + } + + /* class has no component for this CPU */ + if (WARN_ON_ONCE(!ctrl_comp)) + return ERR_PTR(-EINVAL); + + dom = kzalloc_node(sizeof(*dom), GFP_KERNEL, cpu_to_node(cpu)); + if (!dom) + return ERR_PTR(-ENOMEM); + + if (r->alloc_capable) { + dom->ctrl_comp = ctrl_comp; + + ctrl_d = &dom->resctrl_ctrl_dom; + mpam_resctrl_domain_hdr_init(cpu, ctrl_comp, r->rid, &ctrl_d->hdr); + ctrl_d->hdr.type = RESCTRL_CTRL_DOMAIN; + err = resctrl_online_ctrl_domain(r, ctrl_d); + if (err) + goto free_domain; + + mpam_resctrl_domain_insert(&r->ctrl_domains, &ctrl_d->hdr); + } else { + pr_debug("Skipped control domain online - no controls\n"); + } + return dom; + +free_domain: + kfree(dom); + dom = ERR_PTR(err); + + return dom; +} + +static struct mpam_resctrl_dom * +mpam_resctrl_get_domain_from_cpu(int cpu, struct mpam_resctrl_res *res) +{ + struct mpam_resctrl_dom *dom; + struct rdt_resource *r = &res->resctrl_res; + + lockdep_assert_cpus_held(); + + list_for_each_entry_rcu(dom, &r->ctrl_domains, resctrl_ctrl_dom.hdr.list) { + if (cpumask_test_cpu(cpu, &dom->ctrl_comp->affinity)) + return dom; + } + + return NULL; +} + +int mpam_resctrl_online_cpu(unsigned int cpu) +{ + struct mpam_resctrl_res *res; + enum resctrl_res_level rid; + + guard(mutex)(&domain_list_lock); + for_each_mpam_resctrl_control(res, rid) { + struct mpam_resctrl_dom *dom; + struct rdt_resource *r = &res->resctrl_res; + + if (!res->class) + continue; // dummy_resource; + + dom = mpam_resctrl_get_domain_from_cpu(cpu, res); + if (!dom) { + dom = mpam_resctrl_alloc_domain(cpu, res); + if (IS_ERR(dom)) + return PTR_ERR(dom); + } else { + if (r->alloc_capable) { + struct rdt_ctrl_domain *ctrl_d = &dom->resctrl_ctrl_dom; + + mpam_resctrl_online_domain_hdr(cpu, &ctrl_d->hdr); + } + } + } + + resctrl_online_cpu(cpu); + + return 0; +} + +void mpam_resctrl_offline_cpu(unsigned int cpu) +{ + struct mpam_resctrl_res *res; + enum resctrl_res_level rid; + + resctrl_offline_cpu(cpu); + + guard(mutex)(&domain_list_lock); + for_each_mpam_resctrl_control(res, rid) { + struct mpam_resctrl_dom *dom; + struct rdt_ctrl_domain *ctrl_d; + bool ctrl_dom_empty; + struct rdt_resource *r = &res->resctrl_res; + + if (!res->class) + continue; // dummy resource + + dom = mpam_resctrl_get_domain_from_cpu(cpu, res); + if (WARN_ON_ONCE(!dom)) + continue; + + if (r->alloc_capable) { + ctrl_d = &dom->resctrl_ctrl_dom; + ctrl_dom_empty = mpam_resctrl_offline_domain_hdr(cpu, &ctrl_d->hdr); + if (ctrl_dom_empty) + resctrl_offline_ctrl_domain(&res->resctrl_res, ctrl_d); + } else { + ctrl_dom_empty = true; + } + + if (ctrl_dom_empty) + kfree(dom); + } +} + +int mpam_resctrl_setup(void) +{ + int err = 0; + struct mpam_resctrl_res *res; + enum resctrl_res_level rid; + + cpus_read_lock(); + for_each_mpam_resctrl_control(res, rid) { + INIT_LIST_HEAD_RCU(&res->resctrl_res.ctrl_domains); + res->resctrl_res.rid = rid; + } + + /* TODO: pick MPAM classes to map to resctrl resources */ + + /* Initialise the resctrl structures from the classes */ + for_each_mpam_resctrl_control(res, rid) { + if (!res->class) + continue; // dummy resource + + err = mpam_resctrl_control_init(res); + if (err) { + pr_debug("Failed to initialise rid %u\n", rid); + break; + } + } + cpus_read_unlock(); + + if (err) { + pr_debug("Internal error %d - resctrl not supported\n", err); + return err; + } + + if (!resctrl_arch_alloc_capable()) { + pr_debug("No alloc(%u) found - resctrl not supported\n", + resctrl_arch_alloc_capable()); + return -EOPNOTSUPP; + } + + /* TODO: call resctrl_init() */ + + return 0; +} diff --git a/include/linux/arm_mpam.h b/include/linux/arm_mpam.h index 7f00c5285a326..2c7d1413a401f 100644 --- a/include/linux/arm_mpam.h +++ b/include/linux/arm_mpam.h @@ -49,6 +49,9 @@ static inline int mpam_ris_create(struct mpam_msc *msc, u8 ris_idx, } #endif +bool resctrl_arch_alloc_capable(void); +bool resctrl_arch_mon_capable(void); + /** * mpam_register_requestor() - Register a requestor with the MPAM driver * @partid_max: The maximum PARTID value the requestor can generate. From 37be84a88c8659a1e40e0d36e4f7bd98921ef57c Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:45:52 +0000 Subject: [PATCH 179/311] arm_mpam: resctrl: Pick the caches we will use as resctrl resources BugLink: https://bugs.launchpad.net/bugs/2154527 Systems with MPAM support may have a variety of control types at any point of their system layout. We can only expose certain types of control, and only if they exist at particular locations. Start with the well-known caches. These have to be depth 2 or 3 and support MPAM's cache portion bitmap controls, with a number of portions fewer than resctrl's limit. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 52a4edb16121d07734e4e392767d26d286f08c35) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 91 +++++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 2 deletions(-) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 9a30709704142..65bb670dc3fb1 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -65,9 +65,95 @@ struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l) return &mpam_resctrl_controls[l].resctrl_res; } +static bool cache_has_usable_cpor(struct mpam_class *class) +{ + struct mpam_props *cprops = &class->props; + + if (!mpam_has_feature(mpam_feat_cpor_part, cprops)) + return false; + + /* resctrl uses u32 for all bitmap configurations */ + return class->props.cpbm_wd <= 32; +} + +/* Test whether we can export MPAM_CLASS_CACHE:{2,3}? */ +static void mpam_resctrl_pick_caches(void) +{ + struct mpam_class *class; + struct mpam_resctrl_res *res; + + lockdep_assert_cpus_held(); + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(class, &mpam_classes, classes_list, + srcu_read_lock_held(&mpam_srcu)) { + if (class->type != MPAM_CLASS_CACHE) { + pr_debug("class %u is not a cache\n", class->level); + continue; + } + + if (class->level != 2 && class->level != 3) { + pr_debug("class %u is not L2 or L3\n", class->level); + continue; + } + + if (!cache_has_usable_cpor(class)) { + pr_debug("class %u cache misses CPOR\n", class->level); + continue; + } + + if (!cpumask_equal(&class->affinity, cpu_possible_mask)) { + pr_debug("class %u has missing CPUs, mask %*pb != %*pb\n", class->level, + cpumask_pr_args(&class->affinity), + cpumask_pr_args(cpu_possible_mask)); + continue; + } + + if (class->level == 2) + res = &mpam_resctrl_controls[RDT_RESOURCE_L2]; + else + res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; + res->class = class; + } +} + static int mpam_resctrl_control_init(struct mpam_resctrl_res *res) { - /* TODO: initialise the resctrl resources */ + struct mpam_class *class = res->class; + struct rdt_resource *r = &res->resctrl_res; + + switch (r->rid) { + case RDT_RESOURCE_L2: + case RDT_RESOURCE_L3: + r->schema_fmt = RESCTRL_SCHEMA_BITMAP; + r->cache.arch_has_sparse_bitmasks = true; + + r->cache.cbm_len = class->props.cpbm_wd; + /* mpam_devices will reject empty bitmaps */ + r->cache.min_cbm_bits = 1; + + if (r->rid == RDT_RESOURCE_L2) { + r->name = "L2"; + r->ctrl_scope = RESCTRL_L2_CACHE; + r->cdp_capable = true; + } else { + r->name = "L3"; + r->ctrl_scope = RESCTRL_L3_CACHE; + r->cdp_capable = true; + } + + /* + * Which bits are shared with other ...things... Unknown + * devices use partid-0 which uses all the bitmap fields. Until + * we have configured the SMMU and GIC not to do this 'all the + * bits' is the correct answer here. + */ + r->cache.shareable_bits = resctrl_get_default_ctrl(r); + r->alloc_capable = true; + break; + default: + return -EINVAL; + } return 0; } @@ -292,7 +378,8 @@ int mpam_resctrl_setup(void) res->resctrl_res.rid = rid; } - /* TODO: pick MPAM classes to map to resctrl resources */ + /* Find some classes to use for controls */ + mpam_resctrl_pick_caches(); /* Initialise the resctrl structures from the classes */ for_each_mpam_resctrl_control(res, rid) { From 1ff6ef1f1b16a177ddfcb27bbb6175a4f89af96c Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:45:53 +0000 Subject: [PATCH 180/311] arm_mpam: resctrl: Implement resctrl_arch_reset_all_ctrls() BugLink: https://bugs.launchpad.net/bugs/2154527 We already have a helper for resetting an mpam class and component. Hook it up to resctrl_arch_reset_all_ctrls() and the domain offline path. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Shaopeng Tan Reviewed-by: Zeng Heng Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 370d166d878d0c0aa06568d67387a1151a200501) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_devices.c | 2 +- drivers/resctrl/mpam_internal.h | 3 +++ drivers/resctrl/mpam_resctrl.c | 13 +++++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index e35acf8c25d93..90751729e49be 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -2553,7 +2553,7 @@ static void mpam_reset_component_locked(struct mpam_component *comp) } } -static void mpam_reset_class_locked(struct mpam_class *class) +void mpam_reset_class_locked(struct mpam_class *class) { struct mpam_component *comp; diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index 43c8e0f5f7ac5..f063a741aaba2 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -388,6 +388,9 @@ extern u8 mpam_pmg_max; void mpam_enable(struct work_struct *work); void mpam_disable(struct work_struct *work); +/* Reset all the RIS in a class under cpus_read_lock() */ +void mpam_reset_class_locked(struct mpam_class *class); + int mpam_apply_config(struct mpam_component *comp, u16 partid, struct mpam_config *cfg); diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 65bb670dc3fb1..b2217d11561d8 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -170,6 +170,19 @@ static int mpam_resctrl_pick_domain_id(int cpu, struct mpam_component *comp) return comp->comp_id; } +void resctrl_arch_reset_all_ctrls(struct rdt_resource *r) +{ + struct mpam_resctrl_res *res; + + lockdep_assert_cpus_held(); + + if (!mpam_is_enabled()) + return; + + res = container_of(r, struct mpam_resctrl_res, resctrl_res); + mpam_reset_class_locked(res->class); +} + static void mpam_resctrl_domain_hdr_init(int cpu, struct mpam_component *comp, enum resctrl_res_level rid, struct rdt_domain_hdr *hdr) From c0b8f7519bceeceeda5f2a6857f2cfe220aaa0ae Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:45:54 +0000 Subject: [PATCH 181/311] arm_mpam: resctrl: Add resctrl_arch_get_config() BugLink: https://bugs.launchpad.net/bugs/2154527 Implement resctrl_arch_get_config() by testing the live configuration for a CPOR bitmap. For any other configuration type return the default. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 02cc661687886563a0e08ecee51c5ef7d1737237) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index b2217d11561d8..3af57b6f2c1b5 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -170,6 +170,49 @@ static int mpam_resctrl_pick_domain_id(int cpu, struct mpam_component *comp) return comp->comp_id; } +u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d, + u32 closid, enum resctrl_conf_type type) +{ + u32 partid; + struct mpam_config *cfg; + struct mpam_props *cprops; + struct mpam_resctrl_res *res; + struct mpam_resctrl_dom *dom; + enum mpam_device_features configured_by; + + lockdep_assert_cpus_held(); + + if (!mpam_is_enabled()) + return resctrl_get_default_ctrl(r); + + res = container_of(r, struct mpam_resctrl_res, resctrl_res); + dom = container_of(d, struct mpam_resctrl_dom, resctrl_ctrl_dom); + cprops = &res->class->props; + + partid = resctrl_get_config_index(closid, type); + cfg = &dom->ctrl_comp->cfg[partid]; + + switch (r->rid) { + case RDT_RESOURCE_L2: + case RDT_RESOURCE_L3: + configured_by = mpam_feat_cpor_part; + break; + default: + return resctrl_get_default_ctrl(r); + } + + if (!r->alloc_capable || partid >= resctrl_arch_get_num_closid(r) || + !mpam_has_feature(configured_by, cfg)) + return resctrl_get_default_ctrl(r); + + switch (configured_by) { + case mpam_feat_cpor_part: + return cfg->cpbm; + default: + return resctrl_get_default_ctrl(r); + } +} + void resctrl_arch_reset_all_ctrls(struct rdt_resource *r) { struct mpam_resctrl_res *res; From 148294c6adae305fc265c3fbc081510e1936e565 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:45:55 +0000 Subject: [PATCH 182/311] arm_mpam: resctrl: Implement helpers to update configuration BugLink: https://bugs.launchpad.net/bugs/2154527 resctrl has two helpers for updating the configuration. resctrl_arch_update_one() updates a single value, and is used by the software-controller to apply feedback to the bandwidth controls, it has to be called on one of the CPUs in the resctrl:domain. resctrl_arch_update_domains() copies multiple staged configurations, it can be called from anywhere. Both helpers should update any changes to the underlying hardware. Implement resctrl_arch_update_domains() to use resctrl_arch_update_one(). Neither need to be called on a specific CPU as the mpam driver will send IPIs as needed. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 9cd2b522be2cc64fab179d75537d2e8df38d26a6) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 70 ++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 3af57b6f2c1b5..ea60777934ffd 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -213,6 +213,76 @@ u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d, } } +int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d, + u32 closid, enum resctrl_conf_type t, u32 cfg_val) +{ + u32 partid; + struct mpam_config cfg; + struct mpam_props *cprops; + struct mpam_resctrl_res *res; + struct mpam_resctrl_dom *dom; + + lockdep_assert_cpus_held(); + lockdep_assert_irqs_enabled(); + + /* + * No need to check the CPU as mpam_apply_config() doesn't care, and + * resctrl_arch_update_domains() relies on this. + */ + res = container_of(r, struct mpam_resctrl_res, resctrl_res); + dom = container_of(d, struct mpam_resctrl_dom, resctrl_ctrl_dom); + cprops = &res->class->props; + + partid = resctrl_get_config_index(closid, t); + if (!r->alloc_capable || partid >= resctrl_arch_get_num_closid(r)) { + pr_debug("Not alloc capable or computed PARTID out of range\n"); + return -EINVAL; + } + + /* + * Copy the current config to avoid clearing other resources when the + * same component is exposed multiple times through resctrl. + */ + cfg = dom->ctrl_comp->cfg[partid]; + + switch (r->rid) { + case RDT_RESOURCE_L2: + case RDT_RESOURCE_L3: + cfg.cpbm = cfg_val; + mpam_set_feature(mpam_feat_cpor_part, &cfg); + break; + default: + return -EINVAL; + } + + return mpam_apply_config(dom->ctrl_comp, partid, &cfg); +} + +int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid) +{ + int err; + struct rdt_ctrl_domain *d; + + lockdep_assert_cpus_held(); + lockdep_assert_irqs_enabled(); + + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list) { + for (enum resctrl_conf_type t = 0; t < CDP_NUM_TYPES; t++) { + struct resctrl_staged_config *cfg = &d->staged_config[t]; + + if (!cfg->have_new_ctrl) + continue; + + err = resctrl_arch_update_one(r, d, closid, t, + cfg->new_ctrl); + if (err) + return err; + } + } + + return 0; +} + void resctrl_arch_reset_all_ctrls(struct rdt_resource *r) { struct mpam_resctrl_res *res; From 169e20704be79ea2de57dfb066f66467e55be1e4 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:45:56 +0000 Subject: [PATCH 183/311] arm_mpam: resctrl: Add plumbing against arm64 task and cpu hooks BugLink: https://bugs.launchpad.net/bugs/2154527 arm64 provides helpers for changing a task's and a cpu's mpam partid/pmg values. These are used to back a number of resctrl_arch_ functions. Connect them up. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 9d2e1a99fae58ce992f147bdf83b5d9089f70b27) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 58 ++++++++++++++++++++++++++++++++++ include/linux/arm_mpam.h | 5 +++ 2 files changed, 63 insertions(+) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index ea60777934ffd..9cde5b7e644cc 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,8 @@ static struct mpam_resctrl_res mpam_resctrl_controls[RDT_NUM_RESOURCES]; /* The lock for modifying resctrl's domain lists from cpuhp callbacks. */ static DEFINE_MUTEX(domain_list_lock); +static bool cdp_enabled; + bool resctrl_arch_alloc_capable(void) { struct mpam_resctrl_res *res; @@ -57,6 +60,61 @@ u32 resctrl_arch_get_num_closid(struct rdt_resource *ignored) return mpam_partid_max + 1; } +void resctrl_arch_sched_in(struct task_struct *tsk) +{ + lockdep_assert_preemption_disabled(); + + mpam_thread_switch(tsk); +} + +void resctrl_arch_set_cpu_default_closid_rmid(int cpu, u32 closid, u32 rmid) +{ + WARN_ON_ONCE(closid > U16_MAX); + WARN_ON_ONCE(rmid > U8_MAX); + + if (!cdp_enabled) { + mpam_set_cpu_defaults(cpu, closid, closid, rmid, rmid); + } else { + /* + * When CDP is enabled, resctrl halves the closid range and we + * use odd/even partid for one closid. + */ + u32 partid_d = resctrl_get_config_index(closid, CDP_DATA); + u32 partid_i = resctrl_get_config_index(closid, CDP_CODE); + + mpam_set_cpu_defaults(cpu, partid_d, partid_i, rmid, rmid); + } +} + +void resctrl_arch_sync_cpu_closid_rmid(void *info) +{ + struct resctrl_cpu_defaults *r = info; + + lockdep_assert_preemption_disabled(); + + if (r) { + resctrl_arch_set_cpu_default_closid_rmid(smp_processor_id(), + r->closid, r->rmid); + } + + resctrl_arch_sched_in(current); +} + +void resctrl_arch_set_closid_rmid(struct task_struct *tsk, u32 closid, u32 rmid) +{ + WARN_ON_ONCE(closid > U16_MAX); + WARN_ON_ONCE(rmid > U8_MAX); + + if (!cdp_enabled) { + mpam_set_task_partid_pmg(tsk, closid, closid, rmid, rmid); + } else { + u32 partid_d = resctrl_get_config_index(closid, CDP_DATA); + u32 partid_i = resctrl_get_config_index(closid, CDP_CODE); + + mpam_set_task_partid_pmg(tsk, partid_d, partid_i, rmid, rmid); + } +} + struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l) { if (l >= RDT_NUM_RESOURCES) diff --git a/include/linux/arm_mpam.h b/include/linux/arm_mpam.h index 2c7d1413a401f..5a78299ec464b 100644 --- a/include/linux/arm_mpam.h +++ b/include/linux/arm_mpam.h @@ -52,6 +52,11 @@ static inline int mpam_ris_create(struct mpam_msc *msc, u8 ris_idx, bool resctrl_arch_alloc_capable(void); bool resctrl_arch_mon_capable(void); +void resctrl_arch_set_cpu_default_closid(int cpu, u32 closid); +void resctrl_arch_set_closid_rmid(struct task_struct *tsk, u32 closid, u32 rmid); +void resctrl_arch_set_cpu_default_closid_rmid(int cpu, u32 closid, u32 rmid); +void resctrl_arch_sched_in(struct task_struct *tsk); + /** * mpam_register_requestor() - Register a requestor with the MPAM driver * @partid_max: The maximum PARTID value the requestor can generate. From 0bbbf3881a504be0877f301d311147209bacd94b Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:45:57 +0000 Subject: [PATCH 184/311] arm_mpam: resctrl: Add CDP emulation BugLink: https://bugs.launchpad.net/bugs/2154527 Intel RDT's CDP feature allows the cache to use a different control value depending on whether the accesses was for instruction fetch or a data access. MPAM's equivalent feature is the other way up: the CPU assigns a different partid label to traffic depending on whether it was instruction fetch or a data access, which causes the cache to use a different control value based solely on the partid. MPAM can emulate CDP, with the side effect that the alternative partid is seen by all MSC, it can't be enabled per-MSC. Add the resctrl hooks to turn this on or off. Add the helpers that match a closid against a task, which need to be aware that the value written to hardware is not the same as the one resctrl is using. Update the 'arm64_mpam_global_default' variable the arch code uses during context switch to know when the per-cpu value should be used instead. Also, update these per-cpu values and sync the resulting mpam partid/pmg configuration to hardware. resctrl can enable CDP for L2 caches, L3 caches or both. When it is enabled by one and not the other MPAM globally enabled CDP but hides the effect on the other cache resource. This hiding is possible as CPOR is the only supported cache control and that uses a resource bitmap; two partids with the same bitmap act as one. Awkwardly, the MB controls don't implement CDP and CDP can't be hidden as the memory bandwidth control is a maximum per partid which can't be modelled with more partids. If the total maximum is used for both the data and instruction partids then then the maximum may be exceeded and if it is split in two then the one using more bandwidth will hit a lower limit. Hence, hide the MB controls completely if CDP is enabled for any resource. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Cc: Dave Martin Cc: Amit Singh Tomar Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 6789fb99282c0a8e8e84701b7edf456f4a9e71e2) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/arm64/include/asm/mpam.h | 1 + drivers/resctrl/mpam_internal.h | 1 + drivers/resctrl/mpam_resctrl.c | 122 ++++++++++++++++++++++++++++++++ include/linux/arm_mpam.h | 2 + 4 files changed, 126 insertions(+) diff --git a/arch/arm64/include/asm/mpam.h b/arch/arm64/include/asm/mpam.h index 05aa71200f61a..70d396e7b6da8 100644 --- a/arch/arm64/include/asm/mpam.h +++ b/arch/arm64/include/asm/mpam.h @@ -4,6 +4,7 @@ #ifndef __ASM__MPAM_H #define __ASM__MPAM_H +#include #include #include #include diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index f063a741aaba2..2751eeaba302d 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -342,6 +342,7 @@ struct mpam_resctrl_dom { struct mpam_resctrl_res { struct mpam_class *class; struct rdt_resource resctrl_res; + bool cdp_enabled; }; static inline int mpam_alloc_csu_mon(struct mpam_class *class) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 9cde5b7e644cc..2111542f485e1 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -35,6 +35,10 @@ static struct mpam_resctrl_res mpam_resctrl_controls[RDT_NUM_RESOURCES]; /* The lock for modifying resctrl's domain lists from cpuhp callbacks. */ static DEFINE_MUTEX(domain_list_lock); +/* + * MPAM emulates CDP by setting different PARTID in the I/D fields of MPAM0_EL1. + * This applies globally to all traffic the CPU generates. + */ static bool cdp_enabled; bool resctrl_arch_alloc_capable(void) @@ -50,6 +54,74 @@ bool resctrl_arch_alloc_capable(void) return false; } +bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level rid) +{ + return mpam_resctrl_controls[rid].cdp_enabled; +} + +/** + * resctrl_reset_task_closids() - Reset the PARTID/PMG values for all tasks. + * + * At boot, all existing tasks use partid zero for D and I. + * To enable/disable CDP emulation, all these tasks need relabelling. + */ +static void resctrl_reset_task_closids(void) +{ + struct task_struct *p, *t; + + read_lock(&tasklist_lock); + for_each_process_thread(p, t) { + resctrl_arch_set_closid_rmid(t, RESCTRL_RESERVED_CLOSID, + RESCTRL_RESERVED_RMID); + } + read_unlock(&tasklist_lock); +} + +int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable) +{ + u32 partid_i = RESCTRL_RESERVED_CLOSID, partid_d = RESCTRL_RESERVED_CLOSID; + int cpu; + + /* + * resctrl_arch_set_cdp_enabled() is only called with enable set to + * false on error and unmount. + */ + cdp_enabled = enable; + mpam_resctrl_controls[rid].cdp_enabled = enable; + + /* The mbw_max feature can't hide cdp as it's a per-partid maximum. */ + if (cdp_enabled && !mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled) + mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = false; + + if (mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled && + mpam_resctrl_controls[RDT_RESOURCE_MBA].class) + mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = true; + + if (enable) { + if (mpam_partid_max < 1) + return -EINVAL; + + partid_d = resctrl_get_config_index(RESCTRL_RESERVED_CLOSID, CDP_DATA); + partid_i = resctrl_get_config_index(RESCTRL_RESERVED_CLOSID, CDP_CODE); + } + + mpam_set_task_partid_pmg(current, partid_d, partid_i, 0, 0); + WRITE_ONCE(arm64_mpam_global_default, mpam_get_regval(current)); + + resctrl_reset_task_closids(); + + for_each_possible_cpu(cpu) + mpam_set_cpu_defaults(cpu, partid_d, partid_i, 0, 0); + on_each_cpu(resctrl_arch_sync_cpu_closid_rmid, NULL, 1); + + return 0; +} + +static bool mpam_resctrl_hide_cdp(enum resctrl_res_level rid) +{ + return cdp_enabled && !resctrl_arch_get_cdp_enabled(rid); +} + /* * MSC may raise an error interrupt if it sees an out or range partid/pmg, * and go on to truncate the value. Regardless of what the hardware supports, @@ -115,6 +187,30 @@ void resctrl_arch_set_closid_rmid(struct task_struct *tsk, u32 closid, u32 rmid) } } +bool resctrl_arch_match_closid(struct task_struct *tsk, u32 closid) +{ + u64 regval = mpam_get_regval(tsk); + u32 tsk_closid = FIELD_GET(MPAM0_EL1_PARTID_D, regval); + + if (cdp_enabled) + tsk_closid >>= 1; + + return tsk_closid == closid; +} + +/* The task's pmg is not unique, the partid must be considered too */ +bool resctrl_arch_match_rmid(struct task_struct *tsk, u32 closid, u32 rmid) +{ + u64 regval = mpam_get_regval(tsk); + u32 tsk_closid = FIELD_GET(MPAM0_EL1_PARTID_D, regval); + u32 tsk_rmid = FIELD_GET(MPAM0_EL1_PMG_D, regval); + + if (cdp_enabled) + tsk_closid >>= 1; + + return (tsk_closid == closid) && (tsk_rmid == rmid); +} + struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l) { if (l >= RDT_NUM_RESOURCES) @@ -247,6 +343,14 @@ u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d, dom = container_of(d, struct mpam_resctrl_dom, resctrl_ctrl_dom); cprops = &res->class->props; + /* + * When CDP is enabled, but the resource doesn't support it, + * the control is cloned across both partids. + * Pick one at random to read: + */ + if (mpam_resctrl_hide_cdp(r->rid)) + type = CDP_DATA; + partid = resctrl_get_config_index(closid, type); cfg = &dom->ctrl_comp->cfg[partid]; @@ -274,6 +378,7 @@ u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d, int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d, u32 closid, enum resctrl_conf_type t, u32 cfg_val) { + int err; u32 partid; struct mpam_config cfg; struct mpam_props *cprops; @@ -291,6 +396,9 @@ int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d, dom = container_of(d, struct mpam_resctrl_dom, resctrl_ctrl_dom); cprops = &res->class->props; + if (mpam_resctrl_hide_cdp(r->rid)) + t = CDP_DATA; + partid = resctrl_get_config_index(closid, t); if (!r->alloc_capable || partid >= resctrl_arch_get_num_closid(r)) { pr_debug("Not alloc capable or computed PARTID out of range\n"); @@ -313,6 +421,20 @@ int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d, return -EINVAL; } + /* + * When CDP is enabled, but the resource doesn't support it, we need to + * apply the same configuration to the other partid. + */ + if (mpam_resctrl_hide_cdp(r->rid)) { + partid = resctrl_get_config_index(closid, CDP_CODE); + err = mpam_apply_config(dom->ctrl_comp, partid, &cfg); + if (err) + return err; + + partid = resctrl_get_config_index(closid, CDP_DATA); + return mpam_apply_config(dom->ctrl_comp, partid, &cfg); + } + return mpam_apply_config(dom->ctrl_comp, partid, &cfg); } diff --git a/include/linux/arm_mpam.h b/include/linux/arm_mpam.h index 5a78299ec464b..d329b1dc148ba 100644 --- a/include/linux/arm_mpam.h +++ b/include/linux/arm_mpam.h @@ -56,6 +56,8 @@ void resctrl_arch_set_cpu_default_closid(int cpu, u32 closid); void resctrl_arch_set_closid_rmid(struct task_struct *tsk, u32 closid, u32 rmid); void resctrl_arch_set_cpu_default_closid_rmid(int cpu, u32 closid, u32 rmid); void resctrl_arch_sched_in(struct task_struct *tsk); +bool resctrl_arch_match_closid(struct task_struct *tsk, u32 closid); +bool resctrl_arch_match_rmid(struct task_struct *tsk, u32 closid, u32 rmid); /** * mpam_register_requestor() - Register a requestor with the MPAM driver From f61d07ca266e2fddf7b7b9a8bcfd30d068d28511 Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Fri, 13 Mar 2026 14:45:58 +0000 Subject: [PATCH 185/311] arm_mpam: resctrl: Hide CDP emulation behind CONFIG_EXPERT BugLink: https://bugs.launchpad.net/bugs/2154527 When CDP is not enabled, the 'rmid_entry's in the limbo list, rmid_busy_llc, map directly to a (PARTID,PMG) pair and when CDP is enabled the mapping is to two different pairs. As the limbo list is reused between mounts and CDP disabled on unmount this can lead to stale mapping and the limbo handler will then make monitor reads with potentially out of range PARTID. This may then cause an MPAM error interrupt and the driver will disable MPAM. No problems are expected if you just mount the resctrl file system once with CDP enabled and never unmount it. Hide CDP emulation behind CONFIG_EXPERT to protect the unwary. Signed-off-by: Ben Horgan Reviewed-by: Gavin Shan Reviewed-by: Zeng Heng Reviewed-by: James Morse Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Jesse Chick Signed-off-by: James Morse (cherry picked from commit 01a0021f6c39557037bfc41ede7230a0696677ff) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 2111542f485e1..2331e6ddb814b 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -82,6 +82,18 @@ int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable) u32 partid_i = RESCTRL_RESERVED_CLOSID, partid_d = RESCTRL_RESERVED_CLOSID; int cpu; + if (!IS_ENABLED(CONFIG_EXPERT) && enable) { + /* + * If the resctrl fs is mounted more than once, sequentially, + * then CDP can lead to the use of out of range PARTIDs. + */ + pr_warn("CDP not supported\n"); + return -EOPNOTSUPP; + } + + if (enable) + pr_warn("CDP is an expert feature and may cause MPAM to malfunction.\n"); + /* * resctrl_arch_set_cdp_enabled() is only called with enable set to * false on error and unmount. From d9b4fb697ae437a538b818f24bbeda448d498ff2 Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Fri, 13 Mar 2026 14:45:59 +0000 Subject: [PATCH 186/311] arm_mpam: resctrl: Convert to/from MPAMs fixed-point formats BugLink: https://bugs.launchpad.net/bugs/2154527 MPAM uses a fixed-point formats for some hardware controls. Resctrl provides the bandwidth controls as a percentage. Add helpers to convert between these. Ensure bwa_wd is at most 16 to make it clear higher values have no meaning. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Signed-off-by: Dave Martin Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 80d147d293130ee3c8a395cbbea1813e26ab9a1b) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_devices.c | 7 +++++ drivers/resctrl/mpam_resctrl.c | 51 ++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 90751729e49be..506deba05b40c 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -713,6 +713,13 @@ static void mpam_ris_hw_probe(struct mpam_msc_ris *ris) mpam_set_feature(mpam_feat_mbw_part, props); props->bwa_wd = FIELD_GET(MPAMF_MBW_IDR_BWA_WD, mbw_features); + + /* + * The BWA_WD field can represent 0-63, but the control fields it + * describes have a maximum of 16 bits. + */ + props->bwa_wd = min(props->bwa_wd, 16); + if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_MAX, mbw_features)) mpam_set_feature(mpam_feat_mbw_max, props); diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 2331e6ddb814b..240a06df2f079 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -242,6 +243,56 @@ static bool cache_has_usable_cpor(struct mpam_class *class) return class->props.cpbm_wd <= 32; } +/* + * Each fixed-point hardware value architecturally represents a range + * of values: the full range 0% - 100% is split contiguously into + * (1 << cprops->bwa_wd) equal bands. + * + * Although the bwa_bwd fields have 6 bits the maximum valid value is 16 + * as it reports the width of fields that are at most 16 bits. When + * fewer than 16 bits are valid the least significant bits are + * ignored. The implied binary point is kept between bits 15 and 16 and + * so the valid bits are leftmost. + * + * See ARM IHI0099B.a "MPAM system component specification", Section 9.3, + * "The fixed-point fractional format" for more information. + * + * Find the nearest percentage value to the upper bound of the selected band: + */ +static u32 mbw_max_to_percent(u16 mbw_max, struct mpam_props *cprops) +{ + u32 val = mbw_max; + + val >>= 16 - cprops->bwa_wd; + val += 1; + val *= MAX_MBA_BW; + val = DIV_ROUND_CLOSEST(val, 1 << cprops->bwa_wd); + + return val; +} + +/* + * Find the band whose upper bound is closest to the specified percentage. + * + * A round-to-nearest policy is followed here as a balanced compromise + * between unexpected under-commit of the resource (where the total of + * a set of resource allocations after conversion is less than the + * expected total, due to rounding of the individual converted + * percentages) and over-commit (where the total of the converted + * allocations is greater than expected). + */ +static u16 percent_to_mbw_max(u8 pc, struct mpam_props *cprops) +{ + u32 val = pc; + + val <<= cprops->bwa_wd; + val = DIV_ROUND_CLOSEST(val, MAX_MBA_BW); + val = max(val, 1) - 1; + val <<= 16 - cprops->bwa_wd; + + return val; +} + /* Test whether we can export MPAM_CLASS_CACHE:{2,3}? */ static void mpam_resctrl_pick_caches(void) { From f624a0012521572d00ee2925b51bc58a93261af6 Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Fri, 13 Mar 2026 14:46:00 +0000 Subject: [PATCH 187/311] arm_mpam: resctrl: Add rmid index helpers BugLink: https://bugs.launchpad.net/bugs/2154527 Because MPAM's pmg aren't identical to RDT's rmid, resctrl handles some data structures by index. This allows x86 to map indexes to RMID, and MPAM to map them to partid-and-pmg. Add the helpers to do this. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Suggested-by: James Morse Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 3e9b35823aabcb85cc039960256426e50f1fd601) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 16 ++++++++++++++++ include/linux/arm_mpam.h | 3 +++ 2 files changed, 19 insertions(+) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 240a06df2f079..370830ab11197 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -145,6 +145,22 @@ u32 resctrl_arch_get_num_closid(struct rdt_resource *ignored) return mpam_partid_max + 1; } +u32 resctrl_arch_system_num_rmid_idx(void) +{ + return (mpam_pmg_max + 1) * (mpam_partid_max + 1); +} + +u32 resctrl_arch_rmid_idx_encode(u32 closid, u32 rmid) +{ + return closid * (mpam_pmg_max + 1) + rmid; +} + +void resctrl_arch_rmid_idx_decode(u32 idx, u32 *closid, u32 *rmid) +{ + *closid = idx / (mpam_pmg_max + 1); + *rmid = idx % (mpam_pmg_max + 1); +} + void resctrl_arch_sched_in(struct task_struct *tsk) { lockdep_assert_preemption_disabled(); diff --git a/include/linux/arm_mpam.h b/include/linux/arm_mpam.h index d329b1dc148ba..7d23c90f077dc 100644 --- a/include/linux/arm_mpam.h +++ b/include/linux/arm_mpam.h @@ -58,6 +58,9 @@ void resctrl_arch_set_cpu_default_closid_rmid(int cpu, u32 closid, u32 rmid); void resctrl_arch_sched_in(struct task_struct *tsk); bool resctrl_arch_match_closid(struct task_struct *tsk, u32 closid); bool resctrl_arch_match_rmid(struct task_struct *tsk, u32 closid, u32 rmid); +u32 resctrl_arch_rmid_idx_encode(u32 closid, u32 rmid); +void resctrl_arch_rmid_idx_decode(u32 idx, u32 *closid, u32 *rmid); +u32 resctrl_arch_system_num_rmid_idx(void); /** * mpam_register_requestor() - Register a requestor with the MPAM driver From ebe346b93aa4b9d7564893dd633c73a083ab4070 Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Fri, 13 Mar 2026 14:46:01 +0000 Subject: [PATCH 188/311] arm_mpam: resctrl: Wait for cacheinfo to be ready BugLink: https://bugs.launchpad.net/bugs/2154527 In order to calculate the rmid realloc threshold the size of the cache needs to be known. Cache domains will also be named after the cache id. So that this information can be extracted from cacheinfo we need to wait for it to be ready. The cacheinfo information is populated in device_initcall() so we wait for that. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 1c1e2968a860c5af9fca67f1c0e88aab83ace0b3) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 370830ab11197..bf91cff05daf7 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -42,6 +43,13 @@ static DEFINE_MUTEX(domain_list_lock); */ static bool cdp_enabled; +/* + * We use cacheinfo to discover the size of the caches and their id. cacheinfo + * populates this from a device_initcall(). mpam_resctrl_setup() must wait. + */ +static bool cacheinfo_ready; +static DECLARE_WAIT_QUEUE_HEAD(wait_cacheinfo_ready); + bool resctrl_arch_alloc_capable(void) { struct mpam_resctrl_res *res; @@ -757,6 +765,8 @@ int mpam_resctrl_setup(void) struct mpam_resctrl_res *res; enum resctrl_res_level rid; + wait_event(wait_cacheinfo_ready, cacheinfo_ready); + cpus_read_lock(); for_each_mpam_resctrl_control(res, rid) { INIT_LIST_HEAD_RCU(&res->resctrl_res.ctrl_domains); @@ -794,3 +804,12 @@ int mpam_resctrl_setup(void) return 0; } + +static int __init __cacheinfo_ready(void) +{ + cacheinfo_ready = true; + wake_up(&wait_cacheinfo_ready); + + return 0; +} +device_initcall_sync(__cacheinfo_ready); From cb9fa99c6011d55787d8ddafe7f5c5826a6dcf34 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:46:02 +0000 Subject: [PATCH 189/311] arm_mpam: resctrl: Add support for 'MB' resource BugLink: https://bugs.launchpad.net/bugs/2154527 resctrl supports 'MB', as a percentage throttling of traffic from the L3. This is the control that mba_sc uses, so ideally the class chosen should be as close as possible to the counters used for mbm_total. If there is a single L3, it's the last cache, and the topology of the memory matches then the traffic at the memory controller will be equivalent to that at egress of the L3. If these conditions are met allow the memory class to back MB. MB's percentage control should be backed either with the fixed point fraction MBW_MAX or bandwidth portion bitmaps. The bandwidth portion bitmaps is not used as its tricky to pick which bits to use to avoid contention, and may be possible to expose this as something other than a percentage in the future. Tested-by: Shaopeng Tan Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Gavin Shan Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Co-developed-by: Dave Martin Signed-off-by: Dave Martin Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 36528c7681b8093f5f9270d2af7c4326d771f181) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 281 ++++++++++++++++++++++++++++++++- 1 file changed, 280 insertions(+), 1 deletion(-) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index bf91cff05daf7..60d111f7abfd5 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -267,6 +267,33 @@ static bool cache_has_usable_cpor(struct mpam_class *class) return class->props.cpbm_wd <= 32; } +static bool mba_class_use_mbw_max(struct mpam_props *cprops) +{ + return (mpam_has_feature(mpam_feat_mbw_max, cprops) && + cprops->bwa_wd); +} + +static bool class_has_usable_mba(struct mpam_props *cprops) +{ + return mba_class_use_mbw_max(cprops); +} + +/* + * Calculate the worst-case percentage change from each implemented step + * in the control. + */ +static u32 get_mba_granularity(struct mpam_props *cprops) +{ + if (!mba_class_use_mbw_max(cprops)) + return 0; + + /* + * bwa_wd is the number of bits implemented in the 0.xxx + * fixed point fraction. 1 bit is 50%, 2 is 25% etc. + */ + return DIV_ROUND_UP(MAX_MBA_BW, 1 << cprops->bwa_wd); +} + /* * Each fixed-point hardware value architecturally represents a range * of values: the full range 0% - 100% is split contiguously into @@ -317,6 +344,160 @@ static u16 percent_to_mbw_max(u8 pc, struct mpam_props *cprops) return val; } +static u32 get_mba_min(struct mpam_props *cprops) +{ + if (!mba_class_use_mbw_max(cprops)) { + WARN_ON_ONCE(1); + return 0; + } + + return mbw_max_to_percent(0, cprops); +} + +/* Find the L3 cache that has affinity with this CPU */ +static int find_l3_equivalent_bitmask(int cpu, cpumask_var_t tmp_cpumask) +{ + u32 cache_id = get_cpu_cacheinfo_id(cpu, 3); + + lockdep_assert_cpus_held(); + + return mpam_get_cpumask_from_cache_id(cache_id, 3, tmp_cpumask); +} + +/* + * topology_matches_l3() - Is the provided class the same shape as L3 + * @victim: The class we'd like to pretend is L3. + * + * resctrl expects all the world's a Xeon, and all counters are on the + * L3. We allow some mapping counters on other classes. This requires + * that the CPU->domain mapping is the same kind of shape. + * + * Using cacheinfo directly would make this work even if resctrl can't + * use the L3 - but cacheinfo can't tell us anything about offline CPUs. + * Using the L3 resctrl domain list also depends on CPUs being online. + * Using the mpam_class we picked for L3 so we can use its domain list + * assumes that there are MPAM controls on the L3. + * Instead, this path eventually uses the mpam_get_cpumask_from_cache_id() + * helper which can tell us about offline CPUs ... but getting the cache_id + * to start with relies on at least one CPU per L3 cache being online at + * boot. + * + * Walk the victim component list and compare the affinity mask with the + * corresponding L3. The topology matches if each victim:component's affinity + * mask is the same as the CPU's corresponding L3's. These lists/masks are + * computed from firmware tables so don't change at runtime. + */ +static bool topology_matches_l3(struct mpam_class *victim) +{ + int cpu, err; + struct mpam_component *victim_iter; + + lockdep_assert_cpus_held(); + + cpumask_var_t __free(free_cpumask_var) tmp_cpumask = CPUMASK_VAR_NULL; + if (!alloc_cpumask_var(&tmp_cpumask, GFP_KERNEL)) + return false; + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(victim_iter, &victim->components, class_list, + srcu_read_lock_held(&mpam_srcu)) { + if (cpumask_empty(&victim_iter->affinity)) { + pr_debug("class %u has CPU-less component %u - can't match L3!\n", + victim->level, victim_iter->comp_id); + return false; + } + + cpu = cpumask_any_and(&victim_iter->affinity, cpu_online_mask); + if (WARN_ON_ONCE(cpu >= nr_cpu_ids)) + return false; + + cpumask_clear(tmp_cpumask); + err = find_l3_equivalent_bitmask(cpu, tmp_cpumask); + if (err) { + pr_debug("Failed to find L3's equivalent component to class %u component %u\n", + victim->level, victim_iter->comp_id); + return false; + } + + /* Any differing bits in the affinity mask? */ + if (!cpumask_equal(tmp_cpumask, &victim_iter->affinity)) { + pr_debug("class %u component %u has Mismatched CPU mask with L3 equivalent\n" + "L3:%*pbl != victim:%*pbl\n", + victim->level, victim_iter->comp_id, + cpumask_pr_args(tmp_cpumask), + cpumask_pr_args(&victim_iter->affinity)); + + return false; + } + } + + return true; +} + +/* + * Test if the traffic for a class matches that at egress from the L3. For + * MSC at memory controllers this is only possible if there is a single L3 + * as otherwise the counters at the memory can include bandwidth from the + * non-local L3. + */ +static bool traffic_matches_l3(struct mpam_class *class) +{ + int err, cpu; + + lockdep_assert_cpus_held(); + + if (class->type == MPAM_CLASS_CACHE && class->level == 3) + return true; + + if (class->type == MPAM_CLASS_CACHE && class->level != 3) { + pr_debug("class %u is a different cache from L3\n", class->level); + return false; + } + + if (class->type != MPAM_CLASS_MEMORY) { + pr_debug("class %u is neither of type cache or memory\n", class->level); + return false; + } + + cpumask_var_t __free(free_cpumask_var) tmp_cpumask = CPUMASK_VAR_NULL; + if (!alloc_cpumask_var(&tmp_cpumask, GFP_KERNEL)) { + pr_debug("cpumask allocation failed\n"); + return false; + } + + cpu = cpumask_any_and(&class->affinity, cpu_online_mask); + err = find_l3_equivalent_bitmask(cpu, tmp_cpumask); + if (err) { + pr_debug("Failed to find L3 downstream to cpu %d\n", cpu); + return false; + } + + if (!cpumask_equal(tmp_cpumask, cpu_possible_mask)) { + pr_debug("There is more than one L3\n"); + return false; + } + + /* Be strict; the traffic might stop in the intermediate cache. */ + if (get_cpu_cacheinfo_id(cpu, 4) != -1) { + pr_debug("L3 isn't the last level of cache\n"); + return false; + } + + if (num_possible_nodes() > 1) { + pr_debug("There is more than one numa node\n"); + return false; + } + +#ifdef CONFIG_HMEM_REPORTING + if (node_devices[cpu_to_node(cpu)]->cache_dev) { + pr_debug("There is a memory side cache\n"); + return false; + } +#endif + + return true; +} + /* Test whether we can export MPAM_CLASS_CACHE:{2,3}? */ static void mpam_resctrl_pick_caches(void) { @@ -358,9 +539,68 @@ static void mpam_resctrl_pick_caches(void) } } +static void mpam_resctrl_pick_mba(void) +{ + struct mpam_class *class, *candidate_class = NULL; + struct mpam_resctrl_res *res; + + lockdep_assert_cpus_held(); + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(class, &mpam_classes, classes_list, + srcu_read_lock_held(&mpam_srcu)) { + struct mpam_props *cprops = &class->props; + + if (class->level != 3 && class->type == MPAM_CLASS_CACHE) { + pr_debug("class %u is a cache but not the L3\n", class->level); + continue; + } + + if (!class_has_usable_mba(cprops)) { + pr_debug("class %u has no bandwidth control\n", + class->level); + continue; + } + + if (!cpumask_equal(&class->affinity, cpu_possible_mask)) { + pr_debug("class %u has missing CPUs\n", class->level); + continue; + } + + if (!topology_matches_l3(class)) { + pr_debug("class %u topology doesn't match L3\n", + class->level); + continue; + } + + if (!traffic_matches_l3(class)) { + pr_debug("class %u traffic doesn't match L3 egress\n", + class->level); + continue; + } + + /* + * Pick a resource to be MBA that as close as possible to + * the L3. mbm_total counts the bandwidth leaving the L3 + * cache and MBA should correspond as closely as possible + * for proper operation of mba_sc. + */ + if (!candidate_class || class->level < candidate_class->level) + candidate_class = class; + } + + if (candidate_class) { + pr_debug("selected class %u to back MBA\n", + candidate_class->level); + res = &mpam_resctrl_controls[RDT_RESOURCE_MBA]; + res->class = candidate_class; + } +} + static int mpam_resctrl_control_init(struct mpam_resctrl_res *res) { struct mpam_class *class = res->class; + struct mpam_props *cprops = &class->props; struct rdt_resource *r = &res->resctrl_res; switch (r->rid) { @@ -392,6 +632,19 @@ static int mpam_resctrl_control_init(struct mpam_resctrl_res *res) r->cache.shareable_bits = resctrl_get_default_ctrl(r); r->alloc_capable = true; break; + case RDT_RESOURCE_MBA: + r->schema_fmt = RESCTRL_SCHEMA_RANGE; + r->ctrl_scope = RESCTRL_L3_CACHE; + + r->membw.delay_linear = true; + r->membw.throttle_mode = THREAD_THROTTLE_UNDEFINED; + r->membw.min_bw = get_mba_min(cprops); + r->membw.max_bw = MAX_MBA_BW; + r->membw.bw_gran = get_mba_granularity(cprops); + + r->name = "MB"; + r->alloc_capable = true; + break; default: return -EINVAL; } @@ -406,7 +659,17 @@ static int mpam_resctrl_pick_domain_id(int cpu, struct mpam_component *comp) if (class->type == MPAM_CLASS_CACHE) return comp->comp_id; - /* TODO: repaint domain ids to match the L3 domain ids */ + if (topology_matches_l3(class)) { + /* Use the corresponding L3 component ID as the domain ID */ + int id = get_cpu_cacheinfo_id(cpu, 3); + + /* Implies topology_matches_l3() made a mistake */ + if (WARN_ON_ONCE(id == -1)) + return comp->comp_id; + + return id; + } + /* Otherwise, expose the ID used by the firmware table code. */ return comp->comp_id; } @@ -446,6 +709,12 @@ u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d, case RDT_RESOURCE_L3: configured_by = mpam_feat_cpor_part; break; + case RDT_RESOURCE_MBA: + if (mpam_has_feature(mpam_feat_mbw_max, cprops)) { + configured_by = mpam_feat_mbw_max; + break; + } + fallthrough; default: return resctrl_get_default_ctrl(r); } @@ -457,6 +726,8 @@ u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d, switch (configured_by) { case mpam_feat_cpor_part: return cfg->cpbm; + case mpam_feat_mbw_max: + return mbw_max_to_percent(cfg->mbw_max, cprops); default: return resctrl_get_default_ctrl(r); } @@ -504,6 +775,13 @@ int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d, cfg.cpbm = cfg_val; mpam_set_feature(mpam_feat_cpor_part, &cfg); break; + case RDT_RESOURCE_MBA: + if (mpam_has_feature(mpam_feat_mbw_max, cprops)) { + cfg.mbw_max = percent_to_mbw_max(cfg_val, cprops); + mpam_set_feature(mpam_feat_mbw_max, &cfg); + break; + } + fallthrough; default: return -EINVAL; } @@ -775,6 +1053,7 @@ int mpam_resctrl_setup(void) /* Find some classes to use for controls */ mpam_resctrl_pick_caches(); + mpam_resctrl_pick_mba(); /* Initialise the resctrl structures from the classes */ for_each_mpam_resctrl_control(res, rid) { From c7c1e206c6f4f14b6be53294efddef4c2b4d3ca8 Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Fri, 13 Mar 2026 14:46:03 +0000 Subject: [PATCH 190/311] arm_mpam: resctrl: Add kunit test for control format conversions BugLink: https://bugs.launchpad.net/bugs/2154527 resctrl specifies the format of the control schemes, and these don't match the hardware. Some of the conversions are a bit hairy - add some kunit tests. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Signed-off-by: Dave Martin [morse: squashed enough of Dave's fixes in here that it's his patch now!] Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 5dc8f73eaa5dfccb229b9a25c797720e6379f8e0) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 4 + drivers/resctrl/test_mpam_resctrl.c | 315 ++++++++++++++++++++++++++++ 2 files changed, 319 insertions(+) create mode 100644 drivers/resctrl/test_mpam_resctrl.c diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 60d111f7abfd5..f8d4666fbaa85 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -1092,3 +1092,7 @@ static int __init __cacheinfo_ready(void) return 0; } device_initcall_sync(__cacheinfo_ready); + +#ifdef CONFIG_MPAM_KUNIT_TEST +#include "test_mpam_resctrl.c" +#endif diff --git a/drivers/resctrl/test_mpam_resctrl.c b/drivers/resctrl/test_mpam_resctrl.c new file mode 100644 index 0000000000000..b93d6ad87e43f --- /dev/null +++ b/drivers/resctrl/test_mpam_resctrl.c @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Arm Ltd. +/* This file is intended to be included into mpam_resctrl.c */ + +#include +#include +#include +#include +#include + +struct percent_value_case { + u8 pc; + u8 width; + u16 value; +}; + +/* + * Mysterious inscriptions taken from the union of ARM DDI 0598D.b, + * "Arm Architecture Reference Manual Supplement - Memory System + * Resource Partitioning and Monitoring (MPAM), for A-profile + * architecture", Section 9.8, "About the fixed-point fractional + * format" (exact percentage entries only) and ARM IHI0099B.a + * "MPAM system component specification", Section 9.3, + * "The fixed-point fractional format": + */ +static const struct percent_value_case percent_value_cases[] = { + /* Architectural cases: */ + { 1, 8, 1 }, { 1, 12, 0x27 }, { 1, 16, 0x28e }, + { 25, 8, 0x3f }, { 25, 12, 0x3ff }, { 25, 16, 0x3fff }, + { 33, 8, 0x53 }, { 33, 12, 0x546 }, { 33, 16, 0x5479 }, + { 35, 8, 0x58 }, { 35, 12, 0x598 }, { 35, 16, 0x5998 }, + { 45, 8, 0x72 }, { 45, 12, 0x732 }, { 45, 16, 0x7332 }, + { 50, 8, 0x7f }, { 50, 12, 0x7ff }, { 50, 16, 0x7fff }, + { 52, 8, 0x84 }, { 52, 12, 0x850 }, { 52, 16, 0x851d }, + { 55, 8, 0x8b }, { 55, 12, 0x8cb }, { 55, 16, 0x8ccb }, + { 58, 8, 0x93 }, { 58, 12, 0x946 }, { 58, 16, 0x9479 }, + { 75, 8, 0xbf }, { 75, 12, 0xbff }, { 75, 16, 0xbfff }, + { 80, 8, 0xcb }, { 80, 12, 0xccb }, { 80, 16, 0xcccb }, + { 88, 8, 0xe0 }, { 88, 12, 0xe13 }, { 88, 16, 0xe146 }, + { 95, 8, 0xf2 }, { 95, 12, 0xf32 }, { 95, 16, 0xf332 }, + { 100, 8, 0xff }, { 100, 12, 0xfff }, { 100, 16, 0xffff }, +}; + +static void test_percent_value_desc(const struct percent_value_case *param, + char *desc) +{ + snprintf(desc, KUNIT_PARAM_DESC_SIZE, + "pc=%d, width=%d, value=0x%.*x\n", + param->pc, param->width, + DIV_ROUND_UP(param->width, 4), param->value); +} + +KUNIT_ARRAY_PARAM(test_percent_value, percent_value_cases, + test_percent_value_desc); + +struct percent_value_test_info { + u32 pc; /* result of value-to-percent conversion */ + u32 value; /* result of percent-to-value conversion */ + u32 max_value; /* maximum raw value allowed by test params */ + unsigned int shift; /* promotes raw testcase value to 16 bits */ +}; + +/* + * Convert a reference percentage to a fixed-point MAX value and + * vice-versa, based on param (not test->param_value!) + */ +static void __prepare_percent_value_test(struct kunit *test, + struct percent_value_test_info *res, + const struct percent_value_case *param) +{ + struct mpam_props fake_props = { }; + + /* Reject bogus test parameters that would break the tests: */ + KUNIT_ASSERT_GE(test, param->width, 1); + KUNIT_ASSERT_LE(test, param->width, 16); + KUNIT_ASSERT_LT(test, param->value, 1 << param->width); + + mpam_set_feature(mpam_feat_mbw_max, &fake_props); + fake_props.bwa_wd = param->width; + + res->shift = 16 - param->width; + res->max_value = GENMASK_U32(param->width - 1, 0); + res->value = percent_to_mbw_max(param->pc, &fake_props); + res->pc = mbw_max_to_percent(param->value << res->shift, &fake_props); +} + +static void test_get_mba_granularity(struct kunit *test) +{ + int ret; + struct mpam_props fake_props = { }; + + /* Use MBW_MAX */ + mpam_set_feature(mpam_feat_mbw_max, &fake_props); + + fake_props.bwa_wd = 0; + KUNIT_EXPECT_FALSE(test, mba_class_use_mbw_max(&fake_props)); + + fake_props.bwa_wd = 1; + KUNIT_EXPECT_TRUE(test, mba_class_use_mbw_max(&fake_props)); + + /* Architectural maximum: */ + fake_props.bwa_wd = 16; + KUNIT_EXPECT_TRUE(test, mba_class_use_mbw_max(&fake_props)); + + /* No usable control... */ + fake_props.bwa_wd = 0; + ret = get_mba_granularity(&fake_props); + KUNIT_EXPECT_EQ(test, ret, 0); + + fake_props.bwa_wd = 1; + ret = get_mba_granularity(&fake_props); + KUNIT_EXPECT_EQ(test, ret, 50); /* DIV_ROUND_UP(100, 1 << 1)% = 50% */ + + fake_props.bwa_wd = 2; + ret = get_mba_granularity(&fake_props); + KUNIT_EXPECT_EQ(test, ret, 25); /* DIV_ROUND_UP(100, 1 << 2)% = 25% */ + + fake_props.bwa_wd = 3; + ret = get_mba_granularity(&fake_props); + KUNIT_EXPECT_EQ(test, ret, 13); /* DIV_ROUND_UP(100, 1 << 3)% = 13% */ + + fake_props.bwa_wd = 6; + ret = get_mba_granularity(&fake_props); + KUNIT_EXPECT_EQ(test, ret, 2); /* DIV_ROUND_UP(100, 1 << 6)% = 2% */ + + fake_props.bwa_wd = 7; + ret = get_mba_granularity(&fake_props); + KUNIT_EXPECT_EQ(test, ret, 1); /* DIV_ROUND_UP(100, 1 << 7)% = 1% */ + + /* Granularity saturates at 1% */ + fake_props.bwa_wd = 16; /* architectural maximum */ + ret = get_mba_granularity(&fake_props); + KUNIT_EXPECT_EQ(test, ret, 1); /* DIV_ROUND_UP(100, 1 << 16)% = 1% */ +} + +static void test_mbw_max_to_percent(struct kunit *test) +{ + const struct percent_value_case *param = test->param_value; + struct percent_value_test_info res; + + /* + * Since the reference values in percent_value_cases[] all + * correspond to exact percentages, round-to-nearest will + * always give the exact percentage back when the MPAM max + * value has precision of 0.5% or finer. (Always true for the + * reference data, since they all specify 8 bits or more of + * precision. + * + * So, keep it simple and demand an exact match: + */ + __prepare_percent_value_test(test, &res, param); + KUNIT_EXPECT_EQ(test, res.pc, param->pc); +} + +static void test_percent_to_mbw_max(struct kunit *test) +{ + const struct percent_value_case *param = test->param_value; + struct percent_value_test_info res; + + __prepare_percent_value_test(test, &res, param); + + KUNIT_EXPECT_GE(test, res.value, param->value << res.shift); + KUNIT_EXPECT_LE(test, res.value, (param->value + 1) << res.shift); + KUNIT_EXPECT_LE(test, res.value, res.max_value << res.shift); + + /* No flexibility allowed for 0% and 100%! */ + + if (param->pc == 0) + KUNIT_EXPECT_EQ(test, res.value, 0); + + if (param->pc == 100) + KUNIT_EXPECT_EQ(test, res.value, res.max_value << res.shift); +} + +static const void *test_all_bwa_wd_gen_params(struct kunit *test, const void *prev, + char *desc) +{ + uintptr_t param = (uintptr_t)prev; + + if (param > 15) + return NULL; + + param++; + + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "wd=%u\n", (unsigned int)param); + + return (void *)param; +} + +static unsigned int test_get_bwa_wd(struct kunit *test) +{ + uintptr_t param = (uintptr_t)test->param_value; + + KUNIT_ASSERT_GE(test, param, 1); + KUNIT_ASSERT_LE(test, param, 16); + + return param; +} + +static void test_mbw_max_to_percent_limits(struct kunit *test) +{ + struct mpam_props fake_props = {0}; + u32 max_value; + + mpam_set_feature(mpam_feat_mbw_max, &fake_props); + fake_props.bwa_wd = test_get_bwa_wd(test); + max_value = GENMASK(15, 16 - fake_props.bwa_wd); + + KUNIT_EXPECT_EQ(test, mbw_max_to_percent(max_value, &fake_props), + MAX_MBA_BW); + KUNIT_EXPECT_EQ(test, mbw_max_to_percent(0, &fake_props), + get_mba_min(&fake_props)); + + /* + * Rounding policy dependent 0% sanity-check: + * With round-to-nearest, the minimum mbw_max value really + * should map to 0% if there are at least 200 steps. + * (100 steps may be enough for some other rounding policies.) + */ + if (fake_props.bwa_wd >= 8) + KUNIT_EXPECT_EQ(test, mbw_max_to_percent(0, &fake_props), 0); + + if (fake_props.bwa_wd < 8 && + mbw_max_to_percent(0, &fake_props) == 0) + kunit_warn(test, "wd=%d: Testsuite/driver Rounding policy mismatch?", + fake_props.bwa_wd); +} + +/* + * Check that converting a percentage to mbw_max and back again (or, as + * appropriate, vice-versa) always restores the original value: + */ +static void test_percent_max_roundtrip_stability(struct kunit *test) +{ + struct mpam_props fake_props = {0}; + unsigned int shift; + u32 pc, max, pc2, max2; + + mpam_set_feature(mpam_feat_mbw_max, &fake_props); + fake_props.bwa_wd = test_get_bwa_wd(test); + shift = 16 - fake_props.bwa_wd; + + /* + * Converting a valid value from the coarser scale to the finer + * scale and back again must yield the original value: + */ + if (fake_props.bwa_wd >= 7) { + /* More than 100 steps: only test exact pc values: */ + for (pc = get_mba_min(&fake_props); pc <= MAX_MBA_BW; pc++) { + max = percent_to_mbw_max(pc, &fake_props); + pc2 = mbw_max_to_percent(max, &fake_props); + KUNIT_EXPECT_EQ(test, pc2, pc); + } + } else { + /* Fewer than 100 steps: only test exact mbw_max values: */ + for (max = 0; max < 1 << 16; max += 1 << shift) { + pc = mbw_max_to_percent(max, &fake_props); + max2 = percent_to_mbw_max(pc, &fake_props); + KUNIT_EXPECT_EQ(test, max2, max); + } + } +} + +static void test_percent_to_max_rounding(struct kunit *test) +{ + const struct percent_value_case *param = test->param_value; + unsigned int num_rounded_up = 0, total = 0; + struct percent_value_test_info res; + + for (param = percent_value_cases, total = 0; + param < &percent_value_cases[ARRAY_SIZE(percent_value_cases)]; + param++, total++) { + __prepare_percent_value_test(test, &res, param); + if (res.value > param->value << res.shift) + num_rounded_up++; + } + + /* + * The MPAM driver applies a round-to-nearest policy, whereas a + * round-down policy seems to have been applied in the + * reference table from which the test vectors were selected. + * + * For a large and well-distributed suite of test vectors, + * about half should be rounded up and half down compared with + * the reference table. The actual test vectors are few in + * number and probably not very well distributed however, so + * tolerate a round-up rate of between 1/4 and 3/4 before + * crying foul: + */ + + kunit_info(test, "Round-up rate: %u%% (%u/%u)\n", + DIV_ROUND_CLOSEST(num_rounded_up * 100, total), + num_rounded_up, total); + + KUNIT_EXPECT_GE(test, 4 * num_rounded_up, 1 * total); + KUNIT_EXPECT_LE(test, 4 * num_rounded_up, 3 * total); +} + +static struct kunit_case mpam_resctrl_test_cases[] = { + KUNIT_CASE(test_get_mba_granularity), + KUNIT_CASE_PARAM(test_mbw_max_to_percent, test_percent_value_gen_params), + KUNIT_CASE_PARAM(test_percent_to_mbw_max, test_percent_value_gen_params), + KUNIT_CASE_PARAM(test_mbw_max_to_percent_limits, test_all_bwa_wd_gen_params), + KUNIT_CASE(test_percent_to_max_rounding), + KUNIT_CASE_PARAM(test_percent_max_roundtrip_stability, + test_all_bwa_wd_gen_params), + {} +}; + +static struct kunit_suite mpam_resctrl_test_suite = { + .name = "mpam_resctrl_test_suite", + .test_cases = mpam_resctrl_test_cases, +}; + +kunit_test_suites(&mpam_resctrl_test_suite); From 0c8b423e9c49d47147c6a79553a0df2df0ac1f22 Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Fri, 13 Mar 2026 14:46:04 +0000 Subject: [PATCH 191/311] arm_mpam: resctrl: Add monitor initialisation and domain boilerplate BugLink: https://bugs.launchpad.net/bugs/2154527 Add the boilerplate that tells resctrl about the mpam monitors that are available. resctrl expects all (non-telemetry) monitors to be on the L3 and so advertise them there and invent an L3 resctrl resource if required. The L3 cache itself has to exist as the cache ids are used as the domain ids. Bring the resctrl monitor domains online and offline based on the cpus they contain. Support for specific monitor types is left to later. Tested-by: Punit Agrawal Reviewed-by: Zeng Heng Reviewed-by: Jonathan Cameron Signed-off-by: Ben Horgan Reviewed-by: Gavin Shan Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Jesse Chick Signed-off-by: James Morse (cherry picked from commit 264c285999fce128fc52743bce582468b26e9f65) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_internal.h | 15 +++ drivers/resctrl/mpam_resctrl.c | 231 ++++++++++++++++++++++++++++++-- 2 files changed, 235 insertions(+), 11 deletions(-) diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index 2751eeaba302d..301cf5c151bd9 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -336,7 +336,16 @@ struct mpam_msc_ris { struct mpam_resctrl_dom { struct mpam_component *ctrl_comp; + + /* + * There is no single mon_comp because different events may be backed + * by different class/components. mon_comp is indexed by the event + * number. + */ + struct mpam_component *mon_comp[QOS_NUM_EVENTS]; + struct rdt_ctrl_domain resctrl_ctrl_dom; + struct rdt_l3_mon_domain resctrl_mon_dom; }; struct mpam_resctrl_res { @@ -345,6 +354,12 @@ struct mpam_resctrl_res { bool cdp_enabled; }; +struct mpam_resctrl_mon { + struct mpam_class *class; + + /* per-class data that resctrl needs will live here */ +}; + static inline int mpam_alloc_csu_mon(struct mpam_class *class) { struct mpam_props *cprops = &class->props; diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index f8d4666fbaa85..e03d0f400993c 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -34,6 +34,23 @@ static struct mpam_resctrl_res mpam_resctrl_controls[RDT_NUM_RESOURCES]; rid < RDT_NUM_RESOURCES; \ rid++, res = &mpam_resctrl_controls[rid]) +/* + * The classes we've picked to map to resctrl events. + * Resctrl believes all the worlds a Xeon, and these are all on the L3. This + * array lets us find the actual class backing the event counters. e.g. + * the only memory bandwidth counters may be on the memory controller, but to + * make use of them, we pretend they are on L3. Restrict the events considered + * to those supported by MPAM. + * Class pointer may be NULL. + */ +#define MPAM_MAX_EVENT QOS_L3_MBM_TOTAL_EVENT_ID +static struct mpam_resctrl_mon mpam_resctrl_counters[MPAM_MAX_EVENT + 1]; + +#define for_each_mpam_resctrl_mon(mon, eventid) \ + for (eventid = QOS_FIRST_EVENT, mon = &mpam_resctrl_counters[eventid]; \ + eventid <= MPAM_MAX_EVENT; \ + eventid++, mon = &mpam_resctrl_counters[eventid]) + /* The lock for modifying resctrl's domain lists from cpuhp callbacks. */ static DEFINE_MUTEX(domain_list_lock); @@ -63,6 +80,15 @@ bool resctrl_arch_alloc_capable(void) return false; } +bool resctrl_arch_mon_capable(void) +{ + struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; + struct rdt_resource *l3 = &res->resctrl_res; + + /* All monitors are presented as being on the L3 cache */ + return l3->mon_capable; +} + bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level rid) { return mpam_resctrl_controls[rid].cdp_enabled; @@ -89,6 +115,8 @@ static void resctrl_reset_task_closids(void) int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable) { u32 partid_i = RESCTRL_RESERVED_CLOSID, partid_d = RESCTRL_RESERVED_CLOSID; + struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; + struct rdt_resource *l3 = &res->resctrl_res; int cpu; if (!IS_ENABLED(CONFIG_EXPERT) && enable) { @@ -110,6 +138,11 @@ int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable) cdp_enabled = enable; mpam_resctrl_controls[rid].cdp_enabled = enable; + if (enable) + l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx() / 2; + else + l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx(); + /* The mbw_max feature can't hide cdp as it's a per-partid maximum. */ if (cdp_enabled && !mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled) mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = false; @@ -674,6 +707,56 @@ static int mpam_resctrl_pick_domain_id(int cpu, struct mpam_component *comp) return comp->comp_id; } +static int mpam_resctrl_monitor_init(struct mpam_resctrl_mon *mon, + enum resctrl_event_id type) +{ + struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; + struct rdt_resource *l3 = &res->resctrl_res; + + lockdep_assert_cpus_held(); + + /* + * There also needs to be an L3 cache present. + * The check just requires any online CPU and it can't go offline as we + * hold the cpu lock. + */ + if (get_cpu_cacheinfo_id(raw_smp_processor_id(), 3) == -1) + return 0; + + /* + * If there are no MPAM resources on L3, force it into existence. + * topology_matches_l3() already ensures this looks like the L3. + * The domain-ids will be fixed up by mpam_resctrl_domain_hdr_init(). + */ + if (!res->class) { + pr_warn_once("Faking L3 MSC to enable counters.\n"); + res->class = mpam_resctrl_counters[type].class; + } + + /* + * Called multiple times!, once per event type that has a + * monitoring class. + * Setting name is necessary on monitor only platforms. + */ + l3->name = "L3"; + l3->mon_scope = RESCTRL_L3_CACHE; + + /* + * num-rmid is the upper bound for the number of monitoring groups that + * can exist simultaneously, including the default monitoring group for + * each control group. Hence, advertise the whole rmid_idx space even + * though each control group has its own pmg/rmid space. Unfortunately, + * this does mean userspace needs to know the architecture to correctly + * interpret this value. + */ + l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx(); + + if (resctrl_enable_mon_event(type, false, 0, NULL)) + l3->mon_capable = true; + + return 0; +} + u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d, u32 closid, enum resctrl_conf_type type) { @@ -901,11 +984,26 @@ static void mpam_resctrl_domain_insert(struct list_head *list, list_add_tail_rcu(&new->list, pos); } +static struct mpam_component *find_component(struct mpam_class *class, int cpu) +{ + struct mpam_component *comp; + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(comp, &class->components, class_list, + srcu_read_lock_held(&mpam_srcu)) { + if (cpumask_test_cpu(cpu, &comp->affinity)) + return comp; + } + + return NULL; +} + static struct mpam_resctrl_dom * mpam_resctrl_alloc_domain(unsigned int cpu, struct mpam_resctrl_res *res) { int err; struct mpam_resctrl_dom *dom; + struct rdt_l3_mon_domain *mon_d; struct rdt_ctrl_domain *ctrl_d; struct mpam_class *class = res->class; struct mpam_component *comp_iter, *ctrl_comp; @@ -945,8 +1043,56 @@ mpam_resctrl_alloc_domain(unsigned int cpu, struct mpam_resctrl_res *res) } else { pr_debug("Skipped control domain online - no controls\n"); } + + if (r->mon_capable) { + struct mpam_component *any_mon_comp; + struct mpam_resctrl_mon *mon; + enum resctrl_event_id eventid; + + /* + * Even if the monitor domain is backed by a different + * component, the L3 component IDs need to be used... only + * there may be no ctrl_comp for the L3. + * Search each event's class list for a component with + * overlapping CPUs and set up the dom->mon_comp array. + */ + + for_each_mpam_resctrl_mon(mon, eventid) { + struct mpam_component *mon_comp; + + if (!mon->class) + continue; // dummy resource + + mon_comp = find_component(mon->class, cpu); + dom->mon_comp[eventid] = mon_comp; + if (mon_comp) + any_mon_comp = mon_comp; + } + if (!any_mon_comp) { + WARN_ON_ONCE(0); + err = -EFAULT; + goto offline_ctrl_domain; + } + + mon_d = &dom->resctrl_mon_dom; + mpam_resctrl_domain_hdr_init(cpu, any_mon_comp, r->rid, &mon_d->hdr); + mon_d->hdr.type = RESCTRL_MON_DOMAIN; + err = resctrl_online_mon_domain(r, &mon_d->hdr); + if (err) + goto offline_ctrl_domain; + + mpam_resctrl_domain_insert(&r->mon_domains, &mon_d->hdr); + } else { + pr_debug("Skipped monitor domain online - no monitors\n"); + } + return dom; +offline_ctrl_domain: + if (r->alloc_capable) { + mpam_resctrl_offline_domain_hdr(cpu, &ctrl_d->hdr); + resctrl_offline_ctrl_domain(r, ctrl_d); + } free_domain: kfree(dom); dom = ERR_PTR(err); @@ -954,6 +1100,35 @@ mpam_resctrl_alloc_domain(unsigned int cpu, struct mpam_resctrl_res *res) return dom; } +/* + * We know all the monitors are associated with the L3, even if there are no + * controls and therefore no control component. Find the cache-id for the CPU + * and use that to search for existing resctrl domains. + * This relies on mpam_resctrl_pick_domain_id() using the L3 cache-id + * for anything that is not a cache. + */ +static struct mpam_resctrl_dom *mpam_resctrl_get_mon_domain_from_cpu(int cpu) +{ + int cache_id; + struct mpam_resctrl_dom *dom; + struct mpam_resctrl_res *l3 = &mpam_resctrl_controls[RDT_RESOURCE_L3]; + + lockdep_assert_cpus_held(); + + if (!l3->class) + return NULL; + cache_id = get_cpu_cacheinfo_id(cpu, 3); + if (cache_id < 0) + return NULL; + + list_for_each_entry_rcu(dom, &l3->resctrl_res.mon_domains, resctrl_mon_dom.hdr.list) { + if (dom->resctrl_mon_dom.hdr.id == cache_id) + return dom; + } + + return NULL; +} + static struct mpam_resctrl_dom * mpam_resctrl_get_domain_from_cpu(int cpu, struct mpam_resctrl_res *res) { @@ -967,7 +1142,11 @@ mpam_resctrl_get_domain_from_cpu(int cpu, struct mpam_resctrl_res *res) return dom; } - return NULL; + if (r->rid != RDT_RESOURCE_L3) + return NULL; + + /* Search the mon domain list too - needed on monitor only platforms. */ + return mpam_resctrl_get_mon_domain_from_cpu(cpu); } int mpam_resctrl_online_cpu(unsigned int cpu) @@ -994,6 +1173,11 @@ int mpam_resctrl_online_cpu(unsigned int cpu) mpam_resctrl_online_domain_hdr(cpu, &ctrl_d->hdr); } + if (r->mon_capable) { + struct rdt_l3_mon_domain *mon_d = &dom->resctrl_mon_dom; + + mpam_resctrl_online_domain_hdr(cpu, &mon_d->hdr); + } } } @@ -1012,8 +1196,9 @@ void mpam_resctrl_offline_cpu(unsigned int cpu) guard(mutex)(&domain_list_lock); for_each_mpam_resctrl_control(res, rid) { struct mpam_resctrl_dom *dom; + struct rdt_l3_mon_domain *mon_d; struct rdt_ctrl_domain *ctrl_d; - bool ctrl_dom_empty; + bool ctrl_dom_empty, mon_dom_empty; struct rdt_resource *r = &res->resctrl_res; if (!res->class) @@ -1032,7 +1217,16 @@ void mpam_resctrl_offline_cpu(unsigned int cpu) ctrl_dom_empty = true; } - if (ctrl_dom_empty) + if (r->mon_capable) { + mon_d = &dom->resctrl_mon_dom; + mon_dom_empty = mpam_resctrl_offline_domain_hdr(cpu, &mon_d->hdr); + if (mon_dom_empty) + resctrl_offline_mon_domain(&res->resctrl_res, &mon_d->hdr); + } else { + mon_dom_empty = true; + } + + if (ctrl_dom_empty && mon_dom_empty) kfree(dom); } } @@ -1042,12 +1236,15 @@ int mpam_resctrl_setup(void) int err = 0; struct mpam_resctrl_res *res; enum resctrl_res_level rid; + struct mpam_resctrl_mon *mon; + enum resctrl_event_id eventid; wait_event(wait_cacheinfo_ready, cacheinfo_ready); cpus_read_lock(); for_each_mpam_resctrl_control(res, rid) { INIT_LIST_HEAD_RCU(&res->resctrl_res.ctrl_domains); + INIT_LIST_HEAD_RCU(&res->resctrl_res.mon_domains); res->resctrl_res.rid = rid; } @@ -1063,25 +1260,37 @@ int mpam_resctrl_setup(void) err = mpam_resctrl_control_init(res); if (err) { pr_debug("Failed to initialise rid %u\n", rid); - break; + goto internal_error; } } - cpus_read_unlock(); - if (err) { - pr_debug("Internal error %d - resctrl not supported\n", err); - return err; + for_each_mpam_resctrl_mon(mon, eventid) { + if (!mon->class) + continue; // dummy resource + + err = mpam_resctrl_monitor_init(mon, eventid); + if (err) { + pr_debug("Failed to initialise event %u\n", eventid); + goto internal_error; + } } - if (!resctrl_arch_alloc_capable()) { - pr_debug("No alloc(%u) found - resctrl not supported\n", - resctrl_arch_alloc_capable()); + cpus_read_unlock(); + + if (!resctrl_arch_alloc_capable() && !resctrl_arch_mon_capable()) { + pr_debug("No alloc(%u) or monitor(%u) found - resctrl not supported\n", + resctrl_arch_alloc_capable(), resctrl_arch_mon_capable()); return -EOPNOTSUPP; } /* TODO: call resctrl_init() */ return 0; + +internal_error: + cpus_read_unlock(); + pr_debug("Internal error %d - resctrl not supported\n", err); + return err; } static int __init __cacheinfo_ready(void) From 581c4f2dcebb9f6cf1c19022bc64ae15009ebda9 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:46:05 +0000 Subject: [PATCH 192/311] arm_mpam: resctrl: Add support for csu counters BugLink: https://bugs.launchpad.net/bugs/2154527 resctrl exposes a counter via a file named llc_occupancy. This isn't really a counter as its value goes up and down, this is a snapshot of the cache storage usage monitor. Add some picking code which will only find an L3. The resctrl counter file is called llc_occupancy but we don't check it is the last one as it is already identified as L3. Tested-by: Shaopeng Tan Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Gavin Shan Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Co-developed-by: Dave Martin Signed-off-by: Dave Martin Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 1458c4f053355f88cc5d190ca02243d2c60fa010) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 83 ++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index e03d0f400993c..07bb20a01b383 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -311,6 +311,28 @@ static bool class_has_usable_mba(struct mpam_props *cprops) return mba_class_use_mbw_max(cprops); } +static bool cache_has_usable_csu(struct mpam_class *class) +{ + struct mpam_props *cprops; + + if (!class) + return false; + + cprops = &class->props; + + if (!mpam_has_feature(mpam_feat_msmon_csu, cprops)) + return false; + + /* + * CSU counters settle on the value, so we can get away with + * having only one. + */ + if (!cprops->num_csu_mon) + return false; + + return true; +} + /* * Calculate the worst-case percentage change from each implemented step * in the control. @@ -630,6 +652,64 @@ static void mpam_resctrl_pick_mba(void) } } +static void counter_update_class(enum resctrl_event_id evt_id, + struct mpam_class *class) +{ + struct mpam_class *existing_class = mpam_resctrl_counters[evt_id].class; + + if (existing_class) { + if (class->level == 3) { + pr_debug("Existing class is L3 - L3 wins\n"); + return; + } + + if (existing_class->level < class->level) { + pr_debug("Existing class is closer to L3, %u versus %u - closer is better\n", + existing_class->level, class->level); + return; + } + } + + mpam_resctrl_counters[evt_id].class = class; +} + +static void mpam_resctrl_pick_counters(void) +{ + struct mpam_class *class; + + lockdep_assert_cpus_held(); + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(class, &mpam_classes, classes_list, + srcu_read_lock_held(&mpam_srcu)) { + /* The name of the resource is L3... */ + if (class->type == MPAM_CLASS_CACHE && class->level != 3) { + pr_debug("class %u is a cache but not the L3", class->level); + continue; + } + + if (!cpumask_equal(&class->affinity, cpu_possible_mask)) { + pr_debug("class %u does not cover all CPUs", + class->level); + continue; + } + + if (cache_has_usable_csu(class)) { + pr_debug("class %u has usable CSU", + class->level); + + /* CSU counters only make sense on a cache. */ + switch (class->type) { + case MPAM_CLASS_CACHE: + counter_update_class(QOS_L3_OCCUP_EVENT_ID, class); + break; + default: + break; + } + } + } +} + static int mpam_resctrl_control_init(struct mpam_resctrl_res *res) { struct mpam_class *class = res->class; @@ -1264,6 +1344,9 @@ int mpam_resctrl_setup(void) } } + /* Find some classes to use for monitors */ + mpam_resctrl_pick_counters(); + for_each_mpam_resctrl_mon(mon, eventid) { if (!mon->class) continue; // dummy resource From f1df14f2ffec0af2fef87c129b133049c89f99c6 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:46:06 +0000 Subject: [PATCH 193/311] arm_mpam: resctrl: Allow resctrl to allocate monitors BugLink: https://bugs.launchpad.net/bugs/2154527 When resctrl wants to read a domain's 'QOS_L3_OCCUP', it needs to allocate a monitor on the corresponding resource. Monitors are allocated by class instead of component. Add helpers to allocate a CSU monitor. These helper return an out of range value for MBM counters. Allocating a montitor context is expected to block until hardware resources become available. This only makes sense for QOS_L3_OCCUP as unallocated MBM counters are losing data. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 2a3c79c61539779a09928893518c8286d7774b54) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_internal.h | 14 ++++++- drivers/resctrl/mpam_resctrl.c | 67 +++++++++++++++++++++++++++++++++ include/linux/arm_mpam.h | 5 +++ 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index 301cf5c151bd9..85b2b99263601 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -29,6 +29,14 @@ struct platform_device; #define PACKED_FOR_KUNIT #endif +/* + * This 'mon' values must not alias an actual monitor, so must be larger than + * U16_MAX, but not be confused with an errno value, so smaller than + * (u32)-SZ_4K. + * USE_PRE_ALLOCATED is used to avoid confusion with an actual monitor. + */ +#define USE_PRE_ALLOCATED (U16_MAX + 1) + static inline bool mpam_is_enabled(void) { return static_branch_likely(&mpam_enabled); @@ -216,7 +224,11 @@ enum mon_filter_options { }; struct mon_cfg { - u16 mon; + /* + * mon must be large enough to hold out of range values like + * USE_PRE_ALLOCATED + */ + u32 mon; u8 pmg; bool match_pmg; bool csu_exclude_clean; diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 07bb20a01b383..9682ffb151846 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -22,6 +22,8 @@ #include "mpam_internal.h" +DECLARE_WAIT_QUEUE_HEAD(resctrl_mon_ctx_waiters); + /* * The classes we've picked to map to resctrl resources, wrapped * in with their resctrl structure. @@ -289,6 +291,71 @@ struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l) return &mpam_resctrl_controls[l].resctrl_res; } +static int resctrl_arch_mon_ctx_alloc_no_wait(enum resctrl_event_id evtid) +{ + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid]; + + if (!mon->class) + return -EINVAL; + + switch (evtid) { + case QOS_L3_OCCUP_EVENT_ID: + /* With CDP, one monitor gets used for both code/data reads */ + return mpam_alloc_csu_mon(mon->class); + case QOS_L3_MBM_LOCAL_EVENT_ID: + case QOS_L3_MBM_TOTAL_EVENT_ID: + return USE_PRE_ALLOCATED; + default: + return -EOPNOTSUPP; + } +} + +void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r, + enum resctrl_event_id evtid) +{ + DEFINE_WAIT(wait); + int *ret; + + ret = kmalloc_obj(*ret); + if (!ret) + return ERR_PTR(-ENOMEM); + + do { + prepare_to_wait(&resctrl_mon_ctx_waiters, &wait, + TASK_INTERRUPTIBLE); + *ret = resctrl_arch_mon_ctx_alloc_no_wait(evtid); + if (*ret == -ENOSPC) + schedule(); + } while (*ret == -ENOSPC && !signal_pending(current)); + finish_wait(&resctrl_mon_ctx_waiters, &wait); + + return ret; +} + +static void resctrl_arch_mon_ctx_free_no_wait(enum resctrl_event_id evtid, + u32 mon_idx) +{ + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid]; + + if (!mon->class) + return; + + if (evtid == QOS_L3_OCCUP_EVENT_ID) + mpam_free_csu_mon(mon->class, mon_idx); + + wake_up(&resctrl_mon_ctx_waiters); +} + +void resctrl_arch_mon_ctx_free(struct rdt_resource *r, + enum resctrl_event_id evtid, void *arch_mon_ctx) +{ + u32 mon_idx = *(u32 *)arch_mon_ctx; + + kfree(arch_mon_ctx); + + resctrl_arch_mon_ctx_free_no_wait(evtid, mon_idx); +} + static bool cache_has_usable_cpor(struct mpam_class *class) { struct mpam_props *cprops = &class->props; diff --git a/include/linux/arm_mpam.h b/include/linux/arm_mpam.h index 7d23c90f077dc..e1461e32af756 100644 --- a/include/linux/arm_mpam.h +++ b/include/linux/arm_mpam.h @@ -5,6 +5,7 @@ #define __LINUX_ARM_MPAM_H #include +#include #include struct mpam_msc; @@ -62,6 +63,10 @@ u32 resctrl_arch_rmid_idx_encode(u32 closid, u32 rmid); void resctrl_arch_rmid_idx_decode(u32 idx, u32 *closid, u32 *rmid); u32 resctrl_arch_system_num_rmid_idx(void); +struct rdt_resource; +void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r, enum resctrl_event_id evtid); +void resctrl_arch_mon_ctx_free(struct rdt_resource *r, enum resctrl_event_id evtid, void *ctx); + /** * mpam_register_requestor() - Register a requestor with the MPAM driver * @partid_max: The maximum PARTID value the requestor can generate. From 513326c035076d8c8c2a560f5a4d7d49e9933fbd Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:46:07 +0000 Subject: [PATCH 194/311] arm_mpam: resctrl: Add resctrl_arch_rmid_read() BugLink: https://bugs.launchpad.net/bugs/2154527 resctrl uses resctrl_arch_rmid_read() to read counters. CDP emulation means the counter may need reading in three different ways. The helpers behind the resctrl_arch_ functions will be re-used for the ABMC equivalent functions. Add the rounding helper for checking monitor values while we're here. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Jesse Chick Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit fb56b29932ca276df268806ad52ed80f40f99a6e) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 82 ++++++++++++++++++++++++++++++++++ include/linux/arm_mpam.h | 5 +++ 2 files changed, 87 insertions(+) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 9682ffb151846..9a15ddd340f73 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -356,6 +356,88 @@ void resctrl_arch_mon_ctx_free(struct rdt_resource *r, resctrl_arch_mon_ctx_free_no_wait(evtid, mon_idx); } +static int __read_mon(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp, + enum mpam_device_features mon_type, + int mon_idx, + enum resctrl_conf_type cdp_type, u32 closid, u32 rmid, u64 *val) +{ + struct mon_cfg cfg; + + if (!mpam_is_enabled()) + return -EINVAL; + + /* Shift closid to account for CDP */ + closid = resctrl_get_config_index(closid, cdp_type); + + if (irqs_disabled()) { + /* Check if we can access this domain without an IPI */ + return -EIO; + } + + cfg = (struct mon_cfg) { + .mon = mon_idx, + .match_pmg = true, + .partid = closid, + .pmg = rmid, + }; + + return mpam_msmon_read(mon_comp, &cfg, mon_type, val); +} + +static int read_mon_cdp_safe(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp, + enum mpam_device_features mon_type, + int mon_idx, u32 closid, u32 rmid, u64 *val) +{ + if (cdp_enabled) { + u64 code_val = 0, data_val = 0; + int err; + + err = __read_mon(mon, mon_comp, mon_type, mon_idx, + CDP_CODE, closid, rmid, &code_val); + if (err) + return err; + + err = __read_mon(mon, mon_comp, mon_type, mon_idx, + CDP_DATA, closid, rmid, &data_val); + if (err) + return err; + + *val += code_val + data_val; + return 0; + } + + return __read_mon(mon, mon_comp, mon_type, mon_idx, + CDP_NONE, closid, rmid, val); +} + +/* MBWU when not in ABMC mode (not supported), and CSU counters. */ +int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain_hdr *hdr, + u32 closid, u32 rmid, enum resctrl_event_id eventid, + void *arch_priv, u64 *val, void *arch_mon_ctx) +{ + struct mpam_resctrl_dom *l3_dom; + struct mpam_component *mon_comp; + u32 mon_idx = *(u32 *)arch_mon_ctx; + enum mpam_device_features mon_type; + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[eventid]; + + resctrl_arch_rmid_read_context_check(); + + if (eventid >= QOS_NUM_EVENTS || !mon->class) + return -EINVAL; + + l3_dom = container_of(hdr, struct mpam_resctrl_dom, resctrl_mon_dom.hdr); + mon_comp = l3_dom->mon_comp[eventid]; + + if (eventid != QOS_L3_OCCUP_EVENT_ID) + return -EINVAL; + + mon_type = mpam_feat_msmon_csu; + + return read_mon_cdp_safe(mon, mon_comp, mon_type, mon_idx, + closid, rmid, val); +} + static bool cache_has_usable_cpor(struct mpam_class *class) { struct mpam_props *cprops = &class->props; diff --git a/include/linux/arm_mpam.h b/include/linux/arm_mpam.h index e1461e32af756..86d5e326d2bd3 100644 --- a/include/linux/arm_mpam.h +++ b/include/linux/arm_mpam.h @@ -67,6 +67,11 @@ struct rdt_resource; void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r, enum resctrl_event_id evtid); void resctrl_arch_mon_ctx_free(struct rdt_resource *r, enum resctrl_event_id evtid, void *ctx); +static inline unsigned int resctrl_arch_round_mon_val(unsigned int val) +{ + return val; +} + /** * mpam_register_requestor() - Register a requestor with the MPAM driver * @partid_max: The maximum PARTID value the requestor can generate. From 4f9d952fb77a63881ad204a3c7f01128879fb011 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:46:08 +0000 Subject: [PATCH 195/311] arm_mpam: resctrl: Update the rmid reallocation limit BugLink: https://bugs.launchpad.net/bugs/2154527 resctrl's limbo code needs to be told when the data left in a cache is small enough for the partid+pmg value to be re-allocated. x86 uses the cache size divided by the number of rmid users the cache may have. Do the same, but for the smallest cache, and with the number of partid-and-pmg users. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 49b04e401825431529e866470d8d2dcd8e9ef058) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 9a15ddd340f73..f82fff3519df4 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -438,6 +438,42 @@ int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain_hdr *hdr, closid, rmid, val); } +/* + * The rmid realloc threshold should be for the smallest cache exposed to + * resctrl. + */ +static int update_rmid_limits(struct mpam_class *class) +{ + u32 num_unique_pmg = resctrl_arch_system_num_rmid_idx(); + struct mpam_props *cprops = &class->props; + struct cacheinfo *ci; + + lockdep_assert_cpus_held(); + + if (!mpam_has_feature(mpam_feat_msmon_csu, cprops)) + return 0; + + /* + * Assume cache levels are the same size for all CPUs... + * The check just requires any online CPU and it can't go offline as we + * hold the cpu lock. + */ + ci = get_cpu_cacheinfo_level(raw_smp_processor_id(), class->level); + if (!ci || ci->size == 0) { + pr_debug("Could not read cache size for class %u\n", + class->level); + return -EINVAL; + } + + if (!resctrl_rmid_realloc_limit || + ci->size < resctrl_rmid_realloc_limit) { + resctrl_rmid_realloc_limit = ci->size; + resctrl_rmid_realloc_threshold = ci->size / num_unique_pmg; + } + + return 0; +} + static bool cache_has_usable_cpor(struct mpam_class *class) { struct mpam_props *cprops = &class->props; @@ -850,6 +886,9 @@ static void mpam_resctrl_pick_counters(void) /* CSU counters only make sense on a cache. */ switch (class->type) { case MPAM_CLASS_CACHE: + if (update_rmid_limits(class)) + break; + counter_update_class(QOS_L3_OCCUP_EVENT_ID, class); break; default: From 123a6b54294753ce8521a19fcf02abdd00e8a327 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:46:09 +0000 Subject: [PATCH 196/311] arm_mpam: resctrl: Add empty definitions for assorted resctrl functions BugLink: https://bugs.launchpad.net/bugs/2154527 A few resctrl features and hooks need to be provided, but aren't needed or supported on MPAM platforms. resctrl has individual hooks to separately enable and disable the closid/partid and rmid/pmg context switching code. For MPAM this is all the same thing, as the value in struct task_struct is used to cache the value that should be written to hardware. arm64's context switching code is enabled once MPAM is usable, but doesn't touch the hardware unless the value has changed. For now event configuration is not supported, and can be turned off by returning 'false' from resctrl_arch_is_evt_configurable(). The new io_alloc feature is not supported either, always return false from the enable helper to indicate and fail the enable. Add this, and empty definitions for the other hooks. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit efc775eadce2c6e0921c21d9c29a7b6686022281) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 65 ++++++++++++++++++++++++++++++++++ include/linux/arm_mpam.h | 9 +++++ 2 files changed, 74 insertions(+) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index f82fff3519df4..777ecdc2d0f85 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -91,6 +91,71 @@ bool resctrl_arch_mon_capable(void) return l3->mon_capable; } +bool resctrl_arch_is_evt_configurable(enum resctrl_event_id evt) +{ + return false; +} + +void resctrl_arch_mon_event_config_read(void *info) +{ +} + +void resctrl_arch_mon_event_config_write(void *info) +{ +} + +void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_l3_mon_domain *d) +{ +} + +void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + u32 closid, u32 rmid, enum resctrl_event_id eventid) +{ +} + +void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + u32 closid, u32 rmid, int cntr_id, + enum resctrl_event_id eventid) +{ +} + +void resctrl_arch_config_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + enum resctrl_event_id evtid, u32 rmid, u32 closid, + u32 cntr_id, bool assign) +{ +} + +int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + u32 unused, u32 rmid, int cntr_id, + enum resctrl_event_id eventid, u64 *val) +{ + return -EOPNOTSUPP; +} + +bool resctrl_arch_mbm_cntr_assign_enabled(struct rdt_resource *r) +{ + return false; +} + +int resctrl_arch_mbm_cntr_assign_set(struct rdt_resource *r, bool enable) +{ + return -EINVAL; +} + +int resctrl_arch_io_alloc_enable(struct rdt_resource *r, bool enable) +{ + return -EOPNOTSUPP; +} + +bool resctrl_arch_get_io_alloc_enabled(struct rdt_resource *r) +{ + return false; +} + +void resctrl_arch_pre_mount(void) +{ +} + bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level rid) { return mpam_resctrl_controls[rid].cdp_enabled; diff --git a/include/linux/arm_mpam.h b/include/linux/arm_mpam.h index 86d5e326d2bd3..f92a36187a527 100644 --- a/include/linux/arm_mpam.h +++ b/include/linux/arm_mpam.h @@ -67,6 +67,15 @@ struct rdt_resource; void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r, enum resctrl_event_id evtid); void resctrl_arch_mon_ctx_free(struct rdt_resource *r, enum resctrl_event_id evtid, void *ctx); +/* + * The CPU configuration for MPAM is cheap to write, and is only written if it + * has changed. No need for fine grained enables. + */ +static inline void resctrl_arch_enable_mon(void) { } +static inline void resctrl_arch_disable_mon(void) { } +static inline void resctrl_arch_enable_alloc(void) { } +static inline void resctrl_arch_disable_alloc(void) { } + static inline unsigned int resctrl_arch_round_mon_val(unsigned int val) { return val; From 13c8cc5e981a9b793144bb3d4e2b944074cd76a3 Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Fri, 27 Mar 2026 16:30:53 +0100 Subject: [PATCH 197/311] ALSA: usb-audio: Replace hard-coded number with MAX_CHANNELS BugLink: https://bugs.launchpad.net/bugs/2154527 One place in mixer.c still used a hard-coded number 16 instead of MAX_CHANNELS. Replace with it, so that we can extend the max number of channels gracefully. Link: https://lore.kernel.org/F1B104A5-CD6A-4A26-AB46-14BF233C0579@getmailspring.com Tested-by: Phil Willoughby Link: https://patch.msgid.link/20260327153056.691575-1-tiwai@suse.de Signed-off-by: Takashi Iwai (cherry picked from commit ec9a788620be1c11535fe99e9b2779f9eef2b099) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- sound/usb/mixer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sound/usb/mixer.c b/sound/usb/mixer.c index 21191da4c0496..bf6b6b8582ef9 100644 --- a/sound/usb/mixer.c +++ b/sound/usb/mixer.c @@ -1714,7 +1714,7 @@ static void __build_feature_ctl(struct usb_mixer_interface *mixer, cval->master_readonly = readonly_mask; } else { int i, c = 0; - for (i = 0; i < 16; i++) + for (i = 0; i < MAX_CHANNELS; i++) if (ctl_mask & BIT(i)) c++; cval->channels = c; From 42d33183829ef723ea79aeedb7f4e7299fd4ecb3 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:46:10 +0000 Subject: [PATCH 198/311] arm64: mpam: Select ARCH_HAS_CPU_RESCTRL BugLink: https://bugs.launchpad.net/bugs/2154527 Enough MPAM support is present to enable ARCH_HAS_CPU_RESCTRL. Let it rip^Wlink! ARCH_HAS_CPU_RESCTRL indicates resctrl can be enabled. It is enabled by the arch code simply because it has 'arch' in its name. This removes ARM_CPU_RESCTRL as a mimic of X86_CPU_RESCTRL. While here, move the ACPI dependency to the driver's Kconfig file. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Acked-by: Catalin Marinas Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 4aab135bda1661a795e4fe96418bf840833e1119) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/arm64/Kconfig | 2 +- arch/arm64/include/asm/resctrl.h | 2 ++ drivers/resctrl/Kconfig | 7 +++++++ drivers/resctrl/Makefile | 2 +- 4 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/include/asm/resctrl.h diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 00d79552a3c11..241659f285a86 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2018,7 +2018,7 @@ config ARM64_TLB_RANGE config ARM64_MPAM bool "Enable support for MPAM" select ARM64_MPAM_DRIVER - select ACPI_MPAM if ACPI + select ARCH_HAS_CPU_RESCTRL help Memory System Resource Partitioning and Monitoring (MPAM) is an optional extension to the Arm architecture that allows each diff --git a/arch/arm64/include/asm/resctrl.h b/arch/arm64/include/asm/resctrl.h new file mode 100644 index 0000000000000..b506e95cf6e37 --- /dev/null +++ b/arch/arm64/include/asm/resctrl.h @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include diff --git a/drivers/resctrl/Kconfig b/drivers/resctrl/Kconfig index c34e059c6e41f..672abea3b03cc 100644 --- a/drivers/resctrl/Kconfig +++ b/drivers/resctrl/Kconfig @@ -1,6 +1,7 @@ menuconfig ARM64_MPAM_DRIVER bool "MPAM driver" depends on ARM64 && ARM64_MPAM + select ACPI_MPAM if ACPI help Memory System Resource Partitioning and Monitoring (MPAM) driver for System IP, e.g. caches and memory controllers. @@ -22,3 +23,9 @@ config MPAM_KUNIT_TEST If unsure, say N. endif + +config ARM64_MPAM_RESCTRL_FS + bool + default y if ARM64_MPAM_DRIVER && RESCTRL_FS + select RESCTRL_RMID_DEPENDS_ON_CLOSID + select RESCTRL_ASSIGN_FIXED diff --git a/drivers/resctrl/Makefile b/drivers/resctrl/Makefile index 40beaf999582c..4f6d0e81f9b8f 100644 --- a/drivers/resctrl/Makefile +++ b/drivers/resctrl/Makefile @@ -1,5 +1,5 @@ obj-$(CONFIG_ARM64_MPAM_DRIVER) += mpam.o mpam-y += mpam_devices.o -mpam-$(CONFIG_ARM_CPU_RESCTRL) += mpam_resctrl.o +mpam-$(CONFIG_ARM64_MPAM_RESCTRL_FS) += mpam_resctrl.o ccflags-$(CONFIG_ARM64_MPAM_DRIVER_DEBUG) += -DDEBUG From f3169c6a99daeb8cb44f753b111f9102f15f11fa Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:46:11 +0000 Subject: [PATCH 199/311] arm_mpam: resctrl: Call resctrl_init() on platforms that can support resctrl BugLink: https://bugs.launchpad.net/bugs/2154527 Now that MPAM links against resctrl, call resctrl_init() to register the filesystem and setup resctrl's structures. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Peter Newman Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit fb481ec08699e9daf08ab839a79ab37b1bcca94d) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_devices.c | 32 ++++++++++++++--- drivers/resctrl/mpam_internal.h | 4 +++ drivers/resctrl/mpam_resctrl.c | 63 ++++++++++++++++++++++++++++++++- 3 files changed, 94 insertions(+), 5 deletions(-) diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 506deba05b40c..2c65e4c46ed56 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -73,6 +73,14 @@ static DECLARE_WORK(mpam_broken_work, &mpam_disable); /* When mpam is disabled, the printed reason to aid debugging */ static char *mpam_disable_reason; +/* + * Whether resctrl has been setup. Used by cpuhp in preference to + * mpam_is_enabled(). The disable call after an error interrupt makes + * mpam_is_enabled() false before the cpuhp callbacks are made. + * Reads/writes should hold mpam_cpuhp_state_lock, (or be cpuhp callbacks). + */ +static bool mpam_resctrl_enabled; + /* * An MSC is a physical container for controls and monitors, each identified by * their RIS index. These share a base-address, interrupts and some MMIO @@ -1621,7 +1629,7 @@ static int mpam_cpu_online(unsigned int cpu) mpam_reprogram_msc(msc); } - if (mpam_is_enabled()) + if (mpam_resctrl_enabled) return mpam_resctrl_online_cpu(cpu); return 0; @@ -1667,7 +1675,7 @@ static int mpam_cpu_offline(unsigned int cpu) { struct mpam_msc *msc; - if (mpam_is_enabled()) + if (mpam_resctrl_enabled) mpam_resctrl_offline_cpu(cpu); guard(srcu)(&mpam_srcu); @@ -2528,6 +2536,7 @@ static void mpam_enable_once(void) } static_branch_enable(&mpam_enabled); + mpam_resctrl_enabled = true; mpam_register_cpuhp_callbacks(mpam_cpu_online, mpam_cpu_offline, "mpam:online"); @@ -2587,24 +2596,39 @@ static void mpam_reset_class(struct mpam_class *class) void mpam_disable(struct work_struct *ignored) { int idx; + bool do_resctrl_exit; struct mpam_class *class; struct mpam_msc *msc, *tmp; + if (mpam_is_enabled()) + static_branch_disable(&mpam_enabled); + mutex_lock(&mpam_cpuhp_state_lock); if (mpam_cpuhp_state) { cpuhp_remove_state(mpam_cpuhp_state); mpam_cpuhp_state = 0; } + + /* + * Removing the cpuhp state called mpam_cpu_offline() and told resctrl + * all the CPUs are offline. + */ + do_resctrl_exit = mpam_resctrl_enabled; + mpam_resctrl_enabled = false; mutex_unlock(&mpam_cpuhp_state_lock); - static_branch_disable(&mpam_enabled); + if (do_resctrl_exit) + mpam_resctrl_exit(); mpam_unregister_irqs(); idx = srcu_read_lock(&mpam_srcu); list_for_each_entry_srcu(class, &mpam_classes, classes_list, - srcu_read_lock_held(&mpam_srcu)) + srcu_read_lock_held(&mpam_srcu)) { mpam_reset_class(class); + if (do_resctrl_exit) + mpam_resctrl_teardown_class(class); + } srcu_read_unlock(&mpam_srcu, idx); mutex_lock(&mpam_list_lock); diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index 85b2b99263601..68906c6ebfb01 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -431,12 +431,16 @@ int mpam_get_cpumask_from_cache_id(unsigned long cache_id, u32 cache_level, #ifdef CONFIG_RESCTRL_FS int mpam_resctrl_setup(void); +void mpam_resctrl_exit(void); int mpam_resctrl_online_cpu(unsigned int cpu); void mpam_resctrl_offline_cpu(unsigned int cpu); +void mpam_resctrl_teardown_class(struct mpam_class *class); #else static inline int mpam_resctrl_setup(void) { return 0; } +static inline void mpam_resctrl_exit(void) { } static inline int mpam_resctrl_online_cpu(unsigned int cpu) { return 0; } static inline void mpam_resctrl_offline_cpu(unsigned int cpu) { } +static inline void mpam_resctrl_teardown_class(struct mpam_class *class) { } #endif /* CONFIG_RESCTRL_FS */ /* diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 777ecdc2d0f85..a9938006d0e6e 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -69,6 +69,12 @@ static bool cdp_enabled; static bool cacheinfo_ready; static DECLARE_WAIT_QUEUE_HEAD(wait_cacheinfo_ready); +/* + * If resctrl_init() succeeded, resctrl_exit() can be used to remove support + * for the filesystem in the event of an error. + */ +static bool resctrl_enabled; + bool resctrl_arch_alloc_capable(void) { struct mpam_resctrl_res *res; @@ -360,6 +366,9 @@ static int resctrl_arch_mon_ctx_alloc_no_wait(enum resctrl_event_id evtid) { struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid]; + if (!mpam_is_enabled()) + return -EINVAL; + if (!mon->class) return -EINVAL; @@ -402,6 +411,9 @@ static void resctrl_arch_mon_ctx_free_no_wait(enum resctrl_event_id evtid, { struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid]; + if (!mpam_is_enabled()) + return; + if (!mon->class) return; @@ -488,6 +500,9 @@ int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain_hdr *hdr, resctrl_arch_rmid_read_context_check(); + if (!mpam_is_enabled()) + return -EINVAL; + if (eventid >= QOS_NUM_EVENTS || !mon->class) return -EINVAL; @@ -1162,6 +1177,9 @@ int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d, lockdep_assert_cpus_held(); lockdep_assert_irqs_enabled(); + if (!mpam_is_enabled()) + return -EINVAL; + /* * No need to check the CPU as mpam_apply_config() doesn't care, and * resctrl_arch_update_domains() relies on this. @@ -1227,6 +1245,9 @@ int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid) lockdep_assert_cpus_held(); lockdep_assert_irqs_enabled(); + if (!mpam_is_enabled()) + return -EINVAL; + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list) { for (enum resctrl_conf_type t = 0; t < CDP_NUM_TYPES; t++) { struct resctrl_staged_config *cfg = &d->staged_config[t]; @@ -1619,7 +1640,11 @@ int mpam_resctrl_setup(void) return -EOPNOTSUPP; } - /* TODO: call resctrl_init() */ + err = resctrl_init(); + if (err) + return err; + + WRITE_ONCE(resctrl_enabled, true); return 0; @@ -1629,6 +1654,42 @@ int mpam_resctrl_setup(void) return err; } +void mpam_resctrl_exit(void) +{ + if (!READ_ONCE(resctrl_enabled)) + return; + + WRITE_ONCE(resctrl_enabled, false); + resctrl_exit(); +} + +/* + * The driver is detaching an MSC from this class, if resctrl was using it, + * pull on resctrl_exit(). + */ +void mpam_resctrl_teardown_class(struct mpam_class *class) +{ + struct mpam_resctrl_res *res; + enum resctrl_res_level rid; + struct mpam_resctrl_mon *mon; + enum resctrl_event_id eventid; + + might_sleep(); + + for_each_mpam_resctrl_control(res, rid) { + if (res->class == class) { + res->class = NULL; + break; + } + } + for_each_mpam_resctrl_mon(mon, eventid) { + if (mon->class == class) { + mon->class = NULL; + break; + } + } +} + static int __init __cacheinfo_ready(void) { cacheinfo_ready = true; From effed9a02f2805761f2e04b43a01ae6d72e72b60 Mon Sep 17 00:00:00 2001 From: Shanker Donthineni Date: Fri, 13 Mar 2026 14:46:12 +0000 Subject: [PATCH 200/311] arm_mpam: Add quirk framework BugLink: https://bugs.launchpad.net/bugs/2154527 The MPAM specification includes the MPAMF_IIDR, which serves to uniquely identify the MSC implementation through a combination of implementer details, product ID, variant, and revision. Certain hardware issues/errata can be resolved using software workarounds. Introduce a quirk framework to allow workarounds to be enabled based on the MPAMF_IIDR value. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Zeng Heng Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Gavin Shan Signed-off-by: Shanker Donthineni Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Co-developed-by: James Morse Signed-off-by: James Morse (cherry picked from commit fa7745218c9828ac4849ef62bccad684aec0f422) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_devices.c | 32 ++++++++++++++++++++++++++++++++ drivers/resctrl/mpam_internal.h | 25 +++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 2c65e4c46ed56..324c105b28614 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -630,6 +630,30 @@ static struct mpam_msc_ris *mpam_get_or_create_ris(struct mpam_msc *msc, return ERR_PTR(-ENOENT); } +static const struct mpam_quirk mpam_quirks[] = { + { NULL } /* Sentinel */ +}; + +static void mpam_enable_quirks(struct mpam_msc *msc) +{ + const struct mpam_quirk *quirk; + + for (quirk = &mpam_quirks[0]; quirk->iidr_mask; quirk++) { + int err = 0; + + if (quirk->iidr != (msc->iidr & quirk->iidr_mask)) + continue; + + if (quirk->init) + err = quirk->init(msc, quirk); + + if (err) + continue; + + mpam_set_quirk(quirk->workaround, msc); + } +} + /* * IHI009A.a has this nugget: "If a monitor does not support automatic behaviour * of NRDY, software can use this bit for any purpose" - so hardware might not @@ -864,8 +888,11 @@ static int mpam_msc_hw_probe(struct mpam_msc *msc) /* Grab an IDR value to find out how many RIS there are */ mutex_lock(&msc->part_sel_lock); idr = mpam_msc_read_idr(msc); + msc->iidr = mpam_read_partsel_reg(msc, IIDR); mutex_unlock(&msc->part_sel_lock); + mpam_enable_quirks(msc); + msc->ris_max = FIELD_GET(MPAMF_IDR_RIS_MAX, idr); /* Use these values so partid/pmg always starts with a valid value */ @@ -1974,6 +2001,7 @@ static bool mpam_has_cmax_wd_feature(struct mpam_props *props) * resulting safe value must be compatible with both. When merging values in * the tree, all the aliasing resources must be handled first. * On mismatch, parent is modified. + * Quirks on an MSC will apply to all MSC in that class. */ static void __props_mismatch(struct mpam_props *parent, struct mpam_props *child, bool alias) @@ -2093,6 +2121,7 @@ static void __props_mismatch(struct mpam_props *parent, * nobble the class feature, as we can't configure all the resources. * e.g. The L3 cache is composed of two resources with 13 and 17 portion * bitmaps respectively. + * Quirks on an MSC will apply to all MSC in that class. */ static void __class_props_mismatch(struct mpam_class *class, struct mpam_vmsc *vmsc) @@ -2106,6 +2135,9 @@ __class_props_mismatch(struct mpam_class *class, struct mpam_vmsc *vmsc) dev_dbg(dev, "Merging features for class:0x%lx &= vmsc:0x%lx\n", (long)cprops->features, (long)vprops->features); + /* Merge quirks */ + class->quirks |= vmsc->msc->quirks; + /* Take the safe value for any common features */ __props_mismatch(cprops, vprops, false); } diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index 68906c6ebfb01..01858365cd9e7 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -85,6 +85,8 @@ struct mpam_msc { u8 pmg_max; unsigned long ris_idxs; u32 ris_max; + u32 iidr; + u16 quirks; /* * error_irq_lock is taken when registering/unregistering the error @@ -216,6 +218,28 @@ struct mpam_props { #define mpam_set_feature(_feat, x) __set_bit(_feat, (x)->features) #define mpam_clear_feature(_feat, x) __clear_bit(_feat, (x)->features) +/* Workaround bits for msc->quirks */ +enum mpam_device_quirks { + MPAM_QUIRK_LAST +}; + +#define mpam_has_quirk(_quirk, x) ((1 << (_quirk) & (x)->quirks)) +#define mpam_set_quirk(_quirk, x) ((x)->quirks |= (1 << (_quirk))) + +struct mpam_quirk { + int (*init)(struct mpam_msc *msc, const struct mpam_quirk *quirk); + + u32 iidr; + u32 iidr_mask; + + enum mpam_device_quirks workaround; +}; + +#define MPAM_IIDR_MATCH_ONE (FIELD_PREP_CONST(MPAMF_IIDR_PRODUCTID, 0xfff) | \ + FIELD_PREP_CONST(MPAMF_IIDR_VARIANT, 0xf) | \ + FIELD_PREP_CONST(MPAMF_IIDR_REVISION, 0xf) | \ + FIELD_PREP_CONST(MPAMF_IIDR_IMPLEMENTER, 0xfff)) + /* The values for MSMON_CFG_MBWU_FLT.RWBW */ enum mon_filter_options { COUNT_BOTH = 0, @@ -259,6 +283,7 @@ struct mpam_class { struct mpam_props props; u32 nrdy_usec; + u16 quirks; u8 level; enum mpam_class_types type; From e093bcb0d50ed0cee69eac8e27d7dd8d8dd110f5 Mon Sep 17 00:00:00 2001 From: Shanker Donthineni Date: Fri, 13 Mar 2026 14:46:13 +0000 Subject: [PATCH 201/311] arm_mpam: Add workaround for T241-MPAM-1 BugLink: https://bugs.launchpad.net/bugs/2154527 The MPAM bandwidth partitioning controls will not be correctly configured, and hardware will retain default configuration register values, meaning generally that bandwidth will remain unprovisioned. To address the issue, follow the below steps after updating the MBW_MIN and/or MBW_MAX registers. - Perform 64b reads from all 12 bridge MPAM shadow registers at offsets (0x360048 + slice*0x10000 + partid*8). These registers are read-only. - Continue iterating until all 12 shadow register values match in a loop. pr_warn_once if the values fail to match within the loop count 1000. - Perform 64b writes with the value 0x0 to the two spare registers at offsets 0x1b0000 and 0x1c0000. In the hardware, writes to the MPAMCFG_MBW_MAX MPAMCFG_MBW_MIN registers are transformed into broadcast writes to the 12 shadow registers. The final two writes to the spare registers cause a final rank of downstream micro-architectural MPAM registers to be updated from the shadow copies. The intervening loop to read the 12 shadow registers helps avoid a race condition where writes to the spare registers occur before all shadow registers have been updated. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Punit Agrawal Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Gavin Shan Signed-off-by: Shanker Donthineni Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit 70e81fbedc6570b2397e07a645136af0a0eec907) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- Documentation/arch/arm64/silicon-errata.rst | 2 + drivers/resctrl/mpam_devices.c | 88 +++++++++++++++++++++ drivers/resctrl/mpam_internal.h | 9 +++ 3 files changed, 99 insertions(+) diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index 4c300caad9011..a65620f98e3aa 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -247,6 +247,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | T241 GICv3/4.x | T241-FABRIC-4 | N/A | +----------------+-----------------+-----------------+-----------------------------+ +| NVIDIA | T241 MPAM | T241-MPAM-1 | N/A | ++----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ | Freescale/NXP | LS2080A/LS1043A | A-008585 | FSL_ERRATUM_A008585 | +----------------+-----------------+-----------------+-----------------------------+ diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 324c105b28614..ab83987dd6bc1 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -29,6 +29,16 @@ #include "mpam_internal.h" +/* Values for the T241 errata workaround */ +#define T241_CHIPS_MAX 4 +#define T241_CHIP_NSLICES 12 +#define T241_SPARE_REG0_OFF 0x1b0000 +#define T241_SPARE_REG1_OFF 0x1c0000 +#define T241_CHIP_ID(phys) FIELD_GET(GENMASK_ULL(44, 43), phys) +#define T241_SHADOW_REG_OFF(sidx, pid) (0x360048 + (sidx) * 0x10000 + (pid) * 8) +#define SMCCC_SOC_ID_T241 0x036b0241 +static void __iomem *t241_scratch_regs[T241_CHIPS_MAX]; + /* * mpam_list_lock protects the SRCU lists when writing. Once the * mpam_enabled key is enabled these lists are read-only, @@ -630,7 +640,45 @@ static struct mpam_msc_ris *mpam_get_or_create_ris(struct mpam_msc *msc, return ERR_PTR(-ENOENT); } +static int mpam_enable_quirk_nvidia_t241_1(struct mpam_msc *msc, + const struct mpam_quirk *quirk) +{ + s32 soc_id = arm_smccc_get_soc_id_version(); + struct resource *r; + phys_addr_t phys; + + /* + * A mapping to a device other than the MSC is needed, check + * SOC_ID is NVIDIA T241 chip (036b:0241) + */ + if (soc_id < 0 || soc_id != SMCCC_SOC_ID_T241) + return -EINVAL; + + r = platform_get_resource(msc->pdev, IORESOURCE_MEM, 0); + if (!r) + return -EINVAL; + + /* Find the internal registers base addr from the CHIP ID */ + msc->t241_id = T241_CHIP_ID(r->start); + phys = FIELD_PREP(GENMASK_ULL(45, 44), msc->t241_id) | 0x19000000ULL; + + t241_scratch_regs[msc->t241_id] = ioremap(phys, SZ_8M); + if (WARN_ON_ONCE(!t241_scratch_regs[msc->t241_id])) + return -EINVAL; + + pr_info_once("Enabled workaround for NVIDIA T241 erratum T241-MPAM-1\n"); + + return 0; +} + static const struct mpam_quirk mpam_quirks[] = { + { + /* NVIDIA t241 erratum T241-MPAM-1 */ + .init = mpam_enable_quirk_nvidia_t241_1, + .iidr = MPAM_IIDR_NVIDIA_T241, + .iidr_mask = MPAM_IIDR_MATCH_ONE, + .workaround = T241_SCRUB_SHADOW_REGS, + }, { NULL } /* Sentinel */ }; @@ -1378,6 +1426,44 @@ static void mpam_reset_msc_bitmap(struct mpam_msc *msc, u16 reg, u16 wd) __mpam_write_reg(msc, reg, bm); } +static void mpam_apply_t241_erratum(struct mpam_msc_ris *ris, u16 partid) +{ + int sidx, i, lcount = 1000; + void __iomem *regs; + u64 val0, val; + + regs = t241_scratch_regs[ris->vmsc->msc->t241_id]; + + for (i = 0; i < lcount; i++) { + /* Read the shadow register at index 0 */ + val0 = readq_relaxed(regs + T241_SHADOW_REG_OFF(0, partid)); + + /* Check if all the shadow registers have the same value */ + for (sidx = 1; sidx < T241_CHIP_NSLICES; sidx++) { + val = readq_relaxed(regs + + T241_SHADOW_REG_OFF(sidx, partid)); + if (val != val0) + break; + } + if (sidx == T241_CHIP_NSLICES) + break; + } + + if (i == lcount) + pr_warn_once("t241: inconsistent values in shadow regs"); + + /* Write a value zero to spare registers to take effect of MBW conf */ + writeq_relaxed(0, regs + T241_SPARE_REG0_OFF); + writeq_relaxed(0, regs + T241_SPARE_REG1_OFF); +} + +static void mpam_quirk_post_config_change(struct mpam_msc_ris *ris, u16 partid, + struct mpam_config *cfg) +{ + if (mpam_has_quirk(T241_SCRUB_SHADOW_REGS, ris->vmsc->msc)) + mpam_apply_t241_erratum(ris, partid); +} + /* Called via IPI. Call while holding an SRCU reference */ static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid, struct mpam_config *cfg) @@ -1457,6 +1543,8 @@ static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid, mpam_write_partsel_reg(msc, PRI, pri_val); } + mpam_quirk_post_config_change(ris, partid, cfg); + mutex_unlock(&msc->part_sel_lock); } diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index 01858365cd9e7..d9eb342ba2220 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -130,6 +130,9 @@ struct mpam_msc { void __iomem *mapped_hwpage; size_t mapped_hwpage_sz; + /* Values only used on some platforms for quirks */ + u32 t241_id; + struct mpam_garbage garbage; }; @@ -220,6 +223,7 @@ struct mpam_props { /* Workaround bits for msc->quirks */ enum mpam_device_quirks { + T241_SCRUB_SHADOW_REGS, MPAM_QUIRK_LAST }; @@ -240,6 +244,11 @@ struct mpam_quirk { FIELD_PREP_CONST(MPAMF_IIDR_REVISION, 0xf) | \ FIELD_PREP_CONST(MPAMF_IIDR_IMPLEMENTER, 0xfff)) +#define MPAM_IIDR_NVIDIA_T241 (FIELD_PREP_CONST(MPAMF_IIDR_PRODUCTID, 0x241) | \ + FIELD_PREP_CONST(MPAMF_IIDR_VARIANT, 0) | \ + FIELD_PREP_CONST(MPAMF_IIDR_REVISION, 0) | \ + FIELD_PREP_CONST(MPAMF_IIDR_IMPLEMENTER, 0x36b)) + /* The values for MSMON_CFG_MBWU_FLT.RWBW */ enum mon_filter_options { COUNT_BOTH = 0, From dcc1deba2893adf9e5ba2321f2854c3a953f9d5e Mon Sep 17 00:00:00 2001 From: Shanker Donthineni Date: Fri, 13 Mar 2026 14:46:14 +0000 Subject: [PATCH 202/311] arm_mpam: Add workaround for T241-MPAM-4 BugLink: https://bugs.launchpad.net/bugs/2154527 In the T241 implementation of memory-bandwidth partitioning, in the absence of contention for bandwidth, the minimum bandwidth setting can affect the amount of achieved bandwidth. Specifically, the achieved bandwidth in the absence of contention can settle to any value between the values of MPAMCFG_MBW_MIN and MPAMCFG_MBW_MAX. Also, if MPAMCFG_MBW_MIN is set zero (below 0.78125%), once a core enters a throttled state, it will never leave that state. The first issue is not a concern if the MPAM software allows to program MPAMCFG_MBW_MIN through the sysfs interface. This patch ensures program MBW_MIN=1 (0.78125%) whenever MPAMCFG_MBW_MIN=0 is programmed. In the scenario where the resctrl doesn't support the MBW_MIN interface via sysfs, to achieve bandwidth closer to MBW_MAX in the absence of contention, software should configure a relatively narrow gap between MBW_MIN and MBW_MAX. The recommendation is to use a 5% gap to mitigate the problem. Clear the feature MBW_MIN feature from the class to ensure we don't accidentally change behaviour when resctrl adds support for a MBW_MIN interface. Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Fenghua Yu Reviewed-by: Gavin Shan Signed-off-by: Shanker Donthineni Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit a7efe23ed6dd08259ad1b238e9c33bb511666fd4) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- Documentation/arch/arm64/silicon-errata.rst | 2 + drivers/resctrl/mpam_devices.c | 55 +++++++++++++++++++-- drivers/resctrl/mpam_internal.h | 1 + 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index a65620f98e3aa..a4b246655e37e 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -249,6 +249,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | T241 MPAM | T241-MPAM-1 | N/A | +----------------+-----------------+-----------------+-----------------------------+ +| NVIDIA | T241 MPAM | T241-MPAM-4 | N/A | ++----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ | Freescale/NXP | LS2080A/LS1043A | A-008585 | FSL_ERRATUM_A008585 | +----------------+-----------------+-----------------+-----------------------------+ diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index ab83987dd6bc1..7a8623b27f063 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -679,6 +679,12 @@ static const struct mpam_quirk mpam_quirks[] = { .iidr_mask = MPAM_IIDR_MATCH_ONE, .workaround = T241_SCRUB_SHADOW_REGS, }, + { + /* NVIDIA t241 erratum T241-MPAM-4 */ + .iidr = MPAM_IIDR_NVIDIA_T241, + .iidr_mask = MPAM_IIDR_MATCH_ONE, + .workaround = T241_FORCE_MBW_MIN_TO_ONE, + }, { NULL } /* Sentinel */ }; @@ -1464,6 +1470,37 @@ static void mpam_quirk_post_config_change(struct mpam_msc_ris *ris, u16 partid, mpam_apply_t241_erratum(ris, partid); } +static u16 mpam_wa_t241_force_mbw_min_to_one(struct mpam_props *props) +{ + u16 max_hw_value, min_hw_granule, res0_bits; + + res0_bits = 16 - props->bwa_wd; + max_hw_value = ((1 << props->bwa_wd) - 1) << res0_bits; + min_hw_granule = ~max_hw_value; + + return min_hw_granule + 1; +} + +static u16 mpam_wa_t241_calc_min_from_max(struct mpam_props *props, + struct mpam_config *cfg) +{ + u16 val = 0; + u16 max; + u16 delta = ((5 * MPAMCFG_MBW_MAX_MAX) / 100) - 1; + + if (mpam_has_feature(mpam_feat_mbw_max, cfg)) { + max = cfg->mbw_max; + } else { + /* Resetting. Hence, use the ris specific default. */ + max = GENMASK(15, 16 - props->bwa_wd); + } + + if (max > delta) + val = max - delta; + + return val; +} + /* Called via IPI. Call while holding an SRCU reference */ static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid, struct mpam_config *cfg) @@ -1504,9 +1541,18 @@ static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid, mpam_write_partsel_reg(msc, MBW_PBM, cfg->mbw_pbm); } - if (mpam_has_feature(mpam_feat_mbw_min, rprops) && - mpam_has_feature(mpam_feat_mbw_min, cfg)) - mpam_write_partsel_reg(msc, MBW_MIN, 0); + if (mpam_has_feature(mpam_feat_mbw_min, rprops)) { + u16 val = 0; + + if (mpam_has_quirk(T241_FORCE_MBW_MIN_TO_ONE, msc)) { + u16 min = mpam_wa_t241_force_mbw_min_to_one(rprops); + + val = mpam_wa_t241_calc_min_from_max(rprops, cfg); + val = max(val, min); + } + + mpam_write_partsel_reg(msc, MBW_MIN, val); + } if (mpam_has_feature(mpam_feat_mbw_max, rprops)) { if (mpam_has_feature(mpam_feat_mbw_max, cfg)) @@ -2290,6 +2336,9 @@ static void mpam_enable_merge_class_features(struct mpam_component *comp) list_for_each_entry(vmsc, &comp->vmsc, comp_list) __class_props_mismatch(class, vmsc); + + if (mpam_has_quirk(T241_FORCE_MBW_MIN_TO_ONE, class)) + mpam_clear_feature(mpam_feat_mbw_min, &class->props); } /* diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index d9eb342ba2220..f1adbdad39696 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -224,6 +224,7 @@ struct mpam_props { /* Workaround bits for msc->quirks */ enum mpam_device_quirks { T241_SCRUB_SHADOW_REGS, + T241_FORCE_MBW_MIN_TO_ONE, MPAM_QUIRK_LAST }; From 2bb678e9a4c9e3de63ea95a4fd7ed3cd6131474b Mon Sep 17 00:00:00 2001 From: Shanker Donthineni Date: Fri, 13 Mar 2026 14:46:15 +0000 Subject: [PATCH 203/311] arm_mpam: Add workaround for T241-MPAM-6 BugLink: https://bugs.launchpad.net/bugs/2154527 The registers MSMON_MBWU_L and MSMON_MBWU return the number of requests rather than the number of bytes transferred. Bandwidth resource monitoring is performed at the last level cache, where each request arrive in 64Byte granularity. The current implementation returns the number of transactions received at the last level cache but does not provide the value in bytes. Scaling by 64 gives an accurate byte count to match the MPAM specification for the MSMON_MBWU and MSMON_MBWU_L registers. This patch fixes the issue by reporting the actual number of bytes instead of the number of transactions from __ris_msmon_read(). Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Punit Agrawal Tested-by: Peter Newman Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Gavin Shan Signed-off-by: Shanker Donthineni Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit dc48eb1ff27cc3169c3c5cca5eb20645d04d9e22) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- Documentation/arch/arm64/silicon-errata.rst | 2 ++ drivers/resctrl/mpam_devices.c | 26 +++++++++++++++++++-- drivers/resctrl/mpam_internal.h | 1 + 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index a4b246655e37e..1aa3326bb3200 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -251,6 +251,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | T241 MPAM | T241-MPAM-4 | N/A | +----------------+-----------------+-----------------+-----------------------------+ +| NVIDIA | T241 MPAM | T241-MPAM-6 | N/A | ++----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ | Freescale/NXP | LS2080A/LS1043A | A-008585 | FSL_ERRATUM_A008585 | +----------------+-----------------+-----------------+-----------------------------+ diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 7a8623b27f063..8b598f768c2c8 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -685,6 +685,12 @@ static const struct mpam_quirk mpam_quirks[] = { .iidr_mask = MPAM_IIDR_MATCH_ONE, .workaround = T241_FORCE_MBW_MIN_TO_ONE, }, + { + /* NVIDIA t241 erratum T241-MPAM-6 */ + .iidr = MPAM_IIDR_NVIDIA_T241, + .iidr_mask = MPAM_IIDR_MATCH_ONE, + .workaround = T241_MBW_COUNTER_SCALE_64, + }, { NULL } /* Sentinel */ }; @@ -1146,7 +1152,7 @@ static void write_msmon_ctl_flt_vals(struct mon_read *m, u32 ctl_val, } } -static u64 mpam_msmon_overflow_val(enum mpam_device_features type) +static u64 __mpam_msmon_overflow_val(enum mpam_device_features type) { /* TODO: implement scaling counters */ switch (type) { @@ -1161,6 +1167,18 @@ static u64 mpam_msmon_overflow_val(enum mpam_device_features type) } } +static u64 mpam_msmon_overflow_val(enum mpam_device_features type, + struct mpam_msc *msc) +{ + u64 overflow_val = __mpam_msmon_overflow_val(type); + + if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc) && + type != mpam_feat_msmon_mbwu_63counter) + overflow_val *= 64; + + return overflow_val; +} + static void __ris_msmon_read(void *arg) { u64 now; @@ -1251,13 +1269,17 @@ static void __ris_msmon_read(void *arg) now = FIELD_GET(MSMON___VALUE, now); } + if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc) && + m->type != mpam_feat_msmon_mbwu_63counter) + now *= 64; + if (nrdy) break; mbwu_state = &ris->mbwu_state[ctx->mon]; if (overflow) - mbwu_state->correction += mpam_msmon_overflow_val(m->type); + mbwu_state->correction += mpam_msmon_overflow_val(m->type, msc); /* * Include bandwidth consumed before the last hardware reset and diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index f1adbdad39696..8fea28c5fb852 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -225,6 +225,7 @@ struct mpam_props { enum mpam_device_quirks { T241_SCRUB_SHADOW_REGS, T241_FORCE_MBW_MIN_TO_ONE, + T241_MBW_COUNTER_SCALE_64, MPAM_QUIRK_LAST }; From e42f06f1d1a1056f475b3913a07d6409acf61169 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 13 Mar 2026 14:46:16 +0000 Subject: [PATCH 204/311] arm_mpam: Quirk CMN-650's CSU NRDY behaviour BugLink: https://bugs.launchpad.net/bugs/2154527 CMN-650 is afflicted with an erratum where the CSU NRDY bit never clears. This tells us the monitor never finishes scanning the cache. The erratum document says to wait the maximum time, then ignore the field. Add a flag to indicate whether this is the final attempt to read the counter, and when this quirk is applied, ignore the NRDY field. This means accesses to this counter will always retry, even if the counter was previously programmed to the same values. The counter value is not expected to be stable, it drifts up and down with each allocation and eviction. The CSU register provides the value for a point in time. Tested-by: Punit Agrawal Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Jesse Chick Reviewed-by: Zeng Heng Reviewed-by: Gavin Shan Co-developed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: James Morse (cherry picked from commit aeb8595a5f8ba4aac8b5c265a8bcc3f18b473cb5) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- Documentation/arch/arm64/silicon-errata.rst | 3 +++ drivers/resctrl/mpam_devices.c | 12 ++++++++++++ drivers/resctrl/mpam_internal.h | 6 ++++++ 3 files changed, 21 insertions(+) diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index 1aa3326bb3200..65ed6ea33751f 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -214,6 +214,9 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | SI L1 | #4311569 | ARM64_ERRATUM_4311569 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | CMN-650 | #3642720 | N/A | ++----------------+-----------------+-----------------+-----------------------------+ ++----------------+-----------------+-----------------+-----------------------------+ | Broadcom | Brahma-B53 | N/A | ARM64_ERRATUM_845719 | +----------------+-----------------+-----------------+-----------------------------+ | Broadcom | Brahma-B53 | N/A | ARM64_ERRATUM_843419 | diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 8b598f768c2c8..41b14344b16f2 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -691,6 +691,12 @@ static const struct mpam_quirk mpam_quirks[] = { .iidr_mask = MPAM_IIDR_MATCH_ONE, .workaround = T241_MBW_COUNTER_SCALE_64, }, + { + /* ARM CMN-650 CSU erratum 3642720 */ + .iidr = MPAM_IIDR_ARM_CMN_650, + .iidr_mask = MPAM_IIDR_MATCH_ONE, + .workaround = IGNORE_CSU_NRDY, + }, { NULL } /* Sentinel */ }; @@ -1003,6 +1009,7 @@ struct mon_read { enum mpam_device_features type; u64 *val; int err; + bool waited_timeout; }; static bool mpam_ris_has_mbwu_long_counter(struct mpam_msc_ris *ris) @@ -1249,6 +1256,10 @@ static void __ris_msmon_read(void *arg) if (mpam_has_feature(mpam_feat_msmon_csu_hw_nrdy, rprops)) nrdy = now & MSMON___NRDY; now = FIELD_GET(MSMON___VALUE, now); + + if (mpam_has_quirk(IGNORE_CSU_NRDY, msc) && m->waited_timeout) + nrdy = false; + break; case mpam_feat_msmon_mbwu_31counter: case mpam_feat_msmon_mbwu_44counter: @@ -1386,6 +1397,7 @@ int mpam_msmon_read(struct mpam_component *comp, struct mon_cfg *ctx, .ctx = ctx, .type = type, .val = val, + .waited_timeout = true, }; *val = 0; diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index 8fea28c5fb852..1914aefdcba9e 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -226,6 +226,7 @@ enum mpam_device_quirks { T241_SCRUB_SHADOW_REGS, T241_FORCE_MBW_MIN_TO_ONE, T241_MBW_COUNTER_SCALE_64, + IGNORE_CSU_NRDY, MPAM_QUIRK_LAST }; @@ -251,6 +252,11 @@ struct mpam_quirk { FIELD_PREP_CONST(MPAMF_IIDR_REVISION, 0) | \ FIELD_PREP_CONST(MPAMF_IIDR_IMPLEMENTER, 0x36b)) +#define MPAM_IIDR_ARM_CMN_650 (FIELD_PREP_CONST(MPAMF_IIDR_PRODUCTID, 0) | \ + FIELD_PREP_CONST(MPAMF_IIDR_VARIANT, 0) | \ + FIELD_PREP_CONST(MPAMF_IIDR_REVISION, 0) | \ + FIELD_PREP_CONST(MPAMF_IIDR_IMPLEMENTER, 0x43b)) + /* The values for MSMON_CFG_MBWU_FLT.RWBW */ enum mon_filter_options { COUNT_BOTH = 0, From a46feaaf023c3aef38fb7854c623c79637ccd2e5 Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Fri, 13 Mar 2026 14:46:17 +0000 Subject: [PATCH 205/311] arm64: mpam: Add initial MPAM documentation BugLink: https://bugs.launchpad.net/bugs/2154527 MPAM (Memory Partitioning and Monitoring) is now exposed to user-space via resctrl. Add some documentation so the user knows what features to expect. Reviewed-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Acked-by: Catalin Marinas Signed-off-by: Ben Horgan Reviewed-by: Gavin Shan Tested-by: Gavin Shan Tested-by: Shaopeng Tan Tested-by: Jesse Chick Signed-off-by: James Morse (cherry picked from commit 4ce0a2ccc0358f3f746fa50815a599f861fd5d68) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- Documentation/arch/arm64/index.rst | 1 + Documentation/arch/arm64/mpam.rst | 72 ++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 Documentation/arch/arm64/mpam.rst diff --git a/Documentation/arch/arm64/index.rst b/Documentation/arch/arm64/index.rst index af52edc8c0ac6..98052b4ef4a1e 100644 --- a/Documentation/arch/arm64/index.rst +++ b/Documentation/arch/arm64/index.rst @@ -23,6 +23,7 @@ ARM64 Architecture memory memory-tagging-extension mops + mpam perf pointer-authentication ptdump diff --git a/Documentation/arch/arm64/mpam.rst b/Documentation/arch/arm64/mpam.rst new file mode 100644 index 0000000000000..570f51a8d4ebf --- /dev/null +++ b/Documentation/arch/arm64/mpam.rst @@ -0,0 +1,72 @@ +.. SPDX-License-Identifier: GPL-2.0 + +==== +MPAM +==== + +What is MPAM +============ +MPAM (Memory Partitioning and Monitoring) is a feature in the CPUs and memory +system components such as the caches or memory controllers that allow memory +traffic to be labelled, partitioned and monitored. + +Traffic is labelled by the CPU, based on the control or monitor group the +current task is assigned to using resctrl. Partitioning policy can be set +using the schemata file in resctrl, and monitor values read via resctrl. +See Documentation/filesystems/resctrl.rst for more details. + +This allows tasks that share memory system resources, such as caches, to be +isolated from each other according to the partitioning policy (so called noisy +neighbours). + +Supported Platforms +=================== +Use of this feature requires CPU support, support in the memory system +components, and a description from firmware of where the MPAM device controls +are in the MMIO address space. (e.g. the 'MPAM' ACPI table). + +The MMIO device that provides MPAM controls/monitors for a memory system +component is called a memory system component. (MSC). + +Because the user interface to MPAM is via resctrl, only MPAM features that are +compatible with resctrl can be exposed to user-space. + +MSC are considered as a group based on the topology. MSC that correspond with +the L3 cache are considered together, it is not possible to mix MSC between L2 +and L3 to 'cover' a resctrl schema. + +The supported features are: + +* Cache portion bitmap controls (CPOR) on the L2 or L3 caches. To expose + CPOR at L2 or L3, every CPU must have a corresponding CPU cache at this + level that also supports the feature. Mismatched big/little platforms are + not supported as resctrl's controls would then also depend on task + placement. + +* Memory bandwidth maximum controls (MBW_MAX) on or after the L3 cache. + resctrl uses the L3 cache-id to identify where the memory bandwidth + control is applied. For this reason the platform must have an L3 cache + with cache-id's supplied by firmware. (It doesn't need to support MPAM.) + + To be exported as the 'MB' schema, the topology of the group of MSC chosen + must match the topology of the L3 cache so that the cache-id's can be + repainted. For example: Platforms with Memory bandwidth maximum controls + on CPU-less NUMA nodes cannot expose the 'MB' schema to resctrl as these + nodes do not have a corresponding L3 cache. If the memory bandwidth + control is on the memory rather than the L3 then there must be a single + global L3 as otherwise it is unknown which L3 the traffic came from. There + must be no caches between the L3 and the memory so that the two ends of + the path have equivalent traffic. + + When the MPAM driver finds multiple groups of MSC it can use for the 'MB' + schema, it prefers the group closest to the L3 cache. + +* Cache Storage Usage (CSU) counters can expose the 'llc_occupancy' provided + there is at least one CSU monitor on each MSC that makes up the L3 group. + Exposing CSU counters from other caches or devices is not supported. + +Reporting Bugs +============== +If you are not seeing the counters or controls you expect please share the +debug messages produced when enabling dynamic debug and booting with: +dyndbg="file mpam_resctrl.c +pl" From ae957dc0414d44cc3e2b6bb590bfe5018c1a2e30 Mon Sep 17 00:00:00 2001 From: Aaron Tomlin Date: Tue, 24 Mar 2026 20:11:58 -0400 Subject: [PATCH 206/311] fs/resctrl: Report invalid domain ID when parsing io_alloc_cbm BugLink: https://bugs.launchpad.net/bugs/2154527 The last_cmd_status file is intended to report details about the most recent resctrl filesystem operation, specifically to aid in diagnosing failures. However, when parsing io_alloc_cbm, if a user provides a domain ID that does not exist in the resource, the operation fails with -EINVAL without updating last_cmd_status. This results in inconsistent behaviour where the system call returns an error, but last_cmd_status misleadingly reports "ok", leaving the user unaware that the failure was caused by an invalid domain ID. Write an error message to last_cmd_status when the target domain ID cannot be found. Fixes: 28fa2cce7a83 ("fs/resctrl: Introduce interface to modify io_alloc capacity bitmasks") Suggested-by: Reinette Chatre Signed-off-by: Aaron Tomlin Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Reinette Chatre Reviewed-by: Babu Moger Tested-by: Babu Moger Link: https://patch.msgid.link/20260325001159.447075-2-atomlin@atomlin.com (cherry picked from commit d06b8e7c97c3290e61006e30b32beb9e715fab82) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- fs/resctrl/ctrlmondata.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/resctrl/ctrlmondata.c b/fs/resctrl/ctrlmondata.c index cc4237c57cbe4..2ef53161ce119 100644 --- a/fs/resctrl/ctrlmondata.c +++ b/fs/resctrl/ctrlmondata.c @@ -992,6 +992,7 @@ static int resctrl_io_alloc_parse_line(char *line, struct rdt_resource *r, } } + rdt_last_cmd_printf("Invalid domain %lu\n", dom_id); return -EINVAL; } From e87a73419b9589e6c8df5a03b8a83a5ce614a248 Mon Sep 17 00:00:00 2001 From: Aaron Tomlin Date: Tue, 24 Mar 2026 20:11:59 -0400 Subject: [PATCH 207/311] fs/resctrl: Add "*" shorthand to set io_alloc CBM for all domains BugLink: https://bugs.launchpad.net/bugs/2154527 Configuring the io_alloc_cbm interface requires an explicit domain ID for each cache domain. On systems with high core counts and numerous cache clusters, this requirement becomes cumbersome for automation and management tasks that aim to apply a uniform policy. Introduce a wildcard domain ID selector "*" for the io_alloc_cbm interface. This enables users to set the same Capacity Bitmask (CBM) across all cache domains in a single operation. Signed-off-by: Aaron Tomlin Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Reinette Chatre Reviewed-by: Babu Moger Tested-by: Babu Moger Link: https://patch.msgid.link/20260325001159.447075-3-atomlin@atomlin.com (cherry picked from commit d2bf45d067c728b0fe6e8f99a7386b8291e391e3) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- Documentation/filesystems/resctrl.rst | 8 ++++++++ fs/resctrl/ctrlmondata.c | 21 +++++++++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/Documentation/filesystems/resctrl.rst b/Documentation/filesystems/resctrl.rst index ba609f8d4de57..b003bed339fdd 100644 --- a/Documentation/filesystems/resctrl.rst +++ b/Documentation/filesystems/resctrl.rst @@ -215,6 +215,14 @@ related to allocation: # cat /sys/fs/resctrl/info/L3/io_alloc_cbm 0=00ff;1=000f + An ID of "*" configures all domains with the provided CBM. + + Example on a system that does not require a minimum number of consecutive bits in the mask:: + + # echo "*=0" > /sys/fs/resctrl/info/L3/io_alloc_cbm + # cat /sys/fs/resctrl/info/L3/io_alloc_cbm + 0=0;1=0 + When CDP is enabled "io_alloc_cbm" associated with the CDP_DATA and CDP_CODE resources may reflect the same values. For example, values read from and written to /sys/fs/resctrl/info/L3DATA/io_alloc_cbm may be reflected by diff --git a/fs/resctrl/ctrlmondata.c b/fs/resctrl/ctrlmondata.c index 2ef53161ce119..9a7dfc48cb2e2 100644 --- a/fs/resctrl/ctrlmondata.c +++ b/fs/resctrl/ctrlmondata.c @@ -954,25 +954,34 @@ static int resctrl_io_alloc_parse_line(char *line, struct rdt_resource *r, struct resctrl_schema *s, u32 closid) { enum resctrl_conf_type peer_type; + unsigned long dom_id = ULONG_MAX; struct rdt_parse_data data; struct rdt_ctrl_domain *d; + bool update_all = false; char *dom = NULL, *id; - unsigned long dom_id; next: if (!line || line[0] == '\0') return 0; + if (update_all) { + rdt_last_cmd_puts("Configurations after global '*'\n"); + return -EINVAL; + } + dom = strsep(&line, ";"); id = strsep(&dom, "="); - if (!dom || kstrtoul(id, 10, &dom_id)) { + + if (dom && !strcmp(id, "*")) { + update_all = true; + } else if (!dom || kstrtoul(id, 10, &dom_id)) { rdt_last_cmd_puts("Missing '=' or non-numeric domain\n"); return -EINVAL; } dom = strim(dom); list_for_each_entry(d, &r->ctrl_domains, hdr.list) { - if (d->hdr.id == dom_id) { + if (update_all || d->hdr.id == dom_id) { data.buf = dom; data.mode = RDT_MODE_SHAREABLE; data.closid = closid; @@ -988,10 +997,14 @@ static int resctrl_io_alloc_parse_line(char *line, struct rdt_resource *r, &d->staged_config[s->conf_type], sizeof(d->staged_config[0])); } - goto next; + if (!update_all) + goto next; } } + if (update_all) + goto next; + rdt_last_cmd_printf("Invalid domain %lu\n", dom_id); return -EINVAL; } From c9ae624f7612a85349d89c64b37c7c4600866073 Mon Sep 17 00:00:00 2001 From: Reinette Chatre Date: Tue, 7 Apr 2026 09:01:58 -0700 Subject: [PATCH 208/311] MAINTAINERS: Update resctrl entry BugLink: https://bugs.launchpad.net/bugs/2154527 The x86 maintainers handle the resctrl filesystem and x86 architectural resctrl code. Even so, the x86 maintainers are not part of the resctrl section and not returned when scripts/get_maintainer.pl is run on resctrl filesystem code. With patches flowing via x86 maintainers resctrl should also ensure it follows the tip rules. Add the x86 maintainer alias, x86@kernel.org, to the resctrl section to ensure x86 maintainers are included in associated resctrl submissions. Add a reference to the tip tree handbook to make it clear which rules resctrl follows. Signed-off-by: Reinette Chatre Signed-off-by: Borislav Petkov (AMD) Link: https://patch.msgid.link/4c14dd82e81737c6413e10fe097475b1cc0886fc.1775576382.git.reinette.chatre@intel.com (cherry picked from commit c611752be9d73d12fca9b456a0b8f5c8409a2346) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- MAINTAINERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 85bbf2d242458..b9028c49b421e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -22198,11 +22198,13 @@ F: tools/testing/selftests/net/rds/ RDT - RESOURCE ALLOCATION M: Tony Luck M: Reinette Chatre +M: x86@kernel.org R: Dave Martin R: James Morse R: Babu Moger L: linux-kernel@vger.kernel.org S: Supported +P: Documentation/process/maintainer-tip.rst F: Documentation/filesystems/resctrl.rst F: arch/x86/include/asm/resctrl.h F: arch/x86/kernel/cpu/resctrl/ From 51d4d9bad8d4b720d93ea0db41b1cfc81cdba1e0 Mon Sep 17 00:00:00 2001 From: Reinette Chatre Date: Tue, 7 Apr 2026 09:01:59 -0700 Subject: [PATCH 209/311] fs/resctrl: Add missing return value descriptions BugLink: https://bugs.launchpad.net/bugs/2154527 Using the stricter "./tools/docs/kernel-doc -Wall -v" to verify proper formatting of documentation comments includes warnings related to return markup on functions that are omitted during the default verification checks. This stricter verification reports a couple of missing return descriptions in resctrl: Warning: .../fs/resctrl/rdtgroup.c:1536 No description found for return value of 'rdtgroup_cbm_to_size' Warning: .../fs/resctrl/rdtgroup.c:3131 No description found for return value of 'mon_get_kn_priv' Warning: .../fs/resctrl/rdtgroup.c:3523 No description found for return value of 'cbm_ensure_valid' Warning: .../fs/resctrl/monitor.c:238 No description found for return value of 'resctrl_find_cleanest_closid' Add the missing return descriptions. Signed-off-by: Reinette Chatre Signed-off-by: Borislav Petkov (AMD) Link: https://patch.msgid.link/1c50b9f7c73251c007133590986f127e1af57780.1775576382.git.reinette.chatre@intel.com (cherry picked from commit 79727019ce3da234d877ec0cb6a3985f001e2b2d) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- fs/resctrl/monitor.c | 2 ++ fs/resctrl/rdtgroup.c | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c index 49f3f6b846b27..9fd901c78dc66 100644 --- a/fs/resctrl/monitor.c +++ b/fs/resctrl/monitor.c @@ -234,6 +234,8 @@ static struct rmid_entry *resctrl_find_free_rmid(u32 closid) * * When the CLOSID and RMID are independent numbers, the first free CLOSID will * be returned. + * + * Return: Free CLOSID on success, < 0 on failure. */ int resctrl_find_cleanest_closid(void) { diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index 5da305bd36c96..5dfdaa6f9d8ff 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -1519,6 +1519,8 @@ static ssize_t rdtgroup_mode_write(struct kernfs_open_file *of, * * @cbm is unsigned long, even if only 32 bits are used to make the * bitmap functions work correctly. + * + * Return: Size (in bytes) of cache portion represented by CBM, 0 on failure. */ unsigned int rdtgroup_cbm_to_size(struct rdt_resource *r, struct rdt_ctrl_domain *d, unsigned long cbm) @@ -3102,6 +3104,8 @@ static void rmdir_all_sub(void) * @mevt: The type of event file being created. * @do_sum: Whether SNC summing monitors are being created. Only set * when @rid == RDT_RESOURCE_L3. + * + * Return: Pointer to mon_data private data of the event, NULL on failure. */ static struct mon_data *mon_get_kn_priv(enum resctrl_res_level rid, int domid, struct mon_evt *mevt, @@ -3496,6 +3500,8 @@ static int mkdir_mondata_all(struct kernfs_node *parent_kn, * resource group is initialized. The user can follow this with a * modification to the CBM if the default does not satisfy the * requirements. + * + * Return: A CBM that is valid for resource @r. */ static u32 cbm_ensure_valid(u32 _val, struct rdt_resource *r) { From 1fd0f5f167dbe99cbe18b85c902a72d8f79aaaa1 Mon Sep 17 00:00:00 2001 From: Zeng Heng Date: Mon, 13 Apr 2026 17:00:41 +0800 Subject: [PATCH 210/311] arm_mpam: resctrl: Fix MBA CDP alloc_capable handling on unmount BugLink: https://bugs.launchpad.net/bugs/2154527 The code to set MBA's alloc_capable to true appears to be trying to restore alloc_capable on unmount. This can never work because resctrl_arch_set_cdp_enabled() is never invoked with RDT_RESOURCE_MBA as the rid parameter. Consequently, mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled always remains false. The alloc_capable setting in resctrl_arch_set_cdp_enabled() is to re-enable MBA if the caller opts in to separate control values using CDP for this resource. This doesn't happen today. Add a comment to describe this. However a bug remains where MBA allocation is permanently disabled after the mount with CDP option. Remounting without CDP cannot restore the MBA partition capability. Add a check to re-enable MBA when CDP is disabled, which happens on unmount. Fixes: 6789fb99282c ("arm_mpam: resctrl: Add CDP emulation") Signed-off-by: Zeng Heng [ morse: Added comment for existing code, added hunk to fix this bug from Ben H ] Reviewed-by: James Morse Signed-off-by: James Morse (cherry picked from commit f758340da529ccb12531c3f83d5992e912f6c8d5) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index a9938006d0e6e..4205fb2ee312b 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -220,10 +220,18 @@ int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable) if (cdp_enabled && !mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled) mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = false; + /* + * If resctrl has attempted to enable CDP on MBA, re-enable MBA as two + * configurations will be provided so there is no aliasing problem. + */ if (mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled && mpam_resctrl_controls[RDT_RESOURCE_MBA].class) mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = true; + /* On unmount when CDP is disabled, re-enable MBA */ + if (!cdp_enabled && mpam_resctrl_controls[RDT_RESOURCE_MBA].class) + mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = true; + if (enable) { if (mpam_partid_max < 1) return -EINVAL; From 6776a2969fc5bb09e449d3d6130e85687dda98e9 Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Tue, 14 Apr 2026 14:27:56 +0100 Subject: [PATCH 211/311] arm_mpam: resctrl: Fix the check for no monitor components found BugLink: https://bugs.launchpad.net/bugs/2154527 Dan Carpenter reports that, in mpam_resctrl_alloc_domain(), any_mon_comp is used in an 'if' condition when it may be uninitialized. Initialize it to NULL so that the check behaves correctly when no monitor components are found. Reported-by: Dan Carpenter Fixes: 264c285999fc ("arm_mpam: resctrl: Add monitor initialisation and domain boilerplate") Signed-off-by: Ben Horgan Reviewed-by: Gavin Shan Signed-off-by: James Morse (cherry picked from commit 67c0a487efa542cca9477ea84915db2e091f98d0) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 4205fb2ee312b..1b0b37da12afc 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -1407,7 +1407,7 @@ mpam_resctrl_alloc_domain(unsigned int cpu, struct mpam_resctrl_res *res) } if (r->mon_capable) { - struct mpam_component *any_mon_comp; + struct mpam_component *any_mon_comp = NULL; struct mpam_resctrl_mon *mon; enum resctrl_event_id eventid; From beb1649c11cda958897951abd7494a3615f4e2ae Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Tue, 14 Apr 2026 14:27:58 +0100 Subject: [PATCH 212/311] arm_mpam: resctrl: Make resctrl_mon_ctx_waiters static BugLink: https://bugs.launchpad.net/bugs/2154527 resctrl_mon_ctx_waiters is not used outside of this file, so make it static. This fixes the sparse warning: drivers/resctrl/mpam_resctrl.c:25:1: warning: symbol 'resctrl_mon_ctx_waiters' was not declared. Should it be static? Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202603281842.c2K96tJA-lkp@intel.com/ Fixes: 2a3c79c61539 ("arm_mpam: resctrl: Allow resctrl to allocate monitors") Signed-off-by: Ben Horgan Reviewed-by: Gavin Shan Signed-off-by: James Morse (cherry picked from commit 4d5bbbafc170eb21474a37d844211fce6b0f3c51) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 1b0b37da12afc..226ff6f532fab 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -22,7 +22,7 @@ #include "mpam_internal.h" -DECLARE_WAIT_QUEUE_HEAD(resctrl_mon_ctx_waiters); +static DECLARE_WAIT_QUEUE_HEAD(resctrl_mon_ctx_waiters); /* * The classes we've picked to map to resctrl resources, wrapped From df742aab6c98d601d97954d8e07a0265c3cbcb58 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 30 Apr 2026 01:19:15 +0000 Subject: [PATCH 213/311] NVIDIA: SAUCE: Update annotations to set CONFIG_RESCTRL_FS BugLink: https://bugs.launchpad.net/bugs/2154527 Eanble resctrl by CONFIG_RESCTRL_FS=y Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- debian.nvidia/config/annotations | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index b70730cab4cf6..42242c734487b 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -216,6 +216,8 @@ CONFIG_VFIO_CONTAINER note<'LP: #2095028'> CONFIG_VFIO_IOMMU_TYPE1 policy<{'amd64': 'm', 'arm64': '-'}> CONFIG_VFIO_IOMMU_TYPE1 note<'LP: #2095028'> +CONFIG_RESCTRL_FS policy<{'amd64': 'y', 'arm64': 'y'}> +CONFIG_RESCTRL_FS note<'LP: #2122432'> # ---- Annotations without notes ---- From 5bb2cf02ce09b95f5598d402180ada6f60747dd0 Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Wed, 6 May 2026 09:28:49 +0100 Subject: [PATCH 214/311] NVIDIA: SAUCE: fs/resctrl: Tidy up the error path in resctrl_mkdir_event_configs() BugLink: https://bugs.launchpad.net/bugs/2154527 The error path in resctrl_mkdir_event_configs() is unnecessarily complicated. Simplify it to just return directly on error. Signed-off-by: Ben Horgan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Reinette Chatre Reviewed-by: Babu Moger Tested-by: Babu Moger Link: https://lore.kernel.org/r/20260506082855.3694761-1-ben.horgan@arm.com (cherry picked from commit 7625632fed431ddd655e839c302165536553f767 https://gitlab.arm.com/linux-arm/linux-bh.git mpam_abmc_v4) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- fs/resctrl/rdtgroup.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index 5dfdaa6f9d8ff..eca3bb67987d4 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -2331,22 +2331,19 @@ static int resctrl_mkdir_event_configs(struct rdt_resource *r, struct kernfs_nod continue; kn_subdir2 = kernfs_create_dir(kn_subdir, mevt->name, kn_subdir->mode, mevt); - if (IS_ERR(kn_subdir2)) { - ret = PTR_ERR(kn_subdir2); - goto out; - } + if (IS_ERR(kn_subdir2)) + return PTR_ERR(kn_subdir2); ret = rdtgroup_kn_set_ugid(kn_subdir2); if (ret) - goto out; + return ret; ret = rdtgroup_add_files(kn_subdir2, RFTYPE_ASSIGN_CONFIG); if (ret) - break; + return ret; } -out: - return ret; + return 0; } static int rdtgroup_mkdir_info_resdir(void *priv, char *name, From 42d494c484cac4c44755f355d7bc0e96e50f9210 Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Wed, 6 May 2026 09:28:50 +0100 Subject: [PATCH 215/311] NVIDIA: SAUCE: x86,fs/resctrl: Create 'event_filter' files read only if they're not configurable BugLink: https://bugs.launchpad.net/bugs/2154527 When the counter assignment mode is mbm_event resctrl assumes the MBM events are configurable and exposes the 'event_filter' files. These files live at info/L3_MON/event_configs//event_filter and are used to display and set the event configuration. The MPAM architecture has support for configuring the memory bandwidth utilization (MBWU) counters to only count reads or only count writes. However, in MPAM, this event filtering support is optional in the hardware (and not yet implemented in the MPAM driver) but MBM counter assignment is always possible for MPAM MBWU counters. In order to support mbm_event mode with MPAM, create the 'event_filter' files read only if the event configuration can't be changed. A user can still chmod the file and so also return early with an error from event_filter_write(). Introduce a new monitor property, mbm_cntr_configurable, to indicate whether or not assignable MBM counters are configurable. On x86, set this to true whenever mbm_cntr_assignable is true to keep existing behaviour. Signed-off-by: Ben Horgan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Reinette Chatre Reviewed-by: Babu Moger Tested-by: Babu Moger Link: https://lore.kernel.org/20260506082855.3694761-1-ben.horgan@arm.com (cherry picked from commit 94a1206522d11302ae7e7c28d3d494c8f0c9c58e https://gitlab.arm.com/linux-arm/linux-bh.git mpam_abmc_v4) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- Documentation/filesystems/resctrl.rst | 11 +++++------ arch/x86/kernel/cpu/resctrl/monitor.c | 1 + fs/resctrl/internal.h | 2 ++ fs/resctrl/monitor.c | 7 +++++++ fs/resctrl/rdtgroup.c | 11 ++++++++++- include/linux/resctrl.h | 16 +++++++++------- 6 files changed, 34 insertions(+), 14 deletions(-) diff --git a/Documentation/filesystems/resctrl.rst b/Documentation/filesystems/resctrl.rst index b003bed339fdd..2898a51e6f4be 100644 --- a/Documentation/filesystems/resctrl.rst +++ b/Documentation/filesystems/resctrl.rst @@ -427,9 +427,9 @@ with the following files: Two MBM events are supported by default: mbm_local_bytes and mbm_total_bytes. Each MBM event's sub-directory contains a file named "event_filter" that is - used to view and modify which memory transactions the MBM event is configured - with. The file is accessible only when "mbm_event" counter assignment mode is - enabled. + used to view and (if writable) modify which memory transactions the MBM event + is configured with. The file is accessible only when "mbm_event" counter + assignment mode is enabled. List of memory transaction types supported: @@ -454,9 +454,8 @@ with the following files: # cat /sys/fs/resctrl/info/L3_MON/event_configs/mbm_local_bytes/event_filter local_reads,local_non_temporal_writes,local_reads_slow_memory - Modify the event configuration by writing to the "event_filter" file within - the "event_configs" directory. The read/write "event_filter" file contains the - configuration of the event that reflects which memory transactions are counted by it. + The memory transactions the MBM event is configured with can be changed + if "event_filter" is writable. For example:: diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index 9bd87bae49834..794a6fb175e4e 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -454,6 +454,7 @@ int __init rdt_get_l3_mon_config(struct rdt_resource *r) (rdt_cpu_has(X86_FEATURE_CQM_MBM_TOTAL) || rdt_cpu_has(X86_FEATURE_CQM_MBM_LOCAL))) { r->mon.mbm_cntr_assignable = true; + r->mon.mbm_cntr_configurable = true; cpuid_count(0x80000020, 5, &eax, &ebx, &ecx, &edx); r->mon.num_mbm_cntrs = (ebx & GENMASK(15, 0)) + 1; hw_res->mbm_cntr_assign_enabled = true; diff --git a/fs/resctrl/internal.h b/fs/resctrl/internal.h index 1a9b29119f88f..48af75b9dc855 100644 --- a/fs/resctrl/internal.h +++ b/fs/resctrl/internal.h @@ -408,6 +408,8 @@ void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free); void resctrl_file_fflags_init(const char *config, unsigned long fflags); +void resctrl_file_mode_init(const char *config, umode_t mode); + void rdt_staged_configs_clear(void); bool closid_allocated(unsigned int closid); diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c index 9fd901c78dc66..916f7a9d56581 100644 --- a/fs/resctrl/monitor.c +++ b/fs/resctrl/monitor.c @@ -1422,6 +1422,11 @@ ssize_t event_filter_write(struct kernfs_open_file *of, char *buf, size_t nbytes ret = -EINVAL; goto out_unlock; } + if (!r->mon.mbm_cntr_configurable) { + rdt_last_cmd_puts("event_filter is not configurable\n"); + ret = -EPERM; + goto out_unlock; + } ret = resctrl_parse_mem_transactions(buf, &evt_cfg); if (!ret && mevt->evt_cfg != evt_cfg) { @@ -1886,6 +1891,8 @@ int resctrl_l3_mon_resource_init(void) resctrl_file_fflags_init("available_mbm_cntrs", RFTYPE_MON_INFO | RFTYPE_RES_CACHE); resctrl_file_fflags_init("event_filter", RFTYPE_ASSIGN_CONFIG); + if (r->mon.mbm_cntr_configurable) + resctrl_file_mode_init("event_filter", 0644); resctrl_file_fflags_init("mbm_assign_on_mkdir", RFTYPE_MON_INFO | RFTYPE_RES_CACHE); resctrl_file_fflags_init("mbm_L3_assignments", RFTYPE_MON_BASE); diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index eca3bb67987d4..7d800b3060562 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -2022,7 +2022,7 @@ static struct rftype res_common_files[] = { }, { .name = "event_filter", - .mode = 0644, + .mode = 0444, .kf_ops = &rdtgroup_kf_single_ops, .seq_show = event_filter_show, .write = event_filter_write, @@ -2215,6 +2215,15 @@ void resctrl_file_fflags_init(const char *config, unsigned long fflags) rft->fflags = fflags; } +void resctrl_file_mode_init(const char *config, umode_t mode) +{ + struct rftype *rft; + + rft = rdtgroup_get_rftype_by_name(config); + if (rft) + rft->mode = mode; +} + /** * rdtgroup_kn_mode_restrict - Restrict user access to named resctrl file * @r: The resource group with which the file is associated. diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h index 006e57fd7ca58..06e8c72e8660f 100644 --- a/include/linux/resctrl.h +++ b/include/linux/resctrl.h @@ -286,13 +286,14 @@ enum resctrl_schema_fmt { /** * struct resctrl_mon - Monitoring related data of a resctrl resource. - * @num_rmid: Number of RMIDs available. - * @mbm_cfg_mask: Memory transactions that can be tracked when bandwidth - * monitoring events can be configured. - * @num_mbm_cntrs: Number of assignable counters. - * @mbm_cntr_assignable:Is system capable of supporting counter assignment? - * @mbm_assign_on_mkdir:True if counters should automatically be assigned to MBM - * events of monitor groups created via mkdir. + * @num_rmid: Number of RMIDs available. + * @mbm_cfg_mask: Memory transactions that can be tracked when + * bandwidth monitoring events can be configured. + * @num_mbm_cntrs: Number of assignable counters. + * @mbm_cntr_assignable: Is system capable of supporting counter assignment? + * @mbm_assign_on_mkdir: True if counters should automatically be assigned to MBM + * events of monitor groups created via mkdir. + * @mbm_cntr_configurable: True if assignable counters are configurable. */ struct resctrl_mon { u32 num_rmid; @@ -300,6 +301,7 @@ struct resctrl_mon { int num_mbm_cntrs; bool mbm_cntr_assignable; bool mbm_assign_on_mkdir; + bool mbm_cntr_configurable; }; /** From 96189cd02c612547dc34490551e1ac0a2ce607be Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Wed, 6 May 2026 09:28:51 +0100 Subject: [PATCH 216/311] NVIDIA: SAUCE: fs/resctrl: Disallow the software controller when MBM counters are assignable BugLink: https://bugs.launchpad.net/bugs/2154527 The software controller requires that there is one MBM counter per monitor group that is assigned to the event backing the software controller, as per mba_MBps_event. When mbm_event mode is in use, it is not guaranteed that any particular event will have an assigned counter. Currently, only AMD systems support counter assignment, but the MBA delay is non-linear and so the software controller is never supported anyway. On MPAM systems, the MBA delay is linear and so the software controller could be supported. The MPAM driver, unless a need arises, will not support the 'default' mbm_assign_mode and will always use the 'mbm_event' mode for memory bandwidth monitoring. Rather than develop a way to guarantee the counter assignment requirements needed by the software controller, take the pragmatic approach. Don't allow the software controller to be used at the same time as 'mbm_event' mode. As MPAM is the only relevant architecture and it will use 'mbm_event' mode whenever there are assignable MBM counters, for simplicity's sake, don't allow the software controller when the MBM counters are assignable. Implement this by failing the mount if the user requests the software controller, the mba_MBps option, and the MBM counters are assignable. Signed-off-by: Ben Horgan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Reinette Chatre Reviewed-by: Babu Moger Tested-by: Babu Moger Link: https://lore.kernel.org/20260506082855.3694761-1-ben.horgan@arm.com (cherry picked from commit f52abe6502413450b8d0ecaad2555bbe4c6242eb https://gitlab.arm.com/linux-arm/linux-bh.git mpam_abmc_v4) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- fs/resctrl/rdtgroup.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index 7d800b3060562..af2cbab14497e 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -2516,10 +2516,13 @@ static void mba_sc_domain_destroy(struct rdt_resource *r, } /* - * MBA software controller is supported only if - * MBM is supported and MBA is in linear scale, - * and the MBM monitor scope is the same as MBA - * control scope. + * The MBA software controller is supported only if MBM is supported and MBA is + * in linear scale, and the MBM monitor scope is the same as MBA control scope. + * + * The software controller cannot be supported when the MBM counters are + * assignable. There is no guarantee that MBM counters are assigned to the + * event backing the software controller in all monitoring domains of all + * monitoring groups. */ static bool supports_mba_mbps(void) { @@ -2528,7 +2531,8 @@ static bool supports_mba_mbps(void) return (resctrl_is_mbm_enabled() && r->alloc_capable && is_mba_linear() && - r->ctrl_scope == rmbm->mon_scope); + r->ctrl_scope == rmbm->mon_scope && + !rmbm->mon.mbm_cntr_assignable); } /* @@ -2943,7 +2947,7 @@ static int rdt_parse_param(struct fs_context *fc, struct fs_parameter *param) ctx->enable_cdpl2 = true; return 0; case Opt_mba_mbps: - msg = "mba_MBps requires MBM and linear scale MBA at L3 scope"; + msg = "mba_MBps requires MBM (mbm_event mode not supported) and linear scale MBA at L3 scope"; if (!supports_mba_mbps()) return invalfc(fc, msg); ctx->enable_mba_mbps = true; From 3c6f3f07a4bd51fd2e9adabe718506b2ba7d9eca Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Wed, 6 May 2026 09:28:52 +0100 Subject: [PATCH 217/311] NVIDIA: SAUCE: fs/resctrl: Add monitor property 'mbm_cntr_assign_fixed' BugLink: https://bugs.launchpad.net/bugs/2154527 Commit 3b497c3f4f04 ("fs/resctrl: Introduce the interface to display monitoring modes") introduced CONFIG_RESCTRL_ASSIGN_FIXED but left adding the Kconfig entry until it was necessary. The counter assignment mode is fixed in MPAM, even when there are assignable counters, and so addressing this is needed to support MPAM. To avoid the burden of another Kconfig entry, replace CONFIG_RESCTRL_ASSIGN_FIXED with a new property in 'struct resctrl_mon', 'mbm_cntr_assign_fixed' to be set by the architecture. Do not request the architecture to change the counter assignment mode if it does not support doing so. Provide insight to user space about why such a request fails. Signed-off-by: Ben Horgan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Babu Moger Reviewed-by: Reinette Chatre Tested-by: Babu Moger Link: https://lore.kernel.org/20260506082855.3694761-1-ben.horgan@arm.com (cherry picked from commit ee3d4c81d89c92fbeb65807971ac22b3dfa49220 https://gitlab.arm.com/linux-arm/linux-bh.git mpam_abmc_v4) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- fs/resctrl/monitor.c | 8 +++++++- include/linux/resctrl.h | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c index 916f7a9d56581..5fbcc64e50ce7 100644 --- a/fs/resctrl/monitor.c +++ b/fs/resctrl/monitor.c @@ -1456,7 +1456,7 @@ int resctrl_mbm_assign_mode_show(struct kernfs_open_file *of, else seq_puts(s, "[default]\n"); - if (!IS_ENABLED(CONFIG_RESCTRL_ASSIGN_FIXED)) { + if (!r->mon.mbm_cntr_assign_fixed) { if (enabled) seq_puts(s, "default\n"); else @@ -1507,6 +1507,12 @@ ssize_t resctrl_mbm_assign_mode_write(struct kernfs_open_file *of, char *buf, } if (enable != resctrl_arch_mbm_cntr_assign_enabled(r)) { + if (r->mon.mbm_cntr_assign_fixed) { + ret = -EINVAL; + rdt_last_cmd_puts("Counter assignment mode is not configurable\n"); + goto out_unlock; + } + ret = resctrl_arch_mbm_cntr_assign_set(r, enable); if (ret) goto out_unlock; diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h index 06e8c72e8660f..73ff522448a02 100644 --- a/include/linux/resctrl.h +++ b/include/linux/resctrl.h @@ -294,6 +294,7 @@ enum resctrl_schema_fmt { * @mbm_assign_on_mkdir: True if counters should automatically be assigned to MBM * events of monitor groups created via mkdir. * @mbm_cntr_configurable: True if assignable counters are configurable. + * @mbm_cntr_assign_fixed: True if the counter assignment mode is fixed. */ struct resctrl_mon { u32 num_rmid; @@ -302,6 +303,7 @@ struct resctrl_mon { bool mbm_cntr_assignable; bool mbm_assign_on_mkdir; bool mbm_cntr_configurable; + bool mbm_cntr_assign_fixed; }; /** From 9fd6ef124a9e9597e761f539d6d707412233c2b8 Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Wed, 6 May 2026 09:28:53 +0100 Subject: [PATCH 218/311] NVIDIA: SAUCE: fs/resctrl: Continue counter allocation after failure BugLink: https://bugs.launchpad.net/bugs/2154527 In mbm_event mode, with mbm_assign_on_mkdir set to 1, when a user creates a new CTRL_MON or MON group resctrl attempts to allocate counters for each of the supported MBM events on each resctrl domain. As counters are limited, such allocation may fail and when it does counter allocations for the remaining domains are skipped even if the domains have available counters. Because of that, the user needs to view the resource group'smbm_L3_assignments file to get an accurate view of counter assignment in a new resource group and then manually create counters in the skipped domains with available counters. Writes to mbm_L3_assignments using the wildcard format, :*=e, also skip counter allocation in other domains after a counter allocation failure. When handling a request to create counters in all domains it is unnecessary for a counter allocation in one domain to prevent counter allocation in other domains. Always attempt to allocate all the counters requested. [ bp: Massage commit message. ] Signed-off-by: Ben Horgan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Babu Moger Reviewed-by: Reinette Chatre Tested-by: Babu Moger Link: https://lore.kernel.org/20260506082855.3694761-1-ben.horgan@arm.com (cherry picked from commit 3aec86e4ea013c084a232c83754d182c9aaf378e https://gitlab.arm.com/linux-arm/linux-bh.git mpam_abmc_v4) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- fs/resctrl/monitor.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c index 5fbcc64e50ce7..0e6a389a16bf6 100644 --- a/fs/resctrl/monitor.c +++ b/fs/resctrl/monitor.c @@ -1211,9 +1211,10 @@ static int rdtgroup_alloc_assign_cntr(struct rdt_resource *r, struct rdt_l3_mon_ * NULL; otherwise, assign the counter to the specified domain @d. * * If all counters in a domain are already in use, rdtgroup_alloc_assign_cntr() - * will fail. The assignment process will abort at the first failure encountered - * during domain traversal, which may result in the event being only partially - * assigned. + * will fail. When attempting to assign counters to all domains, carry on trying + * to assign counters after a failure since only some domains may have counters + * and the goal is to assign counters where possible. If any counter assignment + * fails, return the error from the last failing assignment. * * Return: * 0 on success, < 0 on failure. @@ -1226,9 +1227,11 @@ static int rdtgroup_assign_cntr_event(struct rdt_l3_mon_domain *d, struct rdtgro if (!d) { list_for_each_entry(d, &r->mon_domains, hdr.list) { - ret = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt); - if (ret) - return ret; + int err; + + err = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt); + if (err) + ret = err; } } else { ret = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt); From 817a9819ba09cfa01129815e49576ae149aaf66c Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Wed, 6 May 2026 09:28:54 +0100 Subject: [PATCH 219/311] NVIDIA: SAUCE: fs/resctrl: Document that automatic counter assignment is best effort BugLink: https://bugs.launchpad.net/bugs/2154527 When using automatic counter assignment it's useful for a user to know which counters they can expect to be assigned on group creation. Document that automatic counter assignment is best effort and how to discover any assignment failures. Suggested-by: Reinette Chatre Signed-off-by: Ben Horgan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Babu Moger Reviewed-by: Reinette Chatre Tested-by: Babu Moger Link: https://lore.kernel.org/20260506082855.3694761-1-ben.horgan@arm.com (cherry picked from commit 9a1646211f8c67c7c98f8109607e3b962aea13eb https://gitlab.arm.com/linux-arm/linux-bh.git mpam_abmc_v4) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- Documentation/filesystems/resctrl.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Documentation/filesystems/resctrl.rst b/Documentation/filesystems/resctrl.rst index 2898a51e6f4be..b388e9193896e 100644 --- a/Documentation/filesystems/resctrl.rst +++ b/Documentation/filesystems/resctrl.rst @@ -479,6 +479,12 @@ with the following files: "1": Auto assignment is enabled. + Automatic counter assignment is done with best effort. If auto + assignment is enabled but there are not enough available counters then + monitor group creation could succeed while one or more events belonging + to the group may not have a counter assigned in all domains. Consult + mbm_L3_assignments for counter assignment states of the new groups. + Example:: # echo 0 > /sys/fs/resctrl/info/L3_MON/mbm_assign_on_mkdir From 11ce8d2ccdf01ee25ee20d828cf02dbbcb4db8ad Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Wed, 6 May 2026 09:28:55 +0100 Subject: [PATCH 220/311] NVIDIA: SAUCE: fs/resctrl: Document tasks file behaviour for task id 0 and idle tasks BugLink: https://bugs.launchpad.net/bugs/2154527 When 0 is written to the tasks file it is interpreted as the current task in rdtgroup_move_task(). Each CPU's idle task has task_struct::pid set to 0 and, on x86, task_struct::closid to RESCTRL_RESERVED_CLOSID and task_struct::rmid to RESCTRL_RESERVED_RMID. Equivalently, on MPAM platforms, thread_info::mpam_partid_pmg is encoded with PARTID and PMG set to RESCTRL_RESERVED_CLOSID and RESCTRL_RESERVED_RMID, respectively. As there is no interface to change these from the default, the resctrl configuration for the idle tasks is fixed and they always behave equivalently to a task in the default tasks file and so take their configuration from the cpus/cpus_list files. On read of the tasks file, show_rdt_tasks() filters out any 0 PID. Hence, a task id of 0 is never shown in the tasks file and the idle tasks are not represented either. Document the user visible behaviour. Signed-off-by: Ben Horgan Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Babu Moger Reviewed-by: Reinette Chatre Tested-by: Babu Moger Link: https://lore.kernel.org/20260506082855.3694761-1-ben.horgan@arm.com (cherry picked from commit 1cfa74c683ea82d37156ccd7ab4f4659056dc701 https://gitlab.arm.com/linux-arm/linux-bh.git mpam_abmc_v4) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- Documentation/filesystems/resctrl.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Documentation/filesystems/resctrl.rst b/Documentation/filesystems/resctrl.rst index b388e9193896e..e4b66af55ffba 100644 --- a/Documentation/filesystems/resctrl.rst +++ b/Documentation/filesystems/resctrl.rst @@ -575,6 +575,11 @@ All groups contain the following files: then the task must already belong to the CTRL_MON parent of this group. The task is removed from any previous MON group. + When writing to this file, a task id of 0 is interpreted as the + task id of the currently running task. On reading the file, a task + id of 0 will never be shown and there is no representation of the + idle tasks. Instead, a CPU's idle task is always considered as a + member of the group owning the CPU. "cpus": Reading this file shows a bitmask of the logical CPUs owned by From ce45a0d8cc21c1577137340d4c6138a701503739 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 5 Dec 2025 21:58:42 +0000 Subject: [PATCH 221/311] NVIDIA: SAUCE: arm_mpam: resctrl: Pick classes for use as MBM counters BugLink: https://bugs.launchpad.net/bugs/2154527 resctrl has two types of bandwidth counters, NUMA-local and global. MPAM can only count globally; either using MSC at the L3 cache or in the memory controllers. When global and local equate to the same thing continue just to call it global. Pick the corresponding MPAM classes to back the MBM counters. As resctrl requires all monitors to be at the L3 cache, we can only use the counters at the memory controllers when they have the same topology as the L3 cache and the traffic they see if the same. In particular, for the bandwidth counters at the memory controllers to be exposed to resctrl it is required there is a single L3 cache and a single NUMA node as otherwise cross NUMA traffic will be counted at the wrong instance. Tested-by: Shaopeng Tan Tested-by: Zeng Heng Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Signed-off-by: James Morse Signed-off-by: Ben Horgan (cherry picked from commit b2bbf3b7f0f73adfa18acd2824be95b62490b9ae https://gitlab.arm.com/linux-arm/linux-bh.git mpam_abmc_v4) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 226ff6f532fab..f70fa65d39e40 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -606,6 +606,16 @@ static bool cache_has_usable_csu(struct mpam_class *class) return true; } +static bool class_has_usable_mbwu(struct mpam_class *class) +{ + struct mpam_props *cprops = &class->props; + + if (!mpam_has_feature(mpam_feat_msmon_mbwu, cprops)) + return false; + + return true; +} + /* * Calculate the worst-case percentage change from each implemented step * in the control. @@ -983,6 +993,22 @@ static void mpam_resctrl_pick_counters(void) break; } } + + if (class_has_usable_mbwu(class) && + topology_matches_l3(class) && + traffic_matches_l3(class)) { + pr_debug("class %u has usable MBWU, and matches L3 topology and traffic\n", + class->level); + + /* + * We can't distinguish traffic by destination so + * we don't know if it's staying on the same NUMA + * node. Hence, we can't calculate mbm_local except + * when we only have one L3 and it's equivalent to + * mbm_total and so always use mbm_total. + */ + counter_update_class(QOS_L3_MBM_TOTAL_EVENT_ID, class); + } } } From c75375d734a0a375ba55f3bcbe88541a0e5ed004 Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Mon, 16 Mar 2026 14:41:00 +0000 Subject: [PATCH 222/311] NVIDIA: SAUCE: arm_mpam: resctrl: Pre-allocate assignable monitors BugLink: https://bugs.launchpad.net/bugs/2154527 MPAM is able to emulate ABMC, i.e. mbm_event mode, by making memory bandwidth monitors assignable. Rather than supporting the 'default' mbm_assign_mode always use 'mbm_event' mode even if there are sufficient memory bandwidth monitors. The per monitor event configuration is only provided by resctrl when in 'mbm_event' mode and so only allowing 'mbm_event' mode will make it easier to support per-monitor event configuration for MPAM. For the moment, the only event supported is mbm_total_event with no bandwidth type configuration. The 'mbm_assign_mode' file will still show 'default' when there is no support for memory bandwidth monitoring. The monitors need to be allocated from the driver, and mapped to whichever control/monitor group resctrl wants to use them with. Add a second array to hold the monitor values indexed by resctrl's cntr_id. When CDP is in use, two monitors are needed so the available number of counters halves. Platforms with one monitor will have zero monitors when CDP is in use. Co-developed-by: James Morse Signed-off-by: James Morse Signed-off-by: Ben Horgan (cherry picked from commit 4766d7e28b303ef2ce83a7fe28c4b3af4918bb78 https://gitlab.arm.com/linux-arm/linux-bh.git mpam_abmc_v4) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_internal.h | 6 +- drivers/resctrl/mpam_resctrl.c | 139 +++++++++++++++++++++++++++++++- 2 files changed, 141 insertions(+), 4 deletions(-) diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index 1914aefdcba9e..7a166b395b5af 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -411,7 +411,11 @@ struct mpam_resctrl_res { struct mpam_resctrl_mon { struct mpam_class *class; - /* per-class data that resctrl needs will live here */ + /* Array of allocated MBWU monitors, indexed by (closid, rmid). */ + int *mbwu_idx_to_mon; + + /* Array of assigned MBWU monitors, indexed by idx argument. */ + int *assigned_counters; }; static inline int mpam_alloc_csu_mon(struct mpam_class *class) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index f70fa65d39e40..a13eb232a19de 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -75,6 +75,8 @@ static DECLARE_WAIT_QUEUE_HEAD(wait_cacheinfo_ready); */ static bool resctrl_enabled; +static unsigned int l3_num_allocated_mbwu = ~0; + bool resctrl_arch_alloc_capable(void) { struct mpam_resctrl_res *res; @@ -140,7 +142,7 @@ int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, bool resctrl_arch_mbm_cntr_assign_enabled(struct rdt_resource *r) { - return false; + return (r == &mpam_resctrl_controls[RDT_RESOURCE_L3].resctrl_res); } int resctrl_arch_mbm_cntr_assign_set(struct rdt_resource *r, bool enable) @@ -185,6 +187,18 @@ static void resctrl_reset_task_closids(void) read_unlock(&tasklist_lock); } +static void mpam_resctrl_monitor_sync_abmc_vals(struct rdt_resource *l3) +{ + l3->mon.num_mbm_cntrs = l3_num_allocated_mbwu; + if (cdp_enabled) + l3->mon.num_mbm_cntrs /= 2; + + /* + * Continue as normal even if enabling cdp causes there to be + * zero counters. This avoid giving resctrl mixed messages. + */ +} + int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable) { u32 partid_i = RESCTRL_RESERVED_CLOSID, partid_d = RESCTRL_RESERVED_CLOSID; @@ -244,6 +258,7 @@ int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable) WRITE_ONCE(arm64_mpam_global_default, mpam_get_regval(current)); resctrl_reset_task_closids(); + mpam_resctrl_monitor_sync_abmc_vals(l3); for_each_possible_cpu(cpu) mpam_set_cpu_defaults(cpu, partid_d, partid_i, 0, 0); @@ -613,6 +628,9 @@ static bool class_has_usable_mbwu(struct mpam_class *class) if (!mpam_has_feature(mpam_feat_msmon_mbwu, cprops)) return false; + if (!cprops->num_mbwu_mon) + return false; + return true; } @@ -935,6 +953,52 @@ static void mpam_resctrl_pick_mba(void) } } +static void __free_mbwu_mon(struct mpam_class *class, int *array, + u16 num_mbwu_mon) +{ + for (int i = 0; i < num_mbwu_mon; i++) { + if (array[i] < 0) + continue; + + mpam_free_mbwu_mon(class, array[i]); + array[i] = ~0; + } +} + +static int __alloc_mbwu_mon(struct mpam_class *class, int *array, + u16 num_mbwu_mon) +{ + for (int i = 0; i < num_mbwu_mon; i++) { + int mbwu_mon = mpam_alloc_mbwu_mon(class); + + if (mbwu_mon < 0) { + __free_mbwu_mon(class, array, num_mbwu_mon); + return mbwu_mon; + } + array[i] = mbwu_mon; + } + + l3_num_allocated_mbwu = min(l3_num_allocated_mbwu, num_mbwu_mon); + + return 0; +} + +static int *__alloc_mbwu_array(struct mpam_class *class, u16 num_mbwu_mon) +{ + int err; + + int *array __free(kvfree) = kvmalloc_objs(*array, num_mbwu_mon); + if (!array) + return ERR_PTR(-ENOMEM); + + memset(array, -1, num_mbwu_mon * sizeof(*array)); + + err = __alloc_mbwu_mon(class, array, num_mbwu_mon); + if (err) + return ERR_PTR(err); + return_ptr(array); +} + static void counter_update_class(enum resctrl_event_id evt_id, struct mpam_class *class) { @@ -1089,6 +1153,43 @@ static int mpam_resctrl_pick_domain_id(int cpu, struct mpam_component *comp) return comp->comp_id; } +/* + * This must run after all event counters have been picked so that any free + * running counters have already been allocated. + */ +static int mpam_resctrl_monitor_init_abmc(struct mpam_resctrl_mon *mon) +{ + struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; + size_t num_rmid = resctrl_arch_system_num_rmid_idx(); + struct rdt_resource *l3 = &res->resctrl_res; + struct mpam_class *class = mon->class; + u16 num_mbwu_mon; + int *cntrs; + + int *rmid_array __free(kvfree) = kvmalloc_objs(*rmid_array, num_rmid); + if (!rmid_array) { + pr_debug("Failed to allocate RMID array\n"); + return -ENOMEM; + } + memset(rmid_array, -1, num_rmid * sizeof(*rmid_array)); + + num_mbwu_mon = class->props.num_mbwu_mon; + cntrs = __alloc_mbwu_array(mon->class, num_mbwu_mon); + if (IS_ERR(cntrs)) + return PTR_ERR(cntrs); + mon->assigned_counters = cntrs; + mon->mbwu_idx_to_mon = no_free_ptr(rmid_array); + + l3->mon.mbm_cntr_assignable = true; + l3->mon.mbm_assign_on_mkdir = true; + l3->mon.mbm_cntr_configurable = false; + l3->mon.mbm_cntr_assign_fixed = true; + + mpam_resctrl_monitor_sync_abmc_vals(l3); + + return 0; +} + static int mpam_resctrl_monitor_init(struct mpam_resctrl_mon *mon, enum resctrl_event_id type) { @@ -1133,8 +1234,21 @@ static int mpam_resctrl_monitor_init(struct mpam_resctrl_mon *mon, */ l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx(); - if (resctrl_enable_mon_event(type, false, 0, NULL)) - l3->mon_capable = true; + if (type == QOS_L3_MBM_TOTAL_EVENT_ID) { + int err; + + err = mpam_resctrl_monitor_init_abmc(mon); + if (err) + return err; + + static_assert(MAX_EVT_CONFIG_BITS == 0x7f); + l3->mon.mbm_cfg_mask = MAX_EVT_CONFIG_BITS; + } + + if (!resctrl_enable_mon_event(type, false, 0, NULL)) + return -EINVAL; + + l3->mon_capable = true; return 0; } @@ -1697,6 +1811,23 @@ void mpam_resctrl_exit(void) resctrl_exit(); } +static void mpam_resctrl_teardown_mon(struct mpam_resctrl_mon *mon, struct mpam_class *class) +{ + u32 num_mbwu_mon = l3_num_allocated_mbwu; + + if (!mon->mbwu_idx_to_mon) + return; + + if (mon->assigned_counters) { + __free_mbwu_mon(class, mon->assigned_counters, num_mbwu_mon); + kvfree(mon->assigned_counters); + mon->assigned_counters = NULL; + } + + kvfree(mon->mbwu_idx_to_mon); + mon->mbwu_idx_to_mon = NULL; +} + /* * The driver is detaching an MSC from this class, if resctrl was using it, * pull on resctrl_exit(). @@ -1719,6 +1850,8 @@ void mpam_resctrl_teardown_class(struct mpam_class *class) for_each_mpam_resctrl_mon(mon, eventid) { if (mon->class == class) { mon->class = NULL; + + mpam_resctrl_teardown_mon(mon, class); break; } } From e49e1a7f7d046ee1a7ac274266e9944614995687 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 5 Dec 2025 21:58:46 +0000 Subject: [PATCH 223/311] NVIDIA: SAUCE: arm_mpam: resctrl: Add resctrl_arch_config_cntr() for ABMC use BugLink: https://bugs.launchpad.net/bugs/2154527 ABMC, mbm_event mode, has a helper resctrl_arch_config_cntr() for changing the mapping between 'cntr_id' and a CLOSID/RMID pair. Add the helper. For MPAM this is done by updating the mon->mbwu_idx_to_mon[] array, and as usual CDP means it needs doing in three different ways. Reviewed-by: Jonathan Cameron Signed-off-by: James Morse Signed-off-by: Ben Horgan (cherry picked from commit 47b7baaa2f246e6018b467984e94a827bf5e875b https://gitlab.arm.com/linux-arm/linux-bh.git mpam_abmc_v4) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 44 +++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index a13eb232a19de..1f9a8ae157ca2 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -127,12 +127,6 @@ void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d { } -void resctrl_arch_config_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, - enum resctrl_event_id evtid, u32 rmid, u32 closid, - u32 cntr_id, bool assign) -{ -} - int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 unused, u32 rmid, int cntr_id, enum resctrl_event_id eventid, u64 *val) @@ -1076,6 +1070,44 @@ static void mpam_resctrl_pick_counters(void) } } +static void __config_cntr(struct mpam_resctrl_mon *mon, u32 cntr_id, + enum resctrl_conf_type cdp_type, u32 closid, u32 rmid, + bool assign) +{ + u32 mbwu_idx, mon_idx = resctrl_get_config_index(cntr_id, cdp_type); + + WARN_ON_ONCE(mon_idx >= l3_num_allocated_mbwu); + + closid = resctrl_get_config_index(closid, cdp_type); + mbwu_idx = resctrl_arch_rmid_idx_encode(closid, rmid); + + if (assign) + mon->mbwu_idx_to_mon[mbwu_idx] = mon->assigned_counters[mon_idx]; + else + mon->mbwu_idx_to_mon[mbwu_idx] = -1; +} + +void resctrl_arch_config_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + enum resctrl_event_id evtid, u32 rmid, u32 closid, + u32 cntr_id, bool assign) +{ + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid]; + + if (!mon->mbwu_idx_to_mon || !mon->assigned_counters) { + pr_debug("monitor arrays not allocated\n"); + return; + } + + if (cdp_enabled) { + __config_cntr(mon, cntr_id, CDP_CODE, closid, rmid, assign); + __config_cntr(mon, cntr_id, CDP_DATA, closid, rmid, assign); + } else { + __config_cntr(mon, cntr_id, CDP_NONE, closid, rmid, assign); + } + + resctrl_arch_reset_rmid(r, d, closid, rmid, evtid); +} + static int mpam_resctrl_control_init(struct mpam_resctrl_res *res) { struct mpam_class *class = res->class; From 02d82b5acf33345263af6045d6cc6a2fee298273 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 5 Dec 2025 21:58:49 +0000 Subject: [PATCH 224/311] NVIDIA: SAUCE: arm_mpam: resctrl: Add resctrl_arch_cntr_read() & resctrl_arch_reset_cntr() BugLink: https://bugs.launchpad.net/bugs/2154527 When used in 'mbm_event' mode, ABMC emulation, resctrl uses arch hooks to read and reset the memory bandwidth utilization (MBWU) counters. Add these. Reviewed-by: Jonathan Cameron Signed-off-by: James Morse Signed-off-by: Ben Horgan (cherry picked from commit ed60492bde504769ec90e3dfa8090892bfbf803a https://gitlab.arm.com/linux-arm/linux-bh.git mpam_abmc_v4) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 99 +++++++++++++++++++++++++++++----- 1 file changed, 86 insertions(+), 13 deletions(-) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 1f9a8ae157ca2..d236ecd38aa35 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -121,19 +121,6 @@ void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_l3_mon_domain *d { } -void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, - u32 closid, u32 rmid, int cntr_id, - enum resctrl_event_id eventid) -{ -} - -int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, - u32 unused, u32 rmid, int cntr_id, - enum resctrl_event_id eventid, u64 *val) -{ - return -EOPNOTSUPP; -} - bool resctrl_arch_mbm_cntr_assign_enabled(struct rdt_resource *r) { return (r == &mpam_resctrl_controls[RDT_RESOURCE_L3].resctrl_res); @@ -463,6 +450,14 @@ static int __read_mon(struct mpam_resctrl_mon *mon, struct mpam_component *mon_c /* Shift closid to account for CDP */ closid = resctrl_get_config_index(closid, cdp_type); + if (mon_idx == USE_PRE_ALLOCATED) { + int mbwu_idx = resctrl_arch_rmid_idx_encode(closid, rmid); + + mon_idx = mon->mbwu_idx_to_mon[mbwu_idx]; + if (mon_idx == -1) + return -ENOENT; + } + if (irqs_disabled()) { /* Check if we can access this domain without an IPI */ return -EIO; @@ -535,6 +530,84 @@ int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain_hdr *hdr, closid, rmid, val); } +/* MBWU counters when in ABMC mode */ +int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + u32 closid, u32 rmid, int mon_idx, + enum resctrl_event_id eventid, u64 *val) +{ + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[eventid]; + struct mpam_resctrl_dom *l3_dom; + struct mpam_component *mon_comp; + + if (!mpam_is_enabled()) + return -EINVAL; + + if (eventid == QOS_L3_OCCUP_EVENT_ID || !mon->class) + return -EINVAL; + + l3_dom = container_of(d, struct mpam_resctrl_dom, resctrl_mon_dom); + mon_comp = l3_dom->mon_comp[eventid]; + + return read_mon_cdp_safe(mon, mon_comp, mpam_feat_msmon_mbwu, + USE_PRE_ALLOCATED, closid, rmid, val); +} + +static void __reset_mon(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp, + int mon_idx, + enum resctrl_conf_type cdp_type, u32 closid, u32 rmid) +{ + struct mon_cfg cfg = { }; + + if (!mpam_is_enabled()) + return; + + /* Shift closid to account for CDP */ + closid = resctrl_get_config_index(closid, cdp_type); + + if (mon_idx == USE_PRE_ALLOCATED) { + int mbwu_idx = resctrl_arch_rmid_idx_encode(closid, rmid); + + mon_idx = mon->mbwu_idx_to_mon[mbwu_idx]; + } + + if (mon_idx == -1) + return; + cfg.mon = mon_idx; + mpam_msmon_reset_mbwu(mon_comp, &cfg); +} + +static void reset_mon_cdp_safe(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp, + int mon_idx, u32 closid, u32 rmid) +{ + if (cdp_enabled) { + __reset_mon(mon, mon_comp, mon_idx, CDP_CODE, closid, rmid); + __reset_mon(mon, mon_comp, mon_idx, CDP_DATA, closid, rmid); + } else { + __reset_mon(mon, mon_comp, mon_idx, CDP_NONE, closid, rmid); + } +} + +/* Reset an assigned counter */ +void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + u32 closid, u32 rmid, int cntr_id, + enum resctrl_event_id eventid) +{ + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[eventid]; + struct mpam_resctrl_dom *l3_dom; + struct mpam_component *mon_comp; + + if (!mpam_is_enabled()) + return; + + if (eventid == QOS_L3_OCCUP_EVENT_ID || !mon->class) + return; + + l3_dom = container_of(d, struct mpam_resctrl_dom, resctrl_mon_dom); + mon_comp = l3_dom->mon_comp[eventid]; + + reset_mon_cdp_safe(mon, mon_comp, USE_PRE_ALLOCATED, closid, rmid); +} + /* * The rmid realloc threshold should be for the smallest cache exposed to * resctrl. From e27edf048042ac9698db037232c17defdf5d8ab7 Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Wed, 11 Mar 2026 16:40:29 +0000 Subject: [PATCH 225/311] NVIDIA: SAUCE: arm64: mpam: Add memory bandwidth usage (MBWU) documentation BugLink: https://bugs.launchpad.net/bugs/2154527 Memory bandwidth monitoring make uses of MBWU monitors and is now exposed to the user via resctrl. Add some documentation so the user knows what to expect. Co-developed-by: James Morse Signed-off-by: James Morse Signed-off-by: Ben Horgan (cherry picked from commit 3219a44691c2ce434e6721201b030e9e58424feb https://gitlab.arm.com/linux-arm/linux-bh.git mpam_abmc_v4) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- Documentation/arch/arm64/mpam.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Documentation/arch/arm64/mpam.rst b/Documentation/arch/arm64/mpam.rst index 570f51a8d4ebf..208ff17068c4b 100644 --- a/Documentation/arch/arm64/mpam.rst +++ b/Documentation/arch/arm64/mpam.rst @@ -65,6 +65,23 @@ The supported features are: there is at least one CSU monitor on each MSC that makes up the L3 group. Exposing CSU counters from other caches or devices is not supported. +* Memory Bandwidth Usage (MBWU) on or after the L3 cache. resctrl uses the + L3 cache-id to identify where the memory bandwidth is measured. For this + reason the platform must have an L3 cache with cache-id's supplied by + firmware. (It doesn't need to support MPAM.) + + Memory bandwidth monitoring makes use of MBWU monitors in each MSC that + makes up the L3 group. If the memory bandwidth monitoring is on the memory + rather than the L3 then there must be a single global L3 as otherwise it + is unknown which L3 the traffic came from. + + To expose 'mbm_total_bytes', the topology of the group of MSC chosen must + match the topology of the L3 cache so that the cache-id's can be + repainted. For example: Platforms with Memory bandwidth monitors on + CPU-less NUMA nodes cannot expose 'mbm_total_bytes' as these nodes do not + have a corresponding L3 cache. 'mbm_local_bytes' is not exposed as MPAM + cannot distinguish local traffic from global traffic. + Reporting Bugs ============== If you are not seeing the counters or controls you expect please share the From 8a4c80a337a9b10fd95d1b2ff025233fb1440a86 Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Fri, 15 Aug 2025 15:43:56 +0100 Subject: [PATCH 226/311] NVIDIA: SAUCE: arm_mpam: Add resctrl_arch_round_bw() BugLink: https://bugs.launchpad.net/bugs/2154527 Add the required hook to pre-round a userspace memory bandwidth allocation percentage value to a value acceptable to the driver backend. For MPAM, no rounding is needed because the driver has all the information necessary for rounding the value when resctrl_arch_update_one() is called. So, just "round" the value to itself here. Signed-off-by: Dave Martin Signed-off-by: James Morse (cherry picked from commit 935611d607afe707a00b0311fdbb500b8acdd654 https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) [fenghuay: - Resolve minor conflicts in `include/linux/arm_mpam.h`; ] Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- include/linux/arm_mpam.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/include/linux/arm_mpam.h b/include/linux/arm_mpam.h index f92a36187a527..4ccf32fe07fd5 100644 --- a/include/linux/arm_mpam.h +++ b/include/linux/arm_mpam.h @@ -5,6 +5,7 @@ #define __LINUX_ARM_MPAM_H #include +#include #include #include @@ -76,6 +77,19 @@ static inline void resctrl_arch_disable_mon(void) { } static inline void resctrl_arch_enable_alloc(void) { } static inline void resctrl_arch_disable_alloc(void) { } +struct resctrl_schema; + +struct rdt_resource; +static inline u32 resctrl_arch_round_bw(u32 val, + const struct rdt_resource *r __always_unused) +{ + /* + * Do nothing: for MPAM, resctrl_arch_update_one() has the necessary + * context to round the incoming value correctly. + */ + return val; +} + static inline unsigned int resctrl_arch_round_mon_val(unsigned int val) { return val; From 3cd60da60da6fa56c74a4e29003652707a0cfb06 Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Fri, 15 Aug 2025 15:43:55 +0100 Subject: [PATCH 227/311] NVIDIA: SAUCE: fs/resctrl,x86/resctrl: Factor mba rounding to be per-arch BugLink: https://bugs.launchpad.net/bugs/2154527 The control value parser for the MB resource currently coerces the memory bandwidth percentage value from userspace to be an exact multiple of the bw_gran parameter. On MPAM systems, this results in somewhat worse-than-worst-case rounding, since bw_gran is in general only an approximation to the actual hardware granularity, and the hardware bandwidth allocation control value is not natively a percentage. Allow the arch to provide its own conversion that is appropriate for the hardware, and move the existing conversion to x86. This will avoid accumulated error from rounding the value twice on MPAM systems. Clarify the documentation, but avoid overly exact promises. Clamping to bw_min and bw_max still feels generic: leave it in the core code, for now. No functional change. Signed-off-by: Dave Martin Signed-off-by: James Morse (cherry picked from commit cabdc680e1dde14521ab2a61ff32b525b3ba334e https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- Documentation/filesystems/resctrl.rst | 7 +++---- arch/x86/kernel/cpu/resctrl/ctrlmondata.c | 6 ++++++ fs/resctrl/ctrlmondata.c | 2 +- include/linux/resctrl.h | 2 ++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Documentation/filesystems/resctrl.rst b/Documentation/filesystems/resctrl.rst index e4b66af55ffba..730c6d662e040 100644 --- a/Documentation/filesystems/resctrl.rst +++ b/Documentation/filesystems/resctrl.rst @@ -236,12 +236,11 @@ with respect to allocation: user can request. "bandwidth_gran": - The granularity in which the memory bandwidth + The approximate granularity in which the memory bandwidth percentage is allocated. The allocated b/w percentage is rounded off to the next - control step available on the hardware. The - available bandwidth control steps are: - min_bandwidth + N * bandwidth_gran. + control step available on the hardware. The available + steps are at least as small as this value. "delay_linear": Indicates if the delay scale is linear or diff --git a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c index b20e705606b8f..d539e56c2b1f0 100644 --- a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c +++ b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c @@ -16,9 +16,15 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include "internal.h" +u32 resctrl_arch_round_bw(u32 val, const struct rdt_resource *r) +{ + return roundup(val, (unsigned long)r->membw.bw_gran); +} + int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d, u32 closid, enum resctrl_conf_type t, u32 cfg_val) { diff --git a/fs/resctrl/ctrlmondata.c b/fs/resctrl/ctrlmondata.c index 9a7dfc48cb2e2..0c02451c687b2 100644 --- a/fs/resctrl/ctrlmondata.c +++ b/fs/resctrl/ctrlmondata.c @@ -71,7 +71,7 @@ static bool bw_validate(char *buf, u32 *data, struct rdt_resource *r) return false; } - *data = roundup(bw, (unsigned long)r->membw.bw_gran); + *data = resctrl_arch_round_bw(bw, r); return true; } diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h index 73ff522448a02..f7faa509ebd52 100644 --- a/include/linux/resctrl.h +++ b/include/linux/resctrl.h @@ -504,6 +504,8 @@ bool resctrl_arch_mbm_cntr_assign_enabled(struct rdt_resource *r); */ int resctrl_arch_mbm_cntr_assign_set(struct rdt_resource *r, bool enable); +u32 resctrl_arch_round_bw(u32 val, const struct rdt_resource *r); + /* * Update the ctrl_val and apply this config right now. * Must be called on one of the domain's CPUs. From ff0bc8df8d632cf16507a8e2ef36c64113c1dfcc Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 15 Mar 2024 16:46:12 +0000 Subject: [PATCH 228/311] NVIDIA: SAUCE: x86/resctrl: Add stub to allow other architecture to disable monitor overflow BugLink: https://bugs.launchpad.net/bugs/2154527 Resctrl has an overflow handler that runs on each domain every second to ensure that any overflow of the hardware counter is accounted for. MPAM can have counters as large as 63 bits, in which case there is no need to check for overflow. To allow other architectures to disable this, add a helper that reports whether counters can overflow. Signed-off-by: James Morse (cherry picked from commit 6a4360b3e0339ffc510b68d7a7d22941030f0604 https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/x86/include/asm/resctrl.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/x86/include/asm/resctrl.h b/arch/x86/include/asm/resctrl.h index 575f8408a9e7c..40a74a0617345 100644 --- a/arch/x86/include/asm/resctrl.h +++ b/arch/x86/include/asm/resctrl.h @@ -191,6 +191,11 @@ static inline void resctrl_arch_mon_ctx_free(struct rdt_resource *r, enum resctrl_event_id evtid, void *ctx) { } +static inline bool resctrl_arch_mon_can_overflow(void) +{ + return true; +} + void resctrl_cpu_detect(struct cpuinfo_x86 *c); #else From 8468136f197fd274c7987e9e45df3b5e46121f98 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 15 Mar 2024 17:32:53 +0000 Subject: [PATCH 229/311] NVIDIA: SAUCE: arm_mpam: resctrl: Determine if any exposed counter can overflow BugLink: https://bugs.launchpad.net/bugs/2154527 Resctrl has an overflow handler that runs on each domain every second to ensure that any overflow of the hardware counter is accounted for. MPAM can have counters as large as 63 bits, in which case there is no need to check for overflow. To allow the overflow handler to be disabled, determine if an overflow can happen. If a class is not implemented, or has the 63bit counter, it can't overflow. Signed-off-by: James Morse (cherry picked from commit 0f6aefdf5164dd6be3bd8c6cd82b6257fadbeab2 https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) [fenghuay: - Resolve minor conflicts in `drivers/resctrl/mpam_resctrl.c`; - Remove overflow check on QOS_L3_MBM_LOCAL_EVENT_ID since it's not supported in MPAM anymore. ] Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 18 ++++++++++++++++++ include/linux/arm_mpam.h | 1 + 2 files changed, 19 insertions(+) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index d236ecd38aa35..f3b31687c12df 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -437,6 +437,24 @@ void resctrl_arch_mon_ctx_free(struct rdt_resource *r, resctrl_arch_mon_ctx_free_no_wait(evtid, mon_idx); } +static bool __resctrl_arch_mon_can_overflow(enum resctrl_event_id eventid) +{ + struct mpam_props *cprops; + struct mpam_class *class = mpam_resctrl_counters[eventid].class; + + if (!class) + return false; + + /* No need to worry about a 63 bit counter overflowing */ + cprops = &class->props; + return !mpam_has_feature(mpam_feat_msmon_mbwu_63counter, cprops); +} + +bool resctrl_arch_mon_can_overflow(void) +{ + return __resctrl_arch_mon_can_overflow(QOS_L3_MBM_TOTAL_EVENT_ID); +} + static int __read_mon(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp, enum mpam_device_features mon_type, int mon_idx, diff --git a/include/linux/arm_mpam.h b/include/linux/arm_mpam.h index 4ccf32fe07fd5..b066d57e1a085 100644 --- a/include/linux/arm_mpam.h +++ b/include/linux/arm_mpam.h @@ -53,6 +53,7 @@ static inline int mpam_ris_create(struct mpam_msc *msc, u8 ris_idx, bool resctrl_arch_alloc_capable(void); bool resctrl_arch_mon_capable(void); +bool resctrl_arch_mon_can_overflow(void); void resctrl_arch_set_cpu_default_closid(int cpu, u32 closid); void resctrl_arch_set_closid_rmid(struct task_struct *tsk, u32 closid, u32 rmid); From 91f16e097391b8e25d67d18ff8ff9ee6fab40206 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 15 Mar 2024 17:36:02 +0000 Subject: [PATCH 230/311] NVIDIA: SAUCE: fs/restrl: Allow the overflow handler to be disabled BugLink: https://bugs.launchpad.net/bugs/2154527 Resctrl has an overflow handler that runs on each domain every second to ensure that any overflow of the hardware counter is accounted for. MPAM can have counters as large as 63 bits, in which case there is no need to check for overflow. Call the new arch helpers to determine this. Signed-off-by: James Morse (cherry picked from commit 72e375a4611a0eb5355e5a171a67a419ffd53522 https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- fs/resctrl/monitor.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c index 0e6a389a16bf6..2378017fb3b7e 100644 --- a/fs/resctrl/monitor.c +++ b/fs/resctrl/monitor.c @@ -895,8 +895,10 @@ void mbm_setup_overflow_handler(struct rdt_l3_mon_domain *dom, unsigned long del /* * When a domain comes online there is no guarantee the filesystem is * mounted. If not, there is no need to catch counter overflow. + * Some architecture may have ~64bit counters, and can ignore overflow. */ - if (!resctrl_mounted || !resctrl_arch_mon_capable()) + if (!resctrl_mounted || !resctrl_arch_mon_capable() || + !resctrl_arch_mon_can_overflow()) return; cpu = cpumask_any_housekeeping(&dom->hdr.cpu_mask, exclude_cpu); dom->mbm_work_cpu = cpu; From 24d897a90b0ced14596c6410e8c9b56d983518f9 Mon Sep 17 00:00:00 2001 From: James Morse Date: Tue, 27 Aug 2024 15:24:08 +0100 Subject: [PATCH 231/311] NVIDIA: SAUCE: arm_mpam: Allow cmax/cmin to be configured BugLink: https://bugs.launchpad.net/bugs/2154527 mpam_reprogram_ris_partid() always resets the CMAX/CMIN controls to their 'unrestricted' value. This prevents the controls from being configured. Add fields in struct mpam_config, and program these values when they are set in the features bitmask. Signed-off-by: James Morse (cherry picked from commit e701b2860ae2c02dc9c2015846d61838904a5b0b https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) [fenghuay: - Resolve minor conflicts in `drivers/resctrl/mpam_devices.c`; - Resolve minor conflicts in `drivers/resctrl/mpam_internal.h`; ] Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_devices.c | 23 +++++++++++++++++++---- drivers/resctrl/mpam_internal.h | 4 ++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 41b14344b16f2..26726b29b5991 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -1598,11 +1598,25 @@ static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid, if (mpam_has_feature(mpam_feat_mbw_prop, rprops)) mpam_write_partsel_reg(msc, MBW_PROP, 0); - if (mpam_has_feature(mpam_feat_cmax_cmax, rprops)) - mpam_write_partsel_reg(msc, CMAX, cmax); + if (mpam_has_feature(mpam_feat_cmax_cmax, rprops)) { + if (mpam_has_feature(mpam_feat_cmax_cmax, cfg)) { + u32 cmax_val = cfg->cmax; - if (mpam_has_feature(mpam_feat_cmax_cmin, rprops)) - mpam_write_partsel_reg(msc, CMIN, 0); + if (cfg->cmax_softlim) + cmax_val |= MPAMCFG_CMAX_SOFTLIM; + mpam_write_partsel_reg(msc, CMAX, cmax_val); + } else { + mpam_write_partsel_reg(msc, CMAX, cmax); + } + } + + if (mpam_has_feature(mpam_feat_cmax_cmin, rprops)) { + if (mpam_has_feature(mpam_feat_cmax_cmin, cfg)) { + mpam_write_partsel_reg(msc, CMIN, cfg->cmin); + } else { + mpam_write_partsel_reg(msc, CMIN, 0); + } + } if (mpam_has_feature(mpam_feat_cmax_cassoc, rprops)) mpam_write_partsel_reg(msc, CASSOC, MPAMCFG_CASSOC_CASSOC); @@ -2887,6 +2901,7 @@ static bool mpam_update_config(struct mpam_config *cfg, bool has_changes = false; maybe_update_config(cfg, mpam_feat_cpor_part, newcfg, cpbm, has_changes); + maybe_update_config(cfg, mpam_feat_cmax_cmax, newcfg, cmax, has_changes); maybe_update_config(cfg, mpam_feat_mbw_part, newcfg, mbw_pbm, has_changes); maybe_update_config(cfg, mpam_feat_mbw_max, newcfg, mbw_max, has_changes); diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index 7a166b395b5af..3a1ed201fe20c 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -320,6 +320,10 @@ struct mpam_config { u32 cpbm; u32 mbw_pbm; u16 mbw_max; + u16 cmax; + u16 cmin; + + bool cmax_softlim; struct mpam_garbage garbage; }; From e5ec6b626ca7c02474a7533a4fe970ca503fd42c Mon Sep 17 00:00:00 2001 From: James Morse Date: Tue, 19 Nov 2024 11:37:26 +0000 Subject: [PATCH 232/311] NVIDIA: SAUCE: arm_mpam: Rename mbw conversion to 'fract16' for code re-use BugLink: https://bugs.launchpad.net/bugs/2154527 Functions like mbw_max_to_percent() convert a value into MPAMs 16 bit fixed point fraction format. These are not only used for memory bandwidth, but cache capcity controls too. Rename these functions to convert to/from a 'fract16', and add helpers for the specific mbw_max/cmax controls. Signed-off-by: James Morse (cherry picked from commit 738f1605fb5c796713a429214270a18ec9c5d6c3 https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) [fenghuay: - Resolve minor conflicts in `drivers/resctrl/mpam_resctrl.c`; - Resolve minor conflicts in `drivers/resctrl/test_mpam_resctrl.c`; ] Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 24 +++++++++++++++++------- drivers/resctrl/test_mpam_resctrl.c | 4 ++-- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index f3b31687c12df..b8dbca07b6bd9 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -751,14 +751,14 @@ static u32 get_mba_granularity(struct mpam_props *cprops) * * Find the nearest percentage value to the upper bound of the selected band: */ -static u32 mbw_max_to_percent(u16 mbw_max, struct mpam_props *cprops) +static u32 fract16_to_percent(u16 fract, u8 wd) { - u32 val = mbw_max; + u32 val = fract; - val >>= 16 - cprops->bwa_wd; + val >>= 16 - wd; val += 1; val *= MAX_MBA_BW; - val = DIV_ROUND_CLOSEST(val, 1 << cprops->bwa_wd); + val = DIV_ROUND_CLOSEST(val, 1 << wd); return val; } @@ -773,18 +773,28 @@ static u32 mbw_max_to_percent(u16 mbw_max, struct mpam_props *cprops) * percentages) and over-commit (where the total of the converted * allocations is greater than expected). */ -static u16 percent_to_mbw_max(u8 pc, struct mpam_props *cprops) +static u16 percent_to_fract16(u8 pc, u8 wd) { u32 val = pc; - val <<= cprops->bwa_wd; + val <<= wd; val = DIV_ROUND_CLOSEST(val, MAX_MBA_BW); val = max(val, 1) - 1; - val <<= 16 - cprops->bwa_wd; + val <<= 16 - wd; return val; } +static u32 mbw_max_to_percent(u16 mbw_max, struct mpam_props *cprops) +{ + return fract16_to_percent(mbw_max, cprops->bwa_wd); +} + +static u16 percent_to_mbw_max(u8 pc, struct mpam_props *cprops) +{ + return percent_to_fract16(pc, cprops->bwa_wd); +} + static u32 get_mba_min(struct mpam_props *cprops) { if (!mba_class_use_mbw_max(cprops)) { diff --git a/drivers/resctrl/test_mpam_resctrl.c b/drivers/resctrl/test_mpam_resctrl.c index b93d6ad87e43f..71d8edf1f7d90 100644 --- a/drivers/resctrl/test_mpam_resctrl.c +++ b/drivers/resctrl/test_mpam_resctrl.c @@ -133,7 +133,7 @@ static void test_get_mba_granularity(struct kunit *test) KUNIT_EXPECT_EQ(test, ret, 1); /* DIV_ROUND_UP(100, 1 << 16)% = 1% */ } -static void test_mbw_max_to_percent(struct kunit *test) +static void test_fract16_to_percent(struct kunit *test) { const struct percent_value_case *param = test->param_value; struct percent_value_test_info res; @@ -298,7 +298,7 @@ static void test_percent_to_max_rounding(struct kunit *test) static struct kunit_case mpam_resctrl_test_cases[] = { KUNIT_CASE(test_get_mba_granularity), - KUNIT_CASE_PARAM(test_mbw_max_to_percent, test_percent_value_gen_params), + KUNIT_CASE_PARAM(test_fract16_to_percent, test_percent_value_gen_params), KUNIT_CASE_PARAM(test_percent_to_mbw_max, test_percent_value_gen_params), KUNIT_CASE_PARAM(test_mbw_max_to_percent_limits, test_all_bwa_wd_gen_params), KUNIT_CASE(test_percent_to_max_rounding), From 5a44ffb89939df9de385684db9384530172f3233 Mon Sep 17 00:00:00 2001 From: James Morse Date: Mon, 18 Nov 2024 18:45:50 +0000 Subject: [PATCH 233/311] NVIDIA: SAUCE: fs/resctrl: Group all the MBA specific properties in a separate struct BugLink: https://bugs.launchpad.net/bugs/2154527 struct resctrl_membw combines parameters that are related to the control value, and parameters that are specific to the MBA resource. To allow the control value parsing and management code to be re-used for other resources, it needs to be separated from the MBA resource. Add struct resctrl_mba that holds all the parameters that are specific to the MBA resource. Signed-off-by: James Morse (cherry picked from commit c1133462aa498d8b75e73b094eb91512d982e067 https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/x86/kernel/cpu/resctrl/core.c | 18 +++++++++--------- drivers/resctrl/mpam_resctrl.c | 4 ++-- fs/resctrl/ctrlmondata.c | 3 ++- fs/resctrl/rdtgroup.c | 18 +++++++++--------- include/linux/resctrl.h | 26 +++++++++++++++++--------- 5 files changed, 39 insertions(+), 30 deletions(-) diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index 7667cf7c4e945..ba3316a41141b 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -212,21 +212,21 @@ static __init bool __get_mem_config_intel(struct rdt_resource *r) hw_res->num_closid = edx.split.cos_max + 1; max_delay = eax.split.max_delay + 1; r->membw.max_bw = MAX_MBA_BW; - r->membw.arch_needs_linear = true; + r->mba.arch_needs_linear = true; if (ecx & MBA_IS_LINEAR) { - r->membw.delay_linear = true; + r->mba.delay_linear = true; r->membw.min_bw = MAX_MBA_BW - max_delay; r->membw.bw_gran = MAX_MBA_BW - max_delay; } else { if (!rdt_get_mb_table(r)) return false; - r->membw.arch_needs_linear = false; + r->mba.arch_needs_linear = false; } if (boot_cpu_has(X86_FEATURE_PER_THREAD_MBA)) - r->membw.throttle_mode = THREAD_THROTTLE_PER_THREAD; + r->mba.throttle_mode = THREAD_THROTTLE_PER_THREAD; else - r->membw.throttle_mode = THREAD_THROTTLE_MAX; + r->mba.throttle_mode = THREAD_THROTTLE_MAX; r->alloc_capable = true; @@ -249,14 +249,14 @@ static __init bool __rdt_get_mem_config_amd(struct rdt_resource *r) r->membw.max_bw = 1 << eax; /* AMD does not use delay */ - r->membw.delay_linear = false; - r->membw.arch_needs_linear = false; + r->mba.delay_linear = false; + r->mba.arch_needs_linear = false; /* * AMD does not use memory delay throttle model to control * the allocation like Intel does. */ - r->membw.throttle_mode = THREAD_THROTTLE_UNDEFINED; + r->mba.throttle_mode = THREAD_THROTTLE_UNDEFINED; r->membw.min_bw = 0; r->membw.bw_gran = 1; @@ -325,7 +325,7 @@ static void mba_wrmsr_amd(struct msr_param *m) */ static u32 delay_bw_map(unsigned long bw, struct rdt_resource *r) { - if (r->membw.delay_linear) + if (r->mba.delay_linear) return MAX_MBA_BW - bw; pr_warn_once("Non Linear delay-bw map not supported but queried\n"); diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index b8dbca07b6bd9..9d4eddcbecf35 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -1248,8 +1248,8 @@ static int mpam_resctrl_control_init(struct mpam_resctrl_res *res) r->schema_fmt = RESCTRL_SCHEMA_RANGE; r->ctrl_scope = RESCTRL_L3_CACHE; - r->membw.delay_linear = true; - r->membw.throttle_mode = THREAD_THROTTLE_UNDEFINED; + r->mba.delay_linear = true; + r->mba.throttle_mode = THREAD_THROTTLE_UNDEFINED; r->membw.min_bw = get_mba_min(cprops); r->membw.max_bw = MAX_MBA_BW; r->membw.bw_gran = get_mba_granularity(cprops); diff --git a/fs/resctrl/ctrlmondata.c b/fs/resctrl/ctrlmondata.c index 0c02451c687b2..1eac8f7dc07ac 100644 --- a/fs/resctrl/ctrlmondata.c +++ b/fs/resctrl/ctrlmondata.c @@ -48,7 +48,8 @@ static bool bw_validate(char *buf, u32 *data, struct rdt_resource *r) /* * Only linear delay values is supported for current Intel SKUs. */ - if (!r->membw.delay_linear && r->membw.arch_needs_linear) { + if (r->rid == RDT_RESOURCE_MBA && + !r->mba.delay_linear && r->mba.arch_needs_linear) { rdt_last_cmd_puts("No support for non-linear MB domains\n"); return false; } diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index af2cbab14497e..6cd75166efa85 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -1197,7 +1197,7 @@ static int rdt_delay_linear_show(struct kernfs_open_file *of, struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); struct rdt_resource *r = s->res; - seq_printf(seq, "%u\n", r->membw.delay_linear); + seq_printf(seq, "%u\n", r->mba.delay_linear); return 0; } @@ -1215,7 +1215,7 @@ static int rdt_thread_throttle_mode_show(struct kernfs_open_file *of, struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); struct rdt_resource *r = s->res; - switch (r->membw.throttle_mode) { + switch (r->mba.throttle_mode) { case THREAD_THROTTLE_PER_THREAD: seq_puts(seq, "per-thread\n"); return 0; @@ -1552,7 +1552,7 @@ bool is_mba_sc(struct rdt_resource *r) if (r->rid != RDT_RESOURCE_MBA) return false; - return r->membw.mba_sc; + return r->mba.mba_sc; } /* @@ -2174,13 +2174,13 @@ static void thread_throttle_mode_init(void) r_mba = resctrl_arch_get_resource(RDT_RESOURCE_MBA); if (r_mba->alloc_capable && - r_mba->membw.throttle_mode != THREAD_THROTTLE_UNDEFINED) - throttle_mode = r_mba->membw.throttle_mode; + r_mba->mba.throttle_mode != THREAD_THROTTLE_UNDEFINED) + throttle_mode = r_mba->mba.throttle_mode; r_smba = resctrl_arch_get_resource(RDT_RESOURCE_SMBA); if (r_smba->alloc_capable && - r_smba->membw.throttle_mode != THREAD_THROTTLE_UNDEFINED) - throttle_mode = r_smba->membw.throttle_mode; + r_smba->mba.throttle_mode != THREAD_THROTTLE_UNDEFINED) + throttle_mode = r_smba->mba.throttle_mode; if (throttle_mode == THREAD_THROTTLE_UNDEFINED) return; @@ -2488,7 +2488,7 @@ mongroup_create_dir(struct kernfs_node *parent_kn, struct rdtgroup *prgrp, static inline bool is_mba_linear(void) { - return resctrl_arch_get_resource(RDT_RESOURCE_MBA)->membw.delay_linear; + return resctrl_arch_get_resource(RDT_RESOURCE_MBA)->mba.delay_linear; } static int mba_sc_domain_allocate(struct rdt_resource *r, struct rdt_ctrl_domain *d) @@ -2550,7 +2550,7 @@ static int set_mba_sc(bool mba_sc) if (!supports_mba_mbps() || mba_sc == is_mba_sc(r)) return -EINVAL; - r->membw.mba_sc = mba_sc; + r->mba.mba_sc = mba_sc; rdtgroup_default.mba_mbps_event = mba_mbps_default_event; diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h index f7faa509ebd52..e77fefdcd0e2f 100644 --- a/include/linux/resctrl.h +++ b/include/linux/resctrl.h @@ -247,22 +247,28 @@ enum membw_throttle_mode { * @min_bw: Minimum memory bandwidth percentage user can request * @max_bw: Maximum memory bandwidth value, used as the reset value * @bw_gran: Granularity at which the memory bandwidth is allocated - * @delay_linear: True if memory B/W delay is in linear scale - * @arch_needs_linear: True if we can't configure non-linear resources - * @throttle_mode: Bandwidth throttling mode when threads request - * different memory bandwidths - * @mba_sc: True if MBA software controller(mba_sc) is enabled - * @mb_map: Mapping of memory B/W percentage to memory B/W delay */ struct resctrl_membw { u32 min_bw; u32 max_bw; u32 bw_gran; - u32 delay_linear; - bool arch_needs_linear; - enum membw_throttle_mode throttle_mode; +}; + +/** + * struct resctrl_mba - Resource properties that are specific to the MBA resource + * @mba_sc: True if MBA software controller(mba_sc) is enabled + * @mb_map: Mapping of memory B/W percentage to memory B/W delay + * @delay_linear: True if control is in linear scale + * @arch_needs_linear: True if we can't configure non-linear resources + * @throttle_mode: Mode when threads request different control values + */ +struct resctrl_mba { bool mba_sc; u32 *mb_map; + bool delay_linear; + bool arch_needs_linear; + enum membw_throttle_mode throttle_mode; + }; struct resctrl_schema; @@ -318,6 +324,7 @@ struct resctrl_mon { * @mon: Monitoring related data. * @ctrl_domains: RCU list of all control domains for this resource * @mon_domains: RCU list of all monitor domains for this resource + * @mba: Properties of the MBA resource * @name: Name to use in "schemata" file. * @schema_fmt: Which format string and parser is used for this schema. * @cdp_capable: Is the CDP feature available on this resource @@ -331,6 +338,7 @@ struct rdt_resource { struct resctrl_cache cache; struct resctrl_membw membw; struct resctrl_mon mon; + struct resctrl_mba mba; struct list_head ctrl_domains; struct list_head mon_domains; char *name; From b750e69206cca86da2b8cc06043f54723771aea5 Mon Sep 17 00:00:00 2001 From: James Morse Date: Tue, 10 Sep 2024 11:33:53 +0100 Subject: [PATCH 234/311] NVIDIA: SAUCE: fs/resctrl: Abstract duplicate domain test to a helper BugLink: https://bugs.launchpad.net/bugs/2154527 parse_cbm() and parse_bw() both test the staged config for an existing entry. These would indicate user-space has provided a schema with a duplicate domain entry. e.g: | L3:0=ffff;1=f00f;0=f00f If new parsers are added this duplicate domain test has to be duplicated. Move it to the caller. Signed-off-by: James Morse (cherry picked from commit 827c80b5ec1b14a0f3d77e12ad13a8fbbf499ccd https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) [fenghuay: - Resolve minor conflicts in `fs/resctrl/ctrlmondata.c`; ] Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- fs/resctrl/ctrlmondata.c | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/fs/resctrl/ctrlmondata.c b/fs/resctrl/ctrlmondata.c index 1eac8f7dc07ac..48ebc0f5bafbb 100644 --- a/fs/resctrl/ctrlmondata.c +++ b/fs/resctrl/ctrlmondata.c @@ -84,12 +84,6 @@ static int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s, u32 closid = data->closid; u32 bw_val; - cfg = &d->staged_config[s->conf_type]; - if (cfg->have_new_ctrl) { - rdt_last_cmd_printf("Duplicate domain %d\n", d->hdr.id); - return -EINVAL; - } - if (!bw_validate(data->buf, &bw_val, r)) return -EINVAL; @@ -98,6 +92,7 @@ static int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s, return 0; } + cfg = &d->staged_config[s->conf_type]; cfg->new_ctrl = bw_val; cfg->have_new_ctrl = true; @@ -165,12 +160,6 @@ static int parse_cbm(struct rdt_parse_data *data, struct resctrl_schema *s, u32 closid = data->closid; u32 cbm_val; - cfg = &d->staged_config[s->conf_type]; - if (cfg->have_new_ctrl) { - rdt_last_cmd_printf("Duplicate domain %d\n", d->hdr.id); - return -EINVAL; - } - /* * Cannot set up more than one pseudo-locked region in a cache * hierarchy. @@ -207,6 +196,7 @@ static int parse_cbm(struct rdt_parse_data *data, struct resctrl_schema *s, } } + cfg = &d->staged_config[s->conf_type]; cfg->new_ctrl = cbm_val; cfg->have_new_ctrl = true; @@ -264,13 +254,18 @@ static int parse_line(char *line, struct resctrl_schema *s, dom = strim(dom); list_for_each_entry(d, &r->ctrl_domains, hdr.list) { if (d->hdr.id == dom_id) { + cfg = &d->staged_config[t]; + if (cfg->have_new_ctrl) { + rdt_last_cmd_printf("Duplicate domain %d\n", d->hdr.id); + return -EINVAL; + } + data.buf = dom; data.closid = rdtgrp->closid; data.mode = rdtgrp->mode; if (parse_ctrlval(&data, s, d)) return -EINVAL; if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { - cfg = &d->staged_config[t]; /* * In pseudo-locking setup mode and just * parsed a valid CBM that should be From 00f07c25ccfae0b2b99e02b70715286a002de6af Mon Sep 17 00:00:00 2001 From: James Morse Date: Tue, 19 Nov 2024 15:02:03 +0000 Subject: [PATCH 235/311] NVIDIA: SAUCE: fs/resctrl: Move MBA supported check to parse_line() instead of parse_bw() BugLink: https://bugs.launchpad.net/bugs/2154527 MBA is only supported on platforms where the delay inserted by the control is linear. Resctrl checks the two properties provided by the arch code match each time it parses part of a new control value. This doesn't need to be done so frequently, and obscures changes to parse_bw() to abstract it for use with other control types. Move this check to the parse_line() caller so it only happens once. Signed-off-by: James Morse (cherry picked from commit 85be43b4b1214a6f88d5643a8973ec6808cec56c https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- fs/resctrl/ctrlmondata.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/fs/resctrl/ctrlmondata.c b/fs/resctrl/ctrlmondata.c index 48ebc0f5bafbb..ec9ea0f607191 100644 --- a/fs/resctrl/ctrlmondata.c +++ b/fs/resctrl/ctrlmondata.c @@ -45,15 +45,6 @@ static bool bw_validate(char *buf, u32 *data, struct rdt_resource *r) int ret; u32 bw; - /* - * Only linear delay values is supported for current Intel SKUs. - */ - if (r->rid == RDT_RESOURCE_MBA && - !r->mba.delay_linear && r->mba.arch_needs_linear) { - rdt_last_cmd_puts("No support for non-linear MB domains\n"); - return false; - } - ret = kstrtou32(buf, 10, &bw); if (ret) { rdt_last_cmd_printf("Invalid MB value %s\n", buf); @@ -242,6 +233,15 @@ static int parse_line(char *line, struct resctrl_schema *s, return -EINVAL; } + /* + * Only linear delay values is supported for current Intel SKUs. + */ + if (r->rid == RDT_RESOURCE_MBA && + !r->mba.delay_linear && r->mba.arch_needs_linear) { + rdt_last_cmd_puts("No support for non-linear MB domains\n"); + return -EINVAL; + } + next: if (!line || line[0] == '\0') return 0; From 8f613230a9a73c016921a6992395cd37dc12bb9c Mon Sep 17 00:00:00 2001 From: James Morse Date: Tue, 19 Nov 2024 15:55:45 +0000 Subject: [PATCH 236/311] NVIDIA: SAUCE: fs/resctrl: Rename resctrl_get_default_ctrl() to include resource BugLink: https://bugs.launchpad.net/bugs/2154527 resctrl_get_default_ctrl() is called by both the architecture code and filesystem code to return the default value for a control. This depends on the schema format. parse_bw() doesn't bother checking the bounds it is given if the resource is in use by mba_sc. This is because the values parsed from user-space are not the same as those the control should take. To make this disparity easier to work with, a second different copy of the schema format is needed, which would need a version of resctrl_get_default_ctrl(). This would let the resctrl change the schema format presented to user-space, provided it converts it to match what the architecture code expects. Rename resctrl_get_default_ctrl() to make it clear it returns the resource default. Signed-off-by: James Morse (cherry picked from commit a4ba73c6546aaf2eb6805ad910b27c55663843e0 https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) [fenghuay: - Resolve minor conflicts in `drivers/resctrl/mpam_resctrl.c`; ] Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/x86/kernel/cpu/resctrl/core.c | 2 +- arch/x86/kernel/cpu/resctrl/rdtgroup.c | 2 +- drivers/resctrl/mpam_resctrl.c | 10 +++++----- fs/resctrl/rdtgroup.c | 4 ++-- include/linux/resctrl.h | 13 ++++++++----- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index ba3316a41141b..244f0d2a93e7a 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -378,7 +378,7 @@ static void setup_default_ctrlval(struct rdt_resource *r, u32 *dc) * For Memory Allocation: Set b/w requested to 100% */ for (i = 0; i < hw_res->num_closid; i++, dc++) - *dc = resctrl_get_default_ctrl(r); + *dc = resctrl_get_resource_default_ctrl(r); } static void ctrl_domain_free(struct rdt_hw_ctrl_domain *hw_dom) diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index 8850264684405..8a017f1111028 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -253,7 +253,7 @@ void resctrl_arch_reset_all_ctrls(struct rdt_resource *r) hw_dom = resctrl_to_arch_ctrl_dom(d); for (i = 0; i < hw_res->num_closid; i++) - hw_dom->ctrl_val[i] = resctrl_get_default_ctrl(r); + hw_dom->ctrl_val[i] = resctrl_get_resource_default_ctrl(r); msr_param.dom = d; smp_call_function_any(&d->hdr.cpu_mask, rdt_ctrl_update, &msr_param, 1); } diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 9d4eddcbecf35..465cf460473d8 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -1241,7 +1241,7 @@ static int mpam_resctrl_control_init(struct mpam_resctrl_res *res) * we have configured the SMMU and GIC not to do this 'all the * bits' is the correct answer here. */ - r->cache.shareable_bits = resctrl_get_default_ctrl(r); + r->cache.shareable_bits = resctrl_get_resource_default_ctrl(r); r->alloc_capable = true; break; case RDT_RESOURCE_MBA: @@ -1399,7 +1399,7 @@ u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d, lockdep_assert_cpus_held(); if (!mpam_is_enabled()) - return resctrl_get_default_ctrl(r); + return resctrl_get_resource_default_ctrl(r); res = container_of(r, struct mpam_resctrl_res, resctrl_res); dom = container_of(d, struct mpam_resctrl_dom, resctrl_ctrl_dom); @@ -1428,12 +1428,12 @@ u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d, } fallthrough; default: - return resctrl_get_default_ctrl(r); + return resctrl_get_resource_default_ctrl(r); } if (!r->alloc_capable || partid >= resctrl_arch_get_num_closid(r) || !mpam_has_feature(configured_by, cfg)) - return resctrl_get_default_ctrl(r); + return resctrl_get_resource_default_ctrl(r); switch (configured_by) { case mpam_feat_cpor_part: @@ -1441,7 +1441,7 @@ u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d, case mpam_feat_mbw_max: return mbw_max_to_percent(cfg->mbw_max, cprops); default: - return resctrl_get_default_ctrl(r); + return resctrl_get_resource_default_ctrl(r); } } diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index 6cd75166efa85..85459a0fb540f 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -1006,7 +1006,7 @@ static int rdt_default_ctrl_show(struct kernfs_open_file *of, struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); struct rdt_resource *r = s->res; - seq_printf(seq, "%x\n", resctrl_get_default_ctrl(r)); + seq_printf(seq, "%x\n", resctrl_get_resource_default_ctrl(r)); return 0; } @@ -3642,7 +3642,7 @@ static void rdtgroup_init_mba(struct rdt_resource *r, u32 closid) } cfg = &d->staged_config[CDP_NONE]; - cfg->new_ctrl = resctrl_get_default_ctrl(r); + cfg->new_ctrl = resctrl_get_resource_default_ctrl(r); cfg->have_new_ctrl = true; } } diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h index e77fefdcd0e2f..7370111c51409 100644 --- a/include/linux/resctrl.h +++ b/include/linux/resctrl.h @@ -326,7 +326,10 @@ struct resctrl_mon { * @mon_domains: RCU list of all monitor domains for this resource * @mba: Properties of the MBA resource * @name: Name to use in "schemata" file. - * @schema_fmt: Which format string and parser is used for this schema. + * @schema_fmt: Which format control parameters should be in for this resource. + * @evt_list: List of monitoring events + * @mbm_cfg_mask: Bandwidth sources that can be tracked when bandwidth + * monitoring events can be configured. * @cdp_capable: Is the CDP feature available on this resource */ struct rdt_resource { @@ -405,11 +408,11 @@ struct resctrl_mon_config_info { void resctrl_arch_sync_cpu_closid_rmid(void *info); /** - * resctrl_get_default_ctrl() - Return the default control value for this - * resource. - * @r: The resource whose default control type is queried. + * resctrl_get_resource_default_ctrl() - Return the default control value for + * this resource. + * @r: The resource whose default control value is queried. */ -static inline u32 resctrl_get_default_ctrl(struct rdt_resource *r) +static inline u32 resctrl_get_resource_default_ctrl(struct rdt_resource *r) { switch (r->schema_fmt) { case RESCTRL_SCHEMA_BITMAP: From 387e29cbd8c31195ad8d1ad88e9cac3ccf4d458d Mon Sep 17 00:00:00 2001 From: James Morse Date: Wed, 20 Nov 2024 12:21:25 +0000 Subject: [PATCH 237/311] NVIDIA: SAUCE: fs/resctrl: Add a schema format to the schema, allowing it to be different BugLink: https://bugs.launchpad.net/bugs/2154527 parse_bw() doesn't bother checking the bounds it is given if the resource is in use by mba_sc. This is because the values parsed from user-space are not the same as those the control should take. To make this disparity easier to work with, a second different copy of the schema format is needed, which would need a version of resctrl_get_default_ctrl(). This would let the resctrl change the schema format presented to user-space, provided it converts it to match what the architecture code expects. Add a second schema format for use with mba_sc. The membw properties are copied and the schema version is used. When mba_sc is enabled the schema copy of these properties is modified. Signed-off-by: James Morse (cherry picked from commit 225d28eb849877c6b97dcdc466d8e1aa67978272 https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) [fenghuay: - Resolve minor conflicts in `fs/resctrl/ctrlmondata.c`; - Resolve minor conflicts in `include/linux/arm_mpam.h`; ] Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/x86/kernel/cpu/resctrl/ctrlmondata.c | 4 ++-- fs/resctrl/ctrlmondata.c | 14 ++++++------ fs/resctrl/rdtgroup.c | 26 +++++++++++++++++------ include/linux/arm_mpam.h | 4 +--- include/linux/resctrl.h | 24 ++++++++++++++++++++- 5 files changed, 52 insertions(+), 20 deletions(-) diff --git a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c index d539e56c2b1f0..91ce05256a004 100644 --- a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c +++ b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c @@ -20,9 +20,9 @@ #include "internal.h" -u32 resctrl_arch_round_bw(u32 val, const struct rdt_resource *r) +u32 resctrl_arch_round_bw(u32 val, const struct resctrl_schema *s) { - return roundup(val, (unsigned long)r->membw.bw_gran); + return roundup(val, (unsigned long)s->membw.bw_gran); } int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d, diff --git a/fs/resctrl/ctrlmondata.c b/fs/resctrl/ctrlmondata.c index ec9ea0f607191..1e51c4a01e785 100644 --- a/fs/resctrl/ctrlmondata.c +++ b/fs/resctrl/ctrlmondata.c @@ -40,7 +40,7 @@ typedef int (ctrlval_parser_t)(struct rdt_parse_data *data, * hardware. The allocated bandwidth percentage is rounded to the next * control step available on the hardware. */ -static bool bw_validate(char *buf, u32 *data, struct rdt_resource *r) +static bool bw_validate(char *buf, u32 *data, struct resctrl_schema *s) { int ret; u32 bw; @@ -52,18 +52,18 @@ static bool bw_validate(char *buf, u32 *data, struct rdt_resource *r) } /* Nothing else to do if software controller is enabled. */ - if (is_mba_sc(r)) { + if (is_mba_sc(s->res)) { *data = bw; return true; } - if (bw < r->membw.min_bw || bw > r->membw.max_bw) { + if (bw < s->membw.min_bw || bw > s->membw.max_bw) { rdt_last_cmd_printf("MB value %u out of range [%d,%d]\n", - bw, r->membw.min_bw, r->membw.max_bw); + bw, s->membw.min_bw, s->membw.max_bw); return false; } - *data = resctrl_arch_round_bw(bw, r); + *data = resctrl_arch_round_bw(bw, s); return true; } @@ -75,7 +75,7 @@ static int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s, u32 closid = data->closid; u32 bw_val; - if (!bw_validate(data->buf, &bw_val, r)) + if (!bw_validate(data->buf, &bw_val, s)) return -EINVAL; if (is_mba_sc(r)) { @@ -215,7 +215,7 @@ static int parse_line(char *line, struct resctrl_schema *s, /* Walking r->domains, ensure it can't race with cpuhp */ lockdep_assert_cpus_held(); - switch (r->schema_fmt) { + switch (s->schema_fmt) { case RESCTRL_SCHEMA_BITMAP: parse_ctrlval = &parse_cbm; break; diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index 85459a0fb540f..058de93843854 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -1004,9 +1004,8 @@ static int rdt_default_ctrl_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); - struct rdt_resource *r = s->res; - seq_printf(seq, "%x\n", resctrl_get_resource_default_ctrl(r)); + seq_printf(seq, "%x\n", resctrl_get_schema_default_ctrl(s)); return 0; } @@ -1147,9 +1146,8 @@ static int rdt_min_bw_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); - struct rdt_resource *r = s->res; - seq_printf(seq, "%u\n", r->membw.min_bw); + seq_printf(seq, "%u\n", s->membw.min_bw); return 0; } @@ -1185,9 +1183,8 @@ static int rdt_bw_gran_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); - struct rdt_resource *r = s->res; - seq_printf(seq, "%u\n", r->membw.bw_gran); + seq_printf(seq, "%u\n", s->membw.bw_gran); return 0; } @@ -2739,7 +2736,22 @@ static int schemata_list_add(struct rdt_resource *r, enum resctrl_conf_type type if (cl > max_name_width) max_name_width = cl; - switch (r->schema_fmt) { + s->schema_fmt = r->schema_fmt; + s->membw = r->membw; + + /* + * When mba_sc() is enabled the format used by user space is different + * to that expected by hardware. The conversion is done by + * update_mba_bw(). + */ + if (is_mba_sc(r)) { + s->schema_fmt = RESCTRL_SCHEMA_RANGE; + s->membw.min_bw = 0; + s->membw.max_bw = MBA_MAX_MBPS; + s->membw.bw_gran = 1; + } + + switch (s->schema_fmt) { case RESCTRL_SCHEMA_BITMAP: s->fmt_str = "%d=%x"; break; diff --git a/include/linux/arm_mpam.h b/include/linux/arm_mpam.h index b066d57e1a085..3aed6fe510151 100644 --- a/include/linux/arm_mpam.h +++ b/include/linux/arm_mpam.h @@ -79,10 +79,8 @@ static inline void resctrl_arch_enable_alloc(void) { } static inline void resctrl_arch_disable_alloc(void) { } struct resctrl_schema; - -struct rdt_resource; static inline u32 resctrl_arch_round_bw(u32 val, - const struct rdt_resource *r __always_unused) + const struct resctrl_schema *s __always_unused) { /* * Do nothing: for MPAM, resctrl_arch_update_one() has the necessary diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h index 7370111c51409..4d84f4b116481 100644 --- a/include/linux/resctrl.h +++ b/include/linux/resctrl.h @@ -362,9 +362,12 @@ struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l); * @list: Member of resctrl_schema_all. * @name: The name to use in the "schemata" file. * @fmt_str: Format string to show domain value. + * @schema_fmt: Which format string and parser is used for this schema. * @conf_type: Whether this schema is specific to code/data. * @res: The resource structure exported by the architecture to describe * the hardware that is configured by this schema. + * @membw The properties of the schema which may be different to the format + * that was specified by the resource, * @num_closid: The number of closid that can be used with this schema. When * features like CDP are enabled, this will be lower than the * hardware supports for the resource. @@ -373,8 +376,10 @@ struct resctrl_schema { struct list_head list; char name[8]; const char *fmt_str; + enum resctrl_schema_fmt schema_fmt; enum resctrl_conf_type conf_type; struct rdt_resource *res; + struct resctrl_membw membw; u32 num_closid; }; @@ -424,6 +429,23 @@ static inline u32 resctrl_get_resource_default_ctrl(struct rdt_resource *r) return WARN_ON_ONCE(1); } +/** + * resctrl_get_schema_default_ctrl() - Return the default control value for + * this schema. + * @s: The schema whose default control value is queried. + */ +static inline u32 resctrl_get_schema_default_ctrl(struct resctrl_schema *s) +{ + switch (s->schema_fmt) { + case RESCTRL_SCHEMA_BITMAP: + return resctrl_get_resource_default_ctrl(s->res); + case RESCTRL_SCHEMA_RANGE: + return s->membw.max_bw; + } + + return WARN_ON_ONCE(1); +} + /* The number of closid supported by this resource regardless of CDP */ u32 resctrl_arch_get_num_closid(struct rdt_resource *r); u32 resctrl_arch_system_num_rmid_idx(void); @@ -515,7 +537,7 @@ bool resctrl_arch_mbm_cntr_assign_enabled(struct rdt_resource *r); */ int resctrl_arch_mbm_cntr_assign_set(struct rdt_resource *r, bool enable); -u32 resctrl_arch_round_bw(u32 val, const struct rdt_resource *r); +u32 resctrl_arch_round_bw(u32 val, const struct resctrl_schema *s); /* * Update the ctrl_val and apply this config right now. From f30efaaabb7a3d062647b1cedfde501cabc2251c Mon Sep 17 00:00:00 2001 From: James Morse Date: Wed, 20 Nov 2024 15:15:54 +0000 Subject: [PATCH 238/311] NVIDIA: SAUCE: fs/resctrl: Add specific schema types for 'range' BugLink: https://bugs.launchpad.net/bugs/2154527 Resctrl allows the architecture code to specify the schema format for a control. Controls can either take a bitmap, or some kind of number. If user-space doesn't know what a control is by its name, it could be told the schema format. 'Some kind of number' isn't useful as the difference between a percentage and a value in MB/s affects how these would be programmed, even if resctrl's parsing code doesn't need to care. Add the types resctrl already has in addition to 'range'. This allows architectures to move over before 'range' is removed. These new schema formats are parsed the same, but will additionally affect which files are visible. Schema formats with a double underscore should not be considered portable between architectures, and are likely to be described to user-space as 'platform defined'. AMDs MBA resource is configured with an absolute bandwidth measured in multiples of one eighth of a GB per second. resctrl needs to be aware of this platform defined format to ensure the existing 'MB' files continue to be shown. Signed-off-by: James Morse (cherry picked from commit bb81e4805d5120058ec44f793780bdf1e775cd5a https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- fs/resctrl/ctrlmondata.c | 3 +++ fs/resctrl/rdtgroup.c | 3 +++ include/linux/resctrl.h | 12 ++++++++++++ 3 files changed, 18 insertions(+) diff --git a/fs/resctrl/ctrlmondata.c b/fs/resctrl/ctrlmondata.c index 1e51c4a01e785..ec925ce6c8773 100644 --- a/fs/resctrl/ctrlmondata.c +++ b/fs/resctrl/ctrlmondata.c @@ -220,6 +220,9 @@ static int parse_line(char *line, struct resctrl_schema *s, parse_ctrlval = &parse_cbm; break; case RESCTRL_SCHEMA_RANGE: + case RESCTRL_SCHEMA_PERCENT: + case RESCTRL_SCHEMA_MBPS: + case RESCTRL_SCHEMA__AMD_MBA: parse_ctrlval = &parse_bw; break; } diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index 058de93843854..e8be2a8765464 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -2756,6 +2756,9 @@ static int schemata_list_add(struct rdt_resource *r, enum resctrl_conf_type type s->fmt_str = "%d=%x"; break; case RESCTRL_SCHEMA_RANGE: + case RESCTRL_SCHEMA_PERCENT: + case RESCTRL_SCHEMA_MBPS: + case RESCTRL_SCHEMA__AMD_MBA: s->fmt_str = "%d=%u"; break; } diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h index 4d84f4b116481..b2bd51e6b8ba0 100644 --- a/include/linux/resctrl.h +++ b/include/linux/resctrl.h @@ -284,10 +284,16 @@ enum resctrl_scope { * enum resctrl_schema_fmt - The format user-space provides for a schema. * @RESCTRL_SCHEMA_BITMAP: The schema is a bitmap in hex. * @RESCTRL_SCHEMA_RANGE: The schema is a decimal number. + * @RESCTRL_SCHEMA_PERCENT: The schema is a percentage. + * @RESCTRL_SCHEMA_MBPS: The schema ia a MBps value. + * @RESCTRL_SCHEMA__AMD_MBA: The schema value is MBA for AMD platforms. */ enum resctrl_schema_fmt { RESCTRL_SCHEMA_BITMAP, RESCTRL_SCHEMA_RANGE, + RESCTRL_SCHEMA_PERCENT, + RESCTRL_SCHEMA_MBPS, + RESCTRL_SCHEMA__AMD_MBA, }; /** @@ -423,6 +429,9 @@ static inline u32 resctrl_get_resource_default_ctrl(struct rdt_resource *r) case RESCTRL_SCHEMA_BITMAP: return BIT_MASK(r->cache.cbm_len) - 1; case RESCTRL_SCHEMA_RANGE: + case RESCTRL_SCHEMA_PERCENT: + case RESCTRL_SCHEMA_MBPS: + case RESCTRL_SCHEMA__AMD_MBA: return r->membw.max_bw; } @@ -440,6 +449,9 @@ static inline u32 resctrl_get_schema_default_ctrl(struct resctrl_schema *s) case RESCTRL_SCHEMA_BITMAP: return resctrl_get_resource_default_ctrl(s->res); case RESCTRL_SCHEMA_RANGE: + case RESCTRL_SCHEMA_PERCENT: + case RESCTRL_SCHEMA_MBPS: + case RESCTRL_SCHEMA__AMD_MBA: return s->membw.max_bw; } From 5ad4b526cffff26d7e4b8a349bd05e9219f811ef Mon Sep 17 00:00:00 2001 From: James Morse Date: Wed, 20 Nov 2024 15:19:37 +0000 Subject: [PATCH 239/311] NVIDIA: SAUCE: x86/resctrl: Move over to specifying MBA control formats BugLink: https://bugs.launchpad.net/bugs/2154527 Resctrl specifies the schema format for MB and SMBA in rdt_resources_all[]. Intel platforms take a percentage for MB, AMD platforms take an absolute value which isn't MB/s. Currently these are both treated as a 'range'. Adding support for additional types of control shows that user-space needs to be told what the control formats are. Today users of resctrl must already know if their platform is Intel or AMD to know how the MB resource will behave. The MPAM support exposes new control types that take a 'percentage'. The Intel MB resource is also configured by a percentage, so should be able to expose this to user-space. Remove the static configuration for schema_fmt in rdt_resources_all[] and specify it with the other control properties in __get_mem_config_intel() or __get_mem_config_amd(). Signed-off-by: James Morse (cherry picked from commit 3323499e5df777ad2eb10be5c7dc29ae5358c93d https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) [fenghuay: - Resolve minor conflicts in `arch/x86/kernel/cpu/resctrl/core.c`; ] Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/x86/kernel/cpu/resctrl/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index 244f0d2a93e7a..00bc2e3ed0ff1 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -88,7 +88,6 @@ struct rdt_hw_resource rdt_resources_all[RDT_NUM_RESOURCES] = { .name = "MB", .ctrl_scope = RESCTRL_L3_CACHE, .ctrl_domains = ctrl_domain_init(RDT_RESOURCE_MBA), - .schema_fmt = RESCTRL_SCHEMA_RANGE, }, }, [RDT_RESOURCE_SMBA] = @@ -97,7 +96,6 @@ struct rdt_hw_resource rdt_resources_all[RDT_NUM_RESOURCES] = { .name = "SMBA", .ctrl_scope = RESCTRL_L3_CACHE, .ctrl_domains = ctrl_domain_init(RDT_RESOURCE_SMBA), - .schema_fmt = RESCTRL_SCHEMA_RANGE, }, }, [RDT_RESOURCE_PERF_PKG] = @@ -211,6 +209,7 @@ static __init bool __get_mem_config_intel(struct rdt_resource *r) cpuid_count(0x00000010, 3, &eax.full, &ebx, &ecx, &edx.full); hw_res->num_closid = edx.split.cos_max + 1; max_delay = eax.split.max_delay + 1; + r->schema_fmt = RESCTRL_SCHEMA_PERCENT; r->membw.max_bw = MAX_MBA_BW; r->mba.arch_needs_linear = true; if (ecx & MBA_IS_LINEAR) { @@ -246,6 +245,7 @@ static __init bool __rdt_get_mem_config_amd(struct rdt_resource *r) cpuid_count(0x80000020, subleaf, &eax, &ebx, &ecx, &edx); hw_res->num_closid = edx + 1; + r->schema_fmt = RESCTRL_SCHEMA__AMD_MBA; r->membw.max_bw = 1 << eax; /* AMD does not use delay */ From 53117901a946e7c9ee34ef2886d66a6afefe21cb Mon Sep 17 00:00:00 2001 From: James Morse Date: Wed, 20 Nov 2024 15:49:06 +0000 Subject: [PATCH 240/311] NVIDIA: SAUCE: fs/resctrl: Add additional files for percentage and bitmap controls BugLink: https://bugs.launchpad.net/bugs/2154527 MPAM has cache capacity controls that effectively take a percentage. Resctrl supports percentages, but the collection of files that are exposed to describe this control belong to the MB resource. To find the minimum granularity of the percentage cache capacity controls, user-space is expected to rad the banwdidth_gran file, and know this has nothing to do with bandwidth. The only problem here is the name of the file. Add duplicates of these properties with percentage and bitmap in the name. These will be exposed based on the schema format. The existing files must remain tied to the specific resources so that they remain visible to user-space. Using the same helpers ensures the values will always be the same regardless of the file used. These files are not exposed until the new RFTYPE schema flags are set on a resource 'fflags'. Signed-off-by: James Morse (cherry picked from commit a38c11612e84a927e5b6e2dccf765291a4d498fd https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) [fenghuay: - Resolve minor conflicts in `fs/resctrl/internal.h`; ] Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- fs/resctrl/internal.h | 6 ++++++ fs/resctrl/rdtgroup.c | 28 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/fs/resctrl/internal.h b/fs/resctrl/internal.h index 48af75b9dc855..ec7c968059291 100644 --- a/fs/resctrl/internal.h +++ b/fs/resctrl/internal.h @@ -251,6 +251,7 @@ struct rdtgroup { #define RFTYPE_TOP BIT(6) +/* files that are specific to a type of resource, e.g. throttle_mode */ #define RFTYPE_RES_CACHE BIT(8) #define RFTYPE_RES_MB BIT(9) @@ -261,6 +262,11 @@ struct rdtgroup { #define RFTYPE_RES_PERF_PKG BIT(12) +/* files that are specific to a type of control, e.g. percent_min */ +#define RFTYPE_SCHEMA_BITMAP BIT(13) +#define RFTYPE_SCHEMA_PERCENT BIT(14) +#define RFTYPE_SCHEMA_MBPS BIT(15) + #define RFTYPE_CTRL_INFO (RFTYPE_INFO | RFTYPE_CTRL) #define RFTYPE_MON_INFO (RFTYPE_INFO | RFTYPE_MON) diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index e8be2a8765464..55cb0cc1d27f3 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -1928,6 +1928,13 @@ static struct rftype res_common_files[] = { .kf_ops = &rdtgroup_kf_single_ops, .seq_show = resctrl_num_mbm_cntrs_show, }, + { + .name = "bitmap_mask", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_default_ctrl_show, + .fflags = RFTYPE_CTRL_INFO | RFTYPE_SCHEMA_BITMAP, + }, { .name = "min_cbm_bits", .mode = 0444, @@ -1935,6 +1942,13 @@ static struct rftype res_common_files[] = { .seq_show = rdt_min_cbm_bits_show, .fflags = RFTYPE_CTRL_INFO | RFTYPE_RES_CACHE, }, + { + .name = "bitmaps_min_bits", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_min_cbm_bits_show, + .fflags = RFTYPE_CTRL_INFO | RFTYPE_SCHEMA_BITMAP, + }, { .name = "shareable_bits", .mode = 0444, @@ -1956,6 +1970,13 @@ static struct rftype res_common_files[] = { .seq_show = rdt_min_bw_show, .fflags = RFTYPE_CTRL_INFO | RFTYPE_RES_MB, }, + { + .name = "percent_min", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_min_bw_show, + .fflags = RFTYPE_CTRL_INFO | RFTYPE_SCHEMA_PERCENT, + }, { .name = "bandwidth_gran", .mode = 0444, @@ -1963,6 +1984,13 @@ static struct rftype res_common_files[] = { .seq_show = rdt_bw_gran_show, .fflags = RFTYPE_CTRL_INFO | RFTYPE_RES_MB, }, + { + .name = "percent_gran", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_bw_gran_show, + .fflags = RFTYPE_CTRL_INFO | RFTYPE_SCHEMA_PERCENT, + }, { .name = "delay_linear", .mode = 0444, From 3de4f669f687060a2dfb9fced88efd61571d1ee7 Mon Sep 17 00:00:00 2001 From: James Morse Date: Wed, 20 Nov 2024 16:55:39 +0000 Subject: [PATCH 241/311] NVIDIA: SAUCE: fs/resctrl: Add fflags_from_schema() for files based on schema format BugLink: https://bugs.launchpad.net/bugs/2154527 MPAM has cache capacity controls that effectively take a percentage. Resctrl supports percentages, but the collection of files that are exposed to describe this control belong to the MB resource. New files have been added that are selected based on the schema format. Apply the flags to enable these files based on the schema format. Add a new fflags_from_schema() that is used for controls. Signed-off-by: James Morse (cherry picked from commit db005687c69b453ea63389314ba791dc9df18e1a https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) [fenghuay: - Resolve minor conflicts in `fs/resctrl/rdtgroup.c`; ] Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- fs/resctrl/rdtgroup.c | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index 55cb0cc1d27f3..b4476ed278c17 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -2433,7 +2433,35 @@ static unsigned long fflags_from_resource(struct rdt_resource *r) return RFTYPE_RES_PERF_PKG; } - return WARN_ON_ONCE(1); + return 0; +} + +static u32 fflags_from_schema(struct resctrl_schema *s) +{ + struct rdt_resource *r = s->res; + u32 fflags = 0; + + /* Some resources are configured purely from their rid */ + fflags |= fflags_from_resource(r); + if (fflags) + return fflags; + + switch (s->schema_fmt) { + case RESCTRL_SCHEMA_BITMAP: + fflags |= RFTYPE_SCHEMA_BITMAP; + break; + case RESCTRL_SCHEMA_PERCENT: + fflags |= RFTYPE_SCHEMA_PERCENT; + break; + case RESCTRL_SCHEMA_MBPS: + fflags |= RFTYPE_SCHEMA_MBPS; + break; + case RESCTRL_SCHEMA__AMD_MBA: + /* No standard files are exposed */ + break; + } + + return fflags; } static int rdtgroup_create_info_dir(struct kernfs_node *parent_kn) @@ -2456,7 +2484,7 @@ static int rdtgroup_create_info_dir(struct kernfs_node *parent_kn) /* loop over enabled controls, these are all alloc_capable */ list_for_each_entry(s, &resctrl_schema_all, list) { r = s->res; - fflags = fflags_from_resource(r) | RFTYPE_CTRL_INFO; + fflags = fflags_from_schema(s) | RFTYPE_CTRL_INFO; ret = rdtgroup_mkdir_info_resdir(s, s->name, fflags); if (ret) goto out_destroy; From 44d2efc73922febe2037329a3e603b2800457e50 Mon Sep 17 00:00:00 2001 From: James Morse Date: Tue, 10 Sep 2024 18:13:37 +0100 Subject: [PATCH 242/311] NVIDIA: SAUCE: fs/resctrl: Expose the schema format to user-space BugLink: https://bugs.launchpad.net/bugs/2154527 If more schemas are added to resctrl, user-space needs to know how to configure them. To allow user-space to configure schema it doesn't know about, it would be helpful to tell user-space the format, e.g. percentage. Add a file under info that describes the schema format. Percentages and 'mbps' are implicitly decimal, bitmaps are expected to be in hex. Signed-off-by: James Morse (forward ported from commit f0ae6915fc22fa0a7affd46f61e0fe4a7673df06 https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) [fenghuay: - Resolve minor conflicts in `fs/resctrl/rdtgroup.c`; - Add RESCTRL_SCHEMA_RANGE in resctrl_schema_format_show(); ] Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- fs/resctrl/rdtgroup.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index b4476ed278c17..f7848f20599a1 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -1687,6 +1687,33 @@ static int mbm_local_bytes_config_show(struct kernfs_open_file *of, return 0; } +static int resctrl_schema_format_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); + + switch (s->schema_fmt) { + case RESCTRL_SCHEMA_BITMAP: + seq_puts(seq, "bitmap\n"); + break; + case RESCTRL_SCHEMA_PERCENT: + seq_puts(seq, "percentage\n"); + break; + case RESCTRL_SCHEMA_MBPS: + seq_puts(seq, "mbps\n"); + break; + case RESCTRL_SCHEMA_RANGE: + seq_puts(seq, "range\n"); + break; + /* The way these schema behave isn't discoverable from resctrl */ + case RESCTRL_SCHEMA__AMD_MBA: + seq_puts(seq, "platform\n"); + break; + } + + return 0; +} + static void mbm_config_write_domain(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 evtid, u32 val) { @@ -2143,6 +2170,14 @@ static struct rftype res_common_files[] = { .seq_show = rdtgroup_closid_show, .fflags = RFTYPE_CTRL_BASE | RFTYPE_DEBUG, }, + { + .name = "schema_format", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = resctrl_schema_format_show, + .fflags = RFTYPE_CTRL_INFO, + }, + }; static int rdtgroup_add_files(struct kernfs_node *kn, unsigned long fflags) From 14a1d9ce3126f51a2fa1d6bafe46a57d3563a092 Mon Sep 17 00:00:00 2001 From: James Morse Date: Tue, 19 Nov 2024 12:35:13 +0000 Subject: [PATCH 243/311] NVIDIA: SAUCE: fs/resctrl: Add L2 and L3 'MAX' resource schema BugLink: https://bugs.launchpad.net/bugs/2154527 MPAM can have both cache portion and cache capacity controls on any cache that supports MPAM. Cache portion bitmaps can be exposed via resctrl if they are implemented on L2 or L3. The cache capacity controls can not be used to isolate portions, which is in implicit in the L2 or L3 bitmap provided by user-space. These controls need to be configured with something more like a percentage. Add the resource enum entries for these two resources. No additional resctrl code is needed because the architecture code will specify this resource takes a 'percentage', re-using the support previously used only for the MB resource. Signed-off-by: James Morse (cherry picked from commit 2e9f961c2cad4bdcc49f1a598ee131725129337f https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) [fenghuay: - Resolve minor conflicts in `include/linux/resctrl.h`; ] Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- include/linux/resctrl.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h index b2bd51e6b8ba0..010897e1b91cc 100644 --- a/include/linux/resctrl.h +++ b/include/linux/resctrl.h @@ -54,6 +54,8 @@ enum resctrl_res_level { RDT_RESOURCE_MBA, RDT_RESOURCE_SMBA, RDT_RESOURCE_PERF_PKG, + RDT_RESOURCE_L3_MAX, + RDT_RESOURCE_L2_MAX, /* Must be the last */ RDT_NUM_RESOURCES, From b02e81e59f07613e8626d60ca744c781f2effa9b Mon Sep 17 00:00:00 2001 From: James Morse Date: Tue, 19 Nov 2024 11:51:03 +0000 Subject: [PATCH 244/311] NVIDIA: SAUCE: arm_mpam: resctrl: Add the glue code to convert to/from cmax BugLink: https://bugs.launchpad.net/bugs/2154527 MPAM's maximum cache-capacity controls take a fixed point fraction format. Instead of dumping this on user-space, convert it to a percentage. User-space using resctrl already knows how to handle percentages. Signed-off-by: James Morse (cherry picked from commit 10caa1269560b1006811725d9564f0e859a53e2e https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) [fenghuay: - Resolve minor conflicts in `drivers/resctrl/mpam_resctrl.c`; ] Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_resctrl.c | 67 ++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 7 deletions(-) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 465cf460473d8..2b7a673fca211 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -673,6 +673,13 @@ static bool cache_has_usable_cpor(struct mpam_class *class) return class->props.cpbm_wd <= 32; } +static bool cache_has_usable_cmax(struct mpam_class *class) +{ + struct mpam_props *cprops = &class->props; + + return mpam_has_feature(mpam_feat_cmax_cmax, cprops); +} + static bool mba_class_use_mbw_max(struct mpam_props *cprops) { return (mpam_has_feature(mpam_feat_mbw_max, cprops) && @@ -795,6 +802,11 @@ static u16 percent_to_mbw_max(u8 pc, struct mpam_props *cprops) return percent_to_fract16(pc, cprops->bwa_wd); } +static u16 percent_to_cmax(u8 pc, struct mpam_props *cprops) +{ + return percent_to_fract16(pc, cprops->cmax_wd); +} + static u32 get_mba_min(struct mpam_props *cprops) { if (!mba_class_use_mbw_max(cprops)) { @@ -952,6 +964,7 @@ static bool traffic_matches_l3(struct mpam_class *class) /* Test whether we can export MPAM_CLASS_CACHE:{2,3}? */ static void mpam_resctrl_pick_caches(void) { + bool has_cpor, has_cmax; struct mpam_class *class; struct mpam_resctrl_res *res; @@ -970,7 +983,9 @@ static void mpam_resctrl_pick_caches(void) continue; } - if (!cache_has_usable_cpor(class)) { + has_cpor = cache_has_usable_cpor(class); + has_cmax = cache_has_usable_cmax(class); + if (!has_cpor && !has_cmax) { pr_debug("class %u cache misses CPOR\n", class->level); continue; } @@ -981,12 +996,22 @@ static void mpam_resctrl_pick_caches(void) cpumask_pr_args(cpu_possible_mask)); continue; } - - if (class->level == 2) - res = &mpam_resctrl_controls[RDT_RESOURCE_L2]; - else - res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; - res->class = class; + if (has_cpor) { + pr_debug("pick_caches: Class has CPOR\n"); + if (class->level == 2) + res = &mpam_resctrl_controls[RDT_RESOURCE_L2]; + else + res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; + res->class = class; + } + if (has_cmax) { + pr_debug("pick_caches: Class has CMAX\n"); + if (class->level == 2) + res = &mpam_resctrl_controls[RDT_RESOURCE_L2_MAX]; + else + res = &mpam_resctrl_controls[RDT_RESOURCE_L3_MAX]; + res->class = class; + } } } @@ -1243,6 +1268,23 @@ static int mpam_resctrl_control_init(struct mpam_resctrl_res *res) */ r->cache.shareable_bits = resctrl_get_resource_default_ctrl(r); r->alloc_capable = true; + break; + case RDT_RESOURCE_L2_MAX: + case RDT_RESOURCE_L3_MAX: + r->alloc_capable = true; + r->schema_fmt = RESCTRL_SCHEMA_PERCENT; + r->membw.min_bw = max(100 / (1 << cprops->cmax_wd), 1); + r->membw.bw_gran = max(100 / (1 << cprops->cmax_wd), 1); + r->membw.max_bw = 100; + + if (r->rid == RDT_RESOURCE_L2_MAX) { + r->name = "L2_MAX"; + r->ctrl_scope = RESCTRL_L2_CACHE; + } else { + r->name = "L3_MAX"; + r->ctrl_scope = RESCTRL_L3_CACHE; + } + break; case RDT_RESOURCE_MBA: r->schema_fmt = RESCTRL_SCHEMA_RANGE; @@ -1421,6 +1463,10 @@ u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d, case RDT_RESOURCE_L3: configured_by = mpam_feat_cpor_part; break; + case RDT_RESOURCE_L2_MAX: + case RDT_RESOURCE_L3_MAX: + configured_by = mpam_feat_cmax_cmax; + break; case RDT_RESOURCE_MBA: if (mpam_has_feature(mpam_feat_mbw_max, cprops)) { configured_by = mpam_feat_mbw_max; @@ -1438,6 +1484,8 @@ u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d, switch (configured_by) { case mpam_feat_cpor_part: return cfg->cpbm; + case mpam_feat_cmax_cmax: + return fract16_to_percent(cfg->cmax, cprops->cmax_wd); case mpam_feat_mbw_max: return mbw_max_to_percent(cfg->mbw_max, cprops); default: @@ -1490,6 +1538,11 @@ int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d, cfg.cpbm = cfg_val; mpam_set_feature(mpam_feat_cpor_part, &cfg); break; + case RDT_RESOURCE_L2_MAX: + case RDT_RESOURCE_L3_MAX: + cfg.cmax = percent_to_cmax(cfg_val, cprops); + mpam_set_feature(mpam_feat_cmax_cmax, &cfg); + break; case RDT_RESOURCE_MBA: if (mpam_has_feature(mpam_feat_mbw_max, cprops)) { cfg.mbw_max = percent_to_mbw_max(cfg_val, cprops); From dd732284f76e38493016117dd110aae4e6c335cf Mon Sep 17 00:00:00 2001 From: Shanker Donthineni Date: Wed, 4 Mar 2026 11:53:32 -0600 Subject: [PATCH 245/311] NVIDIA: SAUCE: resctrl/mpam: reset RIS by applying explicit default config BugLink: https://bugs.launchpad.net/bugs/2154527 Reset an RIS by building a default mpam_config and applying it via mpam_reprogram_ris_partid(), like any other config. - mpam_init_reset_cfg(): set features and default values only for controls supported by the RIS (cpor_part, mbw_part, mbw_max, mbw_prop, cmax_cmax, cmax_cmin). Use full masks for CPBM/MBW_PBM and MPAMCFG_* defaults for MBW_MAX, CMAX, CMIN. - mpam_reprogram_ris_partid(): apply cfg for all supported controls (no separate reset path). Signed-off-by: Shanker Donthineni (forward ported from commit e0b6de09b2a78f7aa12400ee756e5e6564118578 https://github.com/NVIDIA/NV-Kernels 24.04_linux-nvidia-6.17-next) [fenghuay: Since upstream code changes, a few changes in previous commit e0b6de09 are irrelevant or are merged in upstream already. Still keep the commit message to keep the history but add the following change log: - reset_cpbm and reset_mbw_pbm are not used. no need to define them; - Resolve minor conflicts in `drivers/resctrl/mpam_devices.c`; - mpam_init_reset_cfg() has been removed from upstream. Empty reset_cfg is used in mpam_reprogram_ris_partid(&reset_cfg) to reset feature values; - Remove changes of fract16_to_percent() and percent_to_fract16() since they don't take wd and don't match definition of fixed-point fractional format defined in MPAM spec. ] Signed-off-by: Fenghua Yu Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_devices.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 26726b29b5991..168a59dada809 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -2629,6 +2629,8 @@ static void mpam_reset_component_cfg(struct mpam_component *comp) comp->cfg[i].mbw_pbm = GENMASK(cprops->mbw_pbm_bits - 1, 0); if (cprops->bwa_wd) comp->cfg[i].mbw_max = GENMASK(15, 16 - cprops->bwa_wd); + if (cprops->cmax_wd) + comp->cfg[i].cmax = MPAMCFG_CMAX_CMAX; } } From 879f7e336d90b8df4ce4bc1927ef32073b3e198e Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Fri, 5 Jun 2026 22:21:24 +0000 Subject: [PATCH 246/311] NVIDIA: SAUCE: arm_mpam: Fix MPAMCFG_MBW_PBM register setting BugLink: https://bugs.launchpad.net/bugs/2154527 MPAMCFG_MBW_PBM is written from cfg if cfg has the MBW partition feature. It is reset when cfg does not have the MBW partition feature. But the register handling is reversed. This may cause an incorrect register setting. For example, during an MPAM reset, reset_cfg is empty (no MBW partition feature set), and cfg->mbw_pbm is 0. Instead of resetting MPAMCFG_MBW_PBM to all 1's, the current logic will set it to cfg->mbw_pbm, which is 0. Fix the issue by swapping the if/else branches. Fixes: a1cb6577f575 ("arm_mpam: Reset when feature configuration bit unset") Reported-by: Matt Ochs Signed-off-by: Fenghua Yu (cherry picked from https://lore.kernel.org/lkml/20260607050925.252475-1-fenghuay@nvidia.com/) Acked-by: Matthew R. Ochs Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- drivers/resctrl/mpam_devices.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 168a59dada809..ab6af4aecd6d1 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -1570,9 +1570,9 @@ static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid, if (mpam_has_feature(mpam_feat_mbw_part, rprops)) { if (mpam_has_feature(mpam_feat_mbw_part, cfg)) - mpam_reset_msc_bitmap(msc, MPAMCFG_MBW_PBM, rprops->mbw_pbm_bits); - else mpam_write_partsel_reg(msc, MBW_PBM, cfg->mbw_pbm); + else + mpam_reset_msc_bitmap(msc, MPAMCFG_MBW_PBM, rprops->mbw_pbm_bits); } if (mpam_has_feature(mpam_feat_mbw_min, rprops)) { From 4af1741cd8852af146e38bcd23899128800255c1 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 9 Jun 2026 11:12:01 +0100 Subject: [PATCH 247/311] arm64: cputype: Add C1-Ultra definitions BugLink: https://bugs.launchpad.net/bugs/2156557 Add cputype definitions for C1-Ultra. These will be used for errata detection in subsequent patches. These values can be found in the C1-Ultra TRM: https://developer.arm.com/documentation/108014/0100/ ... in section A.5.1 ("MIDR_EL1, Main ID Register"). Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Will Deacon Signed-off-by: Will Deacon (cherry picked from commit 60349e64a6c65f9f0aa118af711b3c7e137f07ff linux-next) Signed-off-by: Matthew R. Ochs Acked-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/arm64/include/asm/cputype.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 08860d482e600..70beef0712424 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -97,6 +97,7 @@ #define ARM_CPU_PART_CORTEX_X925 0xD85 #define ARM_CPU_PART_CORTEX_A725 0xD87 #define ARM_CPU_PART_CORTEX_A720AE 0xD89 +#define ARM_CPU_PART_C1_ULTRA 0xD8C #define ARM_CPU_PART_NEOVERSE_N3 0xD8E #define APM_CPU_PART_XGENE 0x000 @@ -188,6 +189,7 @@ #define MIDR_CORTEX_X925 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X925) #define MIDR_CORTEX_A725 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A725) #define MIDR_CORTEX_A720AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A720AE) +#define MIDR_C1_ULTRA MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_ULTRA) #define MIDR_NEOVERSE_N3 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N3) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) From 6f512ce379d0dc738b7b4a11c32b907d74d6ab65 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 9 Jun 2026 11:12:02 +0100 Subject: [PATCH 248/311] arm64: cputype: Add C1-Premium definitions BugLink: https://bugs.launchpad.net/bugs/2156557 Add cputype definitions for C1-Premium. These will be used for errata detection in subsequent patches. These values can be found in the C1-Premium TRM: https://developer.arm.com/documentation/109416/0100/ ... in section A.5.1 ("MIDR_EL1, Main ID Register"). Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Will Deacon Signed-off-by: Will Deacon (backported from commit d28413bfc5a255957241f1df5d7fd0c2cd74fe18 linux-next) [mochs: Minor context adjustment due to absent definitions] Signed-off-by: Matthew R. Ochs Acked-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- arch/arm64/include/asm/cputype.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 70beef0712424..45f96ca01cea5 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -99,6 +99,7 @@ #define ARM_CPU_PART_CORTEX_A720AE 0xD89 #define ARM_CPU_PART_C1_ULTRA 0xD8C #define ARM_CPU_PART_NEOVERSE_N3 0xD8E +#define ARM_CPU_PART_C1_PREMIUM 0xD90 #define APM_CPU_PART_XGENE 0x000 #define APM_CPU_VAR_POTENZA 0x00 @@ -191,6 +192,7 @@ #define MIDR_CORTEX_A720AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A720AE) #define MIDR_C1_ULTRA MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_ULTRA) #define MIDR_NEOVERSE_N3 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N3) +#define MIDR_C1_PREMIUM MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PREMIUM) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) From 4768e0942f54a6f4f2200ead41fff14ede6ce5f7 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 9 Jun 2026 11:12:03 +0100 Subject: [PATCH 249/311] arm64: errata: Mitigate TLBI errata on various Arm CPUs BugLink: https://bugs.launchpad.net/bugs/2156557 A number of CPUs developed by Arm suffer from errata whereby a broadcast TLBI;DSB sequence may complete before the global observation of writes which are translated by an affected TLB entry. These errata ONLY affect the completion of memory accesses which have been translated by an invalidated TLB entry, and these errata DO NOT affect the actual invalidation of TLB entries. TLB entries are removed correctly. This issue has been assigned CVE ID CVE-2025-10263. To mitigate this issue, Arm recommends that software follows any affected TLBI;DSB sequence with an additional TLBI;DSB, which will ensure that all memory write effects affected by the first TLBI have been globally observed. The additional TLBI can use any operation that is broadcast to affected CPUs, and the additional DSB can use any option that is sufficient to complete the additional TLBI. The ARM64_WORKAROUND_REPEAT_TLBI workaround is sufficient to mitigate the issue. Enable this workaround for affected CPUs, and update the silicon errata documentation accordingly. Note that due to the manner in which Arm develops IP and tracks errata, some CPUs share a common erratum number. Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Will Deacon Signed-off-by: Will Deacon (backported from commit cfd391e74134db664feb499d43af286380b10ba8 linux-next) [mochs: Minor context adjustment due to absent definitions] Signed-off-by: Matthew R. Ochs Acked-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- Documentation/arch/arm64/silicon-errata.rst | 42 +++++++++++++++++++++ arch/arm64/Kconfig | 36 ++++++++++++++++++ arch/arm64/kernel/cpu_errata.c | 32 +++++++++++++++- 3 files changed, 108 insertions(+), 2 deletions(-) diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index 65ed6ea33751f..c853204585751 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -128,16 +128,28 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A76 | #3324349 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A76 | #4193800 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A76AE | #4193801 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #1491015 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #1508412 | ARM64_ERRATUM_1508412 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #3324348 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A77 | #4193798 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A78 | #3324344 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A78 | #4193791 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A78AE | #4193793 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A78C | #3324346,3324347| ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A78C | #4193794 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #2119858 | ARM64_ERRATUM_2119858 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #2054223 | ARM64_ERRATUM_2054223 | @@ -146,6 +158,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #3324338 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A710 | #4193788 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A715 | #2645198 | ARM64_ERRATUM_2645198 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A715 | #3456084 | ARM64_ERRATUM_3194386 | @@ -158,20 +172,32 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X1 | #3324344 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X1 | #4193791 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X1C | #3324346 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X1C | #4193792 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #2119858 | ARM64_ERRATUM_2119858 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #2224489 | ARM64_ERRATUM_2224489 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #3324338 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X2 | #4193788 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X3 | #3324335 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X3 | #4193786 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X4 | #3194386 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X4 | #4118414 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X925 | #3324334 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X925 | #4193781 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1188873,1418040| ARM64_ERRATUM_1418040 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1349291 | N/A | @@ -182,6 +208,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #3324349 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-N1 | #4193800 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #2139208 | ARM64_ERRATUM_2139208 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #2067961 | ARM64_ERRATUM_2067961 | @@ -190,18 +218,32 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #3324339 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-N2 | #4193789 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N3 | #3456111 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V1 | #1619801 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V1 | #3324341 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V1 | #4193790 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V2 | #3324336 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V2 | #4193787 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V3 | #3312417 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V3 | #4193784 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V3AE | #3312417 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V3AE | #4193784 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | C1-Premium | #4193780 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | C1-Ultra | #4193780 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-500 | #841119,826419 | ARM_SMMU_MMU_500_CPRE_ERRATA| | | | #562869,1047329 | | +----------------+-----------------+-----------------+-----------------------------+ diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 241659f285a86..4fbb90d6bbb24 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1176,6 +1176,42 @@ config ARM64_ERRATUM_4311569 If unsure, say Y. +config ARM64_ERRATUM_4118414 + bool "Cortex-*/Neoverse-*/C1-*: Completion of affected memory accesses might not be guaranteed by completion of a TLBI" + default y + select ARM64_WORKAROUND_REPEAT_TLBI + help + This option adds a workaround for the following errata: + + * ARM C1-Premium erratum 4193780 + * ARM C1-Ultra erratum 4193780 + * ARM Cortex-A76 erratum 4193800 + * ARM Cortex-A76AE erratum 4193801 + * ARM Cortex-A77 erratum 4193798 + * ARM Cortex-A78 erratum 4193791 + * ARM Cortex-A78AE erratum 4193793 + * ARM Cortex-A78C erratum 4193794 + * ARM Cortex-A710 erratum 4193788 + * ARM Cortex-X1 erratum 4193791 + * ARM Cortex-X1C erratum 4193792 + * ARM Cortex-X2 erratum 4193788 + * ARM Cortex-X3 erratum 4193786 + * ARM Cortex-X4 erratum 4118414 + * ARM Cortex-X925 erratum 4193781 + * ARM Neoverse-N1 erratum 4193800 + * ARM Neoverse-N2 erratum 4193789 + * ARM Neoverse-V1 erratum 4193790 + * ARM Neoverse-V2 erratum 4193787 + * ARM Neoverse-V3 erratum 4193784 + * ARM Neoverse-V3AE erratum 4193784 + + On affected cores, some memory accesses might not be completed by + broadcast TLB invalidation. + + This issue is also known as CVE-2025-10263. + + If unsure, say Y. + config CAVIUM_ERRATUM_22375 bool "Cavium erratum 22375, 24313" default y diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 5c0ab6bfd44a6..1ba9b876ca1e0 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -339,7 +339,35 @@ static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = { ERRATA_MIDR_RANGE(MIDR_CORTEX_A510, 0, 0, 1, 1), }, #endif - {}, +#ifdef CONFIG_ARM64_ERRATUM_4118414 + { + ERRATA_MIDR_RANGE_LIST(((const struct midr_range[]) { + MIDR_ALL_VERSIONS(MIDR_C1_PREMIUM), + MIDR_ALL_VERSIONS(MIDR_C1_ULTRA), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A76), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A76AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A77), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X2), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X3), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X4), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X925), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3AE), + {} + })), + }, +#endif + {} }; #endif @@ -675,7 +703,7 @@ const struct arm64_cpu_capabilities arm64_errata[] = { #endif #ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI { - .desc = "Qualcomm erratum 1009, or ARM erratum 1286807, 2441009", + .desc = "Broken broadcast TLBI completion", .capability = ARM64_WORKAROUND_REPEAT_TLBI, .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, .matches = cpucap_multi_entry_cap_matches, From 67d694878d98367b7bd437acf1aca6889c66d168 Mon Sep 17 00:00:00 2001 From: Shanker Donthineni Date: Tue, 9 Jun 2026 18:40:44 -0500 Subject: [PATCH 250/311] arm64: errata: Mitigate TLBI errata on NVIDIA Olympus CPU BugLink: https://bugs.launchpad.net/bugs/2156557 NVIDIA Olympus cores are affected by the TLBI completion issue tracked as CVE-2025-10263. The existing ARM64_ERRATUM_4118414 handling already uses ARM64_WORKAROUND_REPEAT_TLBI to issue an additional broadcast TLBI;DSB sequence and ensure affected memory write effects are globally observed. Add MIDR_NVIDIA_OLYMPUS to the repeat-TLBI match list so the same mitigation is enabled on affected Olympus systems. Also document the NVIDIA Olympus erratum in the arm64 silicon errata table and list it in the Kconfig help text. Signed-off-by: Shanker Donthineni Cc: Catalin Marinas Cc: Will Deacon Cc: Mark Rutland Acked-by: Mark Rutland Signed-off-by: Will Deacon (cherry picked from commit ec7216f92e4ebd485b1c6dc6aa3f6064b71a5768 linux-next) Signed-off-by: Matthew R. Ochs Acked-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- Documentation/arch/arm64/silicon-errata.rst | 2 ++ arch/arm64/Kconfig | 3 ++- arch/arm64/kernel/cpu_errata.c | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index c853204585751..fa8b12e268712 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -290,6 +290,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | Carmel Core | N/A | NVIDIA_CARMEL_CNP_ERRATUM | +----------------+-----------------+-----------------+-----------------------------+ +| NVIDIA | Olympus core | T410-OLY-1029 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | T241 GICv3/4.x | T241-FABRIC-4 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | T241 MPAM | T241-MPAM-1 | N/A | diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 4fbb90d6bbb24..480eb78619d16 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1177,7 +1177,7 @@ config ARM64_ERRATUM_4311569 If unsure, say Y. config ARM64_ERRATUM_4118414 - bool "Cortex-*/Neoverse-*/C1-*: Completion of affected memory accesses might not be guaranteed by completion of a TLBI" + bool "Various: Completion of affected memory accesses might not be guaranteed by completion of a TLBI" default y select ARM64_WORKAROUND_REPEAT_TLBI help @@ -1204,6 +1204,7 @@ config ARM64_ERRATUM_4118414 * ARM Neoverse-V2 erratum 4193787 * ARM Neoverse-V3 erratum 4193784 * ARM Neoverse-V3AE erratum 4193784 + * NVIDIA Olympus erratum T410-OLY-1029 On affected cores, some memory accesses might not be completed by broadcast TLB invalidation. diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 1ba9b876ca1e0..0b39556c88349 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -363,6 +363,7 @@ static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = { MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3AE), + MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS), {} })), }, From 02d2439621b4a020e8f5b9536938989d464607a6 Mon Sep 17 00:00:00 2001 From: "Matthew R. Ochs" Date: Thu, 11 Jun 2026 15:01:45 -0700 Subject: [PATCH 251/311] NVIDIA: [Config] Enable ARM64_ERRATUM_4118414 CVE-Enable ARM64_ERRATUM_4118414 to mitigate 2025-10263 on NVIDIA platforms. BugLink: https://bugs.launchpad.net/bugs/2156557 Signed-off-by: Matthew R. Ochs Acked-by: Nirmoy Das Acked-by: Carol L Soto Acked-by: Jamie Nguyen Signed-off-by: Brad Figg --- debian.nvidia/config/annotations | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index 42242c734487b..9012bf8df51d0 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -27,6 +27,9 @@ CONFIG_ARM64_ERRATUM_2224489 note<'Required for Grace enablem CONFIG_ARM64_ERRATUM_2253138 policy<{'arm64': 'y'}> CONFIG_ARM64_ERRATUM_2253138 note<'Required for Grace enablement'> +CONFIG_ARM64_ERRATUM_4118414 policy<{'arm64': 'y'}> +CONFIG_ARM64_ERRATUM_4118414 note<'Required for Grace and Vera enablement'> + CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE policy<{'arm64': 'y'}> CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE note<'Required for Grace enablement'> From 85589797ef534b3356a336faee39f9f12575112c Mon Sep 17 00:00:00 2001 From: "Matthew R. Ochs" Date: Tue, 26 May 2026 08:20:21 -0700 Subject: [PATCH 252/311] fuse: back uncached readdir buffers with pages BugLink: https://bugs.launchpad.net/bugs/2156632 Commit dabb90391028 ("fuse: increase readdir buffer size") changed fuse_readdir_uncached() to size its temporary buffer from ctx->count. This is useful for overlayfs and other in-kernel callers that use INT_MAX to indicate an unlimited directory read. The larger buffer is currently supplied as a kvec output argument. For virtiofs, kvec arguments are copied through req->argbuf, which is allocated with kmalloc(..., GFP_ATOMIC). A large uncached readdir buffer can therefore require a multi-megabyte contiguous atomic allocation before the request is queued. Avoid the large bounce-buffer allocation by backing uncached readdir output with pages and setting out_pages. Transports such as virtiofs can then pass the pages as scatter-gather entries instead of copying the output through argbuf. Map the pages with vm_map_ram() only while parsing the returned dirents. The existing parser can then continue to use a linear kernel mapping. [SzM: separate allocation of pages into a helper function] Fixes: dabb90391028 ("fuse: increase readdir buffer size") Cc: stable@vger.kernel.org Signed-off-by: Matthew R. Ochs Signed-off-by: Miklos Szeredi (cherry picked from commit 2fcb1dd15faba5657b825cc5d54423251c70e79a linux-next) Signed-off-by: Matthew R. Ochs Acked-by: Jamie Nguyen Acked-by: Carol L Soto Signed-off-by: Brad Figg --- fs/fuse/readdir.c | 85 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 67 insertions(+), 18 deletions(-) diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index aae657fd56c0e..57a4d6254bc98 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -12,6 +12,7 @@ #include #include #include +#include static bool fuse_use_readdirplus(struct inode *dir, struct dir_context *ctx) { @@ -335,6 +336,43 @@ static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file, return 0; } +static struct page **fuse_readdir_alloc_buf(struct fuse_args_pages *ap, size_t *bufsize) +{ + unsigned int i, nr_alloc, nr_pages = DIV_ROUND_UP(*bufsize, PAGE_SIZE); + struct page **pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL); + + if (!pages) + return NULL; + + nr_alloc = alloc_pages_bulk(GFP_KERNEL, nr_pages, pages); + if (!nr_alloc) + goto free_array; + + if (nr_alloc < nr_pages) { + nr_pages = nr_alloc; + *bufsize = (size_t) nr_pages << PAGE_SHIFT; + } + + ap->folios = fuse_folios_alloc(nr_pages, GFP_KERNEL, &ap->descs); + if (!ap->folios) + goto release_pages; + + for (i = 0; i < nr_pages; i++) { + ap->folios[i] = page_folio(pages[i]); + ap->descs[i].length = min_t(size_t, *bufsize - (size_t)i * PAGE_SIZE, PAGE_SIZE); + } + ap->num_folios = nr_pages; + ap->args.out_pages = true; + + return pages; + +release_pages: + release_pages(pages, nr_pages); +free_array: + kfree(pages); + return NULL; +} + static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) { int plus; @@ -343,18 +381,16 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) struct fuse_mount *fm = get_fuse_mount(inode); struct fuse_conn *fc = fm->fc; struct fuse_io_args ia = {}; - struct fuse_args *args = &ia.ap.args; + struct fuse_args_pages *ap = &ia.ap; void *buf; size_t bufsize = clamp((unsigned int) ctx->count, PAGE_SIZE, fc->max_pages << PAGE_SHIFT); u64 attr_version = 0, evict_ctr = 0; bool locked; + struct page **pages = fuse_readdir_alloc_buf(ap, &bufsize); - buf = kvmalloc(bufsize, GFP_KERNEL); - if (!buf) + if (!pages) return -ENOMEM; - args->out_args[0].value = buf; - plus = fuse_use_readdirplus(inode, ctx); if (plus) { attr_version = fuse_get_attr_version(fm->fc); @@ -364,24 +400,37 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) fuse_read_args_fill(&ia, file, ctx->pos, bufsize, FUSE_READDIR); } locked = fuse_lock_inode(inode); - res = fuse_simple_request(fm, args); + res = fuse_simple_request(fm, &ap->args); fuse_unlock_inode(inode, locked); - if (res >= 0) { - if (!res) { - struct fuse_file *ff = file->private_data; - - if (ff->open_flags & FOPEN_CACHE_DIR) - fuse_readdir_cache_end(file, ctx->pos); - } else if (plus) { - res = parse_dirplusfile(buf, res, file, ctx, attr_version, - evict_ctr); - } else { + if (res < 0) + goto out; + + if (!res) { + struct fuse_file *ff = file->private_data; + + if (ff->open_flags & FOPEN_CACHE_DIR) + fuse_readdir_cache_end(file, ctx->pos); + goto out; + } + + buf = vm_map_ram(pages, ap->num_folios, -1); + if (!buf) { + res = -ENOMEM; + } else { + if (plus) + res = parse_dirplusfile(buf, res, file, ctx, attr_version, evict_ctr); + else res = parse_dirfile(buf, res, file, ctx); - } + + vm_unmap_ram(buf, ap->num_folios); } +out: + kfree(ap->folios); + release_pages(pages, ap->num_folios); + kfree(pages); - kvfree(buf); fuse_invalidate_atime(inode); + return res; } From 5449a2e16a9fa8dda4d58101cab2910adc1dfbdc Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Wed, 20 May 2026 10:03:18 -0700 Subject: [PATCH 253/311] iommu/arm-smmu-v3: Add arm_smmu_kdump_adopt_strtab() for kdump BugLink: https://bugs.launchpad.net/bugs/2156531 When transitioning to a kdump kernel, the primary kernel might have crashed while endpoint devices were actively bus-mastering DMA. Currently, the SMMU driver aggressively resets the hardware during probe by clearing CR0_SMMUEN and setting the Global Bypass Attribute (GBPA) to ABORT. In a kdump scenario, this aggressive reset is highly destructive: a) If GBPA is set to ABORT, in-flight DMA will be aborted, generating fatal PCIe AER or SErrors that may panic the kdump kernel b) If GBPA is set to BYPASS, in-flight DMA targeting some IOVAs will bypass the SMMU and corrupt the physical memory at those 1:1 mapped IOVAs. To safely absorb in-flight DMAs, a kdump kernel will have to leave SMMUEN=1 intact and avoid modifying STRTAB_BASE, allowing HW to continue translating in-flight DMAs reusing the crashed kernel's page tables until the endpoint device drivers probe and quiesce their respective hardware. However, the ARM SMMUv3 architecture specification states that updating the SMMU_STRTAB_BASE register while SMMUEN == 1 is UNPREDICTABLE or ignored. This leaves a kdump kernel no choice but to adopt the stream table from the crashed kernel. Introduce ARM_SMMU_OPT_KDUMP_ADOPT and adopt functions memremapping all the stream tables extracted from STRTAB_BASE and STRTAB_BASE_CFG. Note that the adoption of the crashed kernel's stream table follows certain strict rules, since the old stream table might be compromised. Thus, apply some basic validations against the values read from the registers. If tests fail, it means the stream table cannot be trusted, so toss it entirely. To avoid OOM due to a potentially corrupted stream table, the memremap for l2 tables is done on the kdump kernel's demand. The new option will be set in a following change. Fixes: b63b3439b856 ("iommu/arm-smmu-v3: Abort all transactions if SMMU is enabled in kdump kernel") Cc: stable@vger.kernel.org # v6.12+ Suggested-by: Jason Gunthorpe Signed-off-by: Nicolin Chen (backported from https://lore.kernel.org/linux-iommu/cover.1779265413.git.nicolinc@nvidia.com/#t) Signed-off-by: Jamie Nguyen Acked-by: Seth Forshee Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 254 +++++++++++++++++++- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 1 + 2 files changed, 252 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index d86e888d300b9..0b0abd9d4133b 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1780,16 +1780,70 @@ static void arm_smmu_init_initial_stes(struct arm_smmu_ste *strtab, } } +static int arm_smmu_kdump_adopt_l2_strtab(struct arm_smmu_device *smmu, u32 sid, + phys_addr_t base, u32 span, + struct arm_smmu_strtab_l2 **l2table) +{ + struct arm_smmu_strtab_l2 *table; + size_t size; + + /* + * Only a coherent SMMU is supported at this moment. For a non-coherent + * SMMU that wants to support ARM_SMMU_OPT_KDUMP_ADOPT, try MEMREMAP_WC. + */ + if (WARN_ON(!(smmu->features & ARM_SMMU_FEAT_COHERENCY))) + return -EOPNOTSUPP; + + /* + * Retest the span in case the L1 descriptor has been overwritten since + * the adopt. Reject this master's insert; panic or SMMU-disable would + * either lose the vmcore or cascade aborts. Do not try to fix it, as it + * would break all other SIDs in the same bus (PCI case). The corruption + * blast radius is already bounded to that bus range. + */ + if (span != STRTAB_SPLIT + 1) { + dev_err(smmu->dev, + "kdump: L1[%u] span %u changed since adopt (was %u)\n", + arm_smmu_strtab_l1_idx(sid), span, STRTAB_SPLIT + 1); + return -EINVAL; + } + + size = (1UL << (span - 1)) * sizeof(struct arm_smmu_ste); + + table = devm_memremap(smmu->dev, base, size, MEMREMAP_WB); + if (IS_ERR(table)) { + dev_err(smmu->dev, + "kdump: failed to adopt l2 stream table for SID %u\n", + sid); + return PTR_ERR(table); + } + + *l2table = table; + return 0; +} + static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) { dma_addr_t l2ptr_dma; struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; struct arm_smmu_strtab_l2 **l2table; + u32 l1_idx = arm_smmu_strtab_l1_idx(sid); - l2table = &cfg->l2.l2ptrs[arm_smmu_strtab_l1_idx(sid)]; + l2table = &cfg->l2.l2ptrs[l1_idx]; if (*l2table) return 0; + /* Deferred adoption of the crashed kernel's L2 table */ + if (smmu->options & ARM_SMMU_OPT_KDUMP_ADOPT) { + u64 l2ptr = le64_to_cpu(cfg->l2.l1tab[l1_idx].l2ptr); + phys_addr_t base = l2ptr & STRTAB_L1_DESC_L2PTR_MASK; + u32 span = FIELD_GET(STRTAB_L1_DESC_SPAN, l2ptr); + + if (span && base) + return arm_smmu_kdump_adopt_l2_strtab(smmu, sid, base, + span, l2table); + } + *l2table = dmam_alloc_coherent(smmu->dev, sizeof(**l2table), &l2ptr_dma, GFP_KERNEL); if (!*l2table) { @@ -1801,8 +1855,7 @@ static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) arm_smmu_init_initial_stes((*l2table)->stes, ARRAY_SIZE((*l2table)->stes)); - arm_smmu_write_strtab_l1_desc(&cfg->l2.l1tab[arm_smmu_strtab_l1_idx(sid)], - l2ptr_dma); + arm_smmu_write_strtab_l1_desc(&cfg->l2.l1tab[l1_idx], l2ptr_dma); return 0; } @@ -3939,10 +3992,204 @@ static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) return 0; } +static int arm_smmu_kdump_adopt_strtab_2lvl(struct arm_smmu_device *smmu, + u32 cfg_reg, phys_addr_t base) +{ + u32 log2size = FIELD_GET(STRTAB_BASE_CFG_LOG2SIZE, cfg_reg); + u32 split = FIELD_GET(STRTAB_BASE_CFG_SPLIT, cfg_reg); + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + u32 num_l1_ents; + size_t size; + int i; + + /* + * Only a coherent SMMU is supported at this moment. For a non-coherent + * SMMU that wants to support ARM_SMMU_OPT_KDUMP_ADOPT, try MEMREMAP_WC. + */ + if (WARN_ON(!(smmu->features & ARM_SMMU_FEAT_COHERENCY))) + return -EOPNOTSUPP; + + if (log2size < split || log2size > smmu->sid_bits) { + dev_err(smmu->dev, "kdump: log2size %u out of range [%u, %u]\n", + log2size, split, smmu->sid_bits); + return -EINVAL; + } + if (split != STRTAB_SPLIT) { + dev_err(smmu->dev, + "kdump: unsupported STRTAB_SPLIT %u (expected %u)\n", + split, STRTAB_SPLIT); + return -EINVAL; + } + + num_l1_ents = 1U << (log2size - split); + if (num_l1_ents > STRTAB_MAX_L1_ENTRIES) { + dev_err(smmu->dev, "kdump: l1 entries %u exceeds max %u\n", + num_l1_ents, STRTAB_MAX_L1_ENTRIES); + return -EINVAL; + } + + cfg->l2.num_l1_ents = num_l1_ents; + + size = num_l1_ents * sizeof(struct arm_smmu_strtab_l1); + cfg->l2.l1tab = memremap(base, size, MEMREMAP_WB); + if (!cfg->l2.l1tab) + return -ENOMEM; + + cfg->l2.l2ptrs = + kcalloc(num_l1_ents, sizeof(*cfg->l2.l2ptrs), GFP_KERNEL); + if (!cfg->l2.l2ptrs) + return -ENOMEM; + + for (i = 0; i < num_l1_ents; i++) { + u64 l2ptr = le64_to_cpu(cfg->l2.l1tab[i].l2ptr); + phys_addr_t l2_base = l2ptr & STRTAB_L1_DESC_L2PTR_MASK; + u32 span = FIELD_GET(STRTAB_L1_DESC_SPAN, l2ptr); + + if (!span || !l2_base) + continue; + + if (span != STRTAB_SPLIT + 1) { + dev_err(smmu->dev, + "kdump: L1[%u] unsupported span %u (vs %u)\n", + i, span, STRTAB_SPLIT + 1); + return -EINVAL; + } + + /* + * If the crashed kernel's l1 descriptors are deeply corrupted, + * blindly memremapping every l2 table here could lead to OOM. + * + * Defer the l2 memremap to arm_smmu_init_l2_strtab(), so peak + * memory is bounded by the kdump kernel's actual demand. + */ + } + + return 0; +} + +static int arm_smmu_kdump_adopt_strtab_linear(struct arm_smmu_device *smmu, + u32 cfg_reg, phys_addr_t base) +{ + u32 log2size = FIELD_GET(STRTAB_BASE_CFG_LOG2SIZE, cfg_reg); + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + unsigned int max_log2size; + size_t size; + + /* + * Only a coherent SMMU is supported at this moment. For a non-coherent + * SMMU that wants to support ARM_SMMU_OPT_KDUMP_ADOPT, try MEMREMAP_WC. + */ + if (WARN_ON(!(smmu->features & ARM_SMMU_FEAT_COHERENCY))) + return -EOPNOTSUPP; + + /* Cap the size at what the kdump kernel itself would have allocated */ + if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) + max_log2size = + ilog2(STRTAB_MAX_L1_ENTRIES * STRTAB_NUM_L2_STES); + else + max_log2size = smmu->sid_bits; + + /* cfg->linear.num_ents is unsigned int, so cap log2size at 31 */ + max_log2size = min(max_log2size, 31U); + if (log2size > max_log2size) { + dev_err(smmu->dev, "kdump: unsupported log2size %u (> %u)\n", + log2size, max_log2size); + return -EINVAL; + } + + /* + * We might end up with a num_ents != sid_bits, which is fine. In the + * ARM_SMMU_OPT_KDUMP_ADOPT case, arm_smmu_write_strtab() is bypassed. + */ + cfg->linear.num_ents = 1U << log2size; + + size = cfg->linear.num_ents * sizeof(struct arm_smmu_ste); + cfg->linear.table = memremap(base, size, MEMREMAP_WB); + if (!cfg->linear.table) + return -ENOMEM; + return 0; +} + +static void arm_smmu_kdump_adopt_cleanup(void *data) +{ + struct arm_smmu_device *smmu = data; + u32 cfg_reg = readl_relaxed(smmu->base + ARM_SMMU_STRTAB_BASE_CFG); + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + u32 fmt = FIELD_GET(STRTAB_BASE_CFG_FMT, cfg_reg); + + if (fmt == STRTAB_BASE_CFG_FMT_2LVL) { + kfree(cfg->l2.l2ptrs); + if (cfg->l2.l1tab) + memunmap(cfg->l2.l1tab); + } else if (fmt == STRTAB_BASE_CFG_FMT_LINEAR) { + if (cfg->linear.table) + memunmap(cfg->linear.table); + } +} + +static int arm_smmu_kdump_adopt_strtab(struct arm_smmu_device *smmu) +{ + u32 cfg_reg = readl_relaxed(smmu->base + ARM_SMMU_STRTAB_BASE_CFG); + u64 base_reg = readq_relaxed(smmu->base + ARM_SMMU_STRTAB_BASE); + u32 fmt = FIELD_GET(STRTAB_BASE_CFG_FMT, cfg_reg); + phys_addr_t base = base_reg & STRTAB_BASE_ADDR_MASK; + int ret; + + dev_info(smmu->dev, "kdump: adopting crashed kernel's stream table\n"); + + if (fmt == STRTAB_BASE_CFG_FMT_2LVL) { + /* + * Both kernels run on the same hardware, so it's impossible for + * kdump kernel to see the support for linear stream table only. + */ + if (WARN_ON(!(smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB))) + ret = -EINVAL; + else + ret = arm_smmu_kdump_adopt_strtab_2lvl(smmu, cfg_reg, + base); + } else if (fmt == STRTAB_BASE_CFG_FMT_LINEAR) { + /* + * In case that the old kernel for some reason used the linear + * format, enforce the same format to match the adopted table. + */ + ret = arm_smmu_kdump_adopt_strtab_linear(smmu, cfg_reg, base); + if (!ret) + smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB; + } else { + dev_err(smmu->dev, "kdump: invalid STRTAB format %u\n", fmt); + ret = -EINVAL; + } + + if (ret) { + arm_smmu_kdump_adopt_cleanup(smmu); + goto err; + } + + ret = devm_add_action_or_reset(smmu->dev, arm_smmu_kdump_adopt_cleanup, + smmu); + /* devm_add_action_or_reset ran the cleanup upon failure */ + if (ret) { + dev_warn(smmu->dev, "kdump: failed to set up cleanup action\n"); + goto err; + } + + return 0; + +err: + dev_warn(smmu->dev, "kdump: falling back to full reset\n"); + memset(&smmu->strtab_cfg, 0, sizeof(smmu->strtab_cfg)); + smmu->options &= ~ARM_SMMU_OPT_KDUMP_ADOPT; + return ret; +} + static int arm_smmu_init_strtab(struct arm_smmu_device *smmu) { int ret; + if ((smmu->options & ARM_SMMU_OPT_KDUMP_ADOPT) && + !arm_smmu_kdump_adopt_strtab(smmu)) + goto out; + if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) ret = arm_smmu_init_strtab_2lvl(smmu); else @@ -3950,6 +4197,7 @@ static int arm_smmu_init_strtab(struct arm_smmu_device *smmu) if (ret) return ret; +out: ida_init(&smmu->vmid_map); return 0; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 3c6d65d36164f..cc25d0b3d3e8d 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -774,6 +774,7 @@ struct arm_smmu_device { #define ARM_SMMU_OPT_MSIPOLL (1 << 2) #define ARM_SMMU_OPT_CMDQ_FORCE_SYNC (1 << 3) #define ARM_SMMU_OPT_TEGRA241_CMDQV (1 << 4) +#define ARM_SMMU_OPT_KDUMP_ADOPT (1 << 5) u32 options; struct arm_smmu_cmdq cmdq; From db0f093ae77d7366aaa51fefc1df7305bda711f2 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Wed, 20 May 2026 10:03:19 -0700 Subject: [PATCH 254/311] iommu/arm-smmu-v3: Implement is_attach_deferred() for kdump BugLink: https://bugs.launchpad.net/bugs/2156531 Though the kdump kernel adopts the crashed kernel's stream table, the iommu core will still try to attach each probed device to a default domain, which overwrites the adopted STE and breaks in-flight DMA from that device. Implement an is_attach_deferred() callback to prevent this. For each device that has STE.V=1 and STE.Cfg!=Abort in the adopted table, defer the default domain attachment, until the device driver explicitly requests it. Fixes: b63b3439b856 ("iommu/arm-smmu-v3: Abort all transactions if SMMU is enabled in kdump kernel") Cc: stable@vger.kernel.org # v6.12+ Reviewed-by: Kevin Tian Reviewed-by: Jason Gunthorpe Signed-off-by: Nicolin Chen (backported from https://lore.kernel.org/linux-iommu/cover.1779265413.git.nicolinc@nvidia.com/#t) [jamien: Resolve context conflict around arm_smmu_remove_master() due to the different surrounding arm-smmu-v3 code in this tree.] Signed-off-by: Jamie Nguyen Acked-by: Seth Forshee Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 24 +++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 0b0abd9d4133b..8cb6096a0a339 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3634,6 +3634,29 @@ static void arm_smmu_remove_master(struct arm_smmu_master *master) kfree(master->streams); } +static bool arm_smmu_is_attach_deferred(struct device *dev) +{ + struct arm_smmu_master *master = dev_iommu_priv_get(dev); + struct arm_smmu_device *smmu = master->smmu; + int i; + + if (!(smmu->options & ARM_SMMU_OPT_KDUMP_ADOPT)) + return false; + + for (i = 0; i < master->num_streams; i++) { + struct arm_smmu_ste *ste = + arm_smmu_get_step_for_sid(smmu, master->streams[i].id); + u64 ent0 = le64_to_cpu(ste->data[0]); + + /* Defer only when there might be in-flight DMAs */ + if ((ent0 & STRTAB_STE_0_V) && + FIELD_GET(STRTAB_STE_0_CFG, ent0) != STRTAB_STE_0_CFG_ABORT) + return true; + } + + return false; +} + static struct iommu_device *arm_smmu_probe_device(struct device *dev) { int ret; @@ -3813,6 +3836,7 @@ static const struct iommu_ops arm_smmu_ops = { .hw_info = arm_smmu_hw_info, .domain_alloc_sva = arm_smmu_sva_domain_alloc, .domain_alloc_paging_flags = arm_smmu_domain_alloc_paging_flags, + .is_attach_deferred = arm_smmu_is_attach_deferred, .probe_device = arm_smmu_probe_device, .release_device = arm_smmu_release_device, .device_group = arm_smmu_device_group, From f29a3e8326f47b5d97a6281fae2c06334af4ea88 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Wed, 20 May 2026 10:03:20 -0700 Subject: [PATCH 255/311] iommu/arm-smmu-v3: Do not enable EVTQ/PRIQ interrupts in kdump kernel BugLink: https://bugs.launchpad.net/bugs/2156531 In kdump cases, the crashed kernel's CDs and page tables can be corrupted, which could trigger event spamming. Also, we cannot serve page requests. Skip the IRQ setup for EVTQ/PRIQ in arm_smmu_setup_irqs(). Skip their IRQ handler registration in unique-IRQ and combined-IRQ cases. Fixes: b63b3439b856 ("iommu/arm-smmu-v3: Abort all transactions if SMMU is enabled in kdump kernel") Cc: stable@vger.kernel.org # v6.12+ Reviewed-by: Kevin Tian Signed-off-by: Nicolin Chen (backported from https://lore.kernel.org/linux-iommu/cover.1779265413.git.nicolinc@nvidia.com/#t) Signed-off-by: Jamie Nguyen Acked-by: Seth Forshee Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 58 ++++++++++++++------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 8cb6096a0a339..798aa5f698ef6 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2204,7 +2204,11 @@ static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev) static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev) { - arm_smmu_gerror_handler(irq, dev); + irqreturn_t ret = arm_smmu_gerror_handler(irq, dev); + + /* In kdump, EVTQ/PRIQ are disabled and there is no thread to wake */ + if (is_kdump_kernel()) + return ret; return IRQ_WAKE_THREAD; } @@ -4346,6 +4350,21 @@ static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) arm_smmu_setup_msis(smmu); /* Request interrupt lines */ + irq = smmu->gerr_irq; + if (irq) { + ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler, + 0, "arm-smmu-v3-gerror", smmu); + if (ret < 0) + dev_warn(smmu->dev, "failed to enable gerror irq\n"); + } else { + dev_warn(smmu->dev, + "no gerr irq - errors will not be reported!\n"); + } + + /* No EVTQ/PRIQ interrupts in kdump -- queues are disabled */ + if (is_kdump_kernel()) + return; + irq = smmu->evtq.q.irq; if (irq) { ret = devm_request_threaded_irq(smmu->dev, irq, NULL, @@ -4358,16 +4377,6 @@ static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n"); } - irq = smmu->gerr_irq; - if (irq) { - ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler, - 0, "arm-smmu-v3-gerror", smmu); - if (ret < 0) - dev_warn(smmu->dev, "failed to enable gerror irq\n"); - } else { - dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n"); - } - if (smmu->features & ARM_SMMU_FEAT_PRI) { irq = smmu->priq.q.irq; if (irq) { @@ -4388,7 +4397,7 @@ static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) { int ret, irq; - u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN; + u32 irqen_flags = IRQ_CTRL_GERROR_IRQEN; /* Disable IRQs first */ ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL, @@ -4403,19 +4412,30 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) /* * Cavium ThunderX2 implementation doesn't support unique irq * lines. Use a single irq line for all the SMMUv3 interrupts. + * + * In kdump, EVTQ/PRIQ are disabled, so no threaded handling. */ - ret = devm_request_threaded_irq(smmu->dev, irq, - arm_smmu_combined_irq_handler, - arm_smmu_combined_irq_thread, - IRQF_ONESHOT, - "arm-smmu-v3-combined-irq", smmu); + if (is_kdump_kernel()) + ret = devm_request_irq(smmu->dev, irq, + arm_smmu_combined_irq_handler, 0, + "arm-smmu-v3-combined-irq", + smmu); + else + ret = devm_request_threaded_irq( + smmu->dev, irq, arm_smmu_combined_irq_handler, + arm_smmu_combined_irq_thread, IRQF_ONESHOT, + "arm-smmu-v3-combined-irq", smmu); if (ret < 0) dev_warn(smmu->dev, "failed to enable combined irq\n"); } else arm_smmu_setup_unique_irqs(smmu); - if (smmu->features & ARM_SMMU_FEAT_PRI) - irqen_flags |= IRQ_CTRL_PRIQ_IRQEN; + /* No EVTQ/PRIQ IRQ generation in kdump -- queues are disabled */ + if (!is_kdump_kernel()) { + irqen_flags |= IRQ_CTRL_EVTQ_IRQEN; + if (smmu->features & ARM_SMMU_FEAT_PRI) + irqen_flags |= IRQ_CTRL_PRIQ_IRQEN; + } /* Enable interrupt generation on the SMMU */ ret = arm_smmu_write_reg_sync(smmu, irqen_flags, From a4b79f4bc6345ed9e92d6c3e6711192ca10d8404 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Wed, 20 May 2026 10:03:21 -0700 Subject: [PATCH 256/311] iommu/arm-smmu-v3: Skip EVTQ/PRIQ setup in kdump kernel BugLink: https://bugs.launchpad.net/bugs/2156531 In kdump cases, the crashed kernel's CDs and page tables can be corrupted, which could trigger event spamming. Also, we cannot serve page requests. Skip the EVTQ/PRIQ setup entirely rather than enabling then disabling them. Also add some inline comments explaining that. Fixes: b63b3439b856 ("iommu/arm-smmu-v3: Abort all transactions if SMMU is enabled in kdump kernel") Cc: stable@vger.kernel.org # v6.12+ Suggested-by: Kevin Tian Reviewed-by: Kevin Tian Reviewed-by: Jason Gunthorpe Signed-off-by: Nicolin Chen (backported from https://lore.kernel.org/linux-iommu/cover.1779265413.git.nicolinc@nvidia.com/#t) Signed-off-by: Jamie Nguyen Acked-by: Seth Forshee Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 43 +++++++++++++-------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 798aa5f698ef6..99bee39c0b3c8 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -4544,21 +4544,35 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu) cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL; arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); - /* Event queue */ - writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE); - writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD); - writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS); - - enables |= CR0_EVTQEN; - ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, - ARM_SMMU_CR0ACK); - if (ret) { - dev_err(smmu->dev, "failed to enable event queue\n"); - return ret; + /* + * Event queue + * + * Do not enable in a kdump case, as the crashed kernel's CDs and page + * tables might be corrupted, triggering event spamming. + */ + if (!is_kdump_kernel()) { + writeq_relaxed(smmu->evtq.q.q_base, + smmu->base + ARM_SMMU_EVTQ_BASE); + writel_relaxed(smmu->evtq.q.llq.prod, + smmu->page1 + ARM_SMMU_EVTQ_PROD); + writel_relaxed(smmu->evtq.q.llq.cons, + smmu->page1 + ARM_SMMU_EVTQ_CONS); + + enables |= CR0_EVTQEN; + ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, + ARM_SMMU_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to enable event queue\n"); + return ret; + } } - /* PRI queue */ - if (smmu->features & ARM_SMMU_FEAT_PRI) { + /* + * PRI queue + * + * Do not enable in a kdump case, as we cannot serve page requests. + */ + if (!is_kdump_kernel() && (smmu->features & ARM_SMMU_FEAT_PRI)) { writeq_relaxed(smmu->priq.q.q_base, smmu->base + ARM_SMMU_PRIQ_BASE); writel_relaxed(smmu->priq.q.llq.prod, @@ -4591,9 +4605,6 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu) return ret; } - if (is_kdump_kernel()) - enables &= ~(CR0_EVTQEN | CR0_PRIQEN); - /* Enable the SMMU interface */ enables |= CR0_SMMUEN; ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, From 856a8e2929c44db322d33848da8ff0358d91c325 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Wed, 20 May 2026 10:03:22 -0700 Subject: [PATCH 257/311] iommu/arm-smmu-v3: Retain CR0_SMMUEN during kdump device reset BugLink: https://bugs.launchpad.net/bugs/2156531 When ARM_SMMU_OPT_KDUMP_ADOPT is detected, do not disable SMMUEN and skip the CR1/CR2/STRTAB_BASE update sequence in arm_smmu_device_reset(). Those register writes are all CONSTRAINED UNPREDICTABLE while CR0_SMMUEN==1, so leaving them intact lets in-flight DMAs continue to be translated by the adopted stream table. Initialize 'enables' to 0 so it can carry CR0_SMMUEN in kdump case. Then, preserve that when enabling the command queue. Clear latched gerror bits if necessary. Fixes: b63b3439b856 ("iommu/arm-smmu-v3: Abort all transactions if SMMU is enabled in kdump kernel") Cc: stable@vger.kernel.org # v6.12+ Signed-off-by: Nicolin Chen Reviewed-by: Kevin Tian (backported from https://lore.kernel.org/linux-iommu/cover.1779265413.git.nicolinc@nvidia.com/#t) Signed-off-by: Jamie Nguyen Acked-by: Seth Forshee Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 47 +++++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 99bee39c0b3c8..c8f22a686b10c 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -4484,11 +4484,28 @@ static void arm_smmu_write_strtab(struct arm_smmu_device *smmu) static int arm_smmu_device_reset(struct arm_smmu_device *smmu) { int ret; - u32 reg, enables; + u32 reg, enables = 0; struct arm_smmu_cmdq_ent cmd; - /* Clear CR0 and sync (disables SMMU and queue processing) */ reg = readl_relaxed(smmu->base + ARM_SMMU_CR0); + + /* + * In a kdump case (set when CR0_SMMUEN=1 and !GERROR_SFM_ERR), retain + * CR0_SMMUEN to avoid aborting in-flight DMA, and CR0_ATSCHK to carry + * on the ATS-check policy. + * + * According to spec, updating STRTAB_BASE/CR1/CR2 when CR0_SMMUEN=1 is + * CONSTRAINED UNPREDICTABLE. So, skip those register updates and rely + * on the adopted stream table from the crashed kernel. + */ + if (smmu->options & ARM_SMMU_OPT_KDUMP_ADOPT) { + dev_info(smmu->dev, + "kdump: retaining SMMUEN for in-flight DMA\n"); + enables = reg & (CR0_SMMUEN | CR0_ATSCHK); + goto reset_queues; + } + + /* Clear CR0 and sync (disables SMMU and queue processing) */ if (reg & CR0_SMMUEN) { dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n"); arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0); @@ -4518,12 +4535,36 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu) /* Stream table */ arm_smmu_write_strtab(smmu); +reset_queues: + if (smmu->options & ARM_SMMU_OPT_KDUMP_ADOPT) { + /* Disable queues since arm_smmu_device_disable() was skipped */ + ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, + ARM_SMMU_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to disable queues\n"); + return ret; + } + } + + /* + * GERROR bits are latched. Read after queue disabling so that unhandled + * errors would be visible. Ack everything prior to re-enabling the CMDQ + * as a stale CMDQ_ERR would halt the CMDQ and new command will timeout. + */ + if (is_kdump_kernel()) { + u32 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR); + u32 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN); + + if ((gerror ^ gerrorn) & GERROR_ERR_MASK) + writel(gerror, smmu->base + ARM_SMMU_GERRORN); + } + /* Command queue */ writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE); writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD); writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS); - enables = CR0_CMDQEN; + enables |= CR0_CMDQEN; ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, ARM_SMMU_CR0ACK); if (ret) { From d33a4d41cf44c7569011048fdd81a7fbf49684d7 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Wed, 20 May 2026 10:03:23 -0700 Subject: [PATCH 258/311] iommu/arm-smmu-v3: Skip RMR bypass for kdump adoption BugLink: https://bugs.launchpad.net/bugs/2156531 RMR bypass STEs are installed during SMMUv3 probe for StreamIDs listed by IORT RMR nodes. A normal boot switches the driver to a fresh stream table whose initial STEs abort, so those RMR SIDs need bypass entries before it becomes live. This preserves firmware/guest-owned traffic, including vSMMU guest MSI cases built around RMR-described SIDs. ARM_SMMU_OPT_KDUMP_ADOPT is the opposite case: the driver keeps SMMUEN set and adopts the crashed kernel's stream table, so RMR SIDs already have the only translation state known to be safe for active in-flight DMA. Replacing an adopted STE with bypass can turn translated DMA into physical DMA, then point it at the wrong memory. arm_smmu_make_bypass_ste() also rewrites the STE in place after clearing it first. While the table is live, a concurrent hardware STE fetch can observe V=0 or mixed old/new state. Leaving the adopted STE unmodified keeps the kdump kernel using the crashed kernel's translation. That gives the endpoint driver a chance to probe and quiesce the device. If the old STE was already abort or invalid, installing bypass would create new DMA permission; leaving it alone is a safer failure mode. Later domain setup still gets the RMR direct mappings through the reserved-region path. Fixes: b63b3439b856 ("iommu/arm-smmu-v3: Abort all transactions if SMMU is enabled in kdump kernel") Cc: stable@vger.kernel.org # v6.12+ Assisted-by: Codex:gpt-5.5 Signed-off-by: Nicolin Chen (backported from https://lore.kernel.org/linux-iommu/cover.1779265413.git.nicolinc@nvidia.com/#t) Signed-off-by: Jamie Nguyen Acked-by: Seth Forshee Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index c8f22a686b10c..3b35c510c2085 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -5104,6 +5104,14 @@ static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu) struct list_head rmr_list; struct iommu_resv_region *e; + /* + * Kdump adoption keeps the crashed kernel's table live. Rewriting the + * adopted STE here could expose an in-flight fetch to a transient V=0 + * entry, or change Cfg=translate to Cfg=bypass. Must skip here. + */ + if (smmu->options & ARM_SMMU_OPT_KDUMP_ADOPT) + return; + INIT_LIST_HEAD(&rmr_list); iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); @@ -5120,10 +5128,7 @@ static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu) continue; } - /* - * STE table is not programmed to HW, see - * arm_smmu_initial_bypass_stes() - */ + /* The fresh stream table is not yet live. */ arm_smmu_make_bypass_ste(smmu, arm_smmu_get_step_for_sid(smmu, rmr->sids[i])); } From 52fc084067b63234f40ad154546e374cf9d57686 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Wed, 20 May 2026 10:03:24 -0700 Subject: [PATCH 259/311] iommu/arm-smmu-v3: Detect ARM_SMMU_OPT_KDUMP_ADOPT in probe() BugLink: https://bugs.launchpad.net/bugs/2156531 arm_smmu_device_hw_probe() runs before arm_smmu_init_structures(), so it's natural to decide whether the kdump kernel must adopt the crashed kernel's stream table. Given that memremap is used to adopt the old stream table, set this option only on a coherent SMMU. And make sure SMMU isn't in Service Failure Mode. Fixes: b63b3439b856 ("iommu/arm-smmu-v3: Abort all transactions if SMMU is enabled in kdump kernel") Cc: stable@vger.kernel.org # v6.12+ Reviewed-by: Kevin Tian Reviewed-by: Jason Gunthorpe Signed-off-by: Nicolin Chen (backported from https://lore.kernel.org/linux-iommu/cover.1779265413.git.nicolinc@nvidia.com/#t) [jamien: Resolve context conflict around arm_smmu_device_hw_probe() due to the different probe layout in this tree.] Signed-off-by: Jamie Nguyen Acked-by: Seth Forshee Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 31 +++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 3b35c510c2085..15861f96dd888 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -4726,6 +4726,33 @@ static void arm_smmu_get_httu(struct arm_smmu_device *smmu, u32 reg) hw_features, fw_features); } +static void arm_smmu_device_hw_probe_kdump(struct arm_smmu_device *smmu) +{ + u32 gerror, gerrorn, active; + + /* No adoption if SMMU is disabled (i.e., there is no in-flight DMA) */ + if (!(readl_relaxed(smmu->base + ARM_SMMU_CR0) & CR0_SMMUEN)) + return; + + /* For now, only support a coherent SMMU that works with MEMREMAP_WB */ + if (!(smmu->features & ARM_SMMU_FEAT_COHERENCY)) { + dev_warn(smmu->dev, + "kdump: non-coherent SMMU unsupported; reset to block all DMAs\n"); + return; + } + + gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR); + gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN); + active = gerror ^ gerrorn; + if (active & GERROR_SFM_ERR) { + dev_warn(smmu->dev, + "kdump: SMMU in Service Failure Mode, must reset\n"); + return; + } + + smmu->options |= ARM_SMMU_OPT_KDUMP_ADOPT; +} + static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) { u32 reg; @@ -4940,6 +4967,10 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) dev_info(smmu->dev, "oas %lu-bit (features 0x%08x)\n", smmu->oas, smmu->features); + + if (is_kdump_kernel()) + arm_smmu_device_hw_probe_kdump(smmu); + return 0; } From daccaf10eb0778d1bad68f34947501d82d306afa Mon Sep 17 00:00:00 2001 From: Haiyang Zhang Date: Fri, 5 Jun 2026 14:22:56 -0700 Subject: [PATCH 260/311] net: mana: Add support for PF device 0x00C1 BugLink: https://bugs.launchpad.net/bugs/2156821 Update the device id table to include the new device id 0x00C1. This device's BAR layout is similar to VF's, update the function, mana_gd_init_registers(), accordingly. Signed-off-by: Haiyang Zhang Link: https://patch.msgid.link/20260605212302.2135499-1-haiyangz@linux.microsoft.com Signed-off-by: Jakub Kicinski (backported from commit 53a65db20a4f3fe6c01b1f789f9eae6b1244910f linux-next) [ltrager: minor merge conflicts] Signed-off-by: Lee Trager Acked-by: Jamie Nguyen Acked-by: Matthew R. Ochs Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/net/ethernet/microsoft/mana/gdma_main.c | 7 +++++-- include/net/mana/gdma.h | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index 786186c9a115f..1c5fafb5b108c 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -78,7 +78,7 @@ static void mana_gd_init_registers(struct pci_dev *pdev) { struct gdma_context *gc = pci_get_drvdata(pdev); - if (gc->is_pf) + if (gc->is_pf && !gc->is_pf2) mana_gd_init_pf_regs(pdev); else mana_gd_init_vf_regs(pdev); @@ -1956,7 +1956,7 @@ static void mana_gd_cleanup(struct pci_dev *pdev) static bool mana_is_pf(unsigned short dev_id) { - return dev_id == MANA_PF_DEVICE_ID; + return dev_id == MANA_PF_DEVICE_ID || dev_id == MANA_PF2_DEVICE_ID; } static int mana_gd_probe(struct pci_dev *pdev, const struct pci_device_id *ent) @@ -2003,6 +2003,8 @@ static int mana_gd_probe(struct pci_dev *pdev, const struct pci_device_id *ent) gc->numa_node = dev_to_node(&pdev->dev); gc->is_pf = mana_is_pf(pdev->device); + gc->is_pf2 = (pdev->device == MANA_PF2_DEVICE_ID); + gc->bar0_va = bar0_va; gc->dev = &pdev->dev; xa_init(&gc->irq_contexts); @@ -2176,6 +2178,7 @@ static void mana_gd_shutdown(struct pci_dev *pdev) static const struct pci_device_id mana_id_table[] = { { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF_DEVICE_ID) }, + { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF2_DEVICE_ID) }, { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_VF_DEVICE_ID) }, { } }; diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h index 766f4fb25e266..03e16b07a8bac 100644 --- a/include/net/mana/gdma.h +++ b/include/net/mana/gdma.h @@ -411,6 +411,7 @@ struct gdma_context { u32 test_event_eq_id; bool is_pf; + bool is_pf2; bool in_service; phys_addr_t bar0_pa; @@ -560,6 +561,7 @@ struct gdma_eqe { #define GDMA_SRIOV_REG_CFG_BASE_OFF 0x108 #define MANA_PF_DEVICE_ID 0x00B9 +#define MANA_PF2_DEVICE_ID 0x00C1 #define MANA_VF_DEVICE_ID 0x00BA struct gdma_posted_wqe_info { From c70767dcb88527dc0b8fbf71cdea06cad4cdd5f4 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Wed, 1 Jul 2026 13:44:10 -0500 Subject: [PATCH 261/311] UBUNTU: Start new release Ignore: yes Signed-off-by: Jacob Martin --- debian.nvidia/changelog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debian.nvidia/changelog b/debian.nvidia/changelog index 3bf2cb4618b8d..85c5e287eecf4 100644 --- a/debian.nvidia/changelog +++ b/debian.nvidia/changelog @@ -1,3 +1,11 @@ +linux-nvidia (7.0.0-1015.15) UNRELEASED; urgency=medium + + CHANGELOG: Do not edit directly. Autogenerated at release. + CHANGELOG: Use the printchanges target to see the current changes. + CHANGELOG: Use the insertchanges target to create the final log. + + -- Jacob Martin Wed, 01 Jul 2026 13:44:10 -0500 + linux-nvidia (7.0.0-1014.14) resolute; urgency=medium * resolute/linux-nvidia: 7.0.0-1014.14 -proposed tracker (LP: #2153496) From ce8d54f674e3078a6f93b2026c3136ecebf0e3a7 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Wed, 1 Jul 2026 13:45:50 -0500 Subject: [PATCH 262/311] UBUNTU: link-to-tracker: update tracking bug BugLink: https://bugs.launchpad.net/bugs/2158924 Properties: no-test-build Signed-off-by: Jacob Martin --- debian.nvidia/tracking-bug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian.nvidia/tracking-bug b/debian.nvidia/tracking-bug index 164377f318e42..53ded77eea8a2 100644 --- a/debian.nvidia/tracking-bug +++ b/debian.nvidia/tracking-bug @@ -1 +1 @@ -2153496 d2026.05.20-1 +2158924 d2026.06.30-1 From 8f1a47ef536015c825550434f9206fa05395a0a3 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Wed, 1 Jul 2026 13:58:34 -0500 Subject: [PATCH 263/311] UBUNTU: [Packaging] debian.nvidia/dkms-versions -- update from kernel-versions (adhoc/d2026.06.30) BugLink: https://bugs.launchpad.net/bugs/1786013 Signed-off-by: Jacob Martin --- debian.nvidia/dkms-versions | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debian.nvidia/dkms-versions b/debian.nvidia/dkms-versions index ba40c69369282..4a941b2e62257 100644 --- a/debian.nvidia/dkms-versions +++ b/debian.nvidia/dkms-versions @@ -1,2 +1,4 @@ zfs-linux 2.4.1-1ubuntu5 modulename=zfs debpath=pool/universe/z/%package%/zfs-dkms_%version%_all.deb arch=amd64 arch=arm64 arch=ppc64el arch=riscv64 arch=s390x rprovides=spl-modules rprovides=spl-dkms rprovides=zfs-modules rprovides=zfs-dkms off_series=true v4l2loopback 0.15.3-1ubuntu2 modulename=v4l2loopback debpath=pool/universe/v/%package%/v4l2loopback-dkms_%version%_all.deb arch=amd64 rprovides=v4l2loopback-modules rprovides=v4l2loopback-dkms off_series=true +nvidia-fs 2.29.4-1 modulename=nvidia-fs debpath=pool/universe/n/%package%/nvidia-fs-dkms_%version%_amd64.deb arch=amd64 arch=arm64 rprovides=nvidia-fs-modules rprovides=nvidia-fs-dkms type=standalone +mstflint 4.33.0+1-1.1 modulename=mstflint_access debpath=pool/universe/m/%package%/mstflint-dkms_%version%_all.deb arch=amd64 arch=arm64 rprovides=mstflint-modules rprovides=mstflint-dkms From 6640f145eedce3455c1f9772820e68103dafc0dd Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Wed, 1 Jul 2026 14:00:24 -0500 Subject: [PATCH 264/311] UBUNTU: [Config] nvidia: update configs Ignore: yes Signed-off-by: Jacob Martin --- debian.nvidia/config/annotations | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index 9012bf8df51d0..38ff05b077b62 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -189,6 +189,9 @@ CONFIG_PINCTRL_MT8901 note<'LP: #2117784'> CONFIG_R8127 policy<{'amd64': 'n', 'arm64': 'm'}> CONFIG_R8127 note<'LP: #2109730'> +CONFIG_RESCTRL_FS policy<{'amd64': 'y', 'arm64': 'y'}> +CONFIG_RESCTRL_FS note<'LP: #2122432'> + CONFIG_SAMPLE_CORESIGHT_SYSCFG policy<{'arm64': 'n'}> CONFIG_SAMPLE_CORESIGHT_SYSCFG note<'Required for Grace enablement'> @@ -219,10 +222,12 @@ CONFIG_VFIO_CONTAINER note<'LP: #2095028'> CONFIG_VFIO_IOMMU_TYPE1 policy<{'amd64': 'm', 'arm64': '-'}> CONFIG_VFIO_IOMMU_TYPE1 note<'LP: #2095028'> -CONFIG_RESCTRL_FS policy<{'amd64': 'y', 'arm64': 'y'}> -CONFIG_RESCTRL_FS note<'LP: #2122432'> # ---- Annotations without notes ---- +CONFIG_ARCH_HAS_CPU_RESCTRL policy<{'amd64': 'y', 'arm64': 'y'}> +CONFIG_ARM64_MPAM_RESCTRL_FS policy<{'arm64': 'y'}> CONFIG_BCH policy<{'amd64': 'm', 'arm64': 'y'}> CONFIG_MTD_NAND_CORE policy<{'amd64': 'm', 'arm64': 'y'}> +CONFIG_PROC_CPU_RESCTRL policy<{'amd64': 'y', 'arm64': 'y'}> +CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID policy<{'arm64': 'y'}> From 716feb512c9667427c0a55a66389e8e71d49ba98 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Wed, 1 Jul 2026 14:01:27 -0500 Subject: [PATCH 265/311] UBUNTU: Ubuntu-nvidia-7.0.0-1015.15 Signed-off-by: Jacob Martin --- debian.nvidia/changelog | 153 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 148 insertions(+), 5 deletions(-) diff --git a/debian.nvidia/changelog b/debian.nvidia/changelog index 85c5e287eecf4..05ece2617a2c8 100644 --- a/debian.nvidia/changelog +++ b/debian.nvidia/changelog @@ -1,10 +1,153 @@ -linux-nvidia (7.0.0-1015.15) UNRELEASED; urgency=medium +linux-nvidia (7.0.0-1015.15) resolute; urgency=medium - CHANGELOG: Do not edit directly. Autogenerated at release. - CHANGELOG: Use the printchanges target to see the current changes. - CHANGELOG: Use the insertchanges target to create the final log. + * resolute/linux-nvidia: 7.0.0-1015.15 -proposed tracker (LP: #2158924) - -- Jacob Martin Wed, 01 Jul 2026 13:44:10 -0500 + * Packaging resync (LP: #1786013) + - [Packaging] debian.nvidia/dkms-versions -- update from kernel-versions + (adhoc/d2026.06.30) + + * Backport mana support for PF device 0x00C1 (LP: #2156821) + - net: mana: Add support for PF device 0x00C1 + + * Backport the arm-smmu-v3 kdump adoption series (LP: #2156531) + - iommu/arm-smmu-v3: Add arm_smmu_kdump_adopt_strtab() for kdump + - iommu/arm-smmu-v3: Implement is_attach_deferred() for kdump + - iommu/arm-smmu-v3: Do not enable EVTQ/PRIQ interrupts in kdump kernel + - iommu/arm-smmu-v3: Skip EVTQ/PRIQ setup in kdump kernel + - iommu/arm-smmu-v3: Retain CR0_SMMUEN during kdump device reset + - iommu/arm-smmu-v3: Skip RMR bypass for kdump adoption + - iommu/arm-smmu-v3: Detect ARM_SMMU_OPT_KDUMP_ADOPT in probe() + + * Backport: fuse: back uncached readdir buffers with pages (LP: #2156632) + - fuse: back uncached readdir buffers with pages + + * Backport: Mitigate TLBI errata on various Arm CPUs (LP: #2156557) // CVE- + Enable ARM64_ERRATUM_4118414 to mitigate 2025-10263 on NVIDIA platforms. + - NVIDIA: [Config] Enable ARM64_ERRATUM_4118414 + + * Backport: Mitigate TLBI errata on various Arm CPUs (LP: #2156557) // + CVE-2025-10263. The existing ARM64_ERRATUM_4118414 handling already uses + - arm64: errata: Mitigate TLBI errata on NVIDIA Olympus CPU + + * Backport: Mitigate TLBI errata on various Arm CPUs (LP: #2156557) + - arm64: cputype: Add C1-Ultra definitions + - arm64: cputype: Add C1-Premium definitions + - arm64: errata: Mitigate TLBI errata on various Arm CPUs + + * linux-nvidia: Port MPAM Functionality into Kernel (LP: #2154527) + - arm_mpam: Ensure in_reset_state is false after applying configuration + - arm_mpam: Reset when feature configuration bit unset + - arm64/sysreg: Add MPAMSM_EL1 register + - KVM: arm64: Preserve host MPAM configuration when changing traps + - KVM: arm64: Make MPAMSM_EL1 accesses UNDEF + - arm64: mpam: Context switch the MPAM registers + - arm64: mpam: Re-initialise MPAM regs when CPU comes online + - arm64: mpam: Drop the CONFIG_EXPERT restriction + - arm64: mpam: Advertise the CPUs MPAM limits to the driver + - arm64: mpam: Add cpu_pm notifier to restore MPAM sysregs + - arm64: mpam: Initialise and context switch the MPAMSM_EL1 register + - arm64: mpam: Add helpers to change a task or cpu's MPAM PARTID/PMG + values + - arm_mpam: resctrl: Add boilerplate cpuhp and domain allocation + - arm_mpam: resctrl: Pick the caches we will use as resctrl resources + - arm_mpam: resctrl: Implement resctrl_arch_reset_all_ctrls() + - arm_mpam: resctrl: Add resctrl_arch_get_config() + - arm_mpam: resctrl: Implement helpers to update configuration + - arm_mpam: resctrl: Add plumbing against arm64 task and cpu hooks + - arm_mpam: resctrl: Add CDP emulation + - arm_mpam: resctrl: Hide CDP emulation behind CONFIG_EXPERT + - arm_mpam: resctrl: Convert to/from MPAMs fixed-point formats + - arm_mpam: resctrl: Add rmid index helpers + - arm_mpam: resctrl: Wait for cacheinfo to be ready + - arm_mpam: resctrl: Add support for 'MB' resource + - arm_mpam: resctrl: Add kunit test for control format conversions + - arm_mpam: resctrl: Add monitor initialisation and domain boilerplate + - arm_mpam: resctrl: Add support for csu counters + - arm_mpam: resctrl: Allow resctrl to allocate monitors + - arm_mpam: resctrl: Add resctrl_arch_rmid_read() + - arm_mpam: resctrl: Update the rmid reallocation limit + - arm_mpam: resctrl: Add empty definitions for assorted resctrl functions + - ALSA: usb-audio: Replace hard-coded number with MAX_CHANNELS + - arm64: mpam: Select ARCH_HAS_CPU_RESCTRL + - arm_mpam: resctrl: Call resctrl_init() on platforms that can support + resctrl + - arm_mpam: Add quirk framework + - arm_mpam: Add workaround for T241-MPAM-1 + - arm_mpam: Add workaround for T241-MPAM-4 + - arm_mpam: Add workaround for T241-MPAM-6 + - arm_mpam: Quirk CMN-650's CSU NRDY behaviour + - arm64: mpam: Add initial MPAM documentation + - fs/resctrl: Report invalid domain ID when parsing io_alloc_cbm + - fs/resctrl: Add "*" shorthand to set io_alloc CBM for all domains + - MAINTAINERS: Update resctrl entry + - fs/resctrl: Add missing return value descriptions + - arm_mpam: resctrl: Fix MBA CDP alloc_capable handling on unmount + - arm_mpam: resctrl: Fix the check for no monitor components found + - arm_mpam: resctrl: Make resctrl_mon_ctx_waiters static + - NVIDIA: SAUCE: Update annotations to set CONFIG_RESCTRL_FS + - NVIDIA: SAUCE: fs/resctrl: Tidy up the error path in + resctrl_mkdir_event_configs() + - NVIDIA: SAUCE: x86,fs/resctrl: Create 'event_filter' files read only if + they're not configurable + - NVIDIA: SAUCE: fs/resctrl: Disallow the software controller when MBM + counters are assignable + - NVIDIA: SAUCE: fs/resctrl: Add monitor property 'mbm_cntr_assign_fixed' + - NVIDIA: SAUCE: fs/resctrl: Continue counter allocation after failure + - NVIDIA: SAUCE: fs/resctrl: Document that automatic counter assignment is + best effort + - NVIDIA: SAUCE: fs/resctrl: Document tasks file behaviour for task id 0 + and idle tasks + - NVIDIA: SAUCE: arm_mpam: resctrl: Pick classes for use as MBM counters + - NVIDIA: SAUCE: arm_mpam: resctrl: Pre-allocate assignable monitors + - NVIDIA: SAUCE: arm_mpam: resctrl: Add resctrl_arch_config_cntr() for + ABMC use + - NVIDIA: SAUCE: arm_mpam: resctrl: Add resctrl_arch_cntr_read() & + resctrl_arch_reset_cntr() + - NVIDIA: SAUCE: arm64: mpam: Add memory bandwidth usage (MBWU) + documentation + - NVIDIA: SAUCE: arm_mpam: Add resctrl_arch_round_bw() + - NVIDIA: SAUCE: fs/resctrl,x86/resctrl: Factor mba rounding to be per- + arch + - NVIDIA: SAUCE: x86/resctrl: Add stub to allow other architecture to + disable monitor overflow + - NVIDIA: SAUCE: arm_mpam: resctrl: Determine if any exposed counter can + overflow + - NVIDIA: SAUCE: fs/restrl: Allow the overflow handler to be disabled + - NVIDIA: SAUCE: arm_mpam: Allow cmax/cmin to be configured + - NVIDIA: SAUCE: arm_mpam: Rename mbw conversion to 'fract16' for code re- + use + - NVIDIA: SAUCE: fs/resctrl: Group all the MBA specific properties in a + separate struct + - NVIDIA: SAUCE: fs/resctrl: Abstract duplicate domain test to a helper + - NVIDIA: SAUCE: fs/resctrl: Move MBA supported check to parse_line() + instead of parse_bw() + - NVIDIA: SAUCE: fs/resctrl: Rename resctrl_get_default_ctrl() to include + resource + - NVIDIA: SAUCE: fs/resctrl: Add a schema format to the schema, allowing + it to be different + - NVIDIA: SAUCE: fs/resctrl: Add specific schema types for 'range' + - NVIDIA: SAUCE: x86/resctrl: Move over to specifying MBA control formats + - NVIDIA: SAUCE: fs/resctrl: Add additional files for percentage and + bitmap controls + - NVIDIA: SAUCE: fs/resctrl: Add fflags_from_schema() for files based on + schema format + - NVIDIA: SAUCE: fs/resctrl: Expose the schema format to user-space + - NVIDIA: SAUCE: fs/resctrl: Add L2 and L3 'MAX' resource schema + - NVIDIA: SAUCE: arm_mpam: resctrl: Add the glue code to convert to/from + cmax + - NVIDIA: SAUCE: resctrl/mpam: reset RIS by applying explicit default + config + - NVIDIA: SAUCE: arm_mpam: Fix MPAMCFG_MBW_PBM register setting + + * fs/ntfs3: fix mount failure on 64K page-size kernels (LP: #2155467) + - fs/ntfs3: fix mount failure on 64K page-size kernels + + * linux-nvidia: backport FF-A partition info descriptor size fix + (LP: #2154045) + - firmware: arm_ffa: Bound PARTITION_INFO_GET_REGS copies + - firmware: arm_ffa: Honor partition info descriptor size + + -- Jacob Martin Wed, 01 Jul 2026 14:01:27 -0500 linux-nvidia (7.0.0-1014.14) resolute; urgency=medium From 93251732e2b93f8c625372b85cdd441294e0a6b9 Mon Sep 17 00:00:00 2001 From: David Thompson Date: Thu, 28 May 2026 16:50:17 +0000 Subject: [PATCH 266/311] net: lan743x: avoid netdev-based logging before netdev registration BugLink: https://bugs.launchpad.net/bugs/2156928 This patch updates the lan743x driver to prevent the use of netdev-based logging APIs (such as netdev_dbg) before the network device has been successfully registered. Using netdev-based logging prior to registration results in log messages referencing "(unnamed net_device) (uninitialized)", which can be confusing and less informative. The driver must use netif_msg_ APIs and device-based logging (e.g. dev_dbg) until netdev registration is complete. This ensures log entries are associated with the correct device context and improves log clarity. After registration, netdev-based logging APIs can be used safely. Signed-off-by: David Thompson Link: https://patch.msgid.link/20260528165017.421576-1-davthompson@nvidia.com Signed-off-by: Jakub Kicinski (cherry picked from commit e3c6508a46f56ece0c1550a4fdf1e005afe3d563) Signed-off-by: David Thompson Acked-by: Matthew R. Ochs Acked-by: Nirmoy Das Signed-off-by: Brad Figg --- drivers/net/ethernet/microchip/lan743x_main.c | 48 ++++++++----------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c index f3332417162e6..d40c277af112b 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.c +++ b/drivers/net/ethernet/microchip/lan743x_main.c @@ -108,9 +108,9 @@ static int lan743x_pci_init(struct lan743x_adapter *adapter, if (ret) goto return_error; - netif_info(adapter, probe, adapter->netdev, - "PCI: Vendor ID = 0x%04X, Device ID = 0x%04X\n", - pdev->vendor, pdev->device); + dev_dbg(&adapter->pdev->dev, + "PCI: Vendor ID = 0x%04X, Device ID = 0x%04X\n", + pdev->vendor, pdev->device); bars = pci_select_bars(pdev, IORESOURCE_MEM); if (!test_bit(0, &bars)) goto disable_device; @@ -192,10 +192,10 @@ static int lan743x_csr_init(struct lan743x_adapter *adapter) csr->id_rev = lan743x_csr_read(adapter, ID_REV); csr->fpga_rev = lan743x_csr_read(adapter, FPGA_REV); - netif_info(adapter, probe, adapter->netdev, - "ID_REV = 0x%08X, FPGA_REV = %d.%d\n", - csr->id_rev, FPGA_REV_GET_MAJOR_(csr->fpga_rev), - FPGA_REV_GET_MINOR_(csr->fpga_rev)); + dev_dbg(&adapter->pdev->dev, + "ID_REV = 0x%08X, FPGA_REV = %d.%d\n", + csr->id_rev, FPGA_REV_GET_MAJOR_(csr->fpga_rev), + FPGA_REV_GET_MINOR_(csr->fpga_rev)); if (!ID_REV_IS_VALID_CHIP_ID_(csr->id_rev)) return -ENODEV; @@ -953,8 +953,8 @@ int lan743x_sgmii_read(struct lan743x_adapter *adapter, u8 mmd, u16 addr) u32 val; if (mmd > 31) { - netif_err(adapter, probe, adapter->netdev, - "%s mmd should <= 31\n", __func__); + dev_err(&adapter->pdev->dev, + "%s mmd should <= 31\n", __func__); return -EINVAL; } @@ -983,8 +983,8 @@ static int lan743x_sgmii_write(struct lan743x_adapter *adapter, int ret; if (mmd > 31) { - netif_err(adapter, probe, adapter->netdev, - "%s mmd should <= 31\n", __func__); + dev_err(&adapter->pdev->dev, + "%s mmd should <= 31\n", __func__); return -EINVAL; } mutex_lock(&adapter->sgmii_rw_lock); @@ -1215,8 +1215,7 @@ static void lan743x_mac_set_address(struct lan743x_adapter *adapter, lan743x_csr_write(adapter, MAC_RX_ADDRH, addr_hi); ether_addr_copy(adapter->mac_address, addr); - netif_info(adapter, drv, adapter->netdev, - "MAC address set to %pM\n", addr); + dev_dbg(&adapter->pdev->dev, "MAC address set to %pM\n", addr); } static int lan743x_mac_init(struct lan743x_adapter *adapter) @@ -1370,8 +1369,8 @@ static void lan743x_phy_interface_select(struct lan743x_adapter *adapter) else adapter->phy_interface = PHY_INTERFACE_MODE_RGMII; - netif_dbg(adapter, drv, adapter->netdev, - "selected phy interface: 0x%X\n", adapter->phy_interface); + dev_dbg(&adapter->pdev->dev, + "selected phy interface: 0x%X\n", adapter->phy_interface); } static void lan743x_rfe_open(struct lan743x_adapter *adapter) @@ -3168,7 +3167,7 @@ static int lan743x_phylink_create(struct lan743x_adapter *adapter) } adapter->phylink = pl; - netdev_dbg(netdev, "lan743x phylink created"); + dev_dbg(&adapter->pdev->dev, "lan743x phylink created"); return 0; } @@ -3581,30 +3580,26 @@ static int lan743x_mdiobus_init(struct lan743x_adapter *adapter) adapter->mdiobus->priv = (void *)adapter; if (adapter->is_pci11x1x) { if (adapter->is_sgmii_en) { - netif_dbg(adapter, drv, adapter->netdev, - "SGMII operation\n"); + dev_dbg(&adapter->pdev->dev, "SGMII operation\n"); adapter->mdiobus->read = lan743x_mdiobus_read_c22; adapter->mdiobus->write = lan743x_mdiobus_write_c22; adapter->mdiobus->read_c45 = lan743x_mdiobus_read_c45; adapter->mdiobus->write_c45 = lan743x_mdiobus_write_c45; adapter->mdiobus->name = "lan743x-mdiobus-c45"; - netif_dbg(adapter, drv, adapter->netdev, - "lan743x-mdiobus-c45\n"); + dev_dbg(&adapter->pdev->dev, "lan743x-mdiobus-c45\n"); } else { - netif_dbg(adapter, drv, adapter->netdev, - "RGMII operation\n"); + dev_dbg(&adapter->pdev->dev, "RGMII operation\n"); // Only C22 support when RGMII I/F adapter->mdiobus->read = lan743x_mdiobus_read_c22; adapter->mdiobus->write = lan743x_mdiobus_write_c22; adapter->mdiobus->name = "lan743x-mdiobus"; - netif_dbg(adapter, drv, adapter->netdev, - "lan743x-mdiobus\n"); + dev_dbg(&adapter->pdev->dev, "lan743x-mdiobus\n"); } } else { adapter->mdiobus->read = lan743x_mdiobus_read_c22; adapter->mdiobus->write = lan743x_mdiobus_write_c22; adapter->mdiobus->name = "lan743x-mdiobus"; - netif_dbg(adapter, drv, adapter->netdev, "lan743x-mdiobus\n"); + dev_dbg(&adapter->pdev->dev, "lan743x-mdiobus\n"); } snprintf(adapter->mdiobus->id, MII_BUS_ID_SIZE, @@ -3696,8 +3691,7 @@ static int lan743x_pcidev_probe(struct pci_dev *pdev, ret = lan743x_phylink_create(adapter); if (ret < 0) { - netif_err(adapter, probe, netdev, - "failed to setup phylink (%d)\n", ret); + dev_err(&pdev->dev, "failed to setup phylink (%d)\n", ret); goto cleanup_mdiobus; } From 45ed5d058c59dc5864664a5f2ea55d5665ab6a1f Mon Sep 17 00:00:00 2001 From: David Thompson Date: Fri, 29 May 2026 21:03:00 +0000 Subject: [PATCH 267/311] net: lan743x: permit VLAN-tagged packets up to configured MTU BugLink: https://bugs.launchpad.net/bugs/2156928 VLAN-tagged interfaces on lan743x devices were previously unreachable via SSH and failed to respond to large ping packets (e.g. "ping -s 1469" given MTU=1500). In these scenarios, "ethtool -S" reports non-zero "RX Oversize Frame Errors". According to Microchip AN2948, the MAC_RX FSE (VLAN field size enforcement) bit determines whether frames with VLAN tags exceeding the base MTU plus tag length are discarded. The driver must set the MAC_RX.FSE bit before setting MAC_RX.RXEN to allow VLAN-tagged frames up to the interface MTU, preventing them from being treated as oversized. As a result, both the base and VLAN-tagged interfaces can use the same MTU without receive errors. Fixes: 23f0703c125b ("lan743x: Add main source files for new lan743x driver") Signed-off-by: David Thompson Reviewed-by: Thangaraj Samynathan Reviewed-by: Nicolai Buchwitz Tested-by: Nicolai Buchwitz # lan7430 on arm64 (RevPi Link: https://patch.msgid.link/20260529210300.433135-1-davthompson@nvidia.com Signed-off-by: Jakub Kicinski (cherry picked from commit 8173d22b211f615015f7b35f48ab11a6dd78dc99) Signed-off-by: David Thompson Acked-by: Matthew R. Ochs Acked-by: Nirmoy Das Signed-off-by: Brad Figg --- drivers/net/ethernet/microchip/lan743x_main.c | 32 +++++++++++++++++++ drivers/net/ethernet/microchip/lan743x_main.h | 1 + 2 files changed, 33 insertions(+) diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c index d40c277af112b..1cdce35e14239 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.c +++ b/drivers/net/ethernet/microchip/lan743x_main.c @@ -1218,6 +1218,36 @@ static void lan743x_mac_set_address(struct lan743x_adapter *adapter, dev_dbg(&adapter->pdev->dev, "MAC address set to %pM\n", addr); } +static void lan743x_mac_rx_enable_fse(struct lan743x_adapter *adapter) +{ + u32 mac_rx; + bool rxen; + + mac_rx = lan743x_csr_read(adapter, MAC_RX); + if (mac_rx & MAC_RX_FSE_) + return; + + rxen = mac_rx & MAC_RX_RXEN_; + if (rxen) { + mac_rx &= ~MAC_RX_RXEN_; + lan743x_csr_write(adapter, MAC_RX, mac_rx); + lan743x_csr_wait_for_bit(adapter, MAC_RX, MAC_RX_RXD_, + 1, 1000, 20000, 100); + } + + /* Per AN2948, hardware prevents modification of the FSE bit while the + * MAC receiver is enabled (RXEN bit set). Use separate register write + * to assert the FSE bit before enabling the RXEN bit in MAC_RX + */ + mac_rx |= MAC_RX_FSE_; + lan743x_csr_write(adapter, MAC_RX, mac_rx); + + if (rxen) { + mac_rx |= MAC_RX_RXEN_; + lan743x_csr_write(adapter, MAC_RX, mac_rx); + } +} + static int lan743x_mac_init(struct lan743x_adapter *adapter) { bool mac_address_valid = true; @@ -1257,6 +1287,8 @@ static int lan743x_mac_init(struct lan743x_adapter *adapter) lan743x_mac_set_address(adapter, adapter->mac_address); eth_hw_addr_set(netdev, adapter->mac_address); + lan743x_mac_rx_enable_fse(adapter); + return 0; } diff --git a/drivers/net/ethernet/microchip/lan743x_main.h b/drivers/net/ethernet/microchip/lan743x_main.h index 160d94a7cee66..1573c8f9c9937 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.h +++ b/drivers/net/ethernet/microchip/lan743x_main.h @@ -182,6 +182,7 @@ #define MAC_RX (0x104) #define MAC_RX_MAX_SIZE_SHIFT_ (16) #define MAC_RX_MAX_SIZE_MASK_ (0x3FFF0000) +#define MAC_RX_FSE_ BIT(2) #define MAC_RX_RXD_ BIT(1) #define MAC_RX_RXEN_ BIT(0) From a4da22865288df146768d9218f9fc00c8059710d Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Wed, 27 May 2026 15:55:06 -0700 Subject: [PATCH 268/311] perf/core: out-of-line and export perf_allow_cpu/tracepoint() BugLink: https://bugs.launchpad.net/bugs/2160654 These helpers are static inline in and reach into sysctl_perf_event_paranoid and security_perf_event_open(), neither of which is itself exported. The perf_allow_* trio is therefore asymmetric: built-in callers can use any of the three, but modular code can only call perf_allow_kernel(). Move both bodies into kernel/events/core.c next to perf_allow_kernel() and export them with EXPORT_SYMBOL_GPL, following the shape of commit 5e9629d0ae97 ("drivers/perf: arm_spe: Use perf_allow_kernel() for permissions"). Existing in-tree callers live in built-in arch and tracing code, so the change is invisible to them. Provide !CONFIG_PERF_EVENTS stubs that fall back to perfmon_capable(), so the helpers stay callable when perf is compiled out. Signed-off-by: John Hubbard Reviewed-by: Ashutosh Dixit Link: https://patch.msgid.link/20260527225507.2044027-2-ashutosh.dixit@intel.com Signed-off-by: Ashutosh Dixit (cherry picked from commit d32bf877c0c3ebc345b444cbe009b3f44f9f8073 linux-next) Signed-off-by: Kelsey Steele Acked-by: Jamie Nguyen Acked-by: Carol L Soto Signed-off-by: Brad Figg --- include/linux/perf_event.h | 31 +++++++++++++++---------------- kernel/events/core.c | 18 ++++++++++++++++++ 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 6cdc23fb7e093..f929a7dd324ec 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1797,22 +1797,8 @@ static inline int perf_is_paranoid(void) } extern int perf_allow_kernel(void); - -static inline int perf_allow_cpu(void) -{ - if (sysctl_perf_event_paranoid > 0 && !perfmon_capable()) - return -EACCES; - - return security_perf_event_open(PERF_SECURITY_CPU); -} - -static inline int perf_allow_tracepoint(void) -{ - if (sysctl_perf_event_paranoid > -1 && !perfmon_capable()) - return -EPERM; - - return security_perf_event_open(PERF_SECURITY_TRACEPOINT); -} +extern int perf_allow_cpu(void); +extern int perf_allow_tracepoint(void); extern int perf_exclude_event(struct perf_event *event, struct pt_regs *regs); @@ -2029,6 +2015,19 @@ perf_event_pause(struct perf_event *event, bool reset) { return 0; } static inline int perf_exclude_event(struct perf_event *event, struct pt_regs *regs) { return 0; } +static inline int perf_allow_kernel(void) +{ + return perfmon_capable() ? 0 : -EACCES; +} +static inline int perf_allow_cpu(void) +{ + return perfmon_capable() ? 0 : -EACCES; +} +static inline int perf_allow_tracepoint(void) +{ + return perfmon_capable() ? 0 : -EPERM; +} + #endif /* !CONFIG_PERF_EVENTS */ #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL) diff --git a/kernel/events/core.c b/kernel/events/core.c index b46f849d726dc..eaad93d4493f5 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -14700,6 +14700,24 @@ int perf_allow_kernel(void) } EXPORT_SYMBOL_GPL(perf_allow_kernel); +int perf_allow_cpu(void) +{ + if (sysctl_perf_event_paranoid > 0 && !perfmon_capable()) + return -EACCES; + + return security_perf_event_open(PERF_SECURITY_CPU); +} +EXPORT_SYMBOL_GPL(perf_allow_cpu); + +int perf_allow_tracepoint(void) +{ + if (sysctl_perf_event_paranoid > -1 && !perfmon_capable()) + return -EPERM; + + return security_perf_event_open(PERF_SECURITY_TRACEPOINT); +} +EXPORT_SYMBOL_GPL(perf_allow_tracepoint); + /* * Inherit an event from parent task to child task. * From 42c6222bec070f230f5412a9591b8fa6c1a49ae5 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Wed, 27 May 2026 15:55:07 -0700 Subject: [PATCH 269/311] drm/xe: gate observation streams with perf_allow_cpu() BugLink: https://bugs.launchpad.net/bugs/2160654 xe OA and EU-stall paths open-code a partial copy of the system-wide perf CPU-event permission check: if (xe_observation_paranoid && !perfmon_capable()) return -EACCES; This open-coded check skips two things perf_allow_cpu() handles: the graduated kernel.perf_event_paranoid policy that an administrator may have tuned, and the security_perf_event_open() LSM hook. Introduce xe_observation_paranoid_check() to wrap perf_allow_cpu(), and convert the open-coded sites in xe_oa.c and xe_eu_stall.c. The dev.xe.observation_paranoid sysctl still acts as an escape hatch when cleared. xe observation now consults kernel.perf_event_paranoid and the LSM perf hook on every open. Sites that have already configured an LSM perf policy or tuned the paranoid sysctl will see those settings extend to xe. Signed-off-by: John Hubbard Reviewed-by: Ashutosh Dixit Link: https://patch.msgid.link/20260527225507.2044027-3-ashutosh.dixit@intel.com Signed-off-by: Ashutosh Dixit (backported from commit 6680bf0cb7261b7eb62a7226c6845c5c9ce5a009 linux-next) [kelseys: Drop the new int ret declaration and initialize the existing ret variable used by remap_pfn_range() instead. This tree does not contain commit 41255b2f1e03 ("drm/xe/oa: Use drm_gem_mmap_obj for OA buffer mmap"), which removed the legacy mapping loop and its ret variable.] Signed-off-by: Kelsey Steele Acked-by: Jamie Nguyen Acked-by: Carol L Soto Signed-off-by: Brad Figg --- drivers/gpu/drm/xe/xe_eu_stall.c | 5 +++-- drivers/gpu/drm/xe/xe_oa.c | 25 +++++++++++++--------- drivers/gpu/drm/xe/xe_observation.c | 32 ++++++++++++++++++++++++----- drivers/gpu/drm/xe/xe_observation.h | 3 +-- 4 files changed, 46 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c index 39723928a0199..ae6b054f13e55 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.c +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -963,9 +963,10 @@ int xe_eu_stall_stream_open(struct drm_device *dev, u64 data, struct drm_file *f return -ENODEV; } - if (xe_observation_paranoid && !perfmon_capable()) { + ret = xe_observation_paranoid_check(); + if (ret) { drm_dbg(&xe->drm, "Insufficient privileges for EU stall monitoring\n"); - return -EACCES; + return ret; } /* Initialize and set default values */ diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index fa90441d30529..f5b2550d5048d 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -1676,9 +1676,10 @@ static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma) unsigned long start = vma->vm_start; int i, ret; - if (xe_observation_paranoid && !perfmon_capable()) { + ret = xe_observation_paranoid_check(); + if (ret) { drm_dbg(&stream->oa->xe->drm, "Insufficient privilege to map OA buffer\n"); - return -EACCES; + return ret; } /* Can mmap the entire OA buffer or nothing (no partial OA buffer mmaps) */ @@ -2068,10 +2069,12 @@ int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *f privileged_op = true; } - if (privileged_op && xe_observation_paranoid && !perfmon_capable()) { - drm_dbg(&oa->xe->drm, "Insufficient privileges to open xe OA stream\n"); - ret = -EACCES; - goto err_exec_q; + if (privileged_op) { + ret = xe_observation_paranoid_check(); + if (ret) { + drm_dbg(&oa->xe->drm, "Insufficient privileges to open xe OA stream\n"); + goto err_exec_q; + } } if (!param.exec_q && !param.sample) { @@ -2350,9 +2353,10 @@ int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *fi return -ENODEV; } - if (xe_observation_paranoid && !perfmon_capable()) { + err = xe_observation_paranoid_check(); + if (err) { drm_dbg(&oa->xe->drm, "Insufficient privileges to add xe OA config\n"); - return -EACCES; + return err; } err = copy_from_user(¶m, u64_to_user_ptr(data), sizeof(param)); @@ -2452,9 +2456,10 @@ int xe_oa_remove_config_ioctl(struct drm_device *dev, u64 data, struct drm_file return -ENODEV; } - if (xe_observation_paranoid && !perfmon_capable()) { + ret = xe_observation_paranoid_check(); + if (ret) { drm_dbg(&oa->xe->drm, "Insufficient privileges to remove xe OA config\n"); - return -EACCES; + return ret; } ret = get_user(arg, ptr); diff --git a/drivers/gpu/drm/xe/xe_observation.c b/drivers/gpu/drm/xe/xe_observation.c index e3f9b546207e4..39e05b9131a74 100644 --- a/drivers/gpu/drm/xe/xe_observation.c +++ b/drivers/gpu/drm/xe/xe_observation.c @@ -4,6 +4,7 @@ */ #include +#include #include #include @@ -12,9 +13,28 @@ #include "xe_oa.h" #include "xe_observation.h" -u32 xe_observation_paranoid = true; +static u32 xe_observation_paranoid = true; static struct ctl_table_header *sysctl_header; +/** + * xe_observation_paranoid_check - Gate access to xe observation streams. + * + * When the xe-specific observation_paranoid sysctl is enabled (the + * default), defer to perf_allow_cpu() so that access is governed by the + * same policy as system-wide perf CPU events: kernel.perf_event_paranoid + * plus the security_perf_event_open() LSM hook. When the sysctl has been + * cleared by a privileged user, observation is open to all callers. + * + * Return: 0 if access is permitted, a negative errno otherwise. + */ +int xe_observation_paranoid_check(void) +{ + if (!xe_observation_paranoid) + return 0; + + return perf_allow_cpu(); +} + static int xe_oa_ioctl(struct drm_device *dev, struct drm_xe_observation_param *arg, struct drm_file *file) { @@ -83,11 +103,13 @@ static const struct ctl_table observation_ctl_table[] = { }; /** - * xe_observation_sysctl_register - Register xe_observation_paranoid sysctl + * xe_observation_sysctl_register - Register the observation_paranoid sysctl * - * Normally only superuser/root can access observation stream - * data. However, superuser can set xe_observation_paranoid sysctl to 0 to - * allow non-privileged users to also access observation data. + * When dev.xe.observation_paranoid is set (the default), access to + * observation streams follows the system-wide perf_allow_cpu() policy: + * kernel.perf_event_paranoid plus the security_perf_event_open() LSM + * hook. A privileged user can clear the sysctl to bypass that gate and + * allow unprivileged access to observation data. * * Return: always returns 0 */ diff --git a/drivers/gpu/drm/xe/xe_observation.h b/drivers/gpu/drm/xe/xe_observation.h index 17816998e9666..73a03e03c96a7 100644 --- a/drivers/gpu/drm/xe/xe_observation.h +++ b/drivers/gpu/drm/xe/xe_observation.h @@ -11,8 +11,7 @@ struct drm_device; struct drm_file; -extern u32 xe_observation_paranoid; - +int xe_observation_paranoid_check(void); int xe_observation_ioctl(struct drm_device *dev, void *data, struct drm_file *file); int xe_observation_sysctl_register(void); void xe_observation_sysctl_unregister(void); From 35ae8c576d033299fc8169939bc75ac2ef01b99d Mon Sep 17 00:00:00 2001 From: Cosmin Ratiu Date: Thu, 7 May 2026 10:56:04 +0300 Subject: [PATCH 270/311] ipv4: Provide a FIB flushing signal from nexthop removal functions BugLink: https://bugs.launchpad.net/bugs/2158449 Plumb a bool value throughout the various nexthop removal functions, determined in the innermost __remove_nexthop_fib() (which still does the FIB flushing) and propagated up all callers. The next patch will make use of this signal to optimize the removal of multiple nexthops by moving the FIB flushing up the call hierarchy. Signed-off-by: Cosmin Ratiu Reviewed-by: Ido Schimmel Reviewed-by: David Ahern Link: https://patch.msgid.link/20260507075606.322405-2-cratiu@nvidia.com Signed-off-by: Jakub Kicinski (cherry picked from commit 31c777be2a2efd8980a660724955ba795ef751de) Signed-off-by: Benjamin Poirier Acked-by: Omer Barak Acked-by: Aya Levin Acked-by: Jamie Nguyen Acked-by: Carol L Soto Signed-off-by: Brad Figg --- net/ipv4/nexthop.c | 50 +++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/net/ipv4/nexthop.c b/net/ipv4/nexthop.c index 2c9036c719b68..2203af3642826 100644 --- a/net/ipv4/nexthop.c +++ b/net/ipv4/nexthop.c @@ -20,7 +20,7 @@ #define NH_RES_DEFAULT_IDLE_TIMER (120 * HZ) #define NH_RES_DEFAULT_UNBALANCED_TIMER 0 /* No forced rebalancing. */ -static void remove_nexthop(struct net *net, struct nexthop *nh, +static bool remove_nexthop(struct net *net, struct nexthop *nh, struct nl_info *nlinfo); #define NH_DEV_HASHBITS 8 @@ -2016,7 +2016,7 @@ static void nh_hthr_group_rebalance(struct nh_group *nhg) } } -static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge, +static bool remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge, struct nl_info *nlinfo, struct list_head *deferred_free) { @@ -2033,10 +2033,8 @@ static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge, newg = nhg->spare; /* last entry, keep it visible and remove the parent */ - if (nhg->num_nh == 1) { - remove_nexthop(net, nhp, nlinfo); - return; - } + if (nhg->num_nh == 1) + return remove_nexthop(net, nhp, nlinfo); newg->has_v4 = false; newg->is_multipath = nhg->is_multipath; @@ -2093,22 +2091,26 @@ static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge, if (nlinfo) nexthop_notify(RTM_NEWNEXTHOP, nhp, nlinfo); + + return false; } -static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh, +static bool remove_nexthop_from_groups(struct net *net, struct nexthop *nh, struct nl_info *nlinfo) { struct nh_grp_entry *nhge, *tmp; LIST_HEAD(deferred_free); + bool need_flush = false; /* If there is nothing to do, let's avoid the costly call to * synchronize_net() */ if (list_empty(&nh->grp_list)) - return; + return false; list_for_each_entry_safe(nhge, tmp, &nh->grp_list, nh_list) - remove_nh_grp_entry(net, nhge, nlinfo, &deferred_free); + need_flush |= remove_nh_grp_entry(net, nhge, nlinfo, + &deferred_free); /* make sure all see the newly published array before releasing rtnl */ synchronize_net(); @@ -2118,6 +2120,8 @@ static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh, list_del(&nhge->nh_list); free_percpu(nhge->stats); } + + return need_flush; } static void remove_nexthop_group(struct nexthop *nh, struct nl_info *nlinfo) @@ -2142,17 +2146,15 @@ static void remove_nexthop_group(struct nexthop *nh, struct nl_info *nlinfo) } /* not called for nexthop replace */ -static void __remove_nexthop_fib(struct net *net, struct nexthop *nh) +static bool __remove_nexthop_fib(struct net *net, struct nexthop *nh) { + bool need_flush = !list_empty(&nh->fi_list); struct fib6_info *f6i; - bool do_flush = false; struct fib_info *fi; - list_for_each_entry(fi, &nh->fi_list, nh_list) { + list_for_each_entry(fi, &nh->fi_list, nh_list) fi->fib_flags |= RTNH_F_DEAD; - do_flush = true; - } - if (do_flush) + if (need_flush) fib_flush(net); spin_lock_bh(&nh->lock); @@ -2173,12 +2175,14 @@ static void __remove_nexthop_fib(struct net *net, struct nexthop *nh) } spin_unlock_bh(&nh->lock); + + return need_flush; } -static void __remove_nexthop(struct net *net, struct nexthop *nh, +static bool __remove_nexthop(struct net *net, struct nexthop *nh, struct nl_info *nlinfo) { - __remove_nexthop_fib(net, nh); + bool need_flush = __remove_nexthop_fib(net, nh); if (nh->is_group) { remove_nexthop_group(nh, nlinfo); @@ -2189,13 +2193,17 @@ static void __remove_nexthop(struct net *net, struct nexthop *nh, if (nhi->fib_nhc.nhc_dev) hlist_del(&nhi->dev_hash); - remove_nexthop_from_groups(net, nh, nlinfo); + need_flush |= remove_nexthop_from_groups(net, nh, nlinfo); } + + return need_flush; } -static void remove_nexthop(struct net *net, struct nexthop *nh, +static bool remove_nexthop(struct net *net, struct nexthop *nh, struct nl_info *nlinfo) { + bool need_flush; + call_nexthop_notifiers(net, NEXTHOP_EVENT_DEL, nh, NULL); /* remove from the tree */ @@ -2204,10 +2212,12 @@ static void remove_nexthop(struct net *net, struct nexthop *nh, if (nlinfo) nexthop_notify(RTM_DELNEXTHOP, nh, nlinfo); - __remove_nexthop(net, nh, nlinfo); + need_flush = __remove_nexthop(net, nh, nlinfo); nh_base_seq_inc(net); nexthop_put(nh); + + return need_flush; } /* if any FIB entries reference this nexthop, any dst entries From c43724b8c23347c6148c9d51af064f75d6c07272 Mon Sep 17 00:00:00 2001 From: Cosmin Ratiu Date: Thu, 7 May 2026 10:56:05 +0300 Subject: [PATCH 271/311] ipv4: Flush the FIB once on multiple nexthop removal BugLink: https://bugs.launchpad.net/bugs/2158449 When a device is going down or when a net namespace is deleted, all nexthops on it are removed, and for each nexthop being removed the FIB table is flushed, which does a full trie traversal looking for entries marked RTNH_F_DEAD and removing them. This is O(N x R), with N being number of dev nexthops and R being number of IPv4 routes. The RTNL is held the entire time. When there are many nexthops to be removed and many routing entries, this can result in the RTNL being held for multiple minutes, which causes unhappiness in other processes trying to acquire the RTNL (e.g. systemd-networkd for DHCP renewals). In a complicated deployment with multiple vxlan devices, each having 16K nexthops and a total of 128K ipv4 routes, this is exactly what happens: nexthop_flush_dev() # loops over 16K nexthops -> remove_nexthop() -> __remove_nexthop() -> __remove_nexthop_fib() # marks fi->fib_flags |= RTNH_F_DEAD -> fib_flush() # for EACH nexthop! -> fib_table_flush() # walks the ENTIRE FIB, 128K entries This patch makes use of the previously added FIB flushing signal to only do a single FIB flush after all nexthops to be removed are marked as RTNH_F_DEAD: - __remove_nexthop_fib() no longer flushes the FIB. - nexthop_flush_dev() and flush_all_nexthops() now keep track whether any nexthop was removed and trigger a FIB flush at the end. - a new wrapper is defined, remove_one_nexthop() which calls remove_nexthop() and flushes if necessary. This is intended for places which must remove a single nexthop and shouldn't worry about the need to trigger a FIB flush. For now, the only caller is rtm_del_nexthop(). - The two direct callers of __remove_nexthop() get a WARN_ON_ONCE, since the nh about to be removed should not have any FIB entries referencing it when replacing or inserting a new one. This dramatically improves performance from O(N x R) to O(N + R). Releasing a nexthop reference in remove_nexthop() now no longer frees it. Instead, it is deleted when the last fib_info pointing to it gets freed via free_fib_info_rcu(). All routing code is already careful not to take into consideration routes marked with RTNH_F_DEAD. Tested with: DEV=eth2 ip link set up dev $DEV ip link add testnh0 link $DEV type macvlan mode bridge ip addr add 198.51.100.1/24 dev testnh0 ip link set testnh0 up seq 1 65536 | \ sed 's/.*/nexthop add id & via 198.51.100.2 dev testnh0/' | \ ip -batch - i=1 for a in $(seq 0 255); do for b in $(seq 0 255); do echo "route add 10.${a}.${b}.0/32 nhid $i" i=$((i + 1)) done done | ip -batch - time ip link set testnh0 down ip link del testnh0 Without this patch: real 0m32.601s user 0m0.000s sys 0m32.511s With this patch: real 0m0.209s user 0m0.000s sys 0m0.153s Signed-off-by: Cosmin Ratiu Reviewed-by: Ido Schimmel Reviewed-by: David Ahern Link: https://patch.msgid.link/20260507075606.322405-3-cratiu@nvidia.com Signed-off-by: Jakub Kicinski (cherry picked from commit 35ce55100c61270eb8234bcc8ac87fec1d8e4ff9) Signed-off-by: Benjamin Poirier Acked-by: Omer Barak Acked-by: Aya Levin Acked-by: Jamie Nguyen Acked-by: Carol L Soto Signed-off-by: Brad Figg --- net/ipv4/nexthop.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/net/ipv4/nexthop.c b/net/ipv4/nexthop.c index 2203af3642826..f0eaea7cb0afc 100644 --- a/net/ipv4/nexthop.c +++ b/net/ipv4/nexthop.c @@ -2154,8 +2154,6 @@ static bool __remove_nexthop_fib(struct net *net, struct nexthop *nh) list_for_each_entry(fi, &nh->fi_list, nh_list) fi->fib_flags |= RTNH_F_DEAD; - if (need_flush) - fib_flush(net); spin_lock_bh(&nh->lock); @@ -2220,6 +2218,13 @@ static bool remove_nexthop(struct net *net, struct nexthop *nh, return need_flush; } +static void remove_one_nexthop(struct net *net, struct nexthop *nh, + struct nl_info *nlinfo) +{ + if (remove_nexthop(net, nh, nlinfo)) + fib_flush(net); +} + /* if any FIB entries reference this nexthop, any dst entries * need to be regenerated */ @@ -2599,7 +2604,7 @@ static int replace_nexthop(struct net *net, struct nexthop *old, if (!err) { nh_rt_cache_flush(net, old, new); - __remove_nexthop(net, new, NULL); + WARN_ON_ONCE(__remove_nexthop(net, new, NULL)); nexthop_put(new); } @@ -2706,6 +2711,7 @@ static void nexthop_flush_dev(struct net_device *dev, unsigned long event) unsigned int hash = nh_dev_hashfn(dev->ifindex); struct net *net = dev_net(dev); struct hlist_head *head = &net->nexthop.devhash[hash]; + bool need_flush = false; struct hlist_node *n; struct nh_info *nhi; @@ -2717,22 +2723,28 @@ static void nexthop_flush_dev(struct net_device *dev, unsigned long event) (event == NETDEV_DOWN || event == NETDEV_CHANGE)) continue; - remove_nexthop(net, nhi->nh_parent, NULL); + need_flush |= remove_nexthop(net, nhi->nh_parent, NULL); } + + if (need_flush) + fib_flush(net); } /* rtnl; called when net namespace is deleted */ static void flush_all_nexthops(struct net *net) { struct rb_root *root = &net->nexthop.rb_root; + bool need_flush = false; struct rb_node *node; struct nexthop *nh; while ((node = rb_first(root))) { nh = rb_entry(node, struct nexthop, rb_node); - remove_nexthop(net, nh, NULL); + need_flush |= remove_nexthop(net, nh, NULL); cond_resched(); } + if (need_flush) + fib_flush(net); } static struct nexthop *nexthop_create_group(struct net *net, @@ -3002,7 +3014,7 @@ static struct nexthop *nexthop_add(struct net *net, struct nh_config *cfg, err = insert_nexthop(net, nh, cfg, extack); if (err) { - __remove_nexthop(net, nh, NULL); + WARN_ON_ONCE(__remove_nexthop(net, nh, NULL)); nexthop_put(nh); nh = ERR_PTR(err); } @@ -3371,7 +3383,7 @@ static int rtm_del_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh, nh = nexthop_find_by_id(net, id); if (nh) - remove_nexthop(net, nh, &nlinfo); + remove_one_nexthop(net, nh, &nlinfo); else err = -ENOENT; From c2b505a25adf903817ce3c91e172b07c95447f9c Mon Sep 17 00:00:00 2001 From: Cosmin Ratiu Date: Thu, 7 May 2026 10:56:06 +0300 Subject: [PATCH 272/311] ipv4: Add __must_check to nexthop removal functions BugLink: https://bugs.launchpad.net/bugs/2158449 These functions return a signal whether FIB flushing is required which must not be ignored. Use the compiler to help with enforcing this requirement in the future. Signed-off-by: Cosmin Ratiu Reviewed-by: Ido Schimmel Reviewed-by: David Ahern Link: https://patch.msgid.link/20260507075606.322405-4-cratiu@nvidia.com Signed-off-by: Jakub Kicinski (cherry picked from commit 5dcbd64e66ba36fc7abd433d9bbba660dc0c473d) Signed-off-by: Benjamin Poirier Acked-by: Omer Barak Acked-by: Aya Levin Acked-by: Jamie Nguyen Acked-by: Carol L Soto Signed-off-by: Brad Figg --- net/ipv4/nexthop.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/net/ipv4/nexthop.c b/net/ipv4/nexthop.c index f0eaea7cb0afc..34a5005fee55e 100644 --- a/net/ipv4/nexthop.c +++ b/net/ipv4/nexthop.c @@ -20,8 +20,8 @@ #define NH_RES_DEFAULT_IDLE_TIMER (120 * HZ) #define NH_RES_DEFAULT_UNBALANCED_TIMER 0 /* No forced rebalancing. */ -static bool remove_nexthop(struct net *net, struct nexthop *nh, - struct nl_info *nlinfo); +static bool __must_check remove_nexthop(struct net *net, struct nexthop *nh, + struct nl_info *nlinfo); #define NH_DEV_HASHBITS 8 #define NH_DEV_HASHSIZE (1U << NH_DEV_HASHBITS) @@ -2016,9 +2016,9 @@ static void nh_hthr_group_rebalance(struct nh_group *nhg) } } -static bool remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge, - struct nl_info *nlinfo, - struct list_head *deferred_free) +static bool __must_check +remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge, + struct nl_info *nlinfo, struct list_head *deferred_free) { struct nh_grp_entry *nhges, *new_nhges; struct nexthop *nhp = nhge->nh_parent; @@ -2095,8 +2095,9 @@ static bool remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge, return false; } -static bool remove_nexthop_from_groups(struct net *net, struct nexthop *nh, - struct nl_info *nlinfo) +static bool __must_check +remove_nexthop_from_groups(struct net *net, struct nexthop *nh, + struct nl_info *nlinfo) { struct nh_grp_entry *nhge, *tmp; LIST_HEAD(deferred_free); @@ -2146,7 +2147,8 @@ static void remove_nexthop_group(struct nexthop *nh, struct nl_info *nlinfo) } /* not called for nexthop replace */ -static bool __remove_nexthop_fib(struct net *net, struct nexthop *nh) +static bool __must_check __remove_nexthop_fib(struct net *net, + struct nexthop *nh) { bool need_flush = !list_empty(&nh->fi_list); struct fib6_info *f6i; @@ -2177,8 +2179,8 @@ static bool __remove_nexthop_fib(struct net *net, struct nexthop *nh) return need_flush; } -static bool __remove_nexthop(struct net *net, struct nexthop *nh, - struct nl_info *nlinfo) +static bool __must_check __remove_nexthop(struct net *net, struct nexthop *nh, + struct nl_info *nlinfo) { bool need_flush = __remove_nexthop_fib(net, nh); @@ -2197,8 +2199,8 @@ static bool __remove_nexthop(struct net *net, struct nexthop *nh, return need_flush; } -static bool remove_nexthop(struct net *net, struct nexthop *nh, - struct nl_info *nlinfo) +static bool __must_check remove_nexthop(struct net *net, struct nexthop *nh, + struct nl_info *nlinfo) { bool need_flush; From ddbfbe6cd87a6ad6388784880e6a1ae592da6fe3 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Fri, 12 Jun 2026 12:00:00 -0700 Subject: [PATCH 273/311] NVIDIA: SAUCE: arm64: drtm: Add DRTM definitions and DLME entry stub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BugLink: https://bugs.launchpad.net/bugs/2161563 Add the ARM DRTM architectural definitions (DEN0113 v1.2) in drtm.h — SMC IDs, return codes, DRTM_PARAMETERS, address-map and DLME-data layouts, the full-range DMA sentinel — and sl_stub.S's sl_entry, the D-CRTM ERET target that recovers the DTB PA from the Preamble->DLME slot and branches to primary_entry. image-vars.h exports sl_entry to the EFI stub via __efistub_. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/include/asm/drtm.h | 139 +++++++++++++++++++++++++++++++++ arch/arm64/kernel/image-vars.h | 4 + arch/arm64/kernel/sl_stub.S | 55 +++++++++++++ 3 files changed, 198 insertions(+) create mode 100644 arch/arm64/include/asm/drtm.h create mode 100644 arch/arm64/kernel/sl_stub.S diff --git a/arch/arm64/include/asm/drtm.h b/arch/arm64/include/asm/drtm.h new file mode 100644 index 0000000000000..a072114c21d1a --- /dev/null +++ b/arch/arm64/include/asm/drtm.h @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * ARM64 DRTM (Dynamic Root of Trust for Measurement) definitions. + * Based on DEN0113 v1.2 — Arm DRTM Architecture Specification. + */ +#ifndef _ASM_ARM64_DRTM_H +#define _ASM_ARM64_DRTM_H + +#ifndef __ASSEMBLY__ +#include +#endif + +/* DRTM SMC Function IDs (DEN0113 v1.2 §3.2-3.10) */ +#define DRTM_SMC_FN_BASE 0xC4000110UL +#define DRTM_SMC_VERSION (DRTM_SMC_FN_BASE + 0x00) +#define DRTM_SMC_FEATURES (DRTM_SMC_FN_BASE + 0x01) +#define DRTM_SMC_UNPROTECT_MEMORY (DRTM_SMC_FN_BASE + 0x03) +#define DRTM_SMC_DYNAMIC_LAUNCH (DRTM_SMC_FN_BASE + 0x04) +#define DRTM_SMC_CLOSE_LOCALITY (DRTM_SMC_FN_BASE + 0x05) +#define DRTM_SMC_GET_ERROR (DRTM_SMC_FN_BASE + 0x06) +#define DRTM_SMC_SET_ERROR (DRTM_SMC_FN_BASE + 0x07) +#define DRTM_SMC_SET_TCB_HASH (DRTM_SMC_FN_BASE + 0x08) +#define DRTM_SMC_LOCK_TCB_HASH (DRTM_SMC_FN_BASE + 0x09) + +/* DRTM Return Codes (DEN0113 v1.2 §3.18, Table 20) */ +#define DRTM_SUCCESS 0 +#define DRTM_NOT_SUPPORTED (-1) +#define DRTM_INVALID_PARAMETERS (-2) +#define DRTM_DENIED (-3) +#define DRTM_INTERNAL_ERROR (-5) + +/* DEN0113 v1.2 Table 9: DRTM_PARAMETERS revision is 2 */ +#define DRTM_PARAMS_REVISION 2 + +/* Launch features */ +#define DRTM_LAUNCH_FEAT_MEM_PROT_ALL (0x0 << 3) + +/* DRTM page size */ +#define DRTM_PAGE_SIZE 0x1000 + +/* + * Preamble->DLME DTB-PA handoff slot: DTB PA written 8 bytes below the + * DLME data region; sl_entry reads it via X0+X1-8 after D-CRTM ERET. + * Private contract (not DEN0113); DTB PA validated before FDT is parsed. + */ +#define SL_DLME_DTB_SLOT_OFFSET (-8) + +/* + * Full-range DMA protection sentinel (DEN0113 v1.2 §3.14 Table 11 + + * §4.6.2): type = NORMAL, start = 0, page count = 2^52 - 1. + */ +#define DRTM_MEM_PROT_FULL_RANGE \ + ((0x0ULL << 55) | (0x0ULL << 52) | ((1ULL << 52) - 1ULL)) + +#ifndef __ASSEMBLY__ +/* + * DRTM_PARAMETERS (DEN0113 v1.2 §3.13, Table 9) + * Passed to DRTM_DYNAMIC_LAUNCH SMC in X1. + */ +struct drtm_parameters { + u16 revision; + u16 reserved; + u32 launch_features; + u64 dlme_region_address; + u64 dlme_region_size; + u64 dlme_image_start; + u64 dlme_entry_point_offset; + u64 dlme_image_size; + u64 dlme_data_offset; + u64 nw_dce_region_address; + u64 nw_dce_region_size; + u64 mem_prot_table_address; + u64 mem_prot_table_size; +} __packed; + +/* Memory Region Descriptor Table (DEN0113 v1.2 §3.14, Table 11) */ +struct drtm_mem_region_hdr { + u16 revision; + u16 reserved; + u32 num_regions; +} __packed; + +struct drtm_mem_region { + u64 start_address; + u64 size_and_type; +} __packed; + +/* + * Address map region types in size_and_type bits [54:52] (DEN0113 v1.2 + * §3.14 Table 11): 0 normal, 1 normal+cacheability, 2 device/MMIO, + * 3 non-volatile, 4 reserved. + */ +#define DRTM_REGION_TYPE_NORMAL 0 +#define DRTM_REGION_TYPE_NORMAL_CACHED 1 +#define DRTM_REGION_TYPE_DEVICE 2 +#define DRTM_REGION_TYPE_NV 3 +#define DRTM_REGION_TYPE_RSVD 4 + +/* + * size_and_type field helpers (DEN0113 v1.2 §3.14 Table 11): page count + * [51:0], region type [54:52], cacheability [56:55] (valid for + * NORMAL_CACHED), reserved [63:57]. + */ +#define DRTM_MEM_REGION_PAGE_COUNT(x) ((x) & ((1ULL << 52) - 1)) +#define DRTM_MEM_REGION_TYPE(x) (((x) >> 52) & 0x7) +#define DRTM_MEM_REGION_CACHEABILITY(x) (((x) >> 55) & 0x3) + +/* DLME Data Header (DEN0113 v1.2 §3.15, Table 14) — populated by D-CRTM */ +struct dlme_data_header { + __le16 version; + __le16 this_hdr_size; + __le32 reserved; + __le64 dlme_data_size; + __le64 protected_regions_size; + __le64 address_map_size; + __le64 drtm_event_log_size; + __le64 tcb_hash_table_size; + __le64 acpi_table_region_size; + __le64 impl_defined_region_size; +}; + +#ifdef CONFIG_ARM64_SECURE_LAUNCH +extern unsigned long sl_dlme_region_pa; +extern unsigned long sl_dlme_data_offset; + +void slaunch_early_init(void); +void slaunch_setup(void); +void slaunch_exit(void); +void slaunch_measure_post_efi(void); +#else +static inline void slaunch_early_init(void) { } +static inline void slaunch_setup(void) { } +static inline void slaunch_exit(void) { } +static inline void slaunch_measure_post_efi(void) { } +#endif + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_ARM64_DRTM_H */ diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index d7b0d12b10155..cca237b3cd424 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -42,6 +42,10 @@ PROVIDE(__efistub_sysfb_primary_display = sysfb_primary_display); #endif PROVIDE(__efistub__ctype = _ctype); +#ifdef CONFIG_ARM64_SECURE_LAUNCH +PROVIDE(__efistub_sl_entry = sl_entry); +#endif + PROVIDE(__pi___memcpy = __pi_memcpy); PROVIDE(__pi___memmove = __pi_memmove); PROVIDE(__pi___memset = __pi_memset); diff --git a/arch/arm64/kernel/sl_stub.S b/arch/arm64/kernel/sl_stub.S new file mode 100644 index 0000000000000..da2f7c46b4397 --- /dev/null +++ b/arch/arm64/kernel/sl_stub.S @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * ARM64 DRTM Secure Launch entry stub — DLME entry point. D-CRTM (TF-A) + * ERETs here per DEN0113 v1.2 §4.6.1 Table 33 with X0 = DLME region PA, + * X1 = data offset, MMU/D-cache off; recovers the DTB PA, branches to + * primary_entry for normal boot. + */ + +#include +#include +#include + + .section ".idmap.text","a" + +/* + * sl_entry — DRTM dynamic launch entry point. Only reached via DRTM + * ERET (DRTM_PARAMETERS entry_point_offset); never during normal boot. + */ +SYM_CODE_START(sl_entry) + /* + * Save DLME region PA for slaunch_setup(). MMU/D-cache off + * (DEN0113 v1.2 §4.6.1 Table 33), so the str hits DRAM; dc ivac + * (not cvac) drops the stale cache line so the kernel re-reads + * the fresh value once caches are on. + */ + adr_l x9, sl_dlme_region_pa + str x0, [x9] + dc ivac, x9 + + /* Same for DLME data offset. */ + adr_l x9, sl_dlme_data_offset + str x1, [x9] + dc ivac, x9 + dsb sy + + /* + * Load DTB PA from the Preamble->DLME slot at + * (region_base + data_offset + SL_DLME_DTB_SLOT_OFFSET). + */ + add x9, x0, x1 + ldr x0, [x9, #SL_DLME_DTB_SLOT_OFFSET] /* X0 = DTB PA */ + + /* Clear X1-X3 per arm64 boot protocol */ + mov x1, xzr + mov x2, xzr + mov x3, xzr + + /* Enter normal kernel boot with X0 = DTB PA */ + b primary_entry +SYM_CODE_END(sl_entry) + + .section .data + .align 3 +SYM_DATA(sl_dlme_region_pa, .quad 0) +SYM_DATA(sl_dlme_data_offset, .quad 0) From 427e66d1098da99270cbde9a828db68284d0051c Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Fri, 12 Jun 2026 12:00:00 -0700 Subject: [PATCH 274/311] NVIDIA: SAUCE: efi/libstub: arm64: Trigger DRTM Secure Launch from EFI stub BugLink: https://bugs.launchpad.net/bugs/2161563 When the cmdline has "drtm=on" and "efi=noruntime", issue DRTM_DYNAMIC_LAUNCH after ExitBootServices so the kernel boots as a DLME. arm64-slaunch.c builds DRTM_PARAMETERS with a full-range DMA table, queries the DLME-data reserve, writes the DTB PA to the Preamble->DLME slot, and launches (no return). The guard fails closed without efi=noruntime; arm64-stub.c/fdt.c/Makefile wire it in. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- drivers/firmware/efi/libstub/Makefile | 1 + drivers/firmware/efi/libstub/arm64-slaunch.c | 252 +++++++++++++++++++ drivers/firmware/efi/libstub/arm64-stub.c | 9 + drivers/firmware/efi/libstub/efistub.h | 9 + drivers/firmware/efi/libstub/fdt.c | 10 + 5 files changed, 281 insertions(+) create mode 100644 drivers/firmware/efi/libstub/arm64-slaunch.c diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index e386ffd009b7e..c1da3b536ece7 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -84,6 +84,7 @@ lib-$(CONFIG_EFI_GENERIC_STUB) += efi-stub.o string.o intrinsics.o systable.o \ lib-$(CONFIG_ARM) += arm32-stub.o lib-$(CONFIG_ARM64) += kaslr.o arm64.o arm64-stub.o smbios.o +lib-$(CONFIG_ARM64_SECURE_LAUNCH) += arm64-slaunch.o lib-$(CONFIG_X86) += x86-stub.o smbios.o lib-$(CONFIG_X86_64) += x86-5lvl.o lib-$(CONFIG_RISCV) += kaslr.o riscv.o riscv-stub.o diff --git a/drivers/firmware/efi/libstub/arm64-slaunch.c b/drivers/firmware/efi/libstub/arm64-slaunch.c new file mode 100644 index 0000000000000..408a286f13a19 --- /dev/null +++ b/drivers/firmware/efi/libstub/arm64-slaunch.c @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * ARM64 DRTM Secure Launch — EFI stub component. Builds DRTM_PARAMETERS + * and issues DRTM_DYNAMIC_LAUNCH when the cmdline has "drtm=on". The SMC + * does not return on success: D-CRTM measures the image and ERETs to + * sl_entry. Copyright (c) 2025-2026, NVIDIA Corporation. + */ + +#include +#include +#include +#include + +#include "efistub.h" + +/* DRTM SMC IDs — duplicated here for EFI stub isolation */ +#define SL_DRTM_SMC_FEATURES 0xC4000111UL +#define SL_DRTM_SMC_DYNAMIC_LAUNCH 0xC4000114UL +#define SL_DRTM_PAGE_SIZE 0x1000 +#define SL_ROUND_UP_4K(x) (((x) + SL_DRTM_PAGE_SIZE - 1) & \ + ~(SL_DRTM_PAGE_SIZE - 1ULL)) + +/* + * Preamble<->DLME DTB-PA convention slot (mirrors SL_DLME_DTB_SLOT_OFFSET + * in arch/arm64/include/asm/drtm.h — keep them in sync). + */ +#define SL_DLME_DTB_SLOT_OFFSET (-8) + +/* From sl_stub.S — accessible via __efistub_ alias in image-vars.h */ +extern char sl_entry[]; + +/* + * DRTM Parameters (DEN0113 v1.2 §3.13 / Table 9) + * Struct must be packed — passed directly to TF-A via SMC. + */ +struct sl_drtm_params { + u16 revision; + u16 reserved; + u32 launch_features; + u64 dlme_region_address; + u64 dlme_region_size; + u64 dlme_image_start; + u64 dlme_entry_point_offset; + u64 dlme_image_size; + u64 dlme_data_offset; + u64 nw_dce_region_address; + u64 nw_dce_region_size; + u64 mem_prot_table_address; + u64 mem_prot_table_size; +} __packed; + +static u64 sl_smc_ret(u64 fn, u64 arg1) +{ + register u64 x0 __asm__("x0") = fn; + register u64 x1 __asm__("x1") = arg1; + register u64 x2 __asm__("x2") = 0; + register u64 x3 __asm__("x3") = 0; + + asm volatile("smc #0" + : "+r"(x0), "+r"(x1), "+r"(x2), "+r"(x3) + : + : "x4", "x5", "x6", "x7", "x8", "x9", "x10", + "x11", "x12", "x13", "x14", "x15", "x16", "x17", + "memory"); + return x0; +} + +static void sl_smc(u64 fn, u64 arg1) +{ + sl_smc_ret(fn, arg1); +} + +/* Read CTR_EL0.DminLine and return cache line size in bytes. */ +static inline unsigned int sl_dcache_line_size(void) +{ + u64 ctr; + + asm volatile("mrs %0, ctr_el0" : "=r"(ctr)); + return 4U << ((ctr >> 16) & 0xfU); +} + +/* + * Clean (cvac) a range to PoC at cache-line granularity so TF-A sees + * the stub's freshly-written value from EL3. + */ +static inline void sl_dc_cvac_range(unsigned long start, unsigned long len) +{ + unsigned int line = sl_dcache_line_size(); + unsigned long mask = (unsigned long)line - 1UL; + unsigned long end = start + len; + unsigned long addr; + + start &= ~mask; + for (addr = start; addr < end; addr += line) + asm volatile("dc cvac, %0" : : "r"(addr) : "memory"); +} + +/* + * DLME data reserve (from DRTM_FEATURES) and whether D-CRTM advertised + * DRTM support. Consumed by arm64-stub.c and the launch gate in fdt.c. + */ +unsigned long sl_dlme_data_reserve; +bool sl_drtm_available; + +/* + * Query DRTM_FEATURES (DEN0113 v1.2 Table 6, feature 0x2) for the minimum + * DLME data size. Best-effort: failure leaves DRTM unavailable (normal + * boot). Called before ExitBootServices. + */ +void efi_slaunch_get_dlme_data_size(void) +{ + register u64 x0 __asm__("x0") = SL_DRTM_SMC_FEATURES; + register u64 x1 __asm__("x1") = (1ULL << 63) | 0x2; + register u64 x2 __asm__("x2") = 0; + register u64 x3 __asm__("x3") = 0; + u32 min_pages; + + asm volatile("smc #0" + : "+r"(x0), "+r"(x1), "+r"(x2), "+r"(x3) + : + : "x4", "x5", "x6", "x7", "x8", "x9", "x10", + "x11", "x12", "x13", "x14", "x15", "x16", "x17", + "memory"); + + /* DRTM_FEATURES success is x0 > 0; x1[31:0] = min DLME data pages. */ + if ((s64)x0 <= 0) + return; + min_pages = (u32)(x1 & 0xFFFFFFFF); + if (min_pages == 0) + return; + + sl_dlme_data_reserve = (unsigned long)min_pages * SL_DRTM_PAGE_SIZE; + sl_drtm_available = true; + efi_info("DRTM: min DLME data size %lu KB (%u pages)\n", + sl_dlme_data_reserve / 1024, min_pages); +} + +/* + * Token-aware cmdline match: true iff `tok` is a standalone + * whitespace-delimited word in `cmdline`, not a substring of another + * option (so "drtm=on" does not match "nodrtm=on" or "root=...drtm=on"). + */ +static bool sl_cmdline_token(const char *cmdline, const char *tok) +{ + size_t toklen = strlen(tok); + const char *p = cmdline; + + while ((p = strstr(p, tok)) != NULL) { + bool start_ok = (p == cmdline) || p[-1] == ' ' || p[-1] == '\t'; + bool end_ok = p[toklen] == '\0' || p[toklen] == ' ' || + p[toklen] == '\t'; + + if (start_ok && end_ok) + return true; + p += toklen; + } + return false; +} + +bool efi_slaunch_enabled(const char *cmdline) +{ + if (!cmdline) + return false; + if (!sl_cmdline_token(cmdline, "drtm=on")) + return false; + + /* + * drtm=on requires efi=noruntime, else the post-DRTM kernel could + * call unmeasured UEFI runtime services. If missing, skip DRTM. + */ + if (!sl_cmdline_token(cmdline, "efi=noruntime")) { + efi_warn("DRTM: drtm=on needs efi=noruntime; skipping DRTM\n"); + return false; + } + return true; +} + +/* + * TF-A requires DRTM_PARAMETERS to be 4KB-aligned; we are past + * ExitBootServices so cannot allocate — use a static buffer. + */ +static struct sl_drtm_params sl_params __aligned(SL_DRTM_PAGE_SIZE); + +void __noreturn efi_slaunch_drtm(unsigned long kernel_addr, + unsigned long fdt_addr) +{ + struct sl_drtm_params *params = &sl_params; + unsigned long image_size, kernel_memsize; + unsigned long dlme_data_offset; + unsigned long sl_entry_offset; + + /* + * Store DTB PA just below the DLME data area (dlme_data_offset - 8); + * sl_entry finds it via X0 + X1 - 8. Direct physical write avoids + * EFI-stub symbol resolution, which can fail under PIC/GOT. + */ + + /* Compute sl_entry offset from kernel image base */ + sl_entry_offset = (unsigned long)sl_entry - (unsigned long)_text; + + /* + * DLME region layout: [_text.._edata] measured image, [_edata.._end] + * BSS, then D-CRTM-populated DLME data after _end. FDT is separate + * (wherever efi_allocate_pages() put it). + */ + image_size = (unsigned long)(_edata - _text); + kernel_memsize = (unsigned long)(_end - _text); + dlme_data_offset = SL_ROUND_UP_4K(kernel_memsize); + + /* + * Write DTB PA into the Preamble->DLME slot at (kernel_addr + + * dlme_data_offset + SL_DLME_DTB_SLOT_OFFSET); sl_entry reads it via + * X0 + X1 + SL_DLME_DTB_SLOT_OFFSET after the D-CRTM ERET. + */ + *(volatile u64 *)(kernel_addr + dlme_data_offset + + SL_DLME_DTB_SLOT_OFFSET) = fdt_addr; + + /* Build DRTM_PARAMETERS */ + params->revision = DRTM_PARAMS_REVISION; + params->reserved = 0; + params->launch_features = 0; /* bits[5:3]=0: complete DMA protection */ + params->dlme_region_address = kernel_addr; + params->dlme_region_size = dlme_data_offset + sl_dlme_data_reserve; + params->dlme_image_start = 0; + params->dlme_entry_point_offset = sl_entry_offset; + params->dlme_image_size = image_size; + params->dlme_data_offset = dlme_data_offset; + params->nw_dce_region_address = 0; + params->nw_dce_region_size = 0; + /* Complete DMA protection: table must be zero (DEN0113 v1.2 Table 9). */ + params->mem_prot_table_address = 0; + params->mem_prot_table_size = 0; + + /* + * Clean to DRAM what the D-CRTM reads after the SMC: the params + * struct and the DTB PA slot (outside TF-A's own DLME flush). + */ + sl_dc_cvac_range((unsigned long)params, sizeof(*params)); + sl_dc_cvac_range(kernel_addr + dlme_data_offset + + SL_DLME_DTB_SLOT_OFFSET, sizeof(u64)); + asm volatile("dsb sy" : : : "memory"); + + /* + * DRTM_DYNAMIC_LAUNCH — does not return on success. + * D-CRTM: measures kernel, populates DLME data, ERETs to sl_entry + */ + sl_smc(SL_DRTM_SMC_DYNAMIC_LAUNCH, (u64)params); + + /* If we reach here, the SMC failed. Halt. */ + for (;;) + asm volatile("wfi"); +} diff --git a/drivers/firmware/efi/libstub/arm64-stub.c b/drivers/firmware/efi/libstub/arm64-stub.c index 2c38693561475..588b2c26617d7 100644 --- a/drivers/firmware/efi/libstub/arm64-stub.c +++ b/drivers/firmware/efi/libstub/arm64-stub.c @@ -36,6 +36,15 @@ efi_status_t handle_kernel_image(unsigned long *image_addr, kernel_codesize = __inittext_end - _text; kernel_memsize = kernel_size + (_end - _edata); *reserve_size = kernel_memsize; + +#ifdef CONFIG_ARM64_SECURE_LAUNCH + /* + * Reserve DLME data space (size from DRTM_FEATURES) after kernel + * BSS for D-CRTM to populate (address map, event log, etc.). + */ + efi_slaunch_get_dlme_data_size(); + *reserve_size += sl_dlme_data_reserve; +#endif *image_addr = (unsigned long)_text; return efi_kaslr_relocate_kernel(image_addr, reserve_addr, reserve_size, diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h index fb532eb61ace7..4e78bd3909aa9 100644 --- a/drivers/firmware/efi/libstub/efistub.h +++ b/drivers/firmware/efi/libstub/efistub.h @@ -1267,4 +1267,13 @@ void arch_accept_memory(phys_addr_t start, phys_addr_t end); efi_status_t efi_zboot_decompress_init(unsigned long *alloc_size); efi_status_t efi_zboot_decompress(u8 *out, unsigned long outlen); +#ifdef CONFIG_ARM64_SECURE_LAUNCH +bool efi_slaunch_enabled(const char *cmdline); +void efi_slaunch_get_dlme_data_size(void); +extern unsigned long sl_dlme_data_reserve; +extern bool sl_drtm_available; +void __noreturn efi_slaunch_drtm(unsigned long kernel_addr, + unsigned long fdt_addr); +#endif + #endif diff --git a/drivers/firmware/efi/libstub/fdt.c b/drivers/firmware/efi/libstub/fdt.c index 6c679da644dd6..994b75372ef07 100644 --- a/drivers/firmware/efi/libstub/fdt.c +++ b/drivers/firmware/efi/libstub/fdt.c @@ -363,6 +363,16 @@ efi_status_t efi_boot_kernel(void *handle, efi_loaded_image_t *image, if (IS_ENABLED(CONFIG_ARM)) efi_handle_post_ebs_state(); +#ifdef CONFIG_ARM64_SECURE_LAUNCH + /* Launch only if requested and the D-CRTM advertised support. */ + if (efi_slaunch_enabled(cmdline_ptr)) { + if (sl_drtm_available) + efi_slaunch_drtm(kernel_addr, fdt_addr); + else + efi_warn("DRTM: firmware lacks DRTM support; booting normally\n"); + } +#endif + efi_enter_kernel(kernel_addr, fdt_addr, fdt_totalsize((void *)fdt_addr)); /* not reached */ } From e8831759448c96382728ed6bf6dceaa3ae1697be Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Fri, 12 Jun 2026 12:00:00 -0700 Subject: [PATCH 275/311] NVIDIA: SAUCE: arm64: drtm: Address-map validation and DTB validation (early init) BugLink: https://bugs.launchpad.net/bugs/2161563 Add arch/arm64/kernel/slaunch.c with the pre-setup_machine_fdt() helpers: slaunch_parse_address_map() stashes the D-CRTM address map, slaunch_assert_full_lockdown() verifies full SMMU lockdown, and slaunch_early_init() validates the DTB PA/magic/extent against the trusted NORMAL regions. Stubs for the later phases and the Makefile wiring (CONFIG_ARM64_SECURE_LAUNCH) are added too. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/include/asm/drtm.h | 6 +- arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/setup.c | 3 + arch/arm64/kernel/slaunch.c | 403 ++++++++++++++++++++++++++++++++++ 4 files changed, 412 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/kernel/slaunch.c diff --git a/arch/arm64/include/asm/drtm.h b/arch/arm64/include/asm/drtm.h index a072114c21d1a..9bf4ff8ce5637 100644 --- a/arch/arm64/include/asm/drtm.h +++ b/arch/arm64/include/asm/drtm.h @@ -73,7 +73,11 @@ struct drtm_parameters { u64 mem_prot_table_size; } __packed; -/* Memory Region Descriptor Table (DEN0113 v1.2 §3.14, Table 11) */ +/* + * Memory Region Descriptor Table (DEN0113 v1.2 §3.14, Table 11): header + * followed by num_regions descriptors, consumed in place from the DLME + * data region (no fixed-size copy, no region-count cap). + */ struct drtm_mem_region_hdr { u16 revision; u16 reserved; diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 74b76bb704523..411e47910a1ee 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -50,6 +50,7 @@ obj-$(CONFIG_HAVE_STATIC_CALL) += static_call.o obj-$(CONFIG_CPU_PM) += sleep.o suspend.o obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_EFI) += efi.o efi-rt-wrapper.o +obj-$(CONFIG_ARM64_SECURE_LAUNCH) += sl_stub.o slaunch.o obj-$(CONFIG_PCI) += pci.o obj-$(CONFIG_ARMV8_DEPRECATED) += armv8_deprecated.o obj-$(CONFIG_ACPI) += acpi.o diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 23c05dc7a8f2a..856463cf6618d 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -289,6 +290,8 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) early_fixmap_init(); early_ioremap_init(); + slaunch_early_init(); + setup_machine_fdt(__fdt_pointer); /* diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c new file mode 100644 index 0000000000000..c10fdc0ed341c --- /dev/null +++ b/arch/arm64/kernel/slaunch.c @@ -0,0 +1,403 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * ARM64 DRTM Secure Launch support. Processes DRTM state when the kernel + * is launched as a DLME via DRTM dynamic launch; called early in + * setup_arch(). Copyright (c) 2025-2026, NVIDIA Corporation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +/* FDT magic number (big-endian 0xd00dfeed at offset 0) */ +#define FDT_HEADER_MAGIC 0xd00dfeed + +/* + * D-CRTM address map in the DLME data region (populated by D-CRTM, + * memblock_reserve'd by slaunch_setup()). Mapped once in early_init; + * pointer kept for all validation. No copy, no region-count cap. + */ +static struct drtm_mem_region *dcrtm_regions; +static u32 dcrtm_num_regions; + +/* + * DLME data extent saved at slaunch_setup() time so that + * slaunch_measure_post_efi() can re-reserve the region after efi_init()'s + * memblock_remove(0, PHYS_ADDR_MAX) wipes our earlier reservation. + */ +static phys_addr_t sl_dlme_data_pa; +static u64 sl_dlme_data_size; + +/* + * Close TPM locality 2 — the DLME's locality, per DEN0113 v1.2 §4.6.1. + * Locality 3 is the DCE's; it's closed by the DCE, not the DLME. + */ +static void __init slaunch_tpm_setup(void) +{ + struct arm_smccc_res res; + + arm_smccc_smc(DRTM_SMC_CLOSE_LOCALITY, 2, 0, 0, 0, 0, 0, 0, &res); + if (res.a0 == (unsigned long)DRTM_NOT_SUPPORTED) + pr_warn("slaunch: CLOSE_LOCALITY not supported (no TPM backend)\n"); + else if (res.a0 != DRTM_SUCCESS) + pr_err("slaunch: CLOSE_LOCALITY failed: %ld\n", (long)res.a0); + else + pr_info("slaunch: TPM locality 2 closed\n"); +} + +/* + * Parse the D-CRTM address map (at header_size + protected_regions_size + * within DLME data). A trusted EL3-populated input describing physical + * memory layout; stored for validating untrusted data (DTB, EFI mmap). + * Returns true on success. + */ +static bool __init slaunch_parse_address_map(phys_addr_t dlme_data_pa, + struct dlme_data_header *hdr) +{ + phys_addr_t map_pa; + u64 map_size; + u64 hdr_size, prot_size; + struct drtm_mem_region_hdr *map_hdr; + struct drtm_mem_region *regions; + u32 num_regions, i; + + /* + * All offset arithmetic below assumes the v1 dlme_data_header + * layout; a later revision would target wrong bytes. Reject != v1. + */ + if (le16_to_cpu(hdr->version) != 1) + panic("slaunch: DLME data header version %u; only v1 supported\n", + le16_to_cpu(hdr->version)); + + hdr_size = le16_to_cpu(hdr->this_hdr_size); + prot_size = le64_to_cpu(hdr->protected_regions_size); + map_size = le64_to_cpu(hdr->address_map_size); + + if (map_size == 0) { + pr_err("slaunch: D-CRTM address map is empty\n"); + return false; + } + + if (map_size < sizeof(struct drtm_mem_region_hdr)) { + pr_err("slaunch: address map too small (%llu bytes)\n", + map_size); + return false; + } + + map_pa = dlme_data_pa + hdr_size + prot_size; + + /* Map temporarily to validate and log */ + map_hdr = early_memremap(map_pa, (size_t)map_size); + if (!map_hdr) { + pr_err("slaunch: failed to map address map at 0x%llx\n", + (u64)map_pa); + return false; + } + + num_regions = le32_to_cpu(map_hdr->num_regions); + pr_info("slaunch: D-CRTM address map: revision=%u, %u regions\n", + le16_to_cpu(map_hdr->revision), num_regions); + + if (sizeof(struct drtm_mem_region_hdr) + + (u64)num_regions * sizeof(struct drtm_mem_region) > map_size) { + pr_err("slaunch: address map regions overflow map size\n"); + early_memunmap(map_hdr, (size_t)map_size); + return false; + } + + /* Log regions for debug */ + regions = (struct drtm_mem_region *)((u8 *)map_hdr + + sizeof(struct drtm_mem_region_hdr)); + for (i = 0; i < num_regions; i++) { + u64 addr = le64_to_cpu(regions[i].start_address); + u64 st = le64_to_cpu(regions[i].size_and_type); + u64 pages = DRTM_MEM_REGION_PAGE_COUNT(st); + u32 type = DRTM_MEM_REGION_TYPE(st); + + pr_info("slaunch: [%u] 0x%012llx - 0x%012llx %s (%llu pages)\n", + i, addr, addr + pages * DRTM_PAGE_SIZE, + type == DRTM_REGION_TYPE_NORMAL ? "NORMAL" : + type == DRTM_REGION_TYPE_NORMAL_CACHED ? "NORMAL_CACHED" : + type == DRTM_REGION_TYPE_DEVICE ? "DEVICE" : + type == DRTM_REGION_TYPE_NV ? "NV" : + type == DRTM_REGION_TYPE_RSVD ? "RSVD" : + "UNKNOWN", pages); + } + + early_memunmap(map_hdr, (size_t)map_size); + + /* + * Map regions and keep the pointer (physical memory is in DLME + * data, memblock_reserve'd later); validators use it directly. + */ + dcrtm_regions = early_memremap_ro(map_pa + sizeof(struct drtm_mem_region_hdr), + num_regions * sizeof(struct drtm_mem_region)); + if (!dcrtm_regions) { + pr_err("slaunch: failed to map address map regions\n"); + return false; + } + dcrtm_num_regions = num_regions; + return true; +} + +/* + * Verify D-CRTM published full-range DMA protection (DEN0113 v1.2 + * §4.6.2). Walks the protected_regions sub-region and requires the + * spec-conformant "single entry, start=0, full-range" encoding; partial + * lockdown breaks the measure-then-parse soundness model. + */ +static void __init slaunch_assert_full_lockdown(phys_addr_t dlme_data_pa, + u64 hdr_size, u64 prot_size) +{ + const struct drtm_mem_region_hdr *phdr; + const struct drtm_mem_region *regs; + phys_addr_t prot_pa; + u32 num; + u64 start, st; + + if (prot_size == 0) + panic("slaunch: DCE published empty protected_regions; cannot verify SMMU lockdown\n"); + if (prot_size < sizeof(*phdr) + sizeof(*regs)) + panic("slaunch: protected_regions size %llu too small for 1 entry\n", + prot_size); + + prot_pa = dlme_data_pa + hdr_size; + phdr = early_memremap_ro(prot_pa, (size_t)prot_size); + if (!phdr) + panic("slaunch: cannot map protected_regions at 0x%llx\n", + (u64)prot_pa); + + num = le32_to_cpu(phdr->num_regions); + regs = (const struct drtm_mem_region *)((const u8 *)phdr + sizeof(*phdr)); + start = le64_to_cpu(regs[0].start_address); + st = le64_to_cpu(regs[0].size_and_type); + + /* + * Strict spec match (DEN0113 v1.2 §3.15 R314110 + §4.6.2): single + * entry, start=0, size_and_type = DRTM_MEM_PROT_FULL_RANGE. Anything + * else is partial lockdown or non-conformant — fatal. + */ + if (num != 1 || start != 0 || st != DRTM_MEM_PROT_FULL_RANGE) + panic("slaunch: SMMU lockdown not full-range (num=%u start=0x%llx st=0x%llx; want 1/0/0x%llx); DRTM secure launch requires full DRAM coverage\n", + num, start, st, (u64)DRTM_MEM_PROT_FULL_RANGE); + + early_memunmap(phdr, (size_t)prot_size); + pr_info("slaunch: SMMU lockdown verified: full NS-DRAM coverage\n"); +} + +/* + * True if [start, start+size) falls entirely within normal memory of + * the D-CRTM address map. Normal = type 0 or type 1 (cacheability); + * DEVICE/NV/RSVD are rejected (DEN0113 v1.2 §3.14 Table 11 + R314100). + * Supports ranges spanning multiple adjacent/overlapping regions. + */ +static bool __init dcrtm_range_in_normal(u64 start, u64 size) +{ + u64 pos = start; + u64 end; + + if (!size || !dcrtm_regions) + return false; + /* Reject ranges that wrap u64 (attacker-controlled inputs). */ + if (check_add_overflow(start, size, &end)) + return false; + + while (pos < end) { + bool advanced = false; + u32 i; + + for (i = 0; i < dcrtm_num_regions; i++) { + u64 st = le64_to_cpu(dcrtm_regions[i].size_and_type); + u32 type = DRTM_MEM_REGION_TYPE(st); + u64 pages = DRTM_MEM_REGION_PAGE_COUNT(st); + u64 rstart = le64_to_cpu(dcrtm_regions[i].start_address); + u64 rsize, rend; + + if (type != DRTM_REGION_TYPE_NORMAL && + type != DRTM_REGION_TYPE_NORMAL_CACHED) + continue; + + /* Skip malformed regions whose size or end wraps. */ + if (check_mul_overflow(pages, (u64)DRTM_PAGE_SIZE, &rsize)) + continue; + if (check_add_overflow(rstart, rsize, &rend)) + continue; + + if (pos >= rstart && pos < rend) { + pos = (rend < end) ? rend : end; + advanced = true; + break; + } + } + + if (!advanced) + return false; + } + return true; +} + +/* + * Check if a range overlaps any non-normal region (DEVICE, NV, RSVD); + * type 0/1 normal memory is skipped (DEN0113 v1.2 §3.14 Table 11). + * Returns the region type if overlap found, -1 otherwise. + */ +static int __init dcrtm_range_overlaps_non_normal(u64 start, u64 size) +{ + u32 i; + u64 end; + + if (!dcrtm_regions) + return -1; + /* Wrapping range — fail closed: pretend it overlaps an RSVD region. */ + if (check_add_overflow(start, size, &end)) + return DRTM_REGION_TYPE_RSVD; + + for (i = 0; i < dcrtm_num_regions; i++) { + u64 st = le64_to_cpu(dcrtm_regions[i].size_and_type); + u32 type = DRTM_MEM_REGION_TYPE(st); + u64 pages = DRTM_MEM_REGION_PAGE_COUNT(st); + u64 rstart = le64_to_cpu(dcrtm_regions[i].start_address); + u64 rsize, rend; + + if (type == DRTM_REGION_TYPE_NORMAL || + type == DRTM_REGION_TYPE_NORMAL_CACHED) + continue; + + /* Malformed region whose size or end wraps: fail closed + * (assume it overlaps). + */ + if (check_mul_overflow(pages, (u64)DRTM_PAGE_SIZE, &rsize)) + return type; + if (check_add_overflow(rstart, rsize, &rend)) + return type; + + if (start < rend && end > rstart) + return type; + } + return -1; +} + +static const char * __init dcrtm_type_name(int type) +{ + switch (type) { + case DRTM_REGION_TYPE_NORMAL: return "NORMAL"; + case DRTM_REGION_TYPE_NORMAL_CACHED: return "NORMAL_CACHED"; + case DRTM_REGION_TYPE_DEVICE: return "DEVICE"; + case DRTM_REGION_TYPE_NV: return "NV"; + case DRTM_REGION_TYPE_RSVD: return "RSVD"; + default: return "UNKNOWN"; + } +} + +/* + * Check if two EFI memory regions overlap. + */ +static bool __init efi_regions_overlap(u64 s1, u64 sz1, u64 s2, u64 sz2) +{ + u64 e1, e2; + + /* Either range wraps u64 -> fail closed (report overlap). */ + if (check_add_overflow(s1, sz1, &e1) || + check_add_overflow(s2, sz2, &e2)) + return true; + return (s1 < e2) && (s2 < e1); +} + +/* + * slaunch_early_init() -- earliest DRTM validation, called BEFORE + * setup_machine_fdt() (after early_ioremap_init). Parses the D-CRTM + * address map, then validates the DTB PA/magic/size against it. Panics + * on any failure: the DTB is untrusted until validated. + */ +void __init slaunch_early_init(void) +{ + struct dlme_data_header *hdr; + phys_addr_t dlme_data_pa; + phys_addr_t dtb_pa; + u32 *dtb_hdr; + u32 fdt_magic, fdt_size; + + if (!sl_dlme_region_pa) + return; + + pr_info("slaunch: DRTM early init -- validating DTB before consumption\n"); + pr_info("slaunch: DLME region PA: 0x%lx, data offset: 0x%lx\n", + sl_dlme_region_pa, sl_dlme_data_offset); + + /* Step 1: Map DLME data header and parse address map */ + dlme_data_pa = sl_dlme_region_pa + sl_dlme_data_offset; + hdr = early_memremap(dlme_data_pa, sizeof(*hdr)); + if (!hdr) + panic("slaunch: failed to map DLME data header at 0x%llx\n", + (u64)dlme_data_pa); + + if (!slaunch_parse_address_map(dlme_data_pa, hdr)) + panic("slaunch: failed to parse D-CRTM address map -- cannot validate untrusted data\n"); + + early_memunmap(hdr, sizeof(*hdr)); + + /* Step 2: Validate DTB PA is in a NORMAL region */ + dtb_pa = __fdt_pointer; + if (!dtb_pa) + panic("slaunch: no DTB physical address available for validation\n"); + + pr_info("slaunch: DTB PA: 0x%llx\n", (u64)dtb_pa); + + /* + * Check the FDT header (magic + totalsize) is in a NORMAL region; + * the full-size check follows once fdt_totalsize is read. + */ + if (!dcrtm_range_in_normal(dtb_pa, sizeof(u32) * 2)) + panic("slaunch: DTB PA 0x%llx is NOT in a D-CRTM NORMAL region\n", + (u64)dtb_pa); + + /* Step 3: Map DTB header and verify FDT magic + size */ + dtb_hdr = early_memremap(dtb_pa, sizeof(u32) * 2); + if (!dtb_hdr) + panic("slaunch: failed to map DTB header at 0x%llx\n", + (u64)dtb_pa); + + fdt_magic = be32_to_cpu(dtb_hdr[0]); + fdt_size = be32_to_cpu(dtb_hdr[1]); + + early_memunmap(dtb_hdr, sizeof(u32) * 2); + + if (fdt_magic != FDT_HEADER_MAGIC) + panic("slaunch: DTB at 0x%llx has invalid FDT magic: 0x%08x (expected 0x%08x)\n", + (u64)dtb_pa, fdt_magic, FDT_HEADER_MAGIC); + + if (fdt_size == 0) + panic("slaunch: DTB at 0x%llx reports zero totalsize\n", + (u64)dtb_pa); + + /* + * Validate the full DTB range is in NORMAL memory; containment also + * caps fdt_size, so an over-large totalsize fails. + */ + if (!dcrtm_range_in_normal(dtb_pa, fdt_size)) + panic("slaunch: DTB range [0x%llx - 0x%llx] extends outside D-CRTM NORMAL region\n", + (u64)dtb_pa, (u64)(dtb_pa + fdt_size)); + + pr_info("slaunch: DTB validated: magic=0x%08x, size=%u bytes, in NORMAL region\n", + fdt_magic, fdt_size); +} + +/* Placeholder; populated by a subsequent patch. */ +void __init slaunch_setup(void) { } + +/* Placeholder; populated by a subsequent patch. */ +void __init slaunch_measure_post_efi(void) { } + +/* Placeholder; populated by a subsequent patch. */ +void slaunch_exit(void) { } From 4898b2698a5c2f980e22540dfa1f54af71ea159b Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Fri, 12 Jun 2026 12:00:00 -0700 Subject: [PATCH 276/311] NVIDIA: SAUCE: arm64: drtm: slaunch_setup, EFI runtime disable, full-lockdown assertion BugLink: https://bugs.launchpad.net/bugs/2161563 Adds slaunch_setup(): CLOSE_LOCALITY, DLME data reservation, full-lockdown assertion, and unconditional EFI runtime services disable. Introduces the sl_efi_info struct, /chosen-EFI reader, and configuration-table size table consumed by the raw-table validators. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/kernel/setup.c | 2 + arch/arm64/kernel/slaunch.c | 128 +++++++++++++++++++++++++++++++++++- 2 files changed, 128 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 856463cf6618d..b8cc26eaf98b2 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -322,6 +322,8 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) */ cpu_uninstall_idmap(); + slaunch_setup(); + xen_early_init(); efi_init(); diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index c10fdc0ed341c..187187c112389 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -393,8 +393,132 @@ void __init slaunch_early_init(void) fdt_magic, fdt_size); } -/* Placeholder; populated by a subsequent patch. */ -void __init slaunch_setup(void) { } +/* + * Validate all untrusted EFI inputs BEFORE efi_init() consumes them + * (it reads systab tables and walks the EFI mmap into memblock). The + * early validators below operate on the raw firmware buffers, not + * efi.memmap, which does not exist yet at this stage. + */ +struct sl_efi_info { + bool present; + u64 systab_pa; + u64 mmap_pa; + u64 mmap_size; + u32 desc_size; + u32 desc_ver; +}; + +/* Forward decls — bodies defined later (or stubbed out by #ifdef + * gating below). + */ +#ifdef CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT +static void __init slaunch_inject_fault(struct sl_efi_info *info); +#else +static inline void slaunch_inject_fault(struct sl_efi_info *info) { } +#endif +#ifdef CONFIG_ARM64_SECURE_LAUNCH_SELFTEST +static void __init slaunch_selftest(void); +#else +static inline void slaunch_selftest(void) { } +#endif + +static void __init slaunch_read_chosen_efi(struct sl_efi_info *info) +{ + const void *fdt = initial_boot_params; + const __be64 *p64; + const __be32 *p32; + int node, len; + + memset(info, 0, sizeof(*info)); + if (!fdt) + return; + node = fdt_path_offset(fdt, "/chosen"); + if (node < 0) + return; + +#define _GET64(name, field) \ + do { \ + p64 = fdt_getprop(fdt, node, name, &len); \ + if (!p64 || len < (int)sizeof(__be64)) \ + return; \ + info->field = be64_to_cpu(*p64); \ + } while (0) +#define _GET32(name, field) \ + do { \ + p32 = fdt_getprop(fdt, node, name, &len); \ + if (!p32 || len < (int)sizeof(__be32)) \ + return; \ + info->field = be32_to_cpu(*p32); \ + } while (0) + _GET64("linux,uefi-system-table", systab_pa); + _GET64("linux,uefi-mmap-start", mmap_pa); + _GET32("linux,uefi-mmap-size", mmap_size); + _GET32("linux,uefi-mmap-desc-size", desc_size); + _GET32("linux,uefi-mmap-desc-ver", desc_ver); +#undef _GET64 +#undef _GET32 + info->present = true; +} +/* + * Process DRTM state early in setup_arch() (address map already parsed + * by slaunch_early_init): reserve DLME data in memblock, CLOSE_LOCALITY, + * assert full-range DMA lockdown, and disable EFI runtime services. + */ +void __init slaunch_setup(void) +{ + struct dlme_data_header *hdr; + phys_addr_t dlme_data_pa; + + if (!sl_dlme_region_pa) + return; + + pr_info("slaunch: DRTM Secure Launch detected\n"); + + /* Map DLME data header for reservation */ + dlme_data_pa = sl_dlme_region_pa + sl_dlme_data_offset; + hdr = early_memremap(dlme_data_pa, sizeof(*hdr)); + if (!hdr) { + pr_err("slaunch: failed to map DLME data header at 0x%llx\n", + (u64)dlme_data_pa); + return; + } + + pr_info("slaunch: DLME data version: %u\n", + le16_to_cpu(hdr->version)); + pr_info("slaunch: DLME data size: %llu\n", + le64_to_cpu(hdr->dlme_data_size)); + pr_info("slaunch: Event log size: %llu\n", + le64_to_cpu(hdr->drtm_event_log_size)); + + /* Enforce full-lockdown assumption. Called from + * slaunch_setup (not slaunch_early_init) so panic prints — + * earlycon is registered by this point. */ + slaunch_assert_full_lockdown(dlme_data_pa, + le16_to_cpu(hdr->this_hdr_size), + le64_to_cpu(hdr->protected_regions_size)); + + /* Reserve DLME data region in memblock so kernel won't reuse it. + * NOTE: efi_init() runs after us and calls memblock_remove(0, + * PHYS_ADDR_MAX) which wipes this reservation. slaunch_validate_efi + * re-reserves using the saved values below. */ + sl_dlme_data_pa = dlme_data_pa; + sl_dlme_data_size = le64_to_cpu(hdr->dlme_data_size); + memblock_reserve(sl_dlme_data_pa, sl_dlme_data_size); + + early_memunmap(hdr, sizeof(*hdr)); + + slaunch_tpm_setup(); + + /* + * Unconditionally disable EFI runtime services: their pointers come + * from untrusted pre-DRTM firmware. Defense-in-depth backstop to the + * stub's efi=noruntime gate. Clearing EFI_RUNTIME_SERVICES and + * runtime_supported_mask makes all runtime dispatch see "unsupported". + */ + clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); + efi.runtime_supported_mask = 0; + pr_info("slaunch: EFI runtime services unconditionally disabled\n"); +} /* Placeholder; populated by a subsequent patch. */ void __init slaunch_measure_post_efi(void) { } From 6a1c71916b337b60c06766945785cc1a70808f2e Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Fri, 12 Jun 2026 12:00:00 -0700 Subject: [PATCH 277/311] NVIDIA: SAUCE: arm64: drtm: Raw EFI table validation (System Table, mmap) BugLink: https://bugs.launchpad.net/bugs/2161563 Adds slaunch_validate_raw_systab(), slaunch_validate_raw_mmap(), and slaunch_validate_efi_early(), wired into slaunch_setup() so untrusted EFI inputs are validated against the D-CRTM address map before efi_init() consumes them. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/kernel/slaunch.c | 221 +++++++++++++++++++++++++++++++++++- 1 file changed, 216 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index 187187c112389..84188ae50861d 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -459,10 +459,207 @@ static void __init slaunch_read_chosen_efi(struct sl_efi_info *info) #undef _GET32 info->present = true; } + +/* + * Per-GUID minimum sizes for known EFI ConfigurationTable entries (spec + * entry-point minimums, e.g. RSDP v2.0+ = 36 B); the validator checks + * NORMAL containment of this size, not a flat 4 KB. Unknown GUIDs use + * SL_CFGTBL_UNKNOWN_BOUND. + */ +struct sl_cfgtbl_size_entry { + efi_guid_t guid; + u32 min_size; +}; + +static const struct sl_cfgtbl_size_entry sl_cfgtbl_sizes[] __initconst = { + { ACPI_20_TABLE_GUID, 36 }, /* RSDP v2.0+ */ + { SMBIOS3_TABLE_GUID, 24 }, /* SMBIOS3 entry point */ + { EFI_RT_PROPERTIES_TABLE_GUID, 8 }, /* version + flags */ + { LINUX_EFI_MEMRESERVE_TABLE_GUID, 32 }, /* header struct */ + { LINUX_EFI_RANDOM_SEED_TABLE_GUID, 32 }, /* header struct */ +}; + +#define SL_CFGTBL_UNKNOWN_BOUND EFI_PAGE_SIZE + +static u32 __init sl_cfgtbl_min_size(const efi_guid_t *guid) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(sl_cfgtbl_sizes); i++) { + if (efi_guidcmp(*guid, sl_cfgtbl_sizes[i].guid) == 0) + return sl_cfgtbl_sizes[i].min_size; + } + return SL_CFGTBL_UNKNOWN_BOUND; +} + +/* Validate System Table + ConfigurationTable pointers against D-CRTM + * map. Runs pre-efi_init, panics on bad input. + */ +static void __init slaunch_validate_raw_systab(u64 systab_pa) +{ + efi_system_table_t *systab; + efi_config_table_t *cfgtbl; + unsigned long tables_pa; + unsigned long nr_tables; + size_t tbl_size; + unsigned long j; + + if (!dcrtm_range_in_normal(systab_pa, sizeof(efi_system_table_t))) + panic("slaunch: EFI System Table PA 0x%llx NOT in NORMAL region\n", + systab_pa); + + systab = early_memremap_ro(systab_pa, sizeof(efi_system_table_t)); + if (!systab) + panic("slaunch: failed to map EFI System Table at 0x%llx\n", + systab_pa); + + nr_tables = systab->nr_tables; + tables_pa = (unsigned long)systab->tables; + early_memunmap(systab, sizeof(efi_system_table_t)); + + if (nr_tables == 0 || !tables_pa) { + pr_info("slaunch: EFI System Table has no ConfigurationTable entries\n"); + return; + } + /* + * Reject nr_tables that overflow when multiplied by the per-entry + * size; NORMAL containment then caps any structurally too-large value. + */ + if (check_mul_overflow(nr_tables, + (unsigned long)sizeof(efi_config_table_t), + &tbl_size)) + panic("slaunch: EFI System Table nr_tables=%lu overflows tbl_size\n", + nr_tables); + + if (!dcrtm_range_in_normal(tables_pa, tbl_size)) + panic("slaunch: EFI ConfigurationTable array at 0x%lx NOT in NORMAL region\n", + tables_pa); + + cfgtbl = early_memremap_ro(tables_pa, tbl_size); + if (!cfgtbl) + panic("slaunch: failed to map ConfigurationTable at 0x%lx\n", + tables_pa); + + for (j = 0; j < nr_tables; j++) { + unsigned long tbl_ptr = (unsigned long)cfgtbl[j].table; + u32 size; + + if (!tbl_ptr) + continue; + size = sl_cfgtbl_min_size(&cfgtbl[j].guid); + if (!dcrtm_range_in_normal(tbl_ptr, size)) + panic("slaunch: EFI ConfigurationTable[%lu] 0x%lx [size %u] NOT in NORMAL region\n", + j, tbl_ptr, size); + } + early_memunmap(cfgtbl, tbl_size); + pr_info("slaunch: early EFI System Table validation PASSED (%lu entries)\n", + nr_tables); +} + +/* + * Validate the raw EFI memory map at /chosen/linux,uefi-mmap-start. + * Walks descriptors at desc-size stride (NOT sizeof(efi_memory_desc_t), + * which can differ for forward compatibility), pre-efi_init. + */ +static void __init slaunch_validate_raw_mmap(const struct sl_efi_info *info) +{ + void *mmap; + u64 offset; + u32 ndesc, nchecked = 0; + + /* Validate desc-size / desc-ver / mmap-size sanity. */ + if (info->desc_ver != 1) + panic("slaunch: linux,uefi-mmap-desc-ver=%u (expected 1)\n", + info->desc_ver); + if (info->desc_size < sizeof(efi_memory_desc_t) || info->desc_size > 128) + panic("slaunch: linux,uefi-mmap-desc-size=%u out of sane range\n", + info->desc_size); + if (info->mmap_size == 0 || + info->mmap_size % info->desc_size != 0) + panic("slaunch: linux,uefi-mmap-size=%llu not a multiple of desc-size=%u\n", + info->mmap_size, info->desc_size); + + /* + * mmap_pa + mmap_size must not wrap u64 and must lie in a NORMAL + * region; containment caps mmap_size, rejecting over-large values. + */ + { + u64 mmap_end; + + if (check_add_overflow(info->mmap_pa, info->mmap_size, &mmap_end)) + panic("slaunch: linux,uefi-mmap [0x%llx + %llu] wraps u64\n", + info->mmap_pa, info->mmap_size); + } + + /* The full mmap buffer must be in NORMAL memory. */ + if (!dcrtm_range_in_normal(info->mmap_pa, info->mmap_size)) + panic("slaunch: linux,uefi-mmap [0x%llx+0x%llx] NOT entirely in NORMAL\n", + info->mmap_pa, info->mmap_size); + + mmap = early_memremap_ro(info->mmap_pa, info->mmap_size); + if (!mmap) + panic("slaunch: failed to map raw EFI mmap at 0x%llx (size %llu)\n", + info->mmap_pa, info->mmap_size); + + ndesc = (u32)(info->mmap_size / info->desc_size); + for (offset = 0; offset < info->mmap_size; offset += info->desc_size) { + efi_memory_desc_t *md = (efi_memory_desc_t *)((u8 *)mmap + offset); + u64 phys = md->phys_addr; + u64 region_size, phys_end; + int otype; + + /* Skip MMIO and EfiReservedMemoryType: secure carveouts are + * reserved in the UEFI map but excluded from the NS-DRAM D-CRTM + * map and never ingested as usable RAM, so need not be NORMAL. */ + if (md->type == EFI_MEMORY_MAPPED_IO || + md->type == EFI_MEMORY_MAPPED_IO_PORT_SPACE || + md->type == EFI_RESERVED_TYPE) + continue; + + /* num_pages * page_size overflow guard. */ + if (check_mul_overflow(md->num_pages, (u64)EFI_PAGE_SIZE, + ®ion_size)) + panic("slaunch: raw EFI mmap[%llu]: type=%u num_pages=%llu overflows u64\n", + offset / info->desc_size, md->type, md->num_pages); + /* phys_addr + region_size address-wrap guard. */ + if (region_size && + check_add_overflow(phys, region_size, &phys_end)) + panic("slaunch: raw EFI mmap[%llu]: [0x%012llx + 0x%llx] type=%u wraps u64\n", + offset / info->desc_size, phys, region_size, + md->type); + + if (!dcrtm_range_in_normal(phys, region_size)) + panic("slaunch: raw EFI mmap[%llu]: region [0x%012llx-0x%012llx] type=%u NOT in NORMAL\n", + offset / info->desc_size, phys, + phys + region_size, md->type); + + otype = dcrtm_range_overlaps_non_normal(phys, region_size); + if (otype >= 0) + panic("slaunch: raw EFI mmap[%llu]: region [0x%012llx-0x%012llx] type=%u OVERLAPS %s\n", + offset / info->desc_size, phys, + phys + region_size, md->type, + dcrtm_type_name(otype)); + nchecked++; + } + early_memunmap(mmap, info->mmap_size); + pr_info("slaunch: early raw EFI mmap validation PASSED (%u of %u descriptors checked)\n", + nchecked, ndesc); +} + +static void __init slaunch_validate_efi_early(const struct sl_efi_info *info) +{ + if (!info->present) { + pr_info("slaunch: /chosen does not have all linux,uefi-* properties — skipping early EFI validation\n"); + return; + } + slaunch_validate_raw_systab(info->systab_pa); + slaunch_validate_raw_mmap(info); +} + /* * Process DRTM state early in setup_arch() (address map already parsed * by slaunch_early_init): reserve DLME data in memblock, CLOSE_LOCALITY, - * assert full-range DMA lockdown, and disable EFI runtime services. + * disable EFI runtime services, and validate the DTB /chosen EFI pointers. */ void __init slaunch_setup(void) { @@ -490,9 +687,10 @@ void __init slaunch_setup(void) pr_info("slaunch: Event log size: %llu\n", le64_to_cpu(hdr->drtm_event_log_size)); - /* Enforce full-lockdown assumption. Called from - * slaunch_setup (not slaunch_early_init) so panic prints — - * earlycon is registered by this point. */ + /* Enforce full-lockdown assumption per DEN0113 v1.2 §4.6.2. + * Called from slaunch_setup (not slaunch_early_init) so panic + * prints — earlycon is registered by this point. + */ slaunch_assert_full_lockdown(dlme_data_pa, le16_to_cpu(hdr->this_hdr_size), le64_to_cpu(hdr->protected_regions_size)); @@ -500,7 +698,8 @@ void __init slaunch_setup(void) /* Reserve DLME data region in memblock so kernel won't reuse it. * NOTE: efi_init() runs after us and calls memblock_remove(0, * PHYS_ADDR_MAX) which wipes this reservation. slaunch_validate_efi - * re-reserves using the saved values below. */ + * re-reserves using the saved values below. + */ sl_dlme_data_pa = dlme_data_pa; sl_dlme_data_size = le64_to_cpu(hdr->dlme_data_size); memblock_reserve(sl_dlme_data_pa, sl_dlme_data_size); @@ -518,6 +717,18 @@ void __init slaunch_setup(void) clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); efi.runtime_supported_mask = 0; pr_info("slaunch: EFI runtime services unconditionally disabled\n"); + + /* + * Validate all untrusted EFI inputs before efi_init() consumes them: + * read /chosen, optionally inject a fault, then run the validators. + */ + { + struct sl_efi_info efi_info; + + slaunch_read_chosen_efi(&efi_info); + slaunch_inject_fault(&efi_info); + slaunch_validate_efi_early(&efi_info); + } } /* Placeholder; populated by a subsequent patch. */ From e1402f1d79c6465f025e07e950ce2d15590ea398 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Fri, 12 Jun 2026 12:00:00 -0700 Subject: [PATCH 278/311] NVIDIA: SAUCE: arm64: drtm: ACPI table measurement and hash-algo verification BugLink: https://bugs.launchpad.net/bugs/2161563 Adds the slaunch_measurement struct, slaunch_measure() helper, slaunch_measure_one_acpi(), slaunch_verify_hash_algo() and slaunch_measure_acpi(). slaunch_measure_post_efi() is wired up to verify the negotiated hash algorithm and measure the ACPI tables into the DLME-side measurement table. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/kernel/setup.c | 6 + arch/arm64/kernel/slaunch.c | 368 +++++++++++++++++++++++++++++++++++- 2 files changed, 372 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index b8cc26eaf98b2..53c6c856f5dbd 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -338,6 +338,12 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) paging_init(); + /* + * Measure ACPI tables before acpi_boot_table_init() consumes them; + * after paging_init() so memblock_alloc() returns linear-mapped ptrs. + */ + slaunch_measure_post_efi(); + acpi_table_upgrade(); /* Parse the ACPI tables for possible boot-time configuration */ diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index 84188ae50861d..c7ca118473037 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -731,8 +731,372 @@ void __init slaunch_setup(void) } } -/* Placeholder; populated by a subsequent patch. */ -void __init slaunch_measure_post_efi(void) { } +/* + * Record an in-memory SHA-256 measurement into slaunch_measurements[] + * for later replay into the DRTM event log on PCR 18. The table is + * memblock_alloc-backed and grows geometrically, so platforms with many + * ACPI tables are not capped (seeded by slaunch_measurements_init()). + */ +struct slaunch_measurement { + char desc[16]; + u8 hash[SHA256_DIGEST_SIZE]; +}; + +static struct slaunch_measurement *slaunch_measurements __initdata; +static unsigned int slaunch_measurement_count __initdata; +static unsigned int slaunch_measurement_capacity __initdata; + +static void __init slaunch_measurements_init(void) +{ + size_t bytes; + + if (slaunch_measurements) + return; + + slaunch_measurement_capacity = 256; + bytes = (size_t)slaunch_measurement_capacity * + sizeof(*slaunch_measurements); + slaunch_measurements = memblock_alloc(bytes, SMP_CACHE_BYTES); + if (!slaunch_measurements) + panic("slaunch: memblock_alloc for measurement table failed (%zu bytes)\n", + bytes); +} + +static void __init slaunch_measurements_reserve(unsigned int needed) +{ + struct slaunch_measurement *new_arr; + unsigned int new_cap; + size_t new_bytes, old_bytes; + + if (needed <= slaunch_measurement_capacity) + return; + + new_cap = slaunch_measurement_capacity ? + slaunch_measurement_capacity : 1; + while (new_cap < needed) + new_cap *= 2; + + new_bytes = (size_t)new_cap * sizeof(*slaunch_measurements); + old_bytes = (size_t)slaunch_measurement_capacity * + sizeof(*slaunch_measurements); + + new_arr = memblock_alloc(new_bytes, SMP_CACHE_BYTES); + if (!new_arr) + panic("slaunch: measurement table grow failed at %u entries\n", + slaunch_measurement_count); + + if (slaunch_measurements && old_bytes) { + memcpy(new_arr, slaunch_measurements, + slaunch_measurement_count * + sizeof(*slaunch_measurements)); + memblock_free(slaunch_measurements, old_bytes); + } + + slaunch_measurements = new_arr; + slaunch_measurement_capacity = new_cap; +} + +static void __init slaunch_measure(const char *desc, const void *data, + size_t size) +{ + u8 hash[SHA256_DIGEST_SIZE]; + struct slaunch_measurement *m; + + sha256(data, size, hash); + + pr_info("slaunch: measured %s (%zu bytes) SHA-256: " + "%*phN\n", desc, size, SHA256_DIGEST_SIZE, hash); + + /* Future revision: also extend this hash into a hardware measurement + * engine (TPM HASH_START or platform-specific equivalent) so the + * measurement is anchored beyond the in-memory event log. + */ + + slaunch_measurements_reserve(slaunch_measurement_count + 1); + + m = &slaunch_measurements[slaunch_measurement_count++]; + strscpy(m->desc, desc, sizeof(m->desc)); + memcpy(m->hash, hash, SHA256_DIGEST_SIZE); +} + +/* + * Measure ACPI tables (RSDP, XSDT, and each XSDT-referenced table) while + * DMA protection is still active (before the slaunch_unprotect_memory + * late_initcall), so no device can tamper with them. The hashes are + * attestation evidence a remote verifier compares to known-good values. + */ +/* + * RSDP layout (ACPI 2.0+): we only need xsdt_physical_address at offset 24 + * and length at offset 20. Avoid depending on for the struct. + */ +/* Minimal ACPI table header — avoid full dependency. + * Full header is 36 bytes; XSDT entries follow after this. + */ +struct slaunch_acpi_hdr { + char signature[4]; + u32 length; + u8 revision; + u8 checksum; + char oem_id[6]; + char oem_table_id[8]; + u32 oem_revision; + u32 creator_id; + u32 creator_revision; +} __packed; + +#define RSDP_SIZE_V1 20 +#define RSDP_OFF_LEN 20 /* u32 length (ACPI 2.0+) */ +#define RSDP_OFF_XSDT 24 /* u64 xsdt_physical_address */ +#define RSDP_MIN_MAP 36 /* enough to read through xsdt_physical_address */ + +/* FADT field offsets (ACPI 6.x §5.2.9) — used to follow indirection + * to DSDT and FACS. Local copies to avoid pulling in . + */ +#define FADT_FIRMWARE_CTRL_OFF 36 /* u32 */ +#define FADT_DSDT_OFF 40 /* u32 */ +#define FADT_X_FIRMWARE_CTRL_OFF 132 /* u64, ACPI 2.0+ */ +#define FADT_X_DSDT_OFF 140 /* u64, ACPI 2.0+ */ + +/* Validate PA + length against D-CRTM map, then measure. Every + * failure is fatal — silent skip breaks attestation soundness. + */ +static void __init slaunch_measure_one_acpi(phys_addr_t pa, const char *desc) +{ + struct slaunch_acpi_hdr *tbl; + u32 tbl_len; + + if (!pa) + panic("slaunch: %s PA is 0 — cannot measure\n", desc); + if (!dcrtm_range_in_normal(pa, sizeof(*tbl))) + panic("slaunch: %s PA 0x%llx (header) NOT in NORMAL region\n", + desc, (u64)pa); + + tbl = early_memremap(pa, sizeof(*tbl)); + if (!tbl) + panic("slaunch: %s header remap failed at 0x%llx\n", + desc, (u64)pa); + tbl_len = tbl->length; + early_memunmap(tbl, sizeof(*tbl)); + + if (tbl_len < sizeof(*tbl)) + panic("slaunch: %s length %u < header size %zu\n", + desc, tbl_len, sizeof(*tbl)); + if (!dcrtm_range_in_normal(pa, tbl_len)) + panic("slaunch: %s [0x%llx+%u] NOT in NORMAL region\n", + desc, (u64)pa, tbl_len); + + tbl = early_memremap(pa, tbl_len); + if (!tbl) + panic("slaunch: %s full remap failed at 0x%llx (size %u)\n", + desc, (u64)pa, tbl_len); + slaunch_measure(desc, tbl, tbl_len); + early_memunmap(tbl, tbl_len); +} + +/* + * Query the D-CRTM TPM hash algorithm (DRTM_FEATURES feature 0x1, + * DEN0113 v1.2 §3.3) and refuse to proceed on mismatch, else the + * event-log digests would not replay against the attester's chain. + * Field layout: firmware_hash_algo [15:0] (0xB SHA-256, 0xC SHA-384). + */ +#define SL_DRTM_FW_HASH_SHA256 0x000B +#define SL_DRTM_FW_HASH_SHA384 0x000C +#define SL_DRTM_FW_HASH_MASK 0xFFFFULL + +static void __init slaunch_verify_hash_algo(void) +{ + struct arm_smccc_res res; + u64 features; + u32 algo; + + /* + * Feature 0x1 = TPM features (bit 63 set per spec). TF-A returns + * a0 = 1 or DRTM_NOT_SUPPORTED, a1 = the tpm_features bitfield. + */ + arm_smccc_smc(DRTM_SMC_FEATURES, (1ULL << 63) | 0x1, + 0, 0, 0, 0, 0, 0, &res); + if ((s64)res.a0 == DRTM_NOT_SUPPORTED) { + pr_warn("slaunch: DRTM_FEATURES(TPM) not supported; assuming SHA-256\n"); + return; + } + + features = res.a1; + algo = features & SL_DRTM_FW_HASH_MASK; + pr_info("slaunch: DCE firmware hash algorithm: 0x%x\n", algo); + + if (algo != SL_DRTM_FW_HASH_SHA256) + panic("slaunch: DCE reports hash algo 0x%x; kernel only implements SHA-256 (0xB). Add SHA-384 path or use a SHA-256 DCE.\n", + algo); +} + +static void __init slaunch_measure_acpi(void) +{ + struct slaunch_acpi_hdr *xsdt; + phys_addr_t rsdp_pa, xsdt_pa; + phys_addr_t dsdt_pa = 0, facs_pa = 0; + u32 rsdp_len, xsdt_len, num_entries, i; + u64 *entry_ptrs; + void *rsdp; + + /* Every failure below is fatal. The "kernel acts on unmeasured + * bytes" case breaks the attestation-based trust model. + */ + rsdp_pa = efi.acpi20; + if (rsdp_pa == EFI_INVALID_TABLE_ADDR || !rsdp_pa) + panic("slaunch: no ACPI RSDP in EFI System Table (DRTM requires ACPI)\n"); + + if (!dcrtm_range_in_normal(rsdp_pa, RSDP_MIN_MAP)) + panic("slaunch: RSDP PA 0x%llx NOT in NORMAL region\n", + (u64)rsdp_pa); + + rsdp = early_memremap(rsdp_pa, RSDP_MIN_MAP); + if (!rsdp) + panic("slaunch: RSDP header remap failed at 0x%llx\n", + (u64)rsdp_pa); + + rsdp_len = *(u32 *)((u8 *)rsdp + RSDP_OFF_LEN); + if (!rsdp_len) + rsdp_len = RSDP_SIZE_V1; + xsdt_pa = *(u64 *)((u8 *)rsdp + RSDP_OFF_XSDT); + early_memunmap(rsdp, RSDP_MIN_MAP); + + if (!dcrtm_range_in_normal(rsdp_pa, rsdp_len)) + panic("slaunch: RSDP [0x%llx+%u] NOT in NORMAL region\n", + (u64)rsdp_pa, rsdp_len); + rsdp = early_memremap(rsdp_pa, rsdp_len); + if (!rsdp) + panic("slaunch: RSDP full remap failed at 0x%llx (size %u)\n", + (u64)rsdp_pa, rsdp_len); + slaunch_measure("RSDP", rsdp, rsdp_len); + early_memunmap(rsdp, rsdp_len); + + if (!dcrtm_range_in_normal(xsdt_pa, sizeof(*xsdt))) + panic("slaunch: XSDT PA 0x%llx NOT in NORMAL region\n", xsdt_pa); + xsdt = early_memremap(xsdt_pa, sizeof(*xsdt)); + if (!xsdt) + panic("slaunch: XSDT header remap failed at 0x%llx\n", xsdt_pa); + xsdt_len = xsdt->length; + early_memunmap(xsdt, sizeof(*xsdt)); + + if (xsdt_len < sizeof(*xsdt)) + panic("slaunch: XSDT length %u < header size %zu\n", + xsdt_len, sizeof(*xsdt)); + if (!dcrtm_range_in_normal(xsdt_pa, xsdt_len)) + panic("slaunch: XSDT [0x%llx+%u] NOT in NORMAL region\n", + xsdt_pa, xsdt_len); + xsdt = early_memremap(xsdt_pa, xsdt_len); + if (!xsdt) + panic("slaunch: XSDT full remap failed at 0x%llx (size %u)\n", + xsdt_pa, xsdt_len); + slaunch_measure("XSDT", xsdt, xsdt_len); + + /* Walk XSDT entries. Each is a 64-bit PA to a top-level ACPI table. */ + num_entries = (xsdt_len - sizeof(*xsdt)) / sizeof(u64); + entry_ptrs = (u64 *)((u8 *)xsdt + sizeof(*xsdt)); + + for (i = 0; i < num_entries; i++) { + struct slaunch_acpi_hdr *tbl; + u64 tbl_pa = entry_ptrs[i]; + u32 tbl_len; + char desc[32]; + + if (!dcrtm_range_in_normal(tbl_pa, sizeof(*tbl))) + panic("slaunch: XSDT entry[%u] PA 0x%llx (hdr) NOT in NORMAL\n", + i, tbl_pa); + tbl = early_memremap(tbl_pa, sizeof(*tbl)); + if (!tbl) + panic("slaunch: XSDT entry[%u] header remap failed at 0x%llx\n", + i, tbl_pa); + tbl_len = tbl->length; + early_memunmap(tbl, sizeof(*tbl)); + + if (tbl_len < sizeof(*tbl)) + panic("slaunch: XSDT entry[%u] length %u < header size %zu\n", + i, tbl_len, sizeof(*tbl)); + if (!dcrtm_range_in_normal(tbl_pa, tbl_len)) + panic("slaunch: XSDT entry[%u] [0x%llx+%u] NOT in NORMAL\n", + i, tbl_pa, tbl_len); + tbl = early_memremap(tbl_pa, tbl_len); + if (!tbl) + panic("slaunch: XSDT entry[%u] full remap failed at 0x%llx (size %u)\n", + i, tbl_pa, tbl_len); + + snprintf(desc, sizeof(desc), "ACPI:%.4s", tbl->signature); + slaunch_measure(desc, tbl, tbl_len); + + /* Capture FADT indirection while FADT is mapped. Prefer + * 64-bit X_* fields (ACPI 2.0+); fall back to 32-bit + * fields if FADT length is too short to carry them. + */ + if (!memcmp(tbl->signature, "FACP", 4)) { + if (tbl_len >= FADT_X_DSDT_OFF + sizeof(u64)) + memcpy(&dsdt_pa, + (u8 *)tbl + FADT_X_DSDT_OFF, + sizeof(u64)); + if (!dsdt_pa && + tbl_len >= FADT_DSDT_OFF + sizeof(u32)) { + u32 d32; + + memcpy(&d32, + (u8 *)tbl + FADT_DSDT_OFF, + sizeof(u32)); + dsdt_pa = d32; + } + if (tbl_len >= FADT_X_FIRMWARE_CTRL_OFF + sizeof(u64)) + memcpy(&facs_pa, + (u8 *)tbl + FADT_X_FIRMWARE_CTRL_OFF, + sizeof(u64)); + if (!facs_pa && + tbl_len >= FADT_FIRMWARE_CTRL_OFF + sizeof(u32)) { + u32 f32; + + memcpy(&f32, + (u8 *)tbl + FADT_FIRMWARE_CTRL_OFF, + sizeof(u32)); + facs_pa = f32; + } + } + early_memunmap(tbl, tbl_len); + } + + early_memunmap(xsdt, xsdt_len); + + /* + * Follow FADT indirections (not in XSDT): DSDT carries the AML + * executed by ACPICA — mandatory; FACS is optional on HW-reduced + * ACPI. Other indirect tables (BERT/ERST/HEST/EINJ/PCCT) are data, + * not boot-TCB, so they are deliberately not followed. + */ + if (!dsdt_pa) + panic("slaunch: FADT present but X_Dsdt/Dsdt both zero — DSDT cannot be measured\n"); + slaunch_measure_one_acpi(dsdt_pa, "DSDT"); + if (facs_pa) + slaunch_measure_one_acpi(facs_pa, "FACS"); + else + pr_info("slaunch: FACS absent (HW-reduced ACPI) — no measurement needed\n"); +} +/* + * Re-reserve DLME data after memblock teardown, verify the D-CRTM + * hash algorithm, and measure ACPI tables into the DRTM event log. + */ +void __init slaunch_measure_post_efi(void) +{ + if (!sl_dlme_region_pa) + return; + + if (sl_dlme_data_size) + memblock_reserve(sl_dlme_data_pa, sl_dlme_data_size); + + slaunch_measurements_init(); + slaunch_verify_hash_algo(); + slaunch_measure_acpi(); + + if (dcrtm_regions) { + early_memunmap(dcrtm_regions, + dcrtm_num_regions * sizeof(*dcrtm_regions)); + dcrtm_regions = NULL; + } +} /* Placeholder; populated by a subsequent patch. */ void slaunch_exit(void) { } From 16a509b634c31803cf6edc12f70997827993d911 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Fri, 12 Jun 2026 12:00:00 -0700 Subject: [PATCH 279/311] NVIDIA: SAUCE: arm64: drtm: DRTM event log extension (DLME-side TCG_PCR_EVENT2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BugLink: https://bugs.launchpad.net/bugs/2161563 Adds slaunch_ranges_overlap(), sl_evlog_append_event2(), and slaunch_extend_drtm_event_log(); wires the event-log extension call into slaunch_measure_post_efi() so DLME-side measurements append to the canonical event log per DEN0113 v1.2 §3.17 and §4.8.4 on PCR 18. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/kernel/slaunch.c | 172 +++++++++++++++++++++++++++++++++++- 1 file changed, 170 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index c7ca118473037..2c5b17ffc7a46 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -1075,9 +1075,176 @@ static void __init slaunch_measure_acpi(void) else pr_info("slaunch: FACS absent (HW-reduced ACPI) — no measurement needed\n"); } + +/* Half-open range overlap. Fails closed (returns true) on u64 wrap of + * either range — refuse to consume rather than silently miscompute. */ +static bool __init slaunch_ranges_overlap(u64 a_start, u64 a_size, + u64 b_start, u64 b_size) +{ + u64 a_end, b_end; + + if (check_add_overflow(a_start, a_size, &a_end)) + return true; + if (check_add_overflow(b_start, b_size, &b_end)) + return true; + return a_start < b_end && b_start < a_end; +} + +/* + * Extend the DRTM event log with DLME-side measurements. DCE writes a + * TCG log into the DLME data region; we append one TCG_PCR_EVENT2 per + * DLME measurement into the trailing slack and bump drtm_event_log_size, + * so a verifier sees one chain (DEN0113 v1.2 §3.17/§4.8.4; TCG PFP §10.2.2). + */ +#define SL_TPM_ALG_SHA256 0x000B +/* + * TODO: no Arm event type (DEN0113 v1.2 §3.17.2 Table 19, base 0x9000) + * matches a DLME-side ACPI measurement, so generic TCG + * EV_PLATFORM_CONFIG_FLAGS (0x0A) is used; switch to a dedicated + * EVTYPE_ARM_* once one is registered. + */ +#define SL_EV_PLATFORM_CONFIG_FLAGS 0x0000000A +#define SL_DRTM_PCR_INDEX 18 /* DEN0113 v1.2: PCR[18] DLME schema, §4.8.4 Table 40 */ + +/* + * Write one TCG_PCR_EVENT2 at *off in buf (advances *off); -ENOSPC if no + * room within max. Packed LE layout: u32 PCRIndex, EventType, + * digest_count; per digest u16 hashAlg + hash[]; u32 EventSize + Event[]. + */ +static int __init sl_evlog_append_event2(u8 *buf, size_t *off, size_t max, + u32 pcr, u32 type, + const u8 hash[SHA256_DIGEST_SIZE], + const void *event_data, + u32 event_size) +{ + size_t need = 4 + 4 + 4 + 2 + SHA256_DIGEST_SIZE + 4 + event_size; + u8 *p; + + if (*off + need > max) + return -ENOSPC; + + p = buf + *off; + *(__le32 *)(p + 0) = cpu_to_le32(pcr); + *(__le32 *)(p + 4) = cpu_to_le32(type); + *(__le32 *)(p + 8) = cpu_to_le32(1); + *(__le16 *)(p + 12) = cpu_to_le16(SL_TPM_ALG_SHA256); + memcpy(p + 14, hash, SHA256_DIGEST_SIZE); + *(__le32 *)(p + 14 + SHA256_DIGEST_SIZE) = cpu_to_le32(event_size); + if (event_size) + memcpy(p + 14 + SHA256_DIGEST_SIZE + 4, event_data, event_size); + *off += need; + return 0; +} + +static void __init slaunch_extend_drtm_event_log(void) +{ + phys_addr_t dlme_data_pa, evlog_pa; + struct dlme_data_header *hdr; + u64 hdr_size, prot_size, map_size, dlme_data_size; + u64 evlog_size_initial; + size_t evlog_max, evlog_off; + u8 *evlog_va; + unsigned int i; + + if (!sl_dlme_region_pa) + return; + + dlme_data_pa = sl_dlme_region_pa + sl_dlme_data_offset; + + /* Read offsets from the DLME data header. Unmap before we touch + * the event log to avoid fixmap-slot overlap on the same page. + */ + hdr = early_memremap(dlme_data_pa, sizeof(*hdr)); + if (!hdr) + panic("slaunch: event log: cannot map DLME data header\n"); + hdr_size = le16_to_cpu(hdr->this_hdr_size); + prot_size = le64_to_cpu(hdr->protected_regions_size); + map_size = le64_to_cpu(hdr->address_map_size); + dlme_data_size = le64_to_cpu(hdr->dlme_data_size); + evlog_size_initial = le64_to_cpu(hdr->drtm_event_log_size); + early_memunmap(hdr, sizeof(*hdr)); + + if (evlog_size_initial == 0) + panic("slaunch: DCE published empty DRTM event log\n"); + + /* + * Event log runs from after header+protected_regions+address_map to + * the end of the DLME data region: evlog_max is capacity, DCE used + * the first evlog_size_initial bytes, DLME appends into the slack. + * Guard the sub-region arithmetic against u64 wrap/underflow. + */ + { + u64 sub_total; + + if (check_add_overflow(hdr_size, prot_size, &sub_total) || + check_add_overflow(sub_total, map_size, &sub_total)) + panic("slaunch: DLME header sub-region sizes wrap u64 (hdr=%llu prot=%llu map=%llu)\n", + hdr_size, prot_size, map_size); + if (sub_total > sl_dlme_data_size) + panic("slaunch: DLME header sub-region total %llu exceeds dlme_data_size %llu\n", + sub_total, sl_dlme_data_size); + + evlog_pa = dlme_data_pa + sub_total; + evlog_max = (size_t)(sl_dlme_data_size - sub_total); + } + + pr_info("slaunch: DRTM event log buffer: PA 0x%llx, capacity %zu B, DCE used %llu B, slack %zu B\n", + (u64)evlog_pa, evlog_max, evlog_size_initial, + evlog_max - (size_t)evlog_size_initial); + + evlog_va = early_memremap(evlog_pa, evlog_max); + if (!evlog_va) + panic("slaunch: cannot map DRTM event log at 0x%llx (%zu B)\n", + (u64)evlog_pa, evlog_max); + + /* Append DLME events after DCE's events, in-place in DLME data. */ + evlog_off = (size_t)evlog_size_initial; + for (i = 0; i < slaunch_measurement_count; i++) { + const struct slaunch_measurement *m = &slaunch_measurements[i]; + size_t dlen = strnlen(m->desc, sizeof(m->desc)); + + if (sl_evlog_append_event2(evlog_va, &evlog_off, evlog_max, + SL_DRTM_PCR_INDEX, + SL_EV_PLATFORM_CONFIG_FLAGS, + m->hash, m->desc, (u32)dlen)) + panic("slaunch: event log overflow at DLME entry %u (%s) — bump Preamble's sl_dlme_data_reserve\n", + i, m->desc); + } + + pr_info("slaunch: DRTM event log: appended %u DLME entries (%zu B); new total %zu B\n", + slaunch_measurement_count, + evlog_off - (size_t)evlog_size_initial, evlog_off); + + /* Canonical event-log bytes (DCE + DLME) are what a verifier + * replays; exposed for inspection rather than dumped to the log. + */ + pr_info("slaunch: DRTM event log canonical size (DCE + DLME): %zu B\n", + evlog_off); + + early_memunmap(evlog_va, evlog_max); + + /* Update the DLME data header so drtm_event_log_size reflects + * what we just wrote. Verifier reads (PA, size) from the header + * to know what to replay. + */ + hdr = early_memremap(dlme_data_pa, sizeof(*hdr)); + if (!hdr) + panic("slaunch: cannot re-map DLME data header to update size\n"); + hdr->drtm_event_log_size = cpu_to_le64(evlog_off); + early_memunmap(hdr, sizeof(*hdr)); + + pr_info("slaunch: DLME event log entries (PCR %u, EV_PLATFORM_CONFIG_FLAGS):\n", + SL_DRTM_PCR_INDEX); + for (i = 0; i < slaunch_measurement_count; i++) { + const struct slaunch_measurement *m = &slaunch_measurements[i]; + + pr_info("slaunch: [%2u] %-12s SHA-256: %*phN\n", + i, m->desc, SHA256_DIGEST_SIZE, m->hash); + } +} /* - * Re-reserve DLME data after memblock teardown, verify the D-CRTM - * hash algorithm, and measure ACPI tables into the DRTM event log. + * Adds event-log extension: re-reserve DLME data, + * verify hash algo, measure ACPI tables, extend the DRTM event log. */ void __init slaunch_measure_post_efi(void) { @@ -1090,6 +1257,7 @@ void __init slaunch_measure_post_efi(void) slaunch_measurements_init(); slaunch_verify_hash_algo(); slaunch_measure_acpi(); + slaunch_extend_drtm_event_log(); if (dcrtm_regions) { early_memunmap(dcrtm_regions, From 6d6518639046551f478102bf51a3604dd38b175d Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Fri, 12 Jun 2026 12:00:00 -0700 Subject: [PATCH 280/311] NVIDIA: SAUCE: arm64: drtm: Initrd measurement BugLink: https://bugs.launchpad.net/bugs/2161563 Adds slaunch_measure_initrd(). The bootloader-provided initrd extent is validated against the D-CRTM address map and the reserved DLME region, page-aligned, then SHA-256 hashed. The hash is staged in the DLME-side measurement table from slaunch_measure_post_efi() for the event-log extension pass to emit as a TCG_PCR_EVENT2 record. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/kernel/slaunch.c | 85 ++++++++++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index 2c5b17ffc7a46..42365cf7671ba 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -1090,6 +1090,85 @@ static bool __init slaunch_ranges_overlap(u64 a_start, u64 a_size, return a_start < b_end && b_start < a_end; } +#ifdef CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT +static bool __init sl_cmdline_has(const char *tok); +static void __init slaunch_inject_initrd(u64 *start, u64 *size); +#endif + +/* + * Measure the bootloader-supplied initramfs. The buffer sits in + * untrusted DRAM not covered by D-CRTM, so its extent is validated + * before any read and failures are fatal. Absence is legitimate and + * logged (no marker hash); the verifier infers coverage from the event. + */ +static void __init slaunch_measure_initrd(void) +{ + u64 start = phys_initrd_start; + u64 size = phys_initrd_size; + u64 end, dlme_size, off = 0, remaining; + struct sha256_ctx sctx; + u8 hash[SHA256_DIGEST_SIZE]; + struct slaunch_measurement *m; + void *p; + + if (!start || !size) { + pr_info("slaunch: no initrd present, skipping measurement\n"); + return; + } + +#ifdef CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT + slaunch_inject_initrd(&start, &size); +#endif + + if (!IS_ALIGNED(start, PAGE_SIZE)) + panic("slaunch: initrd start 0x%llx not page-aligned\n", + start); + + if (check_add_overflow(start, size, &end)) + panic("slaunch: initrd [0x%llx + %llu] wraps u64\n", + start, size); + + if (!dcrtm_range_in_normal(start, size)) + panic("slaunch: initrd [0x%llx+%llu] NOT in NORMAL region\n", + start, size); + + dlme_size = (sl_dlme_data_pa + sl_dlme_data_size) - sl_dlme_region_pa; + if (slaunch_ranges_overlap(start, size, + sl_dlme_region_pa, dlme_size)) + panic("slaunch: initrd [0x%llx+%llu] overlaps DLME region [0x%llx+%llu]\n", + start, size, (u64)sl_dlme_region_pa, dlme_size); + + /* + * Chunked early_memremap + streaming SHA-256: the initrd can + * exceed a single early_memremap and the linear map is not + * reliable here, so map one page at a time, feed into + * sha256_update, then sha256_final. + */ + sha256_init(&sctx); + remaining = size; + while (remaining > 0) { + size_t chunk = min_t(u64, remaining, (u64)PAGE_SIZE); + + p = early_memremap(start + off, chunk); + if (!p) + panic("slaunch: initrd chunk remap failed at 0x%llx (chunk %zu)\n", + start + off, chunk); + sha256_update(&sctx, p, chunk); + early_memunmap(p, chunk); + off += chunk; + remaining -= chunk; + } + sha256_final(&sctx, hash); + + pr_info("slaunch: measured initrd (%llu bytes) SHA-256: %*phN\n", + size, SHA256_DIGEST_SIZE, hash); + + slaunch_measurements_reserve(slaunch_measurement_count + 1); + m = &slaunch_measurements[slaunch_measurement_count++]; + strscpy(m->desc, "initrd", sizeof(m->desc)); + memcpy(m->hash, hash, SHA256_DIGEST_SIZE); +} + /* * Extend the DRTM event log with DLME-side measurements. DCE writes a * TCG log into the DLME data region; we append one TCG_PCR_EVENT2 per @@ -1243,8 +1322,9 @@ static void __init slaunch_extend_drtm_event_log(void) } } /* - * Adds event-log extension: re-reserve DLME data, - * verify hash algo, measure ACPI tables, extend the DRTM event log. + * Adds initrd measurement: re-reserve DLME data, + * verify hash algo, measure ACPI tables, measure initrd, extend the + * DRTM event log. */ void __init slaunch_measure_post_efi(void) { @@ -1257,6 +1337,7 @@ void __init slaunch_measure_post_efi(void) slaunch_measurements_init(); slaunch_verify_hash_algo(); slaunch_measure_acpi(); + slaunch_measure_initrd(); slaunch_extend_drtm_event_log(); if (dcrtm_regions) { From 61ec9eb8b9785b6fb0b87746e1ac5580e2debfad Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Fri, 12 Jun 2026 12:00:00 -0700 Subject: [PATCH 281/311] NVIDIA: SAUCE: arm64: drtm: Self-test, fault-injection harness, slaunch_exit BugLink: https://bugs.launchpad.net/bugs/2161563 Add the CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT harness (raw mmap mutators, inject_fault, inject_initrd) and the CONFIG_ARM64_SECURE_LAUNCH_SELFTEST self-test, wiring both into the validators and the initrd measurement. Also add the slaunch_unprotect_memory() late_initcall and the slaunch_exit() body for kexec/reboot cleanup. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/kernel/slaunch.c | 275 +++++++++++++++++++++++++++++++++++- 1 file changed, 270 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index 42365cf7671ba..e2ff33fd768b9 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -1321,10 +1321,234 @@ static void __init slaunch_extend_drtm_event_log(void) i, m->desc, SHA256_DIGEST_SIZE, m->hash); } } + +#ifdef CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT /* - * Adds initrd measurement: re-reserve DLME data, - * verify hash algo, measure ACPI tables, measure initrd, extend the - * DRTM event log. + * Negative-test fault injection (slaunch_inject=): mutates the + * raw EFI mmap before slaunch_validate_efi_early so the validator panics + * (a passing negative test). Tokens: mmap_wrap, mmap_pages_overflow, + * mmap_size_huge, systab_nr_tables_huge. Not for production. + */ +static bool __init sl_cmdline_has(const char *tok) +{ + return strstr(boot_command_line, tok); +} + +/* Mutate one descriptor in the raw EFI mmap buffer. Returns true if + * we found a CONVENTIONAL_MEMORY descriptor and applied the mutation. + */ +typedef void (*sl_md_mutator_t)(efi_memory_desc_t *md); + +static bool __init slaunch_inject_raw_mmap(const struct sl_efi_info *info, + sl_md_mutator_t mutate) +{ + void *mmap; + u64 offset; + bool applied = false; + + mmap = early_memremap(info->mmap_pa, info->mmap_size); + if (!mmap) { + pr_warn("slaunch: INJECT: failed to map raw mmap\n"); + return false; + } + for (offset = 0; offset < info->mmap_size; offset += info->desc_size) { + efi_memory_desc_t *md = + (efi_memory_desc_t *)((u8 *)mmap + offset); + + if (md->type != EFI_CONVENTIONAL_MEMORY) + continue; + mutate(md); + applied = true; + break; + } + early_memunmap(mmap, info->mmap_size); + return applied; +} + +static void __init sl_mutate_mmap_wrap(efi_memory_desc_t *md) +{ + /* phys_addr + num_pages*4096 wraps past U64_MAX */ + md->num_pages = (~md->phys_addr / EFI_PAGE_SIZE) + 2; + pr_warn("slaunch: INJECT mmap_wrap on phys=0x%llx -> num_pages=%llu (expect panic 'wraps u64')\n", + md->phys_addr, md->num_pages); +} + +static void __init sl_mutate_mmap_pages_overflow(efi_memory_desc_t *md) +{ + /* num_pages * EFI_PAGE_SIZE itself overflows */ + md->num_pages = (U64_MAX / EFI_PAGE_SIZE) + 2; + pr_warn("slaunch: INJECT mmap_pages_overflow on phys=0x%llx -> num_pages=%llu (expect panic 'overflows u64')\n", + md->phys_addr, md->num_pages); +} + +static void __init slaunch_inject_fault(struct sl_efi_info *info) +{ + if (!info->present) { + /* Nothing to inject into. */ + return; + } + + if (sl_cmdline_has("slaunch_inject=mmap_wrap")) { + if (!slaunch_inject_raw_mmap(info, sl_mutate_mmap_wrap)) + pr_warn("slaunch: INJECT mmap_wrap: no EFI_CONVENTIONAL_MEMORY descriptor found\n"); + return; + } + if (sl_cmdline_has("slaunch_inject=mmap_pages_overflow")) { + if (!slaunch_inject_raw_mmap(info, sl_mutate_mmap_pages_overflow)) + pr_warn("slaunch: INJECT mmap_pages_overflow: no EFI_CONVENTIONAL_MEMORY descriptor found\n"); + return; + } + if (sl_cmdline_has("slaunch_inject=mmap_size_huge")) { + /* + * Inflate the local mmap size (a 48-multiple, so the + * multiple-of-desc guard is not what trips) past any NORMAL + * region so the containment check in validate_raw_mmap fires. + */ + info->mmap_size = (0x10000000000ULL / 48ULL) * 48ULL; + pr_warn("slaunch: INJECT mmap_size_huge: info->mmap_size=%llu (expect panic 'NOT entirely in NORMAL')\n", + info->mmap_size); + return; + } + if (sl_cmdline_has("slaunch_inject=systab_nr_tables_huge")) { + /* + * Inflate nr_tables in-place so nr_tables * sizeof(entry) + * overflows and check_mul_overflow in validate_raw_systab panics. + */ + efi_system_table_t *systab; + + systab = early_memremap(info->systab_pa, + sizeof(efi_system_table_t)); + if (!systab) { + pr_warn("slaunch: INJECT systab_nr_tables_huge: remap failed\n"); + return; + } + systab->nr_tables = + (ULONG_MAX / sizeof(efi_config_table_t)) + 1UL; + pr_warn("slaunch: INJECT systab_nr_tables_huge: nr_tables=%lu (expect panic 'overflows tbl_size')\n", + (unsigned long)systab->nr_tables); + early_memunmap(systab, sizeof(efi_system_table_t)); + return; + } +} + +/* + * Negative-test fault injection for initrd measurement (cmdline + * slaunch_inject=). Mutates the local start/size copies before + * validation, exercising the same path an attacker-controlled DTB would. + * Tokens: initrd_wrap, initrd_size_huge, initrd_outside_normal, _overlap_dlme. + */ +static void __init slaunch_inject_initrd(u64 *start, u64 *size) +{ + if (sl_cmdline_has("slaunch_inject=initrd_wrap")) { + *size = (~(*start)) + 2; + pr_warn("slaunch: INJECT initrd_wrap: start=0x%llx size=%llu (expect panic 'wraps u64')\n", + *start, *size); + return; + } + if (sl_cmdline_has("slaunch_inject=initrd_size_huge")) { + /* + * Force the initrd end past its NORMAL region with a size just + * below the u64 wrap, so check_add_overflow passes and the + * NORMAL-containment check is what panics. + */ + if (*start && U64_MAX - *start > 1ULL) + *size = U64_MAX - *start - 1ULL; + else + *size = (1ULL << 62); + pr_warn("slaunch: INJECT initrd_size_huge: size=%llu (expect panic 'NOT in NORMAL region')\n", + *size); + return; + } + if (sl_cmdline_has("slaunch_inject=initrd_outside_normal")) { + *start = 0x1c090000ULL; + *size = PAGE_SIZE; + pr_warn("slaunch: INJECT initrd_outside_normal: start=0x%llx (expect panic 'NOT in NORMAL region')\n", + *start); + return; + } + if (sl_cmdline_has("slaunch_inject=initrd_overlap_dlme")) { + *start = sl_dlme_region_pa; + *size = PAGE_SIZE; + pr_warn("slaunch: INJECT initrd_overlap_dlme: start=0x%llx (expect panic 'overlaps DLME')\n", + *start); + return; + } +} +#endif /* CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT */ + +#ifdef CONFIG_ARM64_SECURE_LAUNCH_SELFTEST +/* + * Self-test of the validation guards (end of slaunch_measure_post_efi). + * Calls the helpers with crafted wrapping inputs plus valid ones; the + * helpers must reject the bad and accept the good, else panic with a + * "selftest:" prefix that the harness flags as a regression. + */ +static void __init slaunch_selftest(void) +{ + /* T1: dcrtm_range_in_normal MUST reject wrapping start+size */ + if (dcrtm_range_in_normal(0xFFFFFFFFFFFFE000ULL, 0x10000ULL)) + panic("selftest: dcrtm_range_in_normal accepted wrapping range\n"); + + /* T2: dcrtm_range_in_normal MUST reject size==0 */ + if (dcrtm_range_in_normal(0x80000000ULL, 0)) + panic("selftest: dcrtm_range_in_normal accepted size=0\n"); + + /* T3: dcrtm_range_in_normal MUST accept a known-good NORMAL range + * (one page in NS DRAM). Regression check. + */ + if (!dcrtm_range_in_normal(0x80000000ULL, EFI_PAGE_SIZE)) + panic("selftest: dcrtm_range_in_normal rejected known NORMAL range (regression)\n"); + + /* T4: dcrtm_range_overlaps_non_normal MUST fail closed on wrap + * (returns RSVD instead of -1). + */ + if (dcrtm_range_overlaps_non_normal(0xFFFFFFFFFFFFE000ULL, + 0x10000ULL) != DRTM_REGION_TYPE_RSVD) + panic("selftest: dcrtm_range_overlaps_non_normal did not fail closed on wrap\n"); + + /* T5: efi_regions_overlap MUST fail closed (true) on either wrap */ + if (!efi_regions_overlap(0xFFFFFFFFFFFFE000ULL, 0x10000ULL, + 0x80000000ULL, EFI_PAGE_SIZE)) + panic("selftest: efi_regions_overlap did not fail closed on wrap\n"); + + /* T6: efi_regions_overlap on disjoint ranges MUST return false */ + if (efi_regions_overlap(0x80000000ULL, 0x1000ULL, + 0x90000000ULL, 0x1000ULL)) + panic("selftest: efi_regions_overlap reported false overlap on disjoint ranges\n"); + + /* T7: efi_regions_overlap on truly-overlapping ranges MUST return true */ + if (!efi_regions_overlap(0x80000000ULL, 0x10000ULL, + 0x80008000ULL, 0x10000ULL)) + panic("selftest: efi_regions_overlap missed real overlap\n"); + + /* T8: check_mul_overflow catches num_pages * EFI_PAGE_SIZE wrap + * (the macro behind the slaunch_validate_efi guard). + */ + { + u64 out; + + if (!check_mul_overflow((u64)0x10000000000000ULL, + (u64)EFI_PAGE_SIZE, &out)) + panic("selftest: check_mul_overflow missed num_pages*PAGE_SIZE wrap\n"); + } + + pr_info("slaunch: ALL SELFTESTS PASSED (8/8)\n"); +} +#endif /* CONFIG_ARM64_SECURE_LAUNCH_SELFTEST */ + +/* + * All validation of untrusted EFI inputs happens in slaunch_setup() + * via slaunch_validate_efi_early() — before efi_init() ingests them. + * This post-efi_init slot keeps only the jobs that require + * efi_init's outputs: + * + * - Re-reserve DLME data in memblock (efi_init's + * memblock_remove(0, PHYS_ADDR_MAX) wipes the slaunch_setup + * reservation). + * - Measure ACPI tables via efi.acpi20 (which efi_init populated + * from the now pre-validated ConfigurationTable). + * - Run the validation-helper self-test + * (CONFIG_ARM64_SECURE_LAUNCH_SELFTEST). */ void __init slaunch_measure_post_efi(void) { @@ -1335,11 +1559,17 @@ void __init slaunch_measure_post_efi(void) memblock_reserve(sl_dlme_data_pa, sl_dlme_data_size); slaunch_measurements_init(); + slaunch_selftest(); slaunch_verify_hash_algo(); slaunch_measure_acpi(); slaunch_measure_initrd(); slaunch_extend_drtm_event_log(); + /* + * Release the dcrtm_regions early_memremap_ro slot; no further + * validators consume it. NULLing the pointer makes any stray + * post-init caller fail-fast in dcrtm_range_in_normal's guard. + */ if (dcrtm_regions) { early_memunmap(dcrtm_regions, dcrtm_num_regions * sizeof(*dcrtm_regions)); @@ -1347,5 +1577,40 @@ void __init slaunch_measure_post_efi(void) } } -/* Placeholder; populated by a subsequent patch. */ -void slaunch_exit(void) { } +/* + * Release DRTM DMA protection after IOMMU/SMMU drivers have + * established their own DMA isolation. + */ +static int __init slaunch_unprotect_memory(void) +{ + struct arm_smccc_res res; + + if (!sl_dlme_region_pa) + return 0; + + pr_info("slaunch: Calling DRTM_UNPROTECT_MEMORY\n"); + arm_smccc_smc(DRTM_SMC_UNPROTECT_MEMORY, 0, 0, 0, 0, 0, 0, 0, &res); + if (res.a0 != DRTM_SUCCESS) { + pr_err("slaunch: UNPROTECT_MEMORY failed: %ld\n", + (long)res.a0); + return -EIO; + } + + pr_info("slaunch: DMA protection released\n"); + return 0; +} +late_initcall(slaunch_unprotect_memory); + +/* + * Clean up DRTM state before kexec or reboot. Do not call + * DRTM_SET_ERROR(0): per DEN0113 v1.2 §3.8 its argument is the persisted + * error code, zero is reserved, and no "clear errors" semantics exists. + */ +void slaunch_exit(void) +{ + if (!sl_dlme_region_pa) + return; + + pr_info("slaunch: Cleaning DRTM state before kexec/reboot\n"); + sl_dlme_region_pa = 0; +} From 985cd085005c3e6ea7444be0d686e3e1b9039a46 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Fri, 12 Jun 2026 12:00:00 -0700 Subject: [PATCH 282/311] NVIDIA: SAUCE: arm64: drtm: Add CONFIG_ARM64_SECURE_LAUNCH Kconfig option BugLink: https://bugs.launchpad.net/bugs/2161563 Add CONFIG_ARM64_SECURE_LAUNCH and the two test-only siblings (SELFTEST, FAULT_INJECT) so the DRTM boot logic and EFI-stub launcher can be toggled at build time. All three default n; FAULT_INJECT is marked not-for-production because it mutates raw EFI inputs to drive negative tests. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/Kconfig | 55 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 480eb78619d16..15b94370fe218 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2463,6 +2463,61 @@ config EFI allow the kernel to be booted as an EFI application. This is only useful on systems that have UEFI firmware. +config ARM64_SECURE_LAUNCH + bool "ARM64 DRTM Secure Launch support" + depends on ARM64 && EFI + default n + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 + help + Enable support for ARM DRTM (Dynamic Root of Trust for Measurement) + Secure Launch. When enabled, the kernel can act as a DLME, receiving + control after a DRTM dynamic launch with MMU off at EL2. + + The EFI stub triggers DRTM_DYNAMIC_LAUNCH after ExitBootServices + when "drtm=on" is on the kernel command line. D-CRTM measures the + kernel and returns control. The kernel then parses DLME data and + manages DRTM SMC calls during boot. + + If unsure, say N. + +config ARM64_SECURE_LAUNCH_SELFTEST + bool "ARM64 DRTM Secure Launch validation self-tests" + depends on ARM64_SECURE_LAUNCH + default n + help + Build slaunch_selftest() into the kernel. At the end of + slaunch_measure_post_efi() the self-test calls the address-map + validation helpers (dcrtm_range_in_normal, + dcrtm_range_overlaps_non_normal, efi_regions_overlap) with + crafted inputs that previously bypassed validation due to + integer overflow. The helpers must reject the bad inputs and + accept the good ones; mismatch panics with selftest:* message. + + Useful for proving the overflow / wrap guards work as designed. + + If unsure, say N. + +config ARM64_SECURE_LAUNCH_FAULT_INJECT + bool "ARM64 DRTM Secure Launch fault injection" + depends on ARM64_SECURE_LAUNCH + default n + help + Build slaunch_inject_fault() into the kernel. When the kernel + command line contains "slaunch_inject=", mutate the EFI + memory map (or other tainted inputs) in-place before + slaunch_validate_efi_early() runs the real validation. The new + guard is expected to panic with a specific message; the harness + treats that as a passing negative test. + + Tokens implemented: mmap_wrap, mmap_pages_overflow, + mmap_size_huge, systab_nr_tables_huge. + + Useful for proving end-to-end that the validation pipeline + catches the canonical exploits. Intended for development testing only; leave disabled in production builds. + + If unsure, say N. + config COMPRESSED_INSTALL bool "Install compressed image by default" help From cfd5ace98dc496f94f071eed625852af4ebac478 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Fri, 12 Jun 2026 12:00:00 -0700 Subject: [PATCH 283/311] NVIDIA: SAUCE: arm64: drtm: Enable SECURE_LAUNCH by default and fix zboot link BugLink: https://bugs.launchpad.net/bugs/2161563 In an EFI_ZBOOT build the compressed-image link cannot resolve the __efistub_sl_entry alias to the uncompressed kernel, so the link fails. Make ARM64_SECURE_LAUNCH depend on !EFI_ZBOOT, and add a safety-net PROVIDE_HIDDEN __efistub_sl_entry in zboot.lds so a misconfigured build still links. Flip the option to default y so DRTM is exercised on every defconfig build. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/Kconfig | 3 ++- drivers/firmware/efi/libstub/zboot.lds | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 15b94370fe218..3008570f7e19f 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2466,7 +2466,8 @@ config EFI config ARM64_SECURE_LAUNCH bool "ARM64 DRTM Secure Launch support" depends on ARM64 && EFI - default n + depends on !EFI_ZBOOT + default y select CRYPTO_LIB_SHA256 select CRYPTO_LIB_SHA512 help diff --git a/drivers/firmware/efi/libstub/zboot.lds b/drivers/firmware/efi/libstub/zboot.lds index 367907eb7d869..76f0c8031badc 100644 --- a/drivers/firmware/efi/libstub/zboot.lds +++ b/drivers/firmware/efi/libstub/zboot.lds @@ -61,3 +61,13 @@ PROVIDE(__efistub__gzdata_size = PROVIDE(__data_rawsize = ABSOLUTE(_edata - _data)); PROVIDE(__data_size = ABSOLUTE(_end - _data)); PROVIDE(__sbat_size = ABSOLUTE(_esbat - _sbat)); + +/* + * ARM64_SECURE_LAUNCH references these __efistub_ aliases, but sl_entry + * is not reachable in the compressed zboot path, so a drtm=on zboot build + * silently no-ops the DRTM launch. These stubs only let such a build link. + */ +PROVIDE_HIDDEN(__efistub__text = ADDR(.head)); +PROVIDE_HIDDEN(__efistub__end = _end); +PROVIDE_HIDDEN(__efistub__edata = _edata); +PROVIDE_HIDDEN(__efistub_sl_entry = ADDR(.head)); From 2513dabbef682d952a1cb213146fd24d7bbef1b3 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Fri, 12 Jun 2026 12:00:00 -0700 Subject: [PATCH 284/311] NVIDIA: SAUCE: arm64: drtm: Allow SECURE_LAUNCH with EFI_ZBOOT BugLink: https://bugs.launchpad.net/bugs/2161563 Drop the depends-on !EFI_ZBOOT clause on ARM64_SECURE_LAUNCH now that libstub/zboot.lds provides hidden definitions for the __efistub_ aliases used by the DRTM stub. The compressed boot path still does not invoke DRTM_DYNAMIC_LAUNCH; this change only removes the build-time mutual exclusion so distro defconfigs that turn on both options link cleanly. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 3008570f7e19f..45d215999eb88 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2466,7 +2466,6 @@ config EFI config ARM64_SECURE_LAUNCH bool "ARM64 DRTM Secure Launch support" depends on ARM64 && EFI - depends on !EFI_ZBOOT default y select CRYPTO_LIB_SHA256 select CRYPTO_LIB_SHA512 From a24e0236422563305cab1fc4c3eae2ccd78fbcbc Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Fri, 12 Jun 2026 12:00:00 -0700 Subject: [PATCH 285/311] NVIDIA: SAUCE: arm64: drtm: Move EFI-stub writable statics past _edata BugLink: https://bugs.launchpad.net/bugs/2161563 The EFI stub mutates its static buffers (LoadImage memcpy, DRTM params, ImageBase scrub) after the kernel buffer is allocated. With those statics inside [_text, _edata) the D-CRTM-measured bytes diverge from the on-disk Image and offline attestation fails. Relocate the renamed .efistub.data/.bss past _edata (PE rawsize recomputed) so only [_text, _edata) is measured; stub text/rodata stay in PE .text. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/kernel/vmlinux.lds.S | 42 +++++++++++++++++++++++++-- drivers/firmware/efi/libstub/Makefile | 2 +- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 2d1e75263f033..10bd50125a0cd 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -239,7 +239,20 @@ SECTIONS __init_begin = .; __inittext_begin = .; - INIT_TEXT_SECTION(8) + /* + * Inlined INIT_TEXT_SECTION(8) with extra .efistub.text/.rodata + * catchers so the renamed EFI-stub text/rodata stays inside + * [_sinittext, _einittext) and the PE/COFF .text range; otherwise + * the stub entry point lands in PE .data and the loader rejects it. + */ + . = ALIGN(8); + .init.text : { + _sinittext = .; + INIT_TEXT + *(.efistub.text .efistub.text.*) + *(.efistub.rodata .efistub.rodata.*) + _einittext = .; + } __exittext_begin = .; .exit.text : { @@ -270,7 +283,9 @@ SECTIONS INIT_CALLS CON_INITCALL INIT_RAM_FS - *(.init.altinstructions .init.bss) /* from the EFI stub */ + *(.init.altinstructions .init.bss) /* from PI / EFI stub */ + /* .efistub.data / .efistub.bss are placed past _edata + * in their own .efistub PROGBITS block, see below. */ } .exit.data : { EXIT_DATA @@ -327,9 +342,30 @@ SECTIONS } PECOFF_EDATA_PADDING - __pecoff_data_rawsize = ABSOLUTE(. - __initdata_begin); _edata = .; + /* + * EFI-stub writable statics (.efistub.data/.bss) placed AFTER _edata + * so the DRTM measurement bounds [_text, _edata) never cover + * stub-mutable data; __pecoff_data_rawsize is computed past + * __efistub_end. Stub text/rodata stay in PE .text (buckets above). + */ + . = ALIGN(PAGE_SIZE); + __efistub_start = .; + .efistub : ALIGN(PAGE_SIZE) { + *(.efistub.data .efistub.data.*) + *(.efistub.bss .efistub.bss.*) + } + __efistub_end = .; + + /* + * Materialize the trailing PE/COFF FileAlignment padding as a real + * PROGBITS section so `objcopy -O binary` writes the bytes; else the + * PE rawsize claims more than the file holds and the loader rejects it. + */ + .efistub_pad : { BYTE(0); . = ALIGN(PECOFF_FILE_ALIGNMENT); } + __pecoff_data_rawsize = ABSOLUTE(. - __initdata_begin); + /* start of zero-init region */ BSS_SECTION(SBSS_ALIGN, 0, 0) __pi___bss_start = __bss_start; diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index c1da3b536ece7..77b4dd7155c4f 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -137,7 +137,7 @@ STUBCOPY_RELOC-$(CONFIG_ARM) := R_ARM_ABS # a verification pass to see if any absolute relocations exist in any of the # object files. # -STUBCOPY_FLAGS-$(CONFIG_ARM64) += --prefix-alloc-sections=.init \ +STUBCOPY_FLAGS-$(CONFIG_ARM64) += --prefix-alloc-sections=.efistub \ --prefix-symbols=__efistub_ STUBCOPY_RELOC-$(CONFIG_ARM64) := R_AARCH64_ABS From fbc0393acd1dd28eaf2563d41407926d3d361284 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Fri, 12 Jun 2026 12:00:00 -0700 Subject: [PATCH 286/311] NVIDIA: SAUCE: arm64: drtm: Scrub PE/COFF ImageBase field pre-DRTM launch BugLink: https://bugs.launchpad.net/bugs/2161563 UEFI LoadImage patches the PE Optional Header ImageBase with the load PA, so after the stub memcpy's the kernel the D-CRTM-measured bytes diverge from the on-disk Image by 8 bytes. Add efi_slaunch_scrub_imagebase() (called from handle_kernel_image() pre-EBS) to zero and clean the field; it must run pre-EBS because efi_remap_image() marks the header RO and the attribute protocol is gone after EBS. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- drivers/firmware/efi/libstub/arm64-slaunch.c | 59 ++++++++++++++++++++ drivers/firmware/efi/libstub/arm64-stub.c | 16 +++++- drivers/firmware/efi/libstub/efistub.h | 1 + 3 files changed, 73 insertions(+), 3 deletions(-) diff --git a/drivers/firmware/efi/libstub/arm64-slaunch.c b/drivers/firmware/efi/libstub/arm64-slaunch.c index 408a286f13a19..aaa0c51964355 100644 --- a/drivers/firmware/efi/libstub/arm64-slaunch.c +++ b/drivers/firmware/efi/libstub/arm64-slaunch.c @@ -135,6 +135,60 @@ void efi_slaunch_get_dlme_data_size(void) sl_dlme_data_reserve / 1024, min_pages); } +/* + * Zero the PE/COFF Optional Header ImageBase on the relocated kernel + * buffer (pre-EBS): LoadImage patched it with the load PA, diverging the + * D-CRTM-measured bytes from the on-disk Image. Pre-EBS because + * efi_remap_image() marked the header RO; flip RW, zero, restore RO. + */ +void efi_slaunch_scrub_imagebase(unsigned long kernel_addr) +{ + efi_guid_t guid = EFI_MEMORY_ATTRIBUTE_PROTOCOL_GUID; + efi_memory_attribute_protocol_t *memattr; + efi_status_t status; + u32 e_lfanew; + volatile u64 *image_base_ptr; + unsigned long page_base; + + if (!kernel_addr) + return; + + e_lfanew = *(volatile u32 *)((char *)kernel_addr + 0x3c); + image_base_ptr = (volatile u64 *)((char *)kernel_addr + + e_lfanew + 4 + 20 + 0x18); + page_base = (unsigned long)image_base_ptr & ~(SL_DRTM_PAGE_SIZE - 1UL); + + status = efi_bs_call(locate_protocol, &guid, NULL, (void **)&memattr); + if (status != EFI_SUCCESS) { + efi_warn("DRTM: no EFI_MEMORY_ATTRIBUTE_PROTOCOL; " + "skipping PE ImageBase scrub\n"); + return; + } + + status = memattr->clear_memory_attributes(memattr, page_base, + SL_DRTM_PAGE_SIZE, + EFI_MEMORY_RO); + if (status != EFI_SUCCESS) { + efi_warn("DRTM: clear EFI_MEMORY_RO failed for PE header page: 0x%lx\n", + status); + return; + } + + *image_base_ptr = 0; + sl_dc_cvac_range((unsigned long)image_base_ptr, 8); + asm volatile("dsb sy" : : : "memory"); + + status = memattr->set_memory_attributes(memattr, page_base, + SL_DRTM_PAGE_SIZE, + EFI_MEMORY_RO); + if (status != EFI_SUCCESS) + efi_warn("DRTM: restore EFI_MEMORY_RO on PE header page failed: 0x%lx\n", + status); + + efi_info("DRTM: PE ImageBase zeroed at 0x%lx\n", + (unsigned long)image_base_ptr); +} + /* * Token-aware cmdline match: true iff `tok` is a standalone * whitespace-delimited word in `cmdline`, not a substring of another @@ -238,6 +292,11 @@ void __noreturn efi_slaunch_drtm(unsigned long kernel_addr, sl_dc_cvac_range((unsigned long)params, sizeof(*params)); sl_dc_cvac_range(kernel_addr + dlme_data_offset + SL_DLME_DTB_SLOT_OFFSET, sizeof(u64)); + /* + * NOTE: the PE ImageBase scrub happens pre-EBS in + * efi_slaunch_scrub_imagebase(); the memory-attribute protocol used + * to unprotect the header page is unreachable after boot services exit. + */ asm volatile("dsb sy" : : : "memory"); /* diff --git a/drivers/firmware/efi/libstub/arm64-stub.c b/drivers/firmware/efi/libstub/arm64-stub.c index 588b2c26617d7..02ae83f7e8387 100644 --- a/drivers/firmware/efi/libstub/arm64-stub.c +++ b/drivers/firmware/efi/libstub/arm64-stub.c @@ -47,9 +47,19 @@ efi_status_t handle_kernel_image(unsigned long *image_addr, #endif *image_addr = (unsigned long)_text; - return efi_kaslr_relocate_kernel(image_addr, reserve_addr, reserve_size, - kernel_size, kernel_codesize, kernel_memsize, - efi_kaslr_get_phys_seed(image_handle)); + { + efi_status_t st; + + st = efi_kaslr_relocate_kernel(image_addr, reserve_addr, + reserve_size, kernel_size, + kernel_codesize, kernel_memsize, + efi_kaslr_get_phys_seed(image_handle)); +#ifdef CONFIG_ARM64_SECURE_LAUNCH + if (st == EFI_SUCCESS) + efi_slaunch_scrub_imagebase(*image_addr); +#endif + return st; + } } asmlinkage void primary_entry(void); diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h index 4e78bd3909aa9..6b61ae3cb5ce5 100644 --- a/drivers/firmware/efi/libstub/efistub.h +++ b/drivers/firmware/efi/libstub/efistub.h @@ -1270,6 +1270,7 @@ efi_status_t efi_zboot_decompress(u8 *out, unsigned long outlen); #ifdef CONFIG_ARM64_SECURE_LAUNCH bool efi_slaunch_enabled(const char *cmdline); void efi_slaunch_get_dlme_data_size(void); +void efi_slaunch_scrub_imagebase(unsigned long kernel_addr); extern unsigned long sl_dlme_data_reserve; extern bool sl_drtm_available; void __noreturn efi_slaunch_drtm(unsigned long kernel_addr, From f0d552de3cf8c42de0ba3870285f17ae37dc502a Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Fri, 12 Jun 2026 12:00:00 -0700 Subject: [PATCH 287/311] NVIDIA: SAUCE: arm64: drtm: Measure kernel command line into DRTM event log BugLink: https://bugs.launchpad.net/bugs/2161563 Measure boot_command_line into the DRTM event log after the /chosen scan, capturing the cmdline the kernel actually used rather than what UEFI delivered. Per-deployment values (e.g. root=UUID=...) make the hash vary, so verifier policy must account for it; CONFIG_CMDLINE_FORCE with a hardcoded cmdline removes the variance at the cost of flexibility. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/kernel/slaunch.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index e2ff33fd768b9..9de06d6b16341 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -1562,6 +1562,15 @@ void __init slaunch_measure_post_efi(void) slaunch_selftest(); slaunch_verify_hash_algo(); slaunch_measure_acpi(); + + /* + * Measure the effective kernel command line (boot_command_line, the + * post-/chosen cmdline the kernel actually used). Per-deployment + * variance makes the hash vary, so verifier policy must allow for it + * (or pin it via CONFIG_CMDLINE_FORCE). + */ + slaunch_measure("CMDLINE", boot_command_line, strlen(boot_command_line)); + slaunch_measure_initrd(); slaunch_extend_drtm_event_log(); From 51591437d70bb9a454c936154b6af396f03a5739 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Fri, 12 Jun 2026 12:00:00 -0700 Subject: [PATCH 288/311] NVIDIA: SAUCE: arm64: drtm: Validate (not measure) UEFI SRTM TPM event log BugLink: https://bugs.launchpad.net/bugs/2161563 UEFI may publish an SRTM TPM event log via the SRTM-log config tables. Add a per-GUID validator that confirms the full header+body extent is in one NORMAL region and rejects overlap with the DLME region, DTB, or raw mmap. The log is a pre-DRTM (untrusted) artifact, not DRTM evidence, so its extent is validated but it is not measured. Adds three fault-inject tests for the overlap/extent guards. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/kernel/slaunch.c | 267 +++++++++++++++++++++++++++++++++++- 1 file changed, 264 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index 9de06d6b16341..8c96a3b10cb02 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -18,6 +18,7 @@ #include #include +#include #include /* FDT magic number (big-endian 0xd00dfeed at offset 0) */ @@ -39,6 +40,24 @@ static u32 dcrtm_num_regions; static phys_addr_t sl_dlme_data_pa; static u64 sl_dlme_data_size; +/* + * Validated UEFI SRTM TPM event log location, captured in + * slaunch_validate_raw_systab() so the main-vs-final-events "last writer + * wins" selection can prefer the primary log. The SRTM log is a pre-DRTM + * (untrusted) artifact, so it is validated for in-bounds extent only, not + * measured. Zero when firmware published no GUID (e.g. FVP without Tcg2Dxe). + */ +static phys_addr_t sl_srtm_log_pa; +static size_t sl_srtm_log_size; /* includes sizeof(linux_efi_tpm_eventlog) */ + +/* + * Raw EFI memory map extent saved by slaunch_validate_raw_mmap() and + * read by slaunch_validate_srtm_log() so the SRTM log extent can be + * rejected if it overlaps the raw mmap buffer. + */ +static phys_addr_t sl_efi_mmap_pa; +static u64 sl_efi_mmap_size; + /* * Close TPM locality 2 — the DLME's locality, per DEN0113 v1.2 §4.6.1. * Locality 3 is the DCE's; it's closed by the DCE, not the DLME. @@ -492,6 +511,145 @@ static u32 __init sl_cfgtbl_min_size(const efi_guid_t *guid) return SL_CFGTBL_UNKNOWN_BOUND; } +/* Forward decl — definition lives near slaunch_measure_initrd. */ +static bool __init slaunch_ranges_overlap(u64 a_start, u64 a_size, + u64 b_start, u64 b_size); + +/* + * Validate a UEFI SRTM TPM event log Configuration Table entry against + * the D-CRTM address map: cover the full header+body extent and reject + * overlap with DLME/DTB/mmap so firmware cannot alias an input. The log + * is a pre-DRTM artifact, so this validates its extent only (it is not + * measured); on success captures sl_srtm_log_pa/size. + */ +static void __init slaunch_validate_srtm_log(u64 log_pa, + const efi_guid_t *guid) +{ + efi_guid_t main_guid = LINUX_EFI_TPM_EVENT_LOG_GUID; + efi_guid_t final_guid = EFI_TCG2_FINAL_EVENTS_TABLE_GUID; + size_t hdr_size; + u64 body_size; + u64 total_size; + u64 dlme_size; + u64 fdt_size = 0; + phys_addr_t dtb_pa = __fdt_pointer; + bool is_main = (efi_guidcmp(*guid, main_guid) == 0); + bool is_final = (efi_guidcmp(*guid, final_guid) == 0); + + if (!is_main && !is_final) + return; + + if (!log_pa) + panic("slaunch: SRTM log GUID entry has NULL vendor_table\n"); + + hdr_size = is_main ? sizeof(struct linux_efi_tpm_eventlog) + : sizeof(struct efi_tcg2_final_events_table); + + /* Header extent must be in NORMAL before we can read the size + * field. dcrtm_range_in_normal rejects size==0 and u64 wrap. */ + if (!dcrtm_range_in_normal(log_pa, hdr_size)) + panic("slaunch: SRTM log PA 0x%llx (header %zu B) NOT in NORMAL region\n", + log_pa, hdr_size); + + /* Header extent must not alias the DLME region, DTB, or raw EFI + * mmap buffer (an attacker could otherwise coerce the validator + * into mis-interpreting attestation-critical bytes). */ + dlme_size = (sl_dlme_data_pa + sl_dlme_data_size) - sl_dlme_region_pa; + if (slaunch_ranges_overlap(log_pa, hdr_size, + sl_dlme_region_pa, dlme_size)) + panic("slaunch: SRTM log header [0x%llx+%zu] overlaps DLME region [0x%llx+%llu]\n", + log_pa, hdr_size, (u64)sl_dlme_region_pa, dlme_size); + + if (dtb_pa) { + /* Read fdt_totalsize from the (already-validated) DTB. */ + u32 *p = early_memremap(dtb_pa, sizeof(u32) * 2); + + if (p) { + fdt_size = be32_to_cpu(p[1]); + early_memunmap(p, sizeof(u32) * 2); + } + } + if (fdt_size && slaunch_ranges_overlap(log_pa, hdr_size, + dtb_pa, fdt_size)) + panic("slaunch: SRTM log header [0x%llx+%zu] overlaps DTB [0x%llx+%llu]\n", + log_pa, hdr_size, (u64)dtb_pa, fdt_size); + + if (sl_efi_mmap_size && + slaunch_ranges_overlap(log_pa, hdr_size, + sl_efi_mmap_pa, sl_efi_mmap_size)) + panic("slaunch: SRTM log header [0x%llx+%zu] overlaps raw EFI mmap [0x%llx+%llu]\n", + log_pa, hdr_size, (u64)sl_efi_mmap_pa, + sl_efi_mmap_size); + + /* Read the firmware-published body size from the header. */ + if (is_main) { + struct linux_efi_tpm_eventlog *log_tbl; + + log_tbl = early_memremap(log_pa, hdr_size); + if (!log_tbl) + panic("slaunch: SRTM log header remap failed at 0x%llx\n", + log_pa); + body_size = log_tbl->size; + early_memunmap(log_tbl, hdr_size); + } else { + struct efi_tcg2_final_events_table *final_tbl; + + final_tbl = early_memremap(log_pa, hdr_size); + if (!final_tbl) + panic("slaunch: SRTM final-events header remap failed at 0x%llx\n", + log_pa); + /* + * Final-events per-event encoding needs the primary log's + * algorithm-set descriptor we may not have seen, so treat + * events[] as opaque. version must be 1 per TCG2 ACPI spec; + * reject otherwise before nr_events. + */ + if (final_tbl->version != 1) + panic("slaunch: SRTM final-events table version %llu != 1\n", + final_tbl->version); + body_size = 0; + (void)final_tbl->nr_events; + early_memunmap(final_tbl, hdr_size); + } + + if (check_add_overflow(log_pa, (u64)hdr_size + body_size, + &total_size)) + panic("slaunch: SRTM log PA + size wraps u64 (pa 0x%llx + %zu + %llu)\n", + log_pa, hdr_size, body_size); + + total_size = (u64)hdr_size + body_size; + + if (!dcrtm_range_in_normal(log_pa, total_size)) + panic("slaunch: SRTM log [0x%llx+%llu] NOT in NORMAL region (full extent)\n", + log_pa, total_size); + + if (slaunch_ranges_overlap(log_pa, total_size, + sl_dlme_region_pa, dlme_size)) + panic("slaunch: SRTM log [0x%llx+%llu] overlaps DLME region [0x%llx+%llu]\n", + log_pa, total_size, (u64)sl_dlme_region_pa, dlme_size); + if (fdt_size && + slaunch_ranges_overlap(log_pa, total_size, dtb_pa, fdt_size)) + panic("slaunch: SRTM log [0x%llx+%llu] overlaps DTB [0x%llx+%llu]\n", + log_pa, total_size, (u64)dtb_pa, fdt_size); + if (sl_efi_mmap_size && + slaunch_ranges_overlap(log_pa, total_size, + sl_efi_mmap_pa, sl_efi_mmap_size)) + panic("slaunch: SRTM log [0x%llx+%llu] overlaps raw EFI mmap [0x%llx+%llu]\n", + log_pa, total_size, (u64)sl_efi_mmap_pa, + sl_efi_mmap_size); + + /* Last writer wins when both GUIDs are present — both are + * structurally validated above; the main TPM event log is the + * preferred measurement target. */ + if (is_main || !sl_srtm_log_pa) { + sl_srtm_log_pa = log_pa; + sl_srtm_log_size = (size_t)total_size; + } + pr_info("slaunch: SRTM TPM event log validated: PA 0x%llx, %llu B (%s)\n", + log_pa, total_size, + is_main ? "LINUX_EFI_TPM_EVENT_LOG" : "EFI_TCG2_FINAL_EVENTS"); +} + /* Validate System Table + ConfigurationTable pointers against D-CRTM * map. Runs pre-efi_init, panics on bad input. */ @@ -550,6 +708,11 @@ static void __init slaunch_validate_raw_systab(u64 systab_pa) if (!dcrtm_range_in_normal(tbl_ptr, size)) panic("slaunch: EFI ConfigurationTable[%lu] 0x%lx [size %u] NOT in NORMAL region\n", j, tbl_ptr, size); + /* If the entry advertises an SRTM TPM event log (header + * + variable-length body), extend the structural check to + * cover the full body extent (validate-only; the log is a + * pre-DRTM artifact and is not measured). */ + slaunch_validate_srtm_log((u64)tbl_ptr, &cfgtbl[j].guid); } early_memunmap(cfgtbl, tbl_size); pr_info("slaunch: early EFI System Table validation PASSED (%lu entries)\n", @@ -644,6 +807,12 @@ static void __init slaunch_validate_raw_mmap(const struct sl_efi_info *info) early_memunmap(mmap, info->mmap_size); pr_info("slaunch: early raw EFI mmap validation PASSED (%u of %u descriptors checked)\n", nchecked, ndesc); + + /* Remember raw mmap extent for later overlap checks (e.g. the SRTM + * TPM event log validator must reject a published log PA that aliases + * the firmware-provided memory map buffer). */ + sl_efi_mmap_pa = info->mmap_pa; + sl_efi_mmap_size = info->mmap_size; } static void __init slaunch_validate_efi_early(const struct sl_efi_info *info) @@ -652,6 +821,13 @@ static void __init slaunch_validate_efi_early(const struct sl_efi_info *info) pr_info("slaunch: /chosen does not have all linux,uefi-* properties — skipping early EFI validation\n"); return; } + /* + * Stage the raw EFI mmap extent before the systab validator so + * slaunch_validate_srtm_log() can reject an SRTM log PA aliasing it; + * validate_raw_mmap() re-publishes these values after its own checks. + */ + sl_efi_mmap_pa = info->mmap_pa; + sl_efi_mmap_size = info->mmap_size; slaunch_validate_raw_systab(info->systab_pa); slaunch_validate_raw_mmap(info); } @@ -1325,9 +1501,9 @@ static void __init slaunch_extend_drtm_event_log(void) #ifdef CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT /* * Negative-test fault injection (slaunch_inject=): mutates the - * raw EFI mmap before slaunch_validate_efi_early so the validator panics - * (a passing negative test). Tokens: mmap_wrap, mmap_pages_overflow, - * mmap_size_huge, systab_nr_tables_huge. Not for production. + * raw EFI mmap/systab/SRTM-log before slaunch_validate_efi_early so the + * validator panics. Tokens: mmap_*, systab_nr_tables_huge, srtm_log_*. + * Not for production. */ static bool __init sl_cmdline_has(const char *tok) { @@ -1381,6 +1557,89 @@ static void __init sl_mutate_mmap_pages_overflow(efi_memory_desc_t *md) md->phys_addr, md->num_pages); } +/* Overwrite the first non-null EFI ConfigurationTable entry in the raw + * systab's cfgtbl array with a synthetic SRTM-log entry. Returns true + * if we managed to plant the entry. Used by srtm_log_* injectors only; + * not safe for production. */ +static bool __init slaunch_inject_srtm_cfgtbl(const struct sl_efi_info *info, + u64 vendor_pa) +{ + efi_guid_t srtm_guid = LINUX_EFI_TPM_EVENT_LOG_GUID; + efi_system_table_t *systab; + efi_config_table_t *cfgtbl; + unsigned long tables_pa; + unsigned long nr_tables; + size_t tbl_size; + unsigned long j; + bool applied = false; + + systab = early_memremap(info->systab_pa, sizeof(*systab)); + if (!systab) { + pr_warn("slaunch: INJECT srtm_log: systab remap failed\n"); + return false; + } + nr_tables = systab->nr_tables; + tables_pa = (unsigned long)systab->tables; + early_memunmap(systab, sizeof(*systab)); + + if (!nr_tables || !tables_pa) { + pr_warn("slaunch: INJECT srtm_log: systab has no cfgtbl entries\n"); + return false; + } + tbl_size = nr_tables * sizeof(*cfgtbl); + cfgtbl = early_memremap(tables_pa, tbl_size); + if (!cfgtbl) { + pr_warn("slaunch: INJECT srtm_log: cfgtbl remap failed\n"); + return false; + } + for (j = 0; j < nr_tables; j++) { + if (!cfgtbl[j].table) + continue; + cfgtbl[j].guid = srtm_guid; + cfgtbl[j].table = (void *)(unsigned long)vendor_pa; + applied = true; + break; + } + early_memunmap(cfgtbl, tbl_size); + return applied; +} + +static void __init slaunch_inject_srtm_log(const struct sl_efi_info *info) +{ + u64 vendor_pa; + const char *tag = NULL; + + if (sl_cmdline_has("slaunch_inject=srtm_log_overlap_dlme")) { + /* Aim the entry at the start of the DLME region. The header + * extent overlap check in slaunch_validate_srtm_log fires + * before any size field is read. */ + vendor_pa = sl_dlme_region_pa; + tag = "srtm_log_overlap_dlme"; + } else if (sl_cmdline_has("slaunch_inject=srtm_log_outside_normal")) { + /* FVP PL011 UART base — a DEVICE region per DEN0113 v1.2 + * Table 11. dcrtm_range_in_normal must reject. */ + vendor_pa = 0x1c090000ULL; + tag = "srtm_log_outside_normal"; + } else if (sl_cmdline_has("slaunch_inject=srtm_log_overlap_mmap")) { + /* + * Aim the entry at the raw EFI mmap PA: it is NORMAL so the + * range check passes and only the overlap-with-mmap guard in + * slaunch_validate_srtm_log rejects it. + */ + vendor_pa = info->mmap_pa; + tag = "srtm_log_overlap_mmap"; + } else { + return; + } + + if (!slaunch_inject_srtm_cfgtbl(info, vendor_pa)) + pr_warn("slaunch: INJECT %s: could not plant cfgtbl entry\n", + tag); + else + pr_warn("slaunch: INJECT %s: planted SRTM-log entry PA=0x%llx (expect panic)\n", + tag, vendor_pa); +} + static void __init slaunch_inject_fault(struct sl_efi_info *info) { if (!info->present) { @@ -1388,6 +1647,8 @@ static void __init slaunch_inject_fault(struct sl_efi_info *info) return; } + slaunch_inject_srtm_log(info); + if (sl_cmdline_has("slaunch_inject=mmap_wrap")) { if (!slaunch_inject_raw_mmap(info, sl_mutate_mmap_wrap)) pr_warn("slaunch: INJECT mmap_wrap: no EFI_CONVENTIONAL_MEMORY descriptor found\n"); From af54af57f6d133e2e0369b567f8ff04193027003 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Fri, 12 Jun 2026 12:00:00 -0700 Subject: [PATCH 289/311] NVIDIA: SAUCE: arm64: drtm: Validate untrusted inputs before kernel boot consumes them BugLink: https://bugs.launchpad.net/bugs/2161563 - Reserve DLME data after efi_init clears the memblock state - Reject RSDPs shorter than the ACPI 2.0 RSDP layout (36 bytes) - Validate initrd extents in slaunch_setup, before arm64_memblock_init consumes them Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/include/asm/drtm.h | 4 +++ arch/arm64/kernel/setup.c | 2 ++ arch/arm64/kernel/slaunch.c | 64 +++++++++++++++++++++++++---------- 3 files changed, 52 insertions(+), 18 deletions(-) diff --git a/arch/arm64/include/asm/drtm.h b/arch/arm64/include/asm/drtm.h index 9bf4ff8ce5637..80f5fc4a90e6c 100644 --- a/arch/arm64/include/asm/drtm.h +++ b/arch/arm64/include/asm/drtm.h @@ -129,11 +129,15 @@ extern unsigned long sl_dlme_data_offset; void slaunch_early_init(void); void slaunch_setup(void); +void slaunch_validate_initrd(void); +void slaunch_reserve_dlme_data(void); void slaunch_exit(void); void slaunch_measure_post_efi(void); #else static inline void slaunch_early_init(void) { } static inline void slaunch_setup(void) { } +static inline void slaunch_validate_initrd(void) { } +static inline void slaunch_reserve_dlme_data(void) { } static inline void slaunch_exit(void) { } static inline void slaunch_measure_post_efi(void) { } #endif diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 53c6c856f5dbd..371cf27a7bc15 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -323,9 +323,11 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) cpu_uninstall_idmap(); slaunch_setup(); + slaunch_validate_initrd(); xen_early_init(); efi_init(); + slaunch_reserve_dlme_data(); if (!efi_enabled(EFI_BOOT)) { if ((u64)_text % MIN_KIMG_ALIGN) diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index 8c96a3b10cb02..0f554e9c6e500 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -871,14 +871,9 @@ void __init slaunch_setup(void) le16_to_cpu(hdr->this_hdr_size), le64_to_cpu(hdr->protected_regions_size)); - /* Reserve DLME data region in memblock so kernel won't reuse it. - * NOTE: efi_init() runs after us and calls memblock_remove(0, - * PHYS_ADDR_MAX) which wipes this reservation. slaunch_validate_efi - * re-reserves using the saved values below. - */ + /* Stash for slaunch_reserve_dlme_data() to reserve post-efi_init. */ sl_dlme_data_pa = dlme_data_pa; sl_dlme_data_size = le64_to_cpu(hdr->dlme_data_size); - memblock_reserve(sl_dlme_data_pa, sl_dlme_data_size); early_memunmap(hdr, sizeof(*hdr)); @@ -907,6 +902,14 @@ void __init slaunch_setup(void) } } +/* Reserve DLME data after efi_init's memblock_remove(0, PHYS_ADDR_MAX). */ +void __init slaunch_reserve_dlme_data(void) +{ + if (!sl_dlme_data_pa || !sl_dlme_data_size) + return; + memblock_reserve(sl_dlme_data_pa, sl_dlme_data_size); +} + /* * Record an in-memory SHA-256 measurement into slaunch_measurements[] * for later replay into the DRTM event log on PCR 18. The table is @@ -1130,9 +1133,11 @@ static void __init slaunch_measure_acpi(void) panic("slaunch: RSDP header remap failed at 0x%llx\n", (u64)rsdp_pa); + /* ACPI 2.0+ RSDP must be >= 36 bytes so xsdt_pa (offset 24-31) is measured. */ rsdp_len = *(u32 *)((u8 *)rsdp + RSDP_OFF_LEN); - if (!rsdp_len) - rsdp_len = RSDP_SIZE_V1; + if (rsdp_len < RSDP_MIN_MAP) + panic("slaunch: RSDP length %u < %u\n", + rsdp_len, (u32)RSDP_MIN_MAP); xsdt_pa = *(u64 *)((u8 *)rsdp + RSDP_OFF_XSDT); early_memunmap(rsdp, RSDP_MIN_MAP); @@ -1277,20 +1282,15 @@ static void __init slaunch_inject_initrd(u64 *start, u64 *size); * before any read and failures are fatal. Absence is legitimate and * logged (no marker hash); the verifier infers coverage from the event. */ -static void __init slaunch_measure_initrd(void) +/* Returns validated (post-inject) start/size, or false if no initrd. */ +static bool __init slaunch_validate_initrd_extents(u64 *out_start, u64 *out_size) { u64 start = phys_initrd_start; u64 size = phys_initrd_size; - u64 end, dlme_size, off = 0, remaining; - struct sha256_ctx sctx; - u8 hash[SHA256_DIGEST_SIZE]; - struct slaunch_measurement *m; - void *p; + u64 end, dlme_size; - if (!start || !size) { - pr_info("slaunch: no initrd present, skipping measurement\n"); - return; - } + if (!start || !size) + return false; #ifdef CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT slaunch_inject_initrd(&start, &size); @@ -1314,6 +1314,34 @@ static void __init slaunch_measure_initrd(void) panic("slaunch: initrd [0x%llx+%llu] overlaps DLME region [0x%llx+%llu]\n", start, size, (u64)sl_dlme_region_pa, dlme_size); + *out_start = start; + *out_size = size; + return true; +} + +/* Called from slaunch_setup, before arm64_memblock_init consumes the extents. */ +void __init slaunch_validate_initrd(void) +{ + u64 start, size; + + if (!sl_dlme_region_pa) + return; + if (!slaunch_validate_initrd_extents(&start, &size)) + pr_info("slaunch: no initrd present, skipping measurement\n"); +} + +static void __init slaunch_measure_initrd(void) +{ + u64 start, size; + u64 off = 0, remaining; + struct sha256_ctx sctx; + u8 hash[SHA256_DIGEST_SIZE]; + struct slaunch_measurement *m; + void *p; + + if (!slaunch_validate_initrd_extents(&start, &size)) + return; + /* * Chunked early_memremap + streaming SHA-256: the initrd can * exceed a single early_memremap and the linear map is not From d4b48bdc1ab13bd5d4d66229bafb29015b196bb5 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Sat, 13 Jun 2026 20:56:16 -0700 Subject: [PATCH 290/311] NVIDIA: SAUCE: arm64: drtm: Add SHA-384 support for production DCE firmware BugLink: https://bugs.launchpad.net/bugs/2161563 The DLME must hash with the same algorithm the D-CRTM reports via DRTM_FEATURES or the event-log chain cannot be replayed. Generalize the measurement path behind sl_hash_data()/sl_active_algo and add SHA-384 via the kernel library API (sha256()/sha384()), whose scalar paths are safe in setup_arch(). FAULT_INJECT adds sha384_force and sha_algo_bad tokens to exercise this on a SHA-256 FVP. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/Kconfig | 3 +- arch/arm64/kernel/slaunch.c | 226 +++++++++++++++++++++++++++++------- 2 files changed, 185 insertions(+), 44 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 45d215999eb88..9526bb4516c38 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2511,7 +2511,8 @@ config ARM64_SECURE_LAUNCH_FAULT_INJECT treats that as a passing negative test. Tokens implemented: mmap_wrap, mmap_pages_overflow, - mmap_size_huge, systab_nr_tables_huge. + mmap_size_huge, systab_nr_tables_huge, sha384_force, + sha_algo_bad. Useful for proving end-to-end that the validation pipeline catches the canonical exploits. Intended for development testing only; leave disabled in production builds. diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index 0f554e9c6e500..5ddf4bab4b41c 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -21,6 +21,64 @@ #include #include +/* + * Hash algorithm dispatch. The DLME must match the D-CRTM firmware hash + * algo (DRTM_FEATURES feature 0x1) or the event-log chain cannot replay. + * Supports SHA-256 (0x000B) and SHA-384 (0x000C) via the kernel library + * API, whose arch SIMD paths are static-key gated until subsys_initcall. + */ +#define SL_TPM_ALG_SHA256 0x000B +#define SL_TPM_ALG_SHA384 0x000C + +enum sl_hash_algo { + SL_HASH_SHA256 = 0, + SL_HASH_SHA384 = 1, +}; + +#define SL_HASH_MAX_DIGEST_SIZE SHA512_DIGEST_SIZE /* 64 */ + +struct sl_hash_alg_info { + const char *name; + u16 tpm_alg_id; + u8 digest_size; +}; + +static const struct sl_hash_alg_info sl_hash_algs[] = { + [SL_HASH_SHA256] = { + .name = "SHA-256", + .tpm_alg_id = SL_TPM_ALG_SHA256, + .digest_size = SHA256_DIGEST_SIZE, + }, + [SL_HASH_SHA384] = { + .name = "SHA-384", + .tpm_alg_id = SL_TPM_ALG_SHA384, + .digest_size = SHA384_DIGEST_SIZE, + }, +}; + +/* Active algo for the lifetime of this boot. Default SHA-256 until + * slaunch_verify_hash_algo() updates it from DRTM_FEATURES. + */ +static enum sl_hash_algo sl_active_algo __initdata = SL_HASH_SHA256; + +static inline const struct sl_hash_alg_info *sl_active_alg_info(void) +{ + return &sl_hash_algs[sl_active_algo]; +} + +/* + * Hash one contiguous in-RAM buffer with the active algorithm; out must + * hold sl_active_alg_info()->digest_size bytes. Uses sha256()/sha384() + * (safe in setup_arch(); see the dispatch comment above). + */ +static void __init sl_hash_data(const void *data, size_t size, u8 *out) +{ + if (sl_active_algo == SL_HASH_SHA384) + sha384(data, size, out); + else + sha256(data, size, out); +} + /* FDT magic number (big-endian 0xd00dfeed at offset 0) */ #define FDT_HEADER_MAGIC 0xd00dfeed @@ -916,9 +974,16 @@ void __init slaunch_reserve_dlme_data(void) * memblock_alloc-backed and grows geometrically, so platforms with many * ACPI tables are not capped (seeded by slaunch_measurements_init()). */ +/* + * hash is sized for the largest digest (SHA-512, 64 B) so one struct + * serves SHA-256 and SHA-384; digest_size marks the live bytes and + * tpm_alg_id records the algorithm at measurement time. + */ struct slaunch_measurement { char desc[16]; - u8 hash[SHA256_DIGEST_SIZE]; + u8 hash[SL_HASH_MAX_DIGEST_SIZE]; + u8 digest_size; + u16 tpm_alg_id; }; static struct slaunch_measurement *slaunch_measurements __initdata; @@ -978,13 +1043,20 @@ static void __init slaunch_measurements_reserve(unsigned int needed) static void __init slaunch_measure(const char *desc, const void *data, size_t size) { - u8 hash[SHA256_DIGEST_SIZE]; + const struct sl_hash_alg_info *ainfo = sl_active_alg_info(); + u8 hash[SL_HASH_MAX_DIGEST_SIZE]; struct slaunch_measurement *m; - sha256(data, size, hash); + /* + * Hash with the active algorithm via sl_hash_data() (sha256()/ + * sha384()), safe in setup_arch() since the arch SIMD fast-paths + * are static-key gated until subsys_initcall. + */ + sl_hash_data(data, size, hash); - pr_info("slaunch: measured %s (%zu bytes) SHA-256: " - "%*phN\n", desc, size, SHA256_DIGEST_SIZE, hash); + pr_info("slaunch: measured %s (%zu bytes) %s: %*phN\n", + desc, size, ainfo->name, + (int)ainfo->digest_size, hash); /* Future revision: also extend this hash into a hardware measurement * engine (TPM HASH_START or platform-specific equivalent) so the @@ -995,7 +1067,12 @@ static void __init slaunch_measure(const char *desc, const void *data, m = &slaunch_measurements[slaunch_measurement_count++]; strscpy(m->desc, desc, sizeof(m->desc)); - memcpy(m->hash, hash, SHA256_DIGEST_SIZE); + memcpy(m->hash, hash, ainfo->digest_size); + if (ainfo->digest_size < SL_HASH_MAX_DIGEST_SIZE) + memset(m->hash + ainfo->digest_size, 0, + SL_HASH_MAX_DIGEST_SIZE - ainfo->digest_size); + m->digest_size = ainfo->digest_size; + m->tpm_alg_id = ainfo->tpm_alg_id; } /* @@ -1074,19 +1151,22 @@ static void __init slaunch_measure_one_acpi(phys_addr_t pa, const char *desc) /* * Query the D-CRTM TPM hash algorithm (DRTM_FEATURES feature 0x1, - * DEN0113 v1.2 §3.3) and refuse to proceed on mismatch, else the - * event-log digests would not replay against the attester's chain. - * Field layout: firmware_hash_algo [15:0] (0xB SHA-256, 0xC SHA-384). + * DEN0113 v1.2 §3.3), setting sl_active_algo so later measurements use + * the matching path; refuse to proceed on mismatch. Supports 0xB SHA-256 + * / 0xC SHA-384. FAULT_INJECT adds sha384_force / sha_algo_bad tokens. */ -#define SL_DRTM_FW_HASH_SHA256 0x000B -#define SL_DRTM_FW_HASH_SHA384 0x000C #define SL_DRTM_FW_HASH_MASK 0xFFFFULL +#ifdef CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT +static bool __init sl_cmdline_has(const char *tok); +#endif + static void __init slaunch_verify_hash_algo(void) { struct arm_smccc_res res; u64 features; u32 algo; + bool fw_features_supported = true; /* * Feature 0x1 = TPM features (bit 63 set per spec). TF-A returns @@ -1096,16 +1176,46 @@ static void __init slaunch_verify_hash_algo(void) 0, 0, 0, 0, 0, 0, &res); if ((s64)res.a0 == DRTM_NOT_SUPPORTED) { pr_warn("slaunch: DRTM_FEATURES(TPM) not supported; assuming SHA-256\n"); - return; + fw_features_supported = false; + algo = SL_TPM_ALG_SHA256; + } else { + features = res.a1; + algo = features & SL_DRTM_FW_HASH_MASK; } - features = res.a1; - algo = features & SL_DRTM_FW_HASH_MASK; pr_info("slaunch: DCE firmware hash algorithm: 0x%x\n", algo); - if (algo != SL_DRTM_FW_HASH_SHA256) - panic("slaunch: DCE reports hash algo 0x%x; kernel only implements SHA-256 (0xB). Add SHA-384 path or use a SHA-256 DCE.\n", +#ifdef CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT + /* + * Fault-injection override (FAULT_INJECT only): coerce a SHA-256 + * firmware (FVP) into the SHA-384 path to regression-test it without + * a separate TF-A build, plus a "bad algo" token to check the + * dispatch panics on an unsupported algorithm. + */ + if (sl_cmdline_has("slaunch_inject=sha384_force")) { + pr_warn("slaunch: INJECT: overriding hash algo -> SHA-384\n"); + algo = SL_TPM_ALG_SHA384; + } else if (sl_cmdline_has("slaunch_inject=sha_algo_bad")) { + pr_warn("slaunch: INJECT: overriding hash algo -> 0xDEAD (unsupported)\n"); + algo = 0xDEAD; + } +#endif + + switch (algo) { + case SL_TPM_ALG_SHA256: + sl_active_algo = SL_HASH_SHA256; + break; + case SL_TPM_ALG_SHA384: + sl_active_algo = SL_HASH_SHA384; + break; + default: + panic("slaunch: DCE reports hash algo 0x%x; kernel only implements SHA-256 (0xB) and SHA-384 (0xC). Add new algo to sl_hash_algs[] or use a supported DCE.\n", algo); + } + + pr_info("slaunch: DLME will use %s for the measurement chain%s\n", + sl_active_alg_info()->name, + fw_features_supported ? "" : " (DRTM_FEATURES unavailable)"); } static void __init slaunch_measure_acpi(void) @@ -1332,10 +1442,12 @@ void __init slaunch_validate_initrd(void) static void __init slaunch_measure_initrd(void) { + const struct sl_hash_alg_info *ainfo = sl_active_alg_info(); u64 start, size; u64 off = 0, remaining; - struct sha256_ctx sctx; - u8 hash[SHA256_DIGEST_SIZE]; + struct sha256_ctx sctx256; + struct sha384_ctx sctx384; + u8 hash[SL_HASH_MAX_DIGEST_SIZE]; struct slaunch_measurement *m; void *p; @@ -1343,12 +1455,16 @@ static void __init slaunch_measure_initrd(void) return; /* - * Chunked early_memremap + streaming SHA-256: the initrd can - * exceed a single early_memremap and the linear map is not - * reliable here, so map one page at a time, feed into - * sha256_update, then sha256_final. + * Chunked early_memremap + streaming hash with the active algorithm: + * map one page at a time, feed into the streaming context, finalise. + * Both APIs (sha256/sha384_init/update/final) are safe in setup_arch() + * (see the sl_hash_data() comment). */ - sha256_init(&sctx); + if (sl_active_algo == SL_HASH_SHA384) + sha384_init(&sctx384); + else + sha256_init(&sctx256); + remaining = size; while (remaining > 0) { size_t chunk = min_t(u64, remaining, (u64)PAGE_SIZE); @@ -1357,20 +1473,31 @@ static void __init slaunch_measure_initrd(void) if (!p) panic("slaunch: initrd chunk remap failed at 0x%llx (chunk %zu)\n", start + off, chunk); - sha256_update(&sctx, p, chunk); + if (sl_active_algo == SL_HASH_SHA384) + sha384_update(&sctx384, p, chunk); + else + sha256_update(&sctx256, p, chunk); early_memunmap(p, chunk); off += chunk; remaining -= chunk; } - sha256_final(&sctx, hash); + if (sl_active_algo == SL_HASH_SHA384) + sha384_final(&sctx384, hash); + else + sha256_final(&sctx256, hash); - pr_info("slaunch: measured initrd (%llu bytes) SHA-256: %*phN\n", - size, SHA256_DIGEST_SIZE, hash); + pr_info("slaunch: measured initrd (%llu bytes) %s: %*phN\n", + size, ainfo->name, (int)ainfo->digest_size, hash); slaunch_measurements_reserve(slaunch_measurement_count + 1); m = &slaunch_measurements[slaunch_measurement_count++]; strscpy(m->desc, "initrd", sizeof(m->desc)); - memcpy(m->hash, hash, SHA256_DIGEST_SIZE); + memcpy(m->hash, hash, ainfo->digest_size); + if (ainfo->digest_size < SL_HASH_MAX_DIGEST_SIZE) + memset(m->hash + ainfo->digest_size, 0, + SL_HASH_MAX_DIGEST_SIZE - ainfo->digest_size); + m->digest_size = ainfo->digest_size; + m->tpm_alg_id = ainfo->tpm_alg_id; } /* @@ -1379,7 +1506,6 @@ static void __init slaunch_measure_initrd(void) * DLME measurement into the trailing slack and bump drtm_event_log_size, * so a verifier sees one chain (DEN0113 v1.2 §3.17/§4.8.4; TCG PFP §10.2.2). */ -#define SL_TPM_ALG_SHA256 0x000B /* * TODO: no Arm event type (DEN0113 v1.2 §3.17.2 Table 19, base 0x9000) * matches a DLME-side ACPI measurement, so generic TCG @@ -1391,18 +1517,25 @@ static void __init slaunch_measure_initrd(void) /* * Write one TCG_PCR_EVENT2 at *off in buf (advances *off); -ENOSPC if no - * room within max. Packed LE layout: u32 PCRIndex, EventType, - * digest_count; per digest u16 hashAlg + hash[]; u32 EventSize + Event[]. + * room within max. Packed LE: u32 PCRIndex, EventType, digest_count; + * per digest u16 hashAlg + hash[digest_size]; u32 EventSize + Event[]. + * digest_size/alg are variable (SHA-256 32 B, SHA-384 48 B) per record. */ static int __init sl_evlog_append_event2(u8 *buf, size_t *off, size_t max, u32 pcr, u32 type, - const u8 hash[SHA256_DIGEST_SIZE], + u16 tpm_alg_id, + const u8 *hash, u8 digest_size, const void *event_data, u32 event_size) { - size_t need = 4 + 4 + 4 + 2 + SHA256_DIGEST_SIZE + 4 + event_size; + size_t need; u8 *p; + if (digest_size == 0 || digest_size > SL_HASH_MAX_DIGEST_SIZE) + return -EINVAL; + + need = 4 + 4 + 4 + 2 + digest_size + 4 + event_size; + if (*off + need > max) return -ENOSPC; @@ -1410,11 +1543,11 @@ static int __init sl_evlog_append_event2(u8 *buf, size_t *off, size_t max, *(__le32 *)(p + 0) = cpu_to_le32(pcr); *(__le32 *)(p + 4) = cpu_to_le32(type); *(__le32 *)(p + 8) = cpu_to_le32(1); - *(__le16 *)(p + 12) = cpu_to_le16(SL_TPM_ALG_SHA256); - memcpy(p + 14, hash, SHA256_DIGEST_SIZE); - *(__le32 *)(p + 14 + SHA256_DIGEST_SIZE) = cpu_to_le32(event_size); + *(__le16 *)(p + 12) = cpu_to_le16(tpm_alg_id); + memcpy(p + 14, hash, digest_size); + *(__le32 *)(p + 14 + digest_size) = cpu_to_le32(event_size); if (event_size) - memcpy(p + 14 + SHA256_DIGEST_SIZE + 4, event_data, event_size); + memcpy(p + 14 + digest_size + 4, event_data, event_size); *off += need; return 0; } @@ -1489,7 +1622,9 @@ static void __init slaunch_extend_drtm_event_log(void) if (sl_evlog_append_event2(evlog_va, &evlog_off, evlog_max, SL_DRTM_PCR_INDEX, SL_EV_PLATFORM_CONFIG_FLAGS, - m->hash, m->desc, (u32)dlen)) + m->tpm_alg_id, + m->hash, m->digest_size, + m->desc, (u32)dlen)) panic("slaunch: event log overflow at DLME entry %u (%s) — bump Preamble's sl_dlme_data_reserve\n", i, m->desc); } @@ -1520,9 +1655,14 @@ static void __init slaunch_extend_drtm_event_log(void) SL_DRTM_PCR_INDEX); for (i = 0; i < slaunch_measurement_count; i++) { const struct slaunch_measurement *m = &slaunch_measurements[i]; - - pr_info("slaunch: [%2u] %-12s SHA-256: %*phN\n", - i, m->desc, SHA256_DIGEST_SIZE, m->hash); + const char *algo_name = + (m->tpm_alg_id == SL_TPM_ALG_SHA384) ? "SHA-384" : + (m->tpm_alg_id == SL_TPM_ALG_SHA256) ? "SHA-256" : + "UNKNOWN"; + + pr_info("slaunch: [%2u] %-12s %s: %*phN\n", + i, m->desc, algo_name, + (int)m->digest_size, m->hash); } } @@ -1530,8 +1670,8 @@ static void __init slaunch_extend_drtm_event_log(void) /* * Negative-test fault injection (slaunch_inject=): mutates the * raw EFI mmap/systab/SRTM-log before slaunch_validate_efi_early so the - * validator panics. Tokens: mmap_*, systab_nr_tables_huge, srtm_log_*. - * Not for production. + * validator panics. Tokens: mmap_*, systab_nr_tables_huge, srtm_log_*, + * sha384_force, sha_algo_bad. Not for production. */ static bool __init sl_cmdline_has(const char *tok) { From b2e541e6ca72c34f3f51e2e64748e289083d49e3 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Sat, 13 Jun 2026 21:00:19 -0700 Subject: [PATCH 291/311] NVIDIA: SAUCE: arm64: drtm: Kernel-side event log buffer redesign BugLink: https://bugs.launchpad.net/bugs/2161563 The Preamble sizes the DLME-data region to the D-CRTM spec-minimum, so there is no slack in the DCE event log for the kernel to append into. Allocate a separate kernel buffer (sl_kernel_evlog) via memblock, copy the DCE bytes in (read-only), and append DLME TCG_PCR_EVENT2 records there. Capacity is bounded by validated UEFI counts; the DCE region is never mutated and the combined log is exposed via securityfs. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/kernel/slaunch.c | 269 ++++++++++++++++++++++++++++-------- 1 file changed, 215 insertions(+), 54 deletions(-) diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index 5ddf4bab4b41c..f06a78eff856d 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -112,9 +113,33 @@ static size_t sl_srtm_log_size; /* includes sizeof(linux_efi_tpm_eventlog) * Raw EFI memory map extent saved by slaunch_validate_raw_mmap() and * read by slaunch_validate_srtm_log() so the SRTM log extent can be * rejected if it overlaps the raw mmap buffer. + * + * sl_efi_mmap_desc_size and sl_efi_nr_tables are additionally consumed + * by slaunch_extend_drtm_event_log() to compute a runtime upper bound + * for the kernel-side event-log buffer allocation. The bound scales with + * the firmware-reported UEFI input counts (one event per memory-map + * descriptor / ConfigurationTable entry in the upper bound), plus a small + * fixed-event budget for ACPI/CMDLINE/initrd/SRTM. + * + * These values come from pre-DRTM untrusted UEFI inputs; an attacker + * inflating them can only waste memory, not escalate (the kernel image + * hash covers these statics' writes via the past-_edata placement of + * EFI-stub writable globals). */ static phys_addr_t sl_efi_mmap_pa; static u64 sl_efi_mmap_size; +static u32 sl_efi_mmap_desc_size; +static unsigned long sl_efi_nr_tables; + +/* + * Kernel-side DRTM event log buffer (memblock_alloc'd in + * slaunch_measure_post_efi(), not __initdata so securityfs can read it + * at runtime). DCE bytes are copied in first, then DLME TCG_PCR_EVENT2 + * records are appended here; the DCE buffer is never written back. + */ +static u8 *sl_kernel_evlog; +static size_t sl_kernel_evlog_size; +static size_t sl_kernel_evlog_capacity; /* * Close TPM locality 2 — the DLME's locality, per DEN0113 v1.2 §4.6.1. @@ -773,6 +798,12 @@ static void __init slaunch_validate_raw_systab(u64 systab_pa) slaunch_validate_srtm_log((u64)tbl_ptr, &cfgtbl[j].guid); } early_memunmap(cfgtbl, tbl_size); + /* + * Save validated nr_tables so slaunch_extend_drtm_event_log() can + * scale the kernel event-log buffer allocation by the count of + * ConfigurationTable entries (upper bound on per-entry events). + */ + sl_efi_nr_tables = nr_tables; pr_info("slaunch: early EFI System Table validation PASSED (%lu entries)\n", nr_tables); } @@ -868,9 +899,12 @@ static void __init slaunch_validate_raw_mmap(const struct sl_efi_info *info) /* Remember raw mmap extent for later overlap checks (e.g. the SRTM * TPM event log validator must reject a published log PA that aliases - * the firmware-provided memory map buffer). */ - sl_efi_mmap_pa = info->mmap_pa; - sl_efi_mmap_size = info->mmap_size; + * the firmware-provided memory map buffer). desc_size is also saved + * so slaunch_extend_drtm_event_log() can compute the descriptor count + * as part of the kernel event-log buffer sizing formula. */ + sl_efi_mmap_pa = info->mmap_pa; + sl_efi_mmap_size = info->mmap_size; + sl_efi_mmap_desc_size = info->desc_size; } static void __init slaunch_validate_efi_early(const struct sl_efi_info *info) @@ -882,10 +916,12 @@ static void __init slaunch_validate_efi_early(const struct sl_efi_info *info) /* * Stage the raw EFI mmap extent before the systab validator so * slaunch_validate_srtm_log() can reject an SRTM log PA aliasing it; - * validate_raw_mmap() re-publishes these values after its own checks. + * validate_raw_mmap() re-publishes these after its own checks. + * desc_size is pre-staged for the event-log buffer sizing formula. */ - sl_efi_mmap_pa = info->mmap_pa; - sl_efi_mmap_size = info->mmap_size; + sl_efi_mmap_pa = info->mmap_pa; + sl_efi_mmap_size = info->mmap_size; + sl_efi_mmap_desc_size = info->desc_size; slaunch_validate_raw_systab(info->systab_pa); slaunch_validate_raw_mmap(info); } @@ -1501,10 +1537,10 @@ static void __init slaunch_measure_initrd(void) } /* - * Extend the DRTM event log with DLME-side measurements. DCE writes a - * TCG log into the DLME data region; we append one TCG_PCR_EVENT2 per - * DLME measurement into the trailing slack and bump drtm_event_log_size, - * so a verifier sees one chain (DEN0113 v1.2 §3.17/§4.8.4; TCG PFP §10.2.2). + * Extend the DRTM event log with DLME-side measurements. The kernel + * allocates its OWN buffer (sl_kernel_evlog), copies the DCE-published + * bytes in (read-only), and appends one bounds-checked TCG_PCR_EVENT2 + * per measurement; the combined log is exposed via securityfs. */ /* * TODO: no Arm event type (DEN0113 v1.2 §3.17.2 Table 19, base 0x9000) @@ -1515,6 +1551,33 @@ static void __init slaunch_measure_initrd(void) #define SL_EV_PLATFORM_CONFIG_FLAGS 0x0000000A #define SL_DRTM_PCR_INDEX 18 /* DEN0113 v1.2: PCR[18] DLME schema, §4.8.4 Table 40 */ +/* + * Per-event upper bound (TCG PFP §10.2.2): PCRIndex/EventType/count + * (12) + alg_id (2) + max digest (64) + EventSize (4) + desc (16) = 98, + * rounded to 128. Describes one event's shape, not a total-buffer cap. + */ +#define SL_TCG_EVENT2_MAX_BYTES 128 + +/* + * Fixed-event budget: events not scaled by efi_nr_tables/efi_mmap_descs + * (CMDLINE, initrd, SRTM_TPM_LOG) plus headroom. 16 covers the existing + * 4 with margin. + */ +#define SL_DLME_FIXED_EVENTS 16 + +/* + * Constant headroom for small overhead beyond pure TCG_PCR_EVENT2 + * records (future header prefix or padding); 1 KiB, not a buffer cap. + */ +#define SL_KEVLOG_HEADROOM 1024 + +/* + * Sanity cap on the DCE-published event log size the kernel copies: an + * absolute structural bound (1 MiB, well above any realistic log) to + * refuse pathological firmware sizes arithmetic could miss. + */ +#define SL_DCE_EVLOG_MAX (1ULL * 1024 * 1024) + /* * Write one TCG_PCR_EVENT2 at *off in buf (advances *off); -ENOSPC if no * room within max. Packed LE: u32 PCRIndex, EventType, digest_count; @@ -1561,6 +1624,9 @@ static void __init slaunch_extend_drtm_event_log(void) size_t evlog_max, evlog_off; u8 *evlog_va; unsigned int i; + u64 efi_mmap_descs; + size_t kbuf_cap; + u64 per_event_budget; if (!sl_dlme_region_pa) return; @@ -1584,10 +1650,9 @@ static void __init slaunch_extend_drtm_event_log(void) panic("slaunch: DCE published empty DRTM event log\n"); /* - * Event log runs from after header+protected_regions+address_map to - * the end of the DLME data region: evlog_max is capacity, DCE used - * the first evlog_size_initial bytes, DLME appends into the slack. - * Guard the sub-region arithmetic against u64 wrap/underflow. + * DCE event log runs from after header+protected_regions+address_map + * to the end of the DLME data region; guard the sub-region + * arithmetic against u64 wrap/underflow before subtracting. */ { u64 sub_total; @@ -1604,52 +1669,98 @@ static void __init slaunch_extend_drtm_event_log(void) evlog_max = (size_t)(sl_dlme_data_size - sub_total); } - pr_info("slaunch: DRTM event log buffer: PA 0x%llx, capacity %zu B, DCE used %llu B, slack %zu B\n", - (u64)evlog_pa, evlog_max, evlog_size_initial, - evlog_max - (size_t)evlog_size_initial); + if (evlog_size_initial > SL_DCE_EVLOG_MAX) + panic("slaunch: DCE-published event log size %llu exceeds 1 MiB sanity cap\n", + evlog_size_initial); + if (evlog_size_initial > (u64)evlog_max) + panic("slaunch: DCE-published event log size %llu exceeds DLME-data event_log capacity %zu\n", + evlog_size_initial, evlog_max); + + /* + * Allocate the kernel event-log buffer: dce_evlog_size + + * (efi_nr_tables + efi_mmap_descs + SL_DLME_FIXED_EVENTS) * + * SL_TCG_EVENT2_MAX_BYTES + SL_KEVLOG_HEADROOM. Untrusted UEFI counts + * only bound the size; appended bytes are all validated. + */ + efi_mmap_descs = sl_efi_mmap_desc_size ? + sl_efi_mmap_size / sl_efi_mmap_desc_size : 0; + + per_event_budget = (u64)sl_efi_nr_tables + efi_mmap_descs + + SL_DLME_FIXED_EVENTS; + /* + * Multiplication guard for per_event_budget * SL_TCG_EVENT2_MAX_BYTES; + * bounded in practice, but check explicitly before use. + */ + if (per_event_budget > SIZE_MAX / SL_TCG_EVENT2_MAX_BYTES) + panic("slaunch: event log capacity formula overflow (nr_tables=%lu mmap_descs=%llu)\n", + sl_efi_nr_tables, efi_mmap_descs); + + kbuf_cap = (size_t)evlog_size_initial + + (size_t)per_event_budget * SL_TCG_EVENT2_MAX_BYTES + + SL_KEVLOG_HEADROOM; + + sl_kernel_evlog = memblock_alloc(kbuf_cap, SMP_CACHE_BYTES); + if (!sl_kernel_evlog) + panic("slaunch: memblock_alloc for kernel event log buffer failed (%zu bytes)\n", + kbuf_cap); + sl_kernel_evlog_capacity = kbuf_cap; + sl_kernel_evlog_size = 0; + + pr_info("slaunch: kernel event log: VA %p, capacity %zu B (DCE=%llu + (tables=%lu + mmap=%llu + fixed=%u)*%u + headroom=%u)\n", + sl_kernel_evlog, kbuf_cap, evlog_size_initial, + sl_efi_nr_tables, efi_mmap_descs, + SL_DLME_FIXED_EVENTS, SL_TCG_EVENT2_MAX_BYTES, + SL_KEVLOG_HEADROOM); + + pr_info("slaunch: DCE event log: PA 0x%llx, DLME-data sub-region capacity %zu B, DCE-published %llu B\n", + (u64)evlog_pa, evlog_max, evlog_size_initial); - evlog_va = early_memremap(evlog_pa, evlog_max); + /* + * Copy DCE's published event log into the kernel buffer (the only + * read of the DCE buffer; later appends target sl_kernel_evlog). + * Done while DLME-data is still under SMMU DMA protection + * (unprotect is a late_initcall), so no DMA TOCTOU is possible. + */ + evlog_va = early_memremap(evlog_pa, (size_t)evlog_size_initial); if (!evlog_va) - panic("slaunch: cannot map DRTM event log at 0x%llx (%zu B)\n", - (u64)evlog_pa, evlog_max); + panic("slaunch: cannot map DCE event log at 0x%llx (%llu B)\n", + (u64)evlog_pa, evlog_size_initial); + memcpy(sl_kernel_evlog, evlog_va, (size_t)evlog_size_initial); + early_memunmap(evlog_va, (size_t)evlog_size_initial); + sl_kernel_evlog_size = (size_t)evlog_size_initial; - /* Append DLME events after DCE's events, in-place in DLME data. */ - evlog_off = (size_t)evlog_size_initial; + /* + * Append DLME-side measurements into the kernel buffer (NOT the + * DCE buffer). Every sl_evlog_append_event2() call bounds-checks + * against sl_kernel_evlog_capacity. + */ + evlog_off = sl_kernel_evlog_size; for (i = 0; i < slaunch_measurement_count; i++) { const struct slaunch_measurement *m = &slaunch_measurements[i]; size_t dlen = strnlen(m->desc, sizeof(m->desc)); - if (sl_evlog_append_event2(evlog_va, &evlog_off, evlog_max, + if (sl_evlog_append_event2(sl_kernel_evlog, &evlog_off, + sl_kernel_evlog_capacity, SL_DRTM_PCR_INDEX, SL_EV_PLATFORM_CONFIG_FLAGS, m->tpm_alg_id, m->hash, m->digest_size, m->desc, (u32)dlen)) - panic("slaunch: event log overflow at DLME entry %u (%s) — bump Preamble's sl_dlme_data_reserve\n", - i, m->desc); + panic("slaunch: kernel event log overflow at DLME entry %u (%s) — buffer cap=%zu, off=%zu\n", + i, m->desc, sl_kernel_evlog_capacity, evlog_off); } + sl_kernel_evlog_size = evlog_off; - pr_info("slaunch: DRTM event log: appended %u DLME entries (%zu B); new total %zu B\n", - slaunch_measurement_count, - evlog_off - (size_t)evlog_size_initial, evlog_off); + pr_info("slaunch: kernel event log: copied %llu B from DCE, appended %u DLME entries (total %zu B, slack %zu B)\n", + evlog_size_initial, slaunch_measurement_count, + sl_kernel_evlog_size, + sl_kernel_evlog_capacity - sl_kernel_evlog_size); /* Canonical event-log bytes (DCE + DLME) are what a verifier - * replays; exposed for inspection rather than dumped to the log. + * replays; exposed via securityfs rather than dumped to the log. */ pr_info("slaunch: DRTM event log canonical size (DCE + DLME): %zu B\n", - evlog_off); - - early_memunmap(evlog_va, evlog_max); - - /* Update the DLME data header so drtm_event_log_size reflects - * what we just wrote. Verifier reads (PA, size) from the header - * to know what to replay. - */ - hdr = early_memremap(dlme_data_pa, sizeof(*hdr)); - if (!hdr) - panic("slaunch: cannot re-map DLME data header to update size\n"); - hdr->drtm_event_log_size = cpu_to_le64(evlog_off); - early_memunmap(hdr, sizeof(*hdr)); + sl_kernel_evlog_size); pr_info("slaunch: DLME event log entries (PCR %u, EV_PLATFORM_CONFIG_FLAGS):\n", SL_DRTM_PCR_INDEX); @@ -1966,18 +2077,10 @@ static void __init slaunch_selftest(void) #endif /* CONFIG_ARM64_SECURE_LAUNCH_SELFTEST */ /* - * All validation of untrusted EFI inputs happens in slaunch_setup() - * via slaunch_validate_efi_early() — before efi_init() ingests them. - * This post-efi_init slot keeps only the jobs that require - * efi_init's outputs: - * - * - Re-reserve DLME data in memblock (efi_init's - * memblock_remove(0, PHYS_ADDR_MAX) wipes the slaunch_setup - * reservation). - * - Measure ACPI tables via efi.acpi20 (which efi_init populated - * from the now pre-validated ConfigurationTable). - * - Run the validation-helper self-test - * (CONFIG_ARM64_SECURE_LAUNCH_SELFTEST). + * Post-efi_init jobs that need efi_init's outputs: re-reserve DLME data + * (efi_init's memblock_remove wiped the slaunch_setup reservation), + * measure ACPI tables via efi.acpi20, and run the validation-helper + * self-test. Untrusted EFI inputs were already validated in slaunch_setup(). */ void __init slaunch_measure_post_efi(void) { @@ -2015,6 +2118,64 @@ void __init slaunch_measure_post_efi(void) } } +/* + * Securityfs exposure for the kernel-side DRTM event log, mirroring the + * TPM "binary_bios_measurements" file. Same information-disclosure + * surface: measurement hashes are not secrets but the attestation + * primitive a remote verifier fetches. + */ +static ssize_t slaunch_evlog_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + if (!sl_kernel_evlog || !sl_kernel_evlog_size) + return 0; + return simple_read_from_buffer(buf, count, ppos, + sl_kernel_evlog, + sl_kernel_evlog_size); +} + +static const struct file_operations slaunch_evlog_fops = { + .owner = THIS_MODULE, + .read = slaunch_evlog_read, + .llseek = default_llseek, +}; + +static struct dentry *sl_securityfs_dir; +static struct dentry *sl_securityfs_evlog; + +static int __init slaunch_securityfs_init(void) +{ + if (!sl_kernel_evlog || !sl_kernel_evlog_size) + return 0; + + sl_securityfs_dir = securityfs_create_dir("slaunch", NULL); + if (IS_ERR(sl_securityfs_dir)) { + pr_warn("slaunch: securityfs_create_dir failed: %ld\n", + PTR_ERR(sl_securityfs_dir)); + sl_securityfs_dir = NULL; + return 0; + } + + sl_securityfs_evlog = securityfs_create_file("drtm_event_log", + 0440, + sl_securityfs_dir, + NULL, + &slaunch_evlog_fops); + if (IS_ERR(sl_securityfs_evlog)) { + pr_warn("slaunch: securityfs_create_file failed: %ld\n", + PTR_ERR(sl_securityfs_evlog)); + securityfs_remove(sl_securityfs_dir); + sl_securityfs_dir = NULL; + sl_securityfs_evlog = NULL; + return 0; + } + + pr_info("slaunch: securityfs/slaunch/drtm_event_log exposed (%zu B)\n", + sl_kernel_evlog_size); + return 0; +} +late_initcall(slaunch_securityfs_init); + /* * Release DRTM DMA protection after IOMMU/SMMU drivers have * established their own DMA isolation. From 04db89f90670f21b0e105c6a0c34c680d88456e8 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Tue, 16 Jun 2026 19:19:13 -0700 Subject: [PATCH 292/311] NVIDIA: SAUCE: arm64: drtm: refuse ACPI device-map of protected RAM BugLink: https://bugs.launchpad.net/bugs/2161563 After a dynamic launch the EFI memory map is untrusted, yet the ACPI MMIO attribute paths (__acpi_get_mem_attribute, acpi_os_ioremap) still consult it. Add point and range predicates for kernel RAM / the measured DLME and refuse to device-map any such PA; the ioremap path uses the range predicate so a span starting in MMIO cannot reach protected memory. Overlapping raw EFI descriptors warn (non-fatal; the consumer guard is authoritative). Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/include/asm/drtm.h | 4 +++ arch/arm64/kernel/acpi.c | 20 +++++++++++ arch/arm64/kernel/slaunch.c | 66 +++++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+) diff --git a/arch/arm64/include/asm/drtm.h b/arch/arm64/include/asm/drtm.h index 80f5fc4a90e6c..29d65e112c199 100644 --- a/arch/arm64/include/asm/drtm.h +++ b/arch/arm64/include/asm/drtm.h @@ -133,6 +133,8 @@ void slaunch_validate_initrd(void); void slaunch_reserve_dlme_data(void); void slaunch_exit(void); void slaunch_measure_post_efi(void); +bool slaunch_phys_is_protected_ram(phys_addr_t pa); +bool slaunch_phys_range_overlaps_protected_ram(phys_addr_t pa, size_t size); #else static inline void slaunch_early_init(void) { } static inline void slaunch_setup(void) { } @@ -140,6 +142,8 @@ static inline void slaunch_validate_initrd(void) { } static inline void slaunch_reserve_dlme_data(void) { } static inline void slaunch_exit(void) { } static inline void slaunch_measure_post_efi(void) { } +static inline bool slaunch_phys_is_protected_ram(phys_addr_t pa) { return false; } +static inline bool slaunch_phys_range_overlaps_protected_ram(phys_addr_t pa, size_t size) { return false; } #endif #endif /* __ASSEMBLY__ */ diff --git a/arch/arm64/kernel/acpi.c b/arch/arm64/kernel/acpi.c index a9d884fd1d001..d7a56c8fe2e5e 100644 --- a/arch/arm64/kernel/acpi.c +++ b/arch/arm64/kernel/acpi.c @@ -29,6 +29,8 @@ #include #include +#include + #include #include #include @@ -282,6 +284,11 @@ pgprot_t __acpi_get_mem_attribute(phys_addr_t addr) u64 attr; + /* Never let the untrusted EFI attribute device-map kernel RAM or + * the measured DLME; force cacheable for those. */ + if (slaunch_phys_is_protected_ram(addr)) + return PAGE_KERNEL; + attr = efi_mem_attributes(addr); if (attr & EFI_MEMORY_WB) return PAGE_KERNEL; @@ -369,6 +376,14 @@ void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size) fallthrough; default: + /* Refuse a non-RAM descriptor whose span reaches kernel + * RAM or the measured DLME: a device alias of normal + * memory is unsafe. Check the whole range — a span may + * start in MMIO and extend into protected memory. */ + if (slaunch_phys_range_overlaps_protected_ram(phys, size)) { + pr_warn(FW_BUG "DRTM: refusing device-map of kernel RAM/DLME @ %pa\n", &phys); + return NULL; + } if (region->attribute & EFI_MEMORY_WB) prot = PAGE_KERNEL; else if (region->attribute & EFI_MEMORY_WC) @@ -376,6 +391,11 @@ void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size) else if (region->attribute & EFI_MEMORY_WT) prot = __acpi_get_writethrough_mem_attribute(); } + } else if (slaunch_phys_range_overlaps_protected_ram(phys, size)) { + /* Span absent from the EFI map but reaching kernel RAM or the + * DLME: refuse rather than device-map it. */ + pr_warn(FW_BUG "DRTM: refusing device-map of unmapped kernel RAM/DLME @ %pa\n", &phys); + return NULL; } return __ioremap_prot(phys, size, prot); } diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index f06a78eff856d..01a21f043c765 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -416,6 +416,43 @@ static bool __init efi_regions_overlap(u64 s1, u64 sz1, u64 s2, u64 sz2) return (s1 < e2) && (s2 < e1); } +/* True when @pa is kernel RAM or in the measured DLME extent; such a PA + * must never be device-mapped from the untrusted EFI map. False when DRTM + * is inactive; runtime-callable since the DLME handoff values persist. */ +bool slaunch_phys_is_protected_ram(phys_addr_t pa) +{ + if (!sl_dlme_region_pa) + return false; + if (memblock_is_map_memory(pa)) + return true; + return sl_dlme_data_size && + pa >= sl_dlme_region_pa && + pa < sl_dlme_data_pa + sl_dlme_data_size; +} + +/* True if [pa, pa+size) touches kernel RAM or the measured DLME extent. + * A range that wraps is refused. Used by the ACPI ioremap path, where a + * span may start outside protected memory and extend into it. */ +bool slaunch_phys_range_overlaps_protected_ram(phys_addr_t pa, size_t size) +{ + phys_addr_t p, end; + + if (!sl_dlme_region_pa) + return false; + if (check_add_overflow(pa, (phys_addr_t)size, &end)) + return true; + + if (sl_dlme_data_size && + pa < sl_dlme_data_pa + sl_dlme_data_size && + sl_dlme_region_pa < end) + return true; + + for (p = ALIGN_DOWN(pa, PAGE_SIZE); p < end; p += PAGE_SIZE) + if (memblock_is_map_memory(p)) + return true; + return false; +} + /* * slaunch_early_init() -- earliest DRTM validation, called BEFORE * setup_machine_fdt() (after early_ioremap_init). Parses the D-CRTM @@ -893,6 +930,35 @@ static void __init slaunch_validate_raw_mmap(const struct sl_efi_info *info) dcrtm_type_name(otype)); nchecked++; } + + /* Defense-in-depth: warn on any overlapping descriptor pair. + * First-match attribute lookups would let a planted MMIO descriptor + * order-pick the type for a measured PA; the acpi_os_ioremap consumer + * guard blocks the RAM/DLME case regardless, so this is non-fatal. */ + for (offset = 0; offset < info->mmap_size; offset += info->desc_size) { + efi_memory_desc_t *a = (efi_memory_desc_t *)((u8 *)mmap + offset); + u64 asz, off2; + + if (check_mul_overflow(a->num_pages, (u64)EFI_PAGE_SIZE, &asz)) + continue; + for (off2 = offset + info->desc_size; off2 < info->mmap_size; + off2 += info->desc_size) { + efi_memory_desc_t *b = + (efi_memory_desc_t *)((u8 *)mmap + off2); + u64 bsz; + + if (check_mul_overflow(b->num_pages, + (u64)EFI_PAGE_SIZE, &bsz)) + continue; + if (efi_regions_overlap(a->phys_addr, asz, + b->phys_addr, bsz)) + pr_warn("slaunch: raw EFI mmap descriptors [%llu]/[%llu] OVERLAP @ 0x%012llx (types %u/%u)\n", + offset / info->desc_size, + off2 / info->desc_size, + a->phys_addr, a->type, b->type); + } + } + early_memunmap(mmap, info->mmap_size); pr_info("slaunch: early raw EFI mmap validation PASSED (%u of %u descriptors checked)\n", nchecked, ndesc); From 72b983eb928d9348c59d1f23b825cb6a3a369df5 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Tue, 16 Jun 2026 19:19:13 -0700 Subject: [PATCH 293/311] NVIDIA: SAUCE: arm64: drtm: add overlap fault-injection self-test BugLink: https://bugs.launchpad.net/bugs/2161563 Add a fault-injection token (slaunch_inject=mmap_overlap) that forces two conventional descriptors to overlap so the raw-mmap overlap detector is exercised, plus self-test T9 asserting slaunch_phys_is_protected_ram flags the measured DLME region and rejects a non-RAM PA. Both sit behind the existing fault-injection and self-test Kconfig and compile out of production builds. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/kernel/slaunch.c | 51 ++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index 01a21f043c765..bb8f6031b6209 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -1902,6 +1902,41 @@ static void __init sl_mutate_mmap_pages_overflow(efi_memory_desc_t *md) md->phys_addr, md->num_pages); } +/* Force two CONVENTIONAL descriptors to overlap so the overlap + * detector in slaunch_validate_raw_mmap fires its WARN (non-fatal). */ +static void __init slaunch_inject_mmap_overlap(const struct sl_efi_info *info) +{ + void *mmap; + u64 off; + efi_memory_desc_t *first = NULL; + + mmap = early_memremap(info->mmap_pa, info->mmap_size); + if (!mmap) { + pr_warn("slaunch: INJECT mmap_overlap: map failed\n"); + return; + } + for (off = 0; off < info->mmap_size; off += info->desc_size) { + efi_memory_desc_t *md = + (efi_memory_desc_t *)((u8 *)mmap + off); + + if (md->type != EFI_CONVENTIONAL_MEMORY) + continue; + if (!first) { + first = md; + continue; + } + md->phys_addr = first->phys_addr; + if (!md->num_pages) + md->num_pages = 1; + pr_warn("slaunch: INJECT mmap_overlap: 2nd CONV desc -> phys=0x%llx (expect 'OVERLAP')\n", + md->phys_addr); + early_memunmap(mmap, info->mmap_size); + return; + } + pr_warn("slaunch: INJECT mmap_overlap: <2 CONVENTIONAL descriptors\n"); + early_memunmap(mmap, info->mmap_size); +} + /* Overwrite the first non-null EFI ConfigurationTable entry in the raw * systab's cfgtbl array with a synthetic SRTM-log entry. Returns true * if we managed to plant the entry. Used by srtm_log_* injectors only; @@ -2004,6 +2039,10 @@ static void __init slaunch_inject_fault(struct sl_efi_info *info) pr_warn("slaunch: INJECT mmap_pages_overflow: no EFI_CONVENTIONAL_MEMORY descriptor found\n"); return; } + if (sl_cmdline_has("slaunch_inject=mmap_overlap")) { + slaunch_inject_mmap_overlap(info); + return; + } if (sl_cmdline_has("slaunch_inject=mmap_size_huge")) { /* * Inflate the local mmap size (a 48-multiple, so the @@ -2138,7 +2177,17 @@ static void __init slaunch_selftest(void) panic("selftest: check_mul_overflow missed num_pages*PAGE_SIZE wrap\n"); } - pr_info("slaunch: ALL SELFTESTS PASSED (8/8)\n"); + /* T9: slaunch_phys_is_protected_ram is the exact predicate + * acpi_os_ioremap/__acpi_get_mem_attribute gate on. It MUST flag the + * measured DLME region and MUST NOT flag a non-RAM PA. */ + if (sl_dlme_region_pa) { + if (!slaunch_phys_is_protected_ram(sl_dlme_region_pa)) + panic("selftest: slaunch_phys_is_protected_ram missed DLME region\n"); + if (slaunch_phys_is_protected_ram(0)) + panic("selftest: slaunch_phys_is_protected_ram flagged non-RAM PA\n"); + } + + pr_info("slaunch: ALL SELFTESTS PASSED (9/9)\n"); } #endif /* CONFIG_ARM64_SECURE_LAUNCH_SELFTEST */ From f7d0d5c637037daa56ba6c823610561b7c32e979 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Wed, 1 Jul 2026 13:11:06 -0700 Subject: [PATCH 294/311] NVIDIA: SAUCE: arm64: drtm: request Secure-interrupt disable across the launch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BugLink: https://bugs.launchpad.net/bugs/2161563 Set launch_features bit 7 (DEN0113 Table 9) so the D-CRTM disables Secure interrupts during the dynamic launch. The DLME re-enables them via DRTM_ENABLE_SECURE_INTERRUPTS (§3.11) as its first post-launch step, before any failable DLME-data or DTB path could strand them disabled. NOT_SUPPORTED/DENIED mean they were never disabled; any other failure is fatal. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/include/asm/drtm.h | 2 ++ arch/arm64/kernel/slaunch.c | 25 ++++++++++++++++++++ drivers/firmware/efi/libstub/arm64-slaunch.c | 5 +++- 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/drtm.h b/arch/arm64/include/asm/drtm.h index 29d65e112c199..9d9fab0f3e689 100644 --- a/arch/arm64/include/asm/drtm.h +++ b/arch/arm64/include/asm/drtm.h @@ -21,6 +21,7 @@ #define DRTM_SMC_SET_ERROR (DRTM_SMC_FN_BASE + 0x07) #define DRTM_SMC_SET_TCB_HASH (DRTM_SMC_FN_BASE + 0x08) #define DRTM_SMC_LOCK_TCB_HASH (DRTM_SMC_FN_BASE + 0x09) +#define DRTM_SMC_ENABLE_SECURE_INTERRUPTS (DRTM_SMC_FN_BASE + 0x0A) /* 0xC400011A */ /* DRTM Return Codes (DEN0113 v1.2 §3.18, Table 20) */ #define DRTM_SUCCESS 0 @@ -34,6 +35,7 @@ /* Launch features */ #define DRTM_LAUNCH_FEAT_MEM_PROT_ALL (0x0 << 3) +#define DRTM_LAUNCH_FEAT_SEC_INT_DISABLE (0x1 << 7) /* DEN0113 Table 9 */ /* DRTM page size */ #define DRTM_PAGE_SIZE 0x1000 diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index bb8f6031b6209..1f8acecafe569 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -158,6 +158,28 @@ static void __init slaunch_tpm_setup(void) pr_info("slaunch: TPM locality 2 closed\n"); } +/* + * Re-enable Secure interrupts requested disabled via launch_features bit 7 + * (DEN0113 §3.11). Runs as the first post-launch step so no failable init + * path can leave them stranded disabled. NOT_SUPPORTED/DENIED mean they were + * never disabled; any other failure after an explicit request is ambiguous. + */ +static void __init slaunch_enable_secure_ints(void) +{ + struct arm_smccc_res res; + + arm_smccc_smc(DRTM_SMC_ENABLE_SECURE_INTERRUPTS, 0, 0, 0, 0, 0, 0, 0, &res); + if (res.a0 == (unsigned long)DRTM_NOT_SUPPORTED || + res.a0 == (unsigned long)DRTM_DENIED) + pr_info("slaunch: Secure interrupts not disabled by platform (%ld)\n", + (long)res.a0); + else if (res.a0 != DRTM_SUCCESS) + panic("slaunch: ENABLE_SECURE_INTERRUPTS failed: %ld\n", + (long)res.a0); + else + pr_info("slaunch: Secure interrupts re-enabled\n"); +} + /* * Parse the D-CRTM address map (at header_size + protected_regions_size * within DLME data). A trusted EL3-populated input describing physical @@ -470,6 +492,9 @@ void __init slaunch_early_init(void) if (!sl_dlme_region_pa) return; + /* First post-launch action: nothing failable may run before this. */ + slaunch_enable_secure_ints(); + pr_info("slaunch: DRTM early init -- validating DTB before consumption\n"); pr_info("slaunch: DLME region PA: 0x%lx, data offset: 0x%lx\n", sl_dlme_region_pa, sl_dlme_data_offset); diff --git a/drivers/firmware/efi/libstub/arm64-slaunch.c b/drivers/firmware/efi/libstub/arm64-slaunch.c index aaa0c51964355..2faa710db9c0d 100644 --- a/drivers/firmware/efi/libstub/arm64-slaunch.c +++ b/drivers/firmware/efi/libstub/arm64-slaunch.c @@ -272,7 +272,10 @@ void __noreturn efi_slaunch_drtm(unsigned long kernel_addr, /* Build DRTM_PARAMETERS */ params->revision = DRTM_PARAMS_REVISION; params->reserved = 0; - params->launch_features = 0; /* bits[5:3]=0: complete DMA protection */ + /* bits[5:3]=0: complete DMA protection. bit 7: request Secure-interrupt + * disable for the launch window (DEN0113 Table 9); the DLME re-enables + * post-launch via DRTM_ENABLE_SECURE_INTERRUPTS. */ + params->launch_features = DRTM_LAUNCH_FEAT_MEM_PROT_ALL | DRTM_LAUNCH_FEAT_SEC_INT_DISABLE; params->dlme_region_address = kernel_addr; params->dlme_region_size = dlme_data_offset + sl_dlme_data_reserve; params->dlme_image_start = 0; From cc2e6f88d23a775c054bdce1bb3554da62a05232 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Thu, 9 Jul 2026 14:56:19 -0700 Subject: [PATCH 295/311] NVIDIA: SAUCE: arm64: drtm: fail closed if the DLME data header cannot be mapped BugLink: https://bugs.launchpad.net/bugs/2161563 After a Secure Launch slaunch_setup() maps the DLME data header to locate the protected-regions table and DRTM event log, then asserts full lockdown and closes the measurement locality. A failed map took an early return, booting on with those steps skipped. Panic instead, matching the early-init header map, so a launched system never runs with lockdown unverified and the locality open. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/kernel/slaunch.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index 1f8acecafe569..2fc34e42e5ea9 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -1035,11 +1035,12 @@ void __init slaunch_setup(void) /* Map DLME data header for reservation */ dlme_data_pa = sl_dlme_region_pa + sl_dlme_data_offset; hdr = early_memremap(dlme_data_pa, sizeof(*hdr)); - if (!hdr) { - pr_err("slaunch: failed to map DLME data header at 0x%llx\n", - (u64)dlme_data_pa); - return; - } + /* The header locates the protected-regions table and event log that + * the full-lockdown assertion and locality close below need; fail + * closed if it cannot be mapped rather than boot on with them skipped. */ + if (!hdr) + panic("slaunch: failed to map DLME data header at 0x%llx\n", + (u64)dlme_data_pa); pr_info("slaunch: DLME data version: %u\n", le16_to_cpu(hdr->version)); From c8995cdc6d20915f30fb7badf928d2222a1411ad Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Fri, 10 Jul 2026 21:20:14 -0700 Subject: [PATCH 296/311] NVIDIA: SAUCE: arm64: drtm: require ACPI mode after a Secure Launch BugLink: https://bugs.launchpad.net/bugs/2161563 DEN0113 defines the DRTM measurement and attestation chain over ACPI: the ACPI tables are measured into the DRTM event log. A launched system that fell back to device tree runs on unmeasured, unvalidated topology with no valid attestation. Panic once acpi_boot_table_init() has finalised acpi_disabled and before the kernel consumes device-tree topology, so a launched system fails closed rather than booting on. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/kernel/setup.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 371cf27a7bc15..35b92094cf365 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -351,6 +351,14 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) /* Parse the ACPI tables for possible boot-time configuration */ acpi_boot_table_init(); + /* + * DEN0113 defines the DRTM measurement/attestation chain over ACPI. + * A launched system that fell back to device tree runs on unmeasured + * topology with no valid attestation, so fail closed. + */ + if (sl_dlme_region_pa && acpi_disabled) + panic("slaunch: DRTM launch requires ACPI mode, but ACPI is disabled\n"); + if (acpi_disabled) unflatten_device_tree(); From 44096b351c5d3da9ce0813b02cd047b8df25d790 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Sat, 11 Jul 2026 11:06:22 -0700 Subject: [PATCH 297/311] NVIDIA: SAUCE: arm64: drtm: address review feedback (naming, logging, style) BugLink: https://bugs.launchpad.net/bugs/2161563 - cast res.a0 to (long) for all DRTM return-code comparisons - rename SL_ROUND_UP_4K -> SL_ROUND_UP_PAGE - log an efi_err when DRTM dynamic launch fails to occur - use dcrtm_type_name() for the address-map region-type print - rename slaunch_assert_full_lockdown -> slaunch_assert_dma_protection Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/kernel/slaunch.c | 34 +++++++++----------- drivers/firmware/efi/libstub/arm64-slaunch.c | 5 +-- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index 2fc34e42e5ea9..42a0ee09e43a2 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -150,9 +150,9 @@ static void __init slaunch_tpm_setup(void) struct arm_smccc_res res; arm_smccc_smc(DRTM_SMC_CLOSE_LOCALITY, 2, 0, 0, 0, 0, 0, 0, &res); - if (res.a0 == (unsigned long)DRTM_NOT_SUPPORTED) + if ((long)res.a0 == DRTM_NOT_SUPPORTED) pr_warn("slaunch: CLOSE_LOCALITY not supported (no TPM backend)\n"); - else if (res.a0 != DRTM_SUCCESS) + else if ((long)res.a0 != DRTM_SUCCESS) pr_err("slaunch: CLOSE_LOCALITY failed: %ld\n", (long)res.a0); else pr_info("slaunch: TPM locality 2 closed\n"); @@ -169,17 +169,20 @@ static void __init slaunch_enable_secure_ints(void) struct arm_smccc_res res; arm_smccc_smc(DRTM_SMC_ENABLE_SECURE_INTERRUPTS, 0, 0, 0, 0, 0, 0, 0, &res); - if (res.a0 == (unsigned long)DRTM_NOT_SUPPORTED || - res.a0 == (unsigned long)DRTM_DENIED) + if ((long)res.a0 == DRTM_NOT_SUPPORTED || + (long)res.a0 == DRTM_DENIED) pr_info("slaunch: Secure interrupts not disabled by platform (%ld)\n", (long)res.a0); - else if (res.a0 != DRTM_SUCCESS) + else if ((long)res.a0 != DRTM_SUCCESS) panic("slaunch: ENABLE_SECURE_INTERRUPTS failed: %ld\n", (long)res.a0); else pr_info("slaunch: Secure interrupts re-enabled\n"); } +/* Defined after the address-map parser below. */ +static const char *dcrtm_type_name(int type); + /* * Parse the D-CRTM address map (at header_size + protected_regions_size * within DLME data). A trusted EL3-populated input describing physical @@ -251,12 +254,7 @@ static bool __init slaunch_parse_address_map(phys_addr_t dlme_data_pa, pr_info("slaunch: [%u] 0x%012llx - 0x%012llx %s (%llu pages)\n", i, addr, addr + pages * DRTM_PAGE_SIZE, - type == DRTM_REGION_TYPE_NORMAL ? "NORMAL" : - type == DRTM_REGION_TYPE_NORMAL_CACHED ? "NORMAL_CACHED" : - type == DRTM_REGION_TYPE_DEVICE ? "DEVICE" : - type == DRTM_REGION_TYPE_NV ? "NV" : - type == DRTM_REGION_TYPE_RSVD ? "RSVD" : - "UNKNOWN", pages); + dcrtm_type_name(type), pages); } early_memunmap(map_hdr, (size_t)map_size); @@ -281,8 +279,8 @@ static bool __init slaunch_parse_address_map(phys_addr_t dlme_data_pa, * spec-conformant "single entry, start=0, full-range" encoding; partial * lockdown breaks the measure-then-parse soundness model. */ -static void __init slaunch_assert_full_lockdown(phys_addr_t dlme_data_pa, - u64 hdr_size, u64 prot_size) +static void __init slaunch_assert_dma_protection(phys_addr_t dlme_data_pa, + u64 hdr_size, u64 prot_size) { const struct drtm_mem_region_hdr *phdr; const struct drtm_mem_region *regs; @@ -1053,9 +1051,9 @@ void __init slaunch_setup(void) * Called from slaunch_setup (not slaunch_early_init) so panic * prints — earlycon is registered by this point. */ - slaunch_assert_full_lockdown(dlme_data_pa, - le16_to_cpu(hdr->this_hdr_size), - le64_to_cpu(hdr->protected_regions_size)); + slaunch_assert_dma_protection(dlme_data_pa, + le16_to_cpu(hdr->this_hdr_size), + le64_to_cpu(hdr->protected_regions_size)); /* Stash for slaunch_reserve_dlme_data() to reserve post-efi_init. */ sl_dlme_data_pa = dlme_data_pa; @@ -1302,7 +1300,7 @@ static void __init slaunch_verify_hash_algo(void) */ arm_smccc_smc(DRTM_SMC_FEATURES, (1ULL << 63) | 0x1, 0, 0, 0, 0, 0, 0, &res); - if ((s64)res.a0 == DRTM_NOT_SUPPORTED) { + if ((long)res.a0 == DRTM_NOT_SUPPORTED) { pr_warn("slaunch: DRTM_FEATURES(TPM) not supported; assuming SHA-256\n"); fw_features_supported = false; algo = SL_TPM_ALG_SHA256; @@ -2330,7 +2328,7 @@ static int __init slaunch_unprotect_memory(void) pr_info("slaunch: Calling DRTM_UNPROTECT_MEMORY\n"); arm_smccc_smc(DRTM_SMC_UNPROTECT_MEMORY, 0, 0, 0, 0, 0, 0, 0, &res); - if (res.a0 != DRTM_SUCCESS) { + if ((long)res.a0 != DRTM_SUCCESS) { pr_err("slaunch: UNPROTECT_MEMORY failed: %ld\n", (long)res.a0); return -EIO; diff --git a/drivers/firmware/efi/libstub/arm64-slaunch.c b/drivers/firmware/efi/libstub/arm64-slaunch.c index 2faa710db9c0d..034820f04b443 100644 --- a/drivers/firmware/efi/libstub/arm64-slaunch.c +++ b/drivers/firmware/efi/libstub/arm64-slaunch.c @@ -17,7 +17,7 @@ #define SL_DRTM_SMC_FEATURES 0xC4000111UL #define SL_DRTM_SMC_DYNAMIC_LAUNCH 0xC4000114UL #define SL_DRTM_PAGE_SIZE 0x1000 -#define SL_ROUND_UP_4K(x) (((x) + SL_DRTM_PAGE_SIZE - 1) & \ +#define SL_ROUND_UP_PAGE(x) (((x) + SL_DRTM_PAGE_SIZE - 1) & \ ~(SL_DRTM_PAGE_SIZE - 1ULL)) /* @@ -259,7 +259,7 @@ void __noreturn efi_slaunch_drtm(unsigned long kernel_addr, */ image_size = (unsigned long)(_edata - _text); kernel_memsize = (unsigned long)(_end - _text); - dlme_data_offset = SL_ROUND_UP_4K(kernel_memsize); + dlme_data_offset = SL_ROUND_UP_PAGE(kernel_memsize); /* * Write DTB PA into the Preamble->DLME slot at (kernel_addr + @@ -309,6 +309,7 @@ void __noreturn efi_slaunch_drtm(unsigned long kernel_addr, sl_smc(SL_DRTM_SMC_DYNAMIC_LAUNCH, (u64)params); /* If we reach here, the SMC failed. Halt. */ + efi_err("DRTM: dynamic launch did not occur\n"); for (;;) asm volatile("wfi"); } From e20d8d385633f0cf2b14a89bd36f5793ecd1a5dd Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Sat, 11 Jul 2026 17:19:35 -0700 Subject: [PATCH 298/311] NVIDIA: SAUCE: arm64: drtm: reserve a dedicated slot for the DTB-PA handoff BugLink: https://bugs.launchpad.net/bugs/2161563 The Preamble->DLME DTB-PA handoff slot sits at dlme_data_offset - 8. dlme_data_offset was SL_ROUND_UP_PAGE(kernel_memsize), and kernel_memsize is already SEGMENT_ALIGN'd, so the slot fell inside the kernel image tail (early_init_stack/BSS). Push the DLME data region down one page (SL_DLME_DTB_SLOT_GAP) so the slot lands in dedicated scratch below the D-CRTM data. reserve_size grows to match dlme_region_size. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- drivers/firmware/efi/libstub/arm64-slaunch.c | 2 +- drivers/firmware/efi/libstub/arm64-stub.c | 2 +- drivers/firmware/efi/libstub/efistub.h | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/firmware/efi/libstub/arm64-slaunch.c b/drivers/firmware/efi/libstub/arm64-slaunch.c index 034820f04b443..bb907929c27fd 100644 --- a/drivers/firmware/efi/libstub/arm64-slaunch.c +++ b/drivers/firmware/efi/libstub/arm64-slaunch.c @@ -259,7 +259,7 @@ void __noreturn efi_slaunch_drtm(unsigned long kernel_addr, */ image_size = (unsigned long)(_edata - _text); kernel_memsize = (unsigned long)(_end - _text); - dlme_data_offset = SL_ROUND_UP_PAGE(kernel_memsize); + dlme_data_offset = SL_ROUND_UP_PAGE(kernel_memsize) + SL_DLME_DTB_SLOT_GAP; /* * Write DTB PA into the Preamble->DLME slot at (kernel_addr + diff --git a/drivers/firmware/efi/libstub/arm64-stub.c b/drivers/firmware/efi/libstub/arm64-stub.c index 02ae83f7e8387..1e330217298b8 100644 --- a/drivers/firmware/efi/libstub/arm64-stub.c +++ b/drivers/firmware/efi/libstub/arm64-stub.c @@ -43,7 +43,7 @@ efi_status_t handle_kernel_image(unsigned long *image_addr, * BSS for D-CRTM to populate (address map, event log, etc.). */ efi_slaunch_get_dlme_data_size(); - *reserve_size += sl_dlme_data_reserve; + *reserve_size += SL_DLME_DTB_SLOT_GAP + sl_dlme_data_reserve; #endif *image_addr = (unsigned long)_text; diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h index 6b61ae3cb5ce5..c88dd17f24c31 100644 --- a/drivers/firmware/efi/libstub/efistub.h +++ b/drivers/firmware/efi/libstub/efistub.h @@ -1272,6 +1272,10 @@ bool efi_slaunch_enabled(const char *cmdline); void efi_slaunch_get_dlme_data_size(void); void efi_slaunch_scrub_imagebase(unsigned long kernel_addr); extern unsigned long sl_dlme_data_reserve; +/* Page gap below the DLME data region so the Preamble->DLME DTB-PA + * handoff slot (SL_DLME_DTB_SLOT_OFFSET) is dedicated scratch, not the + * kernel image tail. */ +#define SL_DLME_DTB_SLOT_GAP 0x1000 extern bool sl_drtm_available; void __noreturn efi_slaunch_drtm(unsigned long kernel_addr, unsigned long fdt_addr); From 4620f8f2d9c8e1e71d32d0b9c8f64742771aebc5 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Sat, 11 Jul 2026 19:22:41 -0700 Subject: [PATCH 299/311] NVIDIA: SAUCE: arm64: drtm: annotate endianness, drop unused drtm_parameters BugLink: https://bugs.launchpad.net/bugs/2161563 Make the DRTM code sparse- and warning-clean. Annotate the firmware-ABI structs (drtm_mem_region{,_hdr}, sl_drtm_params) __le* and write them via cpu_to_le*(); annotate the big-endian DTB-header reads __be32; cast const away at the early_memunmap(phdr) unmap. Also remove struct drtm_parameters, which duplicates the live sl_drtm_params and is unused. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/include/asm/drtm.h | 30 ++--------- arch/arm64/kernel/slaunch.c | 6 +-- drivers/firmware/efi/libstub/arm64-slaunch.c | 52 ++++++++++---------- 3 files changed, 34 insertions(+), 54 deletions(-) diff --git a/arch/arm64/include/asm/drtm.h b/arch/arm64/include/asm/drtm.h index 9d9fab0f3e689..07654990ead8b 100644 --- a/arch/arm64/include/asm/drtm.h +++ b/arch/arm64/include/asm/drtm.h @@ -55,40 +55,20 @@ ((0x0ULL << 55) | (0x0ULL << 52) | ((1ULL << 52) - 1ULL)) #ifndef __ASSEMBLY__ -/* - * DRTM_PARAMETERS (DEN0113 v1.2 §3.13, Table 9) - * Passed to DRTM_DYNAMIC_LAUNCH SMC in X1. - */ -struct drtm_parameters { - u16 revision; - u16 reserved; - u32 launch_features; - u64 dlme_region_address; - u64 dlme_region_size; - u64 dlme_image_start; - u64 dlme_entry_point_offset; - u64 dlme_image_size; - u64 dlme_data_offset; - u64 nw_dce_region_address; - u64 nw_dce_region_size; - u64 mem_prot_table_address; - u64 mem_prot_table_size; -} __packed; - /* * Memory Region Descriptor Table (DEN0113 v1.2 §3.14, Table 11): header * followed by num_regions descriptors, consumed in place from the DLME * data region (no fixed-size copy, no region-count cap). */ struct drtm_mem_region_hdr { - u16 revision; - u16 reserved; - u32 num_regions; + __le16 revision; + __le16 reserved; + __le32 num_regions; } __packed; struct drtm_mem_region { - u64 start_address; - u64 size_and_type; + __le64 start_address; + __le64 size_and_type; } __packed; /* diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index 42a0ee09e43a2..1878cc986de51 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -314,7 +314,7 @@ static void __init slaunch_assert_dma_protection(phys_addr_t dlme_data_pa, panic("slaunch: SMMU lockdown not full-range (num=%u start=0x%llx st=0x%llx; want 1/0/0x%llx); DRTM secure launch requires full DRAM coverage\n", num, start, st, (u64)DRTM_MEM_PROT_FULL_RANGE); - early_memunmap(phdr, (size_t)prot_size); + early_memunmap((void *)phdr, (size_t)prot_size); pr_info("slaunch: SMMU lockdown verified: full NS-DRAM coverage\n"); } @@ -484,7 +484,7 @@ void __init slaunch_early_init(void) struct dlme_data_header *hdr; phys_addr_t dlme_data_pa; phys_addr_t dtb_pa; - u32 *dtb_hdr; + __be32 *dtb_hdr; u32 fdt_magic, fdt_size; if (!sl_dlme_region_pa) @@ -705,7 +705,7 @@ static void __init slaunch_validate_srtm_log(u64 log_pa, if (dtb_pa) { /* Read fdt_totalsize from the (already-validated) DTB. */ - u32 *p = early_memremap(dtb_pa, sizeof(u32) * 2); + __be32 *p = early_memremap(dtb_pa, sizeof(u32) * 2); if (p) { fdt_size = be32_to_cpu(p[1]); diff --git a/drivers/firmware/efi/libstub/arm64-slaunch.c b/drivers/firmware/efi/libstub/arm64-slaunch.c index bb907929c27fd..a92934173fea3 100644 --- a/drivers/firmware/efi/libstub/arm64-slaunch.c +++ b/drivers/firmware/efi/libstub/arm64-slaunch.c @@ -34,19 +34,19 @@ extern char sl_entry[]; * Struct must be packed — passed directly to TF-A via SMC. */ struct sl_drtm_params { - u16 revision; - u16 reserved; - u32 launch_features; - u64 dlme_region_address; - u64 dlme_region_size; - u64 dlme_image_start; - u64 dlme_entry_point_offset; - u64 dlme_image_size; - u64 dlme_data_offset; - u64 nw_dce_region_address; - u64 nw_dce_region_size; - u64 mem_prot_table_address; - u64 mem_prot_table_size; + __le16 revision; + __le16 reserved; + __le32 launch_features; + __le64 dlme_region_address; + __le64 dlme_region_size; + __le64 dlme_image_start; + __le64 dlme_entry_point_offset; + __le64 dlme_image_size; + __le64 dlme_data_offset; + __le64 nw_dce_region_address; + __le64 nw_dce_region_size; + __le64 mem_prot_table_address; + __le64 mem_prot_table_size; } __packed; static u64 sl_smc_ret(u64 fn, u64 arg1) @@ -270,23 +270,23 @@ void __noreturn efi_slaunch_drtm(unsigned long kernel_addr, SL_DLME_DTB_SLOT_OFFSET) = fdt_addr; /* Build DRTM_PARAMETERS */ - params->revision = DRTM_PARAMS_REVISION; - params->reserved = 0; + params->revision = cpu_to_le16(DRTM_PARAMS_REVISION); + params->reserved = cpu_to_le16(0); /* bits[5:3]=0: complete DMA protection. bit 7: request Secure-interrupt * disable for the launch window (DEN0113 Table 9); the DLME re-enables * post-launch via DRTM_ENABLE_SECURE_INTERRUPTS. */ - params->launch_features = DRTM_LAUNCH_FEAT_MEM_PROT_ALL | DRTM_LAUNCH_FEAT_SEC_INT_DISABLE; - params->dlme_region_address = kernel_addr; - params->dlme_region_size = dlme_data_offset + sl_dlme_data_reserve; - params->dlme_image_start = 0; - params->dlme_entry_point_offset = sl_entry_offset; - params->dlme_image_size = image_size; - params->dlme_data_offset = dlme_data_offset; - params->nw_dce_region_address = 0; - params->nw_dce_region_size = 0; + params->launch_features = cpu_to_le32(DRTM_LAUNCH_FEAT_MEM_PROT_ALL | DRTM_LAUNCH_FEAT_SEC_INT_DISABLE); + params->dlme_region_address = cpu_to_le64(kernel_addr); + params->dlme_region_size = cpu_to_le64(dlme_data_offset + sl_dlme_data_reserve); + params->dlme_image_start = cpu_to_le64(0); + params->dlme_entry_point_offset = cpu_to_le64(sl_entry_offset); + params->dlme_image_size = cpu_to_le64(image_size); + params->dlme_data_offset = cpu_to_le64(dlme_data_offset); + params->nw_dce_region_address = cpu_to_le64(0); + params->nw_dce_region_size = cpu_to_le64(0); /* Complete DMA protection: table must be zero (DEN0113 v1.2 Table 9). */ - params->mem_prot_table_address = 0; - params->mem_prot_table_size = 0; + params->mem_prot_table_address = cpu_to_le64(0); + params->mem_prot_table_size = cpu_to_le64(0); /* * Clean to DRAM what the D-CRTM reads after the SMC: the params From a508e6ab1b78041a7396712d2c3d6cba2bb3b19c Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Sun, 12 Jul 2026 09:14:50 -0700 Subject: [PATCH 300/311] NVIDIA: SAUCE: arm64: drtm: force EFI runtime services off on a DRTM launch BugLink: https://bugs.launchpad.net/bugs/2161563 A DRTM launch must not call unmeasured pre-DRTM UEFI runtime services. Clearing EFI_RUNTIME_SERVICES in slaunch_setup() is not enough: arm_enable_runtime_services() re-enables it unless disable_runtime is set. Add efi_disable_runtime() (init-only) and call it from slaunch_setup() on a detected launch; drop the stub's efi=noruntime requirement so drtm=on alone requests the launch. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/kernel/slaunch.c | 11 ++++++----- drivers/firmware/efi/efi.c | 5 +++++ drivers/firmware/efi/libstub/arm64-slaunch.c | 14 ++++---------- include/linux/efi.h | 2 ++ 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index 1878cc986de51..b8e4123813230 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -1064,14 +1064,15 @@ void __init slaunch_setup(void) slaunch_tpm_setup(); /* - * Unconditionally disable EFI runtime services: their pointers come - * from untrusted pre-DRTM firmware. Defense-in-depth backstop to the - * stub's efi=noruntime gate. Clearing EFI_RUNTIME_SERVICES and - * runtime_supported_mask makes all runtime dispatch see "unsupported". + * A detected DRTM launch forces EFI runtime services off: their + * pointers come from untrusted pre-DRTM firmware. efi_disable_runtime() + * sets disable_runtime so arm_enable_runtime_services() never re-enables + * them; the bit/mask clear is defense in depth. */ + efi_disable_runtime(); clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); efi.runtime_supported_mask = 0; - pr_info("slaunch: EFI runtime services unconditionally disabled\n"); + pr_info("slaunch: EFI runtime services disabled (DRTM launch)\n"); /* * Validate all untrusted EFI inputs before efi_init() consumes them: diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index 533e132e1bd8a..7ce80180802ce 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -96,6 +96,11 @@ bool efi_runtime_disabled(void) return disable_runtime; } +void __init efi_disable_runtime(void) +{ + disable_runtime = true; +} + bool __pure __efi_soft_reserve_enabled(void) { return !efi_enabled(EFI_MEM_NO_SOFT_RESERVE); diff --git a/drivers/firmware/efi/libstub/arm64-slaunch.c b/drivers/firmware/efi/libstub/arm64-slaunch.c index a92934173fea3..fa47fac6a0cef 100644 --- a/drivers/firmware/efi/libstub/arm64-slaunch.c +++ b/drivers/firmware/efi/libstub/arm64-slaunch.c @@ -215,18 +215,12 @@ bool efi_slaunch_enabled(const char *cmdline) { if (!cmdline) return false; - if (!sl_cmdline_token(cmdline, "drtm=on")) - return false; - /* - * drtm=on requires efi=noruntime, else the post-DRTM kernel could - * call unmeasured UEFI runtime services. If missing, skip DRTM. + * A detected DRTM launch forces EFI runtime services off in + * slaunch_setup() (via disable_runtime), so no efi=noruntime token is + * needed here; drtm=on alone requests the launch. */ - if (!sl_cmdline_token(cmdline, "efi=noruntime")) { - efi_warn("DRTM: drtm=on needs efi=noruntime; skipping DRTM\n"); - return false; - } - return true; + return sl_cmdline_token(cmdline, "drtm=on"); } /* diff --git a/include/linux/efi.h b/include/linux/efi.h index 635f7c1d5d66d..88f00b06bbce2 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -1146,8 +1146,10 @@ static inline bool efi_capsule_pending(int *reset_type) { return false; } #ifdef CONFIG_EFI extern bool efi_runtime_disabled(void); +void efi_disable_runtime(void); #else static inline bool efi_runtime_disabled(void) { return true; } +static inline void efi_disable_runtime(void) { } #endif extern void efi_call_virt_check_flags(unsigned long flags, const void *caller); From 4eb49d14dc9d78a16f3b461bb0a8568821b8bd14 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Sun, 12 Jul 2026 09:59:47 -0700 Subject: [PATCH 301/311] NVIDIA: SAUCE: arm64: drtm: rename remaining lockdown references to DMA protection BugLink: https://bugs.launchpad.net/bugs/2161563 slaunch_assert_full_lockdown() was renamed to slaunch_assert_dma_protection(), but "lockdown" and "full-lockdown" still appeared in its comments and panic/info strings, colliding with the kernel lockdown LSM. Rename them to (full-range SMMU) DMA protection for consistency. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/kernel/slaunch.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index b8e4123813230..96e51a1446d6c 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -276,8 +276,8 @@ static bool __init slaunch_parse_address_map(phys_addr_t dlme_data_pa, /* * Verify D-CRTM published full-range DMA protection (DEN0113 v1.2 * §4.6.2). Walks the protected_regions sub-region and requires the - * spec-conformant "single entry, start=0, full-range" encoding; partial - * lockdown breaks the measure-then-parse soundness model. + * spec-conformant "single entry, start=0, full-range" encoding; a partial + * map breaks the measure-then-parse soundness model. */ static void __init slaunch_assert_dma_protection(phys_addr_t dlme_data_pa, u64 hdr_size, u64 prot_size) @@ -289,7 +289,7 @@ static void __init slaunch_assert_dma_protection(phys_addr_t dlme_data_pa, u64 start, st; if (prot_size == 0) - panic("slaunch: DCE published empty protected_regions; cannot verify SMMU lockdown\n"); + panic("slaunch: DCE published empty protected_regions; cannot verify SMMU DMA protection\n"); if (prot_size < sizeof(*phdr) + sizeof(*regs)) panic("slaunch: protected_regions size %llu too small for 1 entry\n", prot_size); @@ -308,14 +308,14 @@ static void __init slaunch_assert_dma_protection(phys_addr_t dlme_data_pa, /* * Strict spec match (DEN0113 v1.2 §3.15 R314110 + §4.6.2): single * entry, start=0, size_and_type = DRTM_MEM_PROT_FULL_RANGE. Anything - * else is partial lockdown or non-conformant — fatal. + * else is partial DMA protection or non-conformant — fatal. */ if (num != 1 || start != 0 || st != DRTM_MEM_PROT_FULL_RANGE) - panic("slaunch: SMMU lockdown not full-range (num=%u start=0x%llx st=0x%llx; want 1/0/0x%llx); DRTM secure launch requires full DRAM coverage\n", + panic("slaunch: SMMU DMA protection not full-range (num=%u start=0x%llx st=0x%llx; want 1/0/0x%llx); DRTM secure launch requires full DRAM coverage\n", num, start, st, (u64)DRTM_MEM_PROT_FULL_RANGE); early_memunmap((void *)phdr, (size_t)prot_size); - pr_info("slaunch: SMMU lockdown verified: full NS-DRAM coverage\n"); + pr_info("slaunch: SMMU DMA protection verified: full NS-DRAM coverage\n"); } /* @@ -1034,7 +1034,7 @@ void __init slaunch_setup(void) dlme_data_pa = sl_dlme_region_pa + sl_dlme_data_offset; hdr = early_memremap(dlme_data_pa, sizeof(*hdr)); /* The header locates the protected-regions table and event log that - * the full-lockdown assertion and locality close below need; fail + * the DMA-protection assertion and locality close below need; fail * closed if it cannot be mapped rather than boot on with them skipped. */ if (!hdr) panic("slaunch: failed to map DLME data header at 0x%llx\n", @@ -1047,7 +1047,7 @@ void __init slaunch_setup(void) pr_info("slaunch: Event log size: %llu\n", le64_to_cpu(hdr->drtm_event_log_size)); - /* Enforce full-lockdown assumption per DEN0113 v1.2 §4.6.2. + /* Enforce full-range DMA-protection assumption per DEN0113 v1.2 §4.6.2. * Called from slaunch_setup (not slaunch_early_init) so panic * prints — earlycon is registered by this point. */ From 46d86f297addbb3b331d62a4ebaa2ac4d4792110 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Sun, 12 Jul 2026 14:52:12 -0700 Subject: [PATCH 302/311] NVIDIA: SAUCE: arm64: drtm: remove unused slaunch_exit() BugLink: https://bugs.launchpad.net/bugs/2161563 slaunch_exit() has no callers. It performed no DRTM teardown: no SMC, no locality change, no DMA-unprotect; it only zeroed sl_dlme_region_pa. The one-shot launch cleanup already lives in slaunch_setup() (close locality) and the slaunch_unprotect_memory() late_initcall, and DEN0113 defines no DLME "exit" operation. Remove the dead function and both declarations. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/include/asm/drtm.h | 2 -- arch/arm64/kernel/slaunch.c | 14 -------------- 2 files changed, 16 deletions(-) diff --git a/arch/arm64/include/asm/drtm.h b/arch/arm64/include/asm/drtm.h index 07654990ead8b..2defb7320b309 100644 --- a/arch/arm64/include/asm/drtm.h +++ b/arch/arm64/include/asm/drtm.h @@ -113,7 +113,6 @@ void slaunch_early_init(void); void slaunch_setup(void); void slaunch_validate_initrd(void); void slaunch_reserve_dlme_data(void); -void slaunch_exit(void); void slaunch_measure_post_efi(void); bool slaunch_phys_is_protected_ram(phys_addr_t pa); bool slaunch_phys_range_overlaps_protected_ram(phys_addr_t pa, size_t size); @@ -122,7 +121,6 @@ static inline void slaunch_early_init(void) { } static inline void slaunch_setup(void) { } static inline void slaunch_validate_initrd(void) { } static inline void slaunch_reserve_dlme_data(void) { } -static inline void slaunch_exit(void) { } static inline void slaunch_measure_post_efi(void) { } static inline bool slaunch_phys_is_protected_ram(phys_addr_t pa) { return false; } static inline bool slaunch_phys_range_overlaps_protected_ram(phys_addr_t pa, size_t size) { return false; } diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c index 96e51a1446d6c..48e139a8fc22e 100644 --- a/arch/arm64/kernel/slaunch.c +++ b/arch/arm64/kernel/slaunch.c @@ -2339,17 +2339,3 @@ static int __init slaunch_unprotect_memory(void) return 0; } late_initcall(slaunch_unprotect_memory); - -/* - * Clean up DRTM state before kexec or reboot. Do not call - * DRTM_SET_ERROR(0): per DEN0113 v1.2 §3.8 its argument is the persisted - * error code, zero is reserved, and no "clear errors" semantics exists. - */ -void slaunch_exit(void) -{ - if (!sl_dlme_region_pa) - return; - - pr_info("slaunch: Cleaning DRTM state before kexec/reboot\n"); - sl_dlme_region_pa = 0; -} From 08765b560f4f3532490017dbdad7f6234e2c06cd Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Sun, 12 Jul 2026 14:52:45 -0700 Subject: [PATCH 303/311] NVIDIA: SAUCE: arm64: drtm: report the DRTM launch decision before ExitBootServices BugLink: https://bugs.launchpad.net/bugs/2161563 The DRTM dynamic launch runs after ExitBootServices, where the EFI console is gone. The old efi_err() on the failure path and the "booting normally" fallback both sat after exit_boot and so could not print. Move both before ExitBootServices: an info breadcrumb ahead of a requested launch (a halt after it means the launch failed), and an error when firmware lacks DRTM support. The breadcrumb is suppressed by "quiet" (and by the default log level on 7.0); the error survives "quiet". The launch SMC's own failure cannot be printed post-EBS without a platform console. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- drivers/firmware/efi/libstub/arm64-slaunch.c | 8 +++++-- drivers/firmware/efi/libstub/fdt.c | 23 ++++++++++++++------ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/drivers/firmware/efi/libstub/arm64-slaunch.c b/drivers/firmware/efi/libstub/arm64-slaunch.c index fa47fac6a0cef..8384f704860a8 100644 --- a/drivers/firmware/efi/libstub/arm64-slaunch.c +++ b/drivers/firmware/efi/libstub/arm64-slaunch.c @@ -302,8 +302,12 @@ void __noreturn efi_slaunch_drtm(unsigned long kernel_addr, */ sl_smc(SL_DRTM_SMC_DYNAMIC_LAUNCH, (u64)params); - /* If we reach here, the SMC failed. Halt. */ - efi_err("DRTM: dynamic launch did not occur\n"); + /* + * The launch SMC returns only on failure (success ERETs to sl_entry). + * Boot services are gone here, so we cannot print; the pre-EBS notice + * in efi_boot_kernel() flagged that reaching this halt means the + * dynamic launch failed. + */ for (;;) asm volatile("wfi"); } diff --git a/drivers/firmware/efi/libstub/fdt.c b/drivers/firmware/efi/libstub/fdt.c index 994b75372ef07..3e79aa64bb830 100644 --- a/drivers/firmware/efi/libstub/fdt.c +++ b/drivers/firmware/efi/libstub/fdt.c @@ -353,6 +353,19 @@ efi_status_t efi_boot_kernel(void *handle, efi_loaded_image_t *image, unsigned long fdt_addr; efi_status_t status; +#ifdef CONFIG_ARM64_SECURE_LAUNCH + /* + * Announce the DRTM decision while boot services can still print: the + * launch runs after ExitBootServices with no console, so a failed + * launch only halts. The info breadcrumb is suppressed by "quiet" (and + * by the default log level on 7.0); the error survives "quiet". + */ + if (efi_slaunch_enabled(cmdline_ptr) && sl_drtm_available) + efi_info("DRTM: launching; a halt after this means it failed\n"); + else if (efi_slaunch_enabled(cmdline_ptr)) + efi_err("DRTM: firmware lacks DRTM support; booting normally\n"); +#endif + status = allocate_new_fdt_and_exit_boot(handle, image, &fdt_addr, cmdline_ptr); if (status != EFI_SUCCESS) { @@ -364,13 +377,9 @@ efi_status_t efi_boot_kernel(void *handle, efi_loaded_image_t *image, efi_handle_post_ebs_state(); #ifdef CONFIG_ARM64_SECURE_LAUNCH - /* Launch only if requested and the D-CRTM advertised support. */ - if (efi_slaunch_enabled(cmdline_ptr)) { - if (sl_drtm_available) - efi_slaunch_drtm(kernel_addr, fdt_addr); - else - efi_warn("DRTM: firmware lacks DRTM support; booting normally\n"); - } + /* Requested + firmware-advertised launch; announced pre-EBS above. */ + if (efi_slaunch_enabled(cmdline_ptr) && sl_drtm_available) + efi_slaunch_drtm(kernel_addr, fdt_addr); #endif efi_enter_kernel(kernel_addr, fdt_addr, fdt_totalsize((void *)fdt_addr)); From 35cf0870ddb7dd7ce146c2f32772a0e07e1f813e Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Sun, 12 Jul 2026 18:23:33 -0700 Subject: [PATCH 304/311] NVIDIA: SAUCE: arm64: drtm: reclaim the EFI stub's writable statics after init BugLink: https://bugs.launchpad.net/bugs/2161563 The EFI stub's writable statics live in the .efistub block placed after _edata (outside the measured image [_text, _edata)), so unlike the stub code in .init.text they are never freed, and their KIMAGE alias stays mapped. Page-align the block, map it as a separately bounded region in the early kernel map (ordinary PTEs at its partial boundary PMDs, no contiguous PTEs), and free + unmap it from free_initmem() like initmem. The reclaim is generic: the .efistub layout exists for all arm64 EFI builds, not only secure launch. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/include/asm/kernel-pgtable.h | 8 +++++++- arch/arm64/include/asm/sections.h | 1 + arch/arm64/kernel/image-vars.h | 10 ++++++++++ arch/arm64/kernel/pi/map_kernel.c | 19 +++++++++++++++++-- arch/arm64/kernel/vmlinux.lds.S | 22 +++++++++++++++------- arch/arm64/mm/init.c | 15 +++++++++++++++ 6 files changed, 65 insertions(+), 10 deletions(-) diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h index 74a4f738c5f52..aba887c561c0a 100644 --- a/arch/arm64/include/asm/kernel-pgtable.h +++ b/arch/arm64/include/asm/kernel-pgtable.h @@ -68,7 +68,13 @@ #define KERNEL_SEGMENT_COUNT 5 #if SWAPPER_BLOCK_SIZE > SEGMENT_ALIGN -#define EARLY_SEGMENT_EXTRA_PAGES (KERNEL_SEGMENT_COUNT + 1) +/* + * The EFI stub's writable statics (.efistub) are mapped as a separate + * segment so they can be unmapped after init; each of its two page-aligned + * boundaries may force one extra page-table page where a swapper-block + * mapping would otherwise be used. + */ +#define EARLY_SEGMENT_EXTRA_PAGES (KERNEL_SEGMENT_COUNT + 1 + 2 * __is_defined(CONFIG_EFI)) /* * The initial ID map consists of the kernel image, mapped as two separate * segments, and may appear misaligned wrt the swapper block size. This means diff --git a/arch/arm64/include/asm/sections.h b/arch/arm64/include/asm/sections.h index 51b0d594239eb..ad402c0ba605e 100644 --- a/arch/arm64/include/asm/sections.h +++ b/arch/arm64/include/asm/sections.h @@ -23,6 +23,7 @@ extern char __irqentry_text_start[], __irqentry_text_end[]; extern char __mmuoff_data_start[], __mmuoff_data_end[]; extern char __entry_tramp_text_start[], __entry_tramp_text_end[]; extern char __relocate_new_kernel_start[], __relocate_new_kernel_end[]; +extern char __efistub_start[], __efistub_end[]; static inline size_t entry_tramp_text_size(void) { diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index cca237b3cd424..a37e1bf22917b 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -46,6 +46,16 @@ PROVIDE(__efistub__ctype = _ctype); PROVIDE(__efistub_sl_entry = sl_entry); #endif +/* + * PI code maps the page-aligned .efistub section as its own segment. + * Plain PROVIDE (not PI_EXPORT_SYM): PI_EXPORT_SYM asserts sym < + * __bss_start, which a boundary symbol equal to __bss_start + * (page-aligned __efistub_end) would fail; these are address bounds, + * not storage. + */ +PROVIDE(__pi___efistub_start = __efistub_start); +PROVIDE(__pi___efistub_end = __efistub_end); + PROVIDE(__pi___memcpy = __pi_memcpy); PROVIDE(__pi___memmove = __pi_memmove); PROVIDE(__pi___memset = __pi_memset); diff --git a/arch/arm64/kernel/pi/map_kernel.c b/arch/arm64/kernel/pi/map_kernel.c index a852264958c36..96aaf51714cd8 100644 --- a/arch/arm64/kernel/pi/map_kernel.c +++ b/arch/arm64/kernel/pi/map_kernel.c @@ -43,6 +43,10 @@ static void __init map_kernel(u64 kaslr_offset, u64 va_offset, int root_level) phys_addr_t pgdp = (phys_addr_t)init_pg_dir + PAGE_SIZE; pgprot_t text_prot = PAGE_KERNEL_ROX; pgprot_t data_prot = PAGE_KERNEL; + void *efistub_start = IS_ENABLED(CONFIG_EFI) ? (void *)__efistub_start + : (void *)_end; + void *efistub_end = IS_ENABLED(CONFIG_EFI) ? (void *)__efistub_end + : (void *)_end; pgprot_t prot; /* @@ -92,8 +96,19 @@ static void __init map_kernel(u64 kaslr_offset, u64 va_offset, int root_level) __inittext_end, prot, false, root_level); map_segment(init_pg_dir, &pgdp, va_offset, __initdata_begin, __initdata_end, data_prot, false, root_level); - map_segment(init_pg_dir, &pgdp, va_offset, _data, _end, data_prot, - true, root_level); + /* + * Map the EFI stub's writable statics (the page-aligned .efistub + * section) as their own segment, without contiguous PTEs, so that no + * block or contiguous mapping straddles a reclaim boundary and + * free_initmem() can unmap [__efistub_start, __efistub_end) at page + * granularity. + */ + map_segment(init_pg_dir, &pgdp, va_offset, _data, efistub_start, + data_prot, true, root_level); + map_segment(init_pg_dir, &pgdp, va_offset, efistub_start, efistub_end, + data_prot, false, root_level); + map_segment(init_pg_dir, &pgdp, va_offset, efistub_end, _end, + data_prot, true, root_level); dsb(ishst); idmap_cpu_replace_ttbr1((phys_addr_t)init_pg_dir); diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 10bd50125a0cd..1a7a0008de790 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -349,21 +349,20 @@ SECTIONS * so the DRTM measurement bounds [_text, _edata) never cover * stub-mutable data; __pecoff_data_rawsize is computed past * __efistub_end. Stub text/rodata stay in PE .text (buckets above). + * The section is padded to a page so [__efistub_start, __efistub_end) + * owns its tail page exclusively and free_initmem() can free and + * unmap it on any page size; PAGE_SIZE is a multiple of the PE + * FileAlignment (0x200), so no separate PE padding is needed, and + * the zero fill is PROGBITS so `objcopy -O binary` writes the bytes. */ . = ALIGN(PAGE_SIZE); __efistub_start = .; .efistub : ALIGN(PAGE_SIZE) { *(.efistub.data .efistub.data.*) *(.efistub.bss .efistub.bss.*) + . = ALIGN(PAGE_SIZE); } __efistub_end = .; - - /* - * Materialize the trailing PE/COFF FileAlignment padding as a real - * PROGBITS section so `objcopy -O binary` writes the bytes; else the - * PE rawsize claims more than the file holds and the loader rejects it. - */ - .efistub_pad : { BYTE(0); . = ALIGN(PECOFF_FILE_ALIGNMENT); } __pecoff_data_rawsize = ABSOLUTE(. - __initdata_begin); /* start of zero-init region */ @@ -446,3 +445,12 @@ ASSERT(KEXEC_CONTROL_PAGE_SIZE >= SZ_4K, "KEXEC_CONTROL_PAGE_SIZE is broken") ASSERT(__relocate_new_kernel_start == arm64_relocate_new_kernel, "kexec control page does not start with arm64_relocate_new_kernel") #endif + +/* + * The EFI stub's writable statics are freed and unmapped after init; both + * boundaries must be page-aligned so the reclaim covers whole pages only. + */ +ASSERT((__efistub_start & (PAGE_SIZE - 1)) == 0, + "__efistub_start is not page aligned") +ASSERT((__efistub_end & (PAGE_SIZE - 1)) == 0, + "__efistub_end is not page aligned") diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index b9b248d24fd10..531fbf5408579 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -403,6 +403,21 @@ void free_initmem(void) * is not supported by kallsyms. */ vunmap_range((u64)__init_begin, (u64)__init_end); + + if (__efistub_start != __efistub_end) { + void *lm_begin = lm_alias(__efistub_start); + void *lm_end = lm_alias(__efistub_end); + + /* + * EFI-stub writable statics: dead after boot; freed and + * unmapped like initmem (the .efistub layout exists for all + * arm64 EFI builds). + */ + memblock_free(lm_begin, lm_end - lm_begin); + free_reserved_area(lm_begin, lm_end, + POISON_FREE_INITMEM, "unused efistub"); + vunmap_range((u64)__efistub_start, (u64)__efistub_end); + } } void dump_mem_limit(void) From 62869e0827f76128b0e1747c8c476eddb0621bd3 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Mon, 13 Jul 2026 04:56:41 -0700 Subject: [PATCH 305/311] NVIDIA: SAUCE: arm64: drtm: probe DRTM_FEATURES only when a launch is requested BugLink: https://bugs.launchpad.net/bugs/2161563 The EFI stub issued the DRTM_FEATURES SMC on every boot to size the DLME data reservation, so a secure-launch-enabled kernel could fault in the stub on platforms without an EL3 monitor. Record the stub's canonical converted command line (a second efi_convert_cmdline() would measure the EFI LoadOptions twice) and gate the probe on the same drtm=on token that gates the launch; grow the reservation only after a successful probe. Unrequested boots issue no SMC and reserve nothing. An explicit drtm=on still requires an EL3 DRTM monitor (noted in the Kconfig help). Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/Kconfig | 4 +++- drivers/firmware/efi/libstub/arm64-slaunch.c | 23 +++++++++++++++++++ drivers/firmware/efi/libstub/arm64-stub.c | 12 +++++++--- drivers/firmware/efi/libstub/efi-stub-entry.c | 5 ++++ drivers/firmware/efi/libstub/efistub.h | 2 ++ 5 files changed, 42 insertions(+), 4 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 9526bb4516c38..81970a0bed85f 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2477,7 +2477,9 @@ config ARM64_SECURE_LAUNCH The EFI stub triggers DRTM_DYNAMIC_LAUNCH after ExitBootServices when "drtm=on" is on the kernel command line. D-CRTM measures the kernel and returns control. The kernel then parses DLME data and - manages DRTM SMC calls during boot. + manages DRTM SMC calls during boot. Booting with drtm=on requires + firmware with an EL3 DRTM monitor; without drtm=on the kernel + boots normally and issues no DRTM calls. If unsure, say N. diff --git a/drivers/firmware/efi/libstub/arm64-slaunch.c b/drivers/firmware/efi/libstub/arm64-slaunch.c index 8384f704860a8..5595fa9100d23 100644 --- a/drivers/firmware/efi/libstub/arm64-slaunch.c +++ b/drivers/firmware/efi/libstub/arm64-slaunch.c @@ -223,6 +223,29 @@ bool efi_slaunch_enabled(const char *cmdline) return sl_cmdline_token(cmdline, "drtm=on"); } +/* Canonical converted command line, recorded once by the stub entry. */ +static const char *sl_cmdline; + +/* + * Record the stub's canonical converted command line for the DRTM gates. + * Reusing it avoids a second efi_convert_cmdline(), which would measure + * the EFI LoadOptions a second time (duplicate PCR 9 event). + */ +void efi_slaunch_set_cmdline(const char *cmdline) +{ + sl_cmdline = cmdline; +} + +/* + * True iff the recorded command line requests a DRTM launch. Gates the + * DRTM_FEATURES probe: an SMC faults on a platform with no EL3 monitor, + * so it must not be issued on boots that never asked for a launch. + */ +bool efi_slaunch_requested(void) +{ + return efi_slaunch_enabled(sl_cmdline); +} + /* * TF-A requires DRTM_PARAMETERS to be 4KB-aligned; we are past * ExitBootServices so cannot allocate — use a static buffer. diff --git a/drivers/firmware/efi/libstub/arm64-stub.c b/drivers/firmware/efi/libstub/arm64-stub.c index 1e330217298b8..72906e86cd6aa 100644 --- a/drivers/firmware/efi/libstub/arm64-stub.c +++ b/drivers/firmware/efi/libstub/arm64-stub.c @@ -40,10 +40,16 @@ efi_status_t handle_kernel_image(unsigned long *image_addr, #ifdef CONFIG_ARM64_SECURE_LAUNCH /* * Reserve DLME data space (size from DRTM_FEATURES) after kernel - * BSS for D-CRTM to populate (address map, event log, etc.). + * BSS for D-CRTM to populate (address map, event log, etc.). Probe + * and reserve only when the command line requests a launch: the + * DRTM_FEATURES SMC would fault on a platform with no EL3 monitor. */ - efi_slaunch_get_dlme_data_size(); - *reserve_size += SL_DLME_DTB_SLOT_GAP + sl_dlme_data_reserve; + if (efi_slaunch_requested()) { + efi_slaunch_get_dlme_data_size(); + if (sl_drtm_available) + *reserve_size += SL_DLME_DTB_SLOT_GAP + + sl_dlme_data_reserve; + } #endif *image_addr = (unsigned long)_text; diff --git a/drivers/firmware/efi/libstub/efi-stub-entry.c b/drivers/firmware/efi/libstub/efi-stub-entry.c index aa85e910fe595..fa3ec3fcc1f52 100644 --- a/drivers/firmware/efi/libstub/efi-stub-entry.c +++ b/drivers/firmware/efi/libstub/efi-stub-entry.c @@ -67,6 +67,11 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, if (status != EFI_SUCCESS) return status; +#ifdef CONFIG_ARM64_SECURE_LAUNCH + /* Record the canonical command line for the DRTM launch gates. */ + efi_slaunch_set_cmdline(cmdline_ptr); +#endif + efi_info("Booting Linux Kernel...\n"); status = handle_kernel_image(&image_addr, &image_size, diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h index c88dd17f24c31..7e9424645cd4f 100644 --- a/drivers/firmware/efi/libstub/efistub.h +++ b/drivers/firmware/efi/libstub/efistub.h @@ -1269,6 +1269,8 @@ efi_status_t efi_zboot_decompress(u8 *out, unsigned long outlen); #ifdef CONFIG_ARM64_SECURE_LAUNCH bool efi_slaunch_enabled(const char *cmdline); +void efi_slaunch_set_cmdline(const char *cmdline); +bool efi_slaunch_requested(void); void efi_slaunch_get_dlme_data_size(void); void efi_slaunch_scrub_imagebase(unsigned long kernel_addr); extern unsigned long sl_dlme_data_reserve; From 281d568dd1b2a4770c6fb68d40b1da059bdd9624 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Mon, 13 Jul 2026 05:14:07 -0700 Subject: [PATCH 306/311] NVIDIA: SAUCE: arm64: drtm: relocate whenever the reservation exceeds the image BugLink: https://bugs.launchpad.net/bugs/2161563 When randomization is unavailable or fails, efi_kaslr_relocate_kernel() could execute the image in place while a DRTM boot's reservation extends beyond kernel_memsize (the DTB slot and DLME data tail). That tail lies beyond PE SizeOfImage, and LoadedImage does not expose the backing allocation's extent, so its coverage cannot be assumed (an EFI memory descriptor can also span coalesced neighboring allocations). Take the in-place path only when nothing beyond the image is reserved; otherwise perform a full relocation, which allocates the whole extent. No change when reserve_size == kernel_memsize. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- drivers/firmware/efi/libstub/kaslr.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/firmware/efi/libstub/kaslr.c b/drivers/firmware/efi/libstub/kaslr.c index 4bc963e999eb9..579e237d47cd5 100644 --- a/drivers/firmware/efi/libstub/kaslr.c +++ b/drivers/firmware/efi/libstub/kaslr.c @@ -128,10 +128,15 @@ efi_status_t efi_kaslr_relocate_kernel(unsigned long *image_addr, if (!check_image_region(*image_addr, kernel_memsize)) { efi_err("FIRMWARE BUG: Image BSS overlaps adjacent EFI memory region\n"); } else if (IS_ALIGNED(*image_addr, min_kimg_align) && - (unsigned long)_end < EFI_ALLOC_LIMIT) { + (unsigned long)_end < EFI_ALLOC_LIMIT && + *reserve_size == kernel_memsize) { /* * Just execute from wherever we were loaded by the - * UEFI PE/COFF loader if the placement is suitable. + * UEFI PE/COFF loader if the placement is suitable + * and nothing beyond the image is reserved: a tail + * (e.g. DRTM DLME data) lies beyond PE SizeOfImage, + * and LoadedImage does not expose the backing + * allocation extent, so relocate to allocate it. */ *reserve_size = 0; return EFI_SUCCESS; From ca2f8d5b468ca6ece4000b913d187c06fa141dd2 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Wed, 15 Jul 2026 11:29:18 -0700 Subject: [PATCH 307/311] NVIDIA: SAUCE: arm64: drtm: randomize the linear map from a trusted RNG BugLink: https://bugs.launchpad.net/bugs/2161563 Under a DRTM Secure Launch the linear/direct map is not randomized (upstream removes it where the CPU PArange < 256 TiB, and the seed would otherwise come from the attacker-visible FDT). Re-randomize memstart_addr over the slack between the linear region and the present-DRAM span, gated by CONFIG_ARM64_SECURE_LAUNCH_KASLR (which gives up memory hotplug). Seed from the PSC-backed firmware TRNG via SMCCC TRNG_RND64 (DEN0098), not CPU RNDR: RNDR is not exposed to NS on all DRTM platforms (NVIDIA Grace reads ID_AA64ISAR0_EL1.RNDR = 0), but EL3 is always present after a Secure Launch. Fail closed if no trusted entropy is available. Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/Kconfig | 18 ++++++++++++++++ arch/arm64/include/asm/drtm.h | 4 ++++ arch/arm64/mm/init.c | 39 +++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 81970a0bed85f..e6e2b37f8988d 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2521,6 +2521,24 @@ config ARM64_SECURE_LAUNCH_FAULT_INJECT If unsure, say N. +config ARM64_SECURE_LAUNCH_KASLR + bool "ARM64 DRTM trusted linear-map KASLR" + depends on ARM64_SECURE_LAUNCH && RANDOMIZE_BASE + depends on !MEMORY_HOTPLUG + default n + help + After a DRTM Secure Launch, re-randomize the kernel linear + (direct) map from the CPU's FEAT_RNG instead of leaving it at a + fixed offset. The randomization slack is measured against present + DRAM only, with no reservation for memory hotplug, so it works + even when the CPU PArange equals the virtual address size. + + Fail-closed: if the CPU lacks FEAT_RNG (RNDR) the kernel panics + rather than booting with a predictable linear map. Requires memory + hotplug to be disabled. + + If unsure, say N. + config COMPRESSED_INSTALL bool "Install compressed image by default" help diff --git a/arch/arm64/include/asm/drtm.h b/arch/arm64/include/asm/drtm.h index 2defb7320b309..8aeab02e24da2 100644 --- a/arch/arm64/include/asm/drtm.h +++ b/arch/arm64/include/asm/drtm.h @@ -116,6 +116,9 @@ void slaunch_reserve_dlme_data(void); void slaunch_measure_post_efi(void); bool slaunch_phys_is_protected_ram(phys_addr_t pa); bool slaunch_phys_range_overlaps_protected_ram(phys_addr_t pa, size_t size); + +/* True iff a DRTM Secure Launch occurred (stub-planted DLME handoff). */ +static inline bool slaunch_active(void) { return sl_dlme_region_pa != 0; } #else static inline void slaunch_early_init(void) { } static inline void slaunch_setup(void) { } @@ -124,6 +127,7 @@ static inline void slaunch_reserve_dlme_data(void) { } static inline void slaunch_measure_post_efi(void) { } static inline bool slaunch_phys_is_protected_ram(phys_addr_t pa) { return false; } static inline bool slaunch_phys_range_overlaps_protected_ram(phys_addr_t pa, size_t size) { return false; } +static inline bool slaunch_active(void) { return false; } #endif #endif /* __ASSEMBLY__ */ diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 531fbf5408579..c256876b77da4 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -40,6 +40,8 @@ #include #include #include +#include +#include #include #include #include @@ -234,6 +236,43 @@ void __init arm64_memblock_init(void) memblock_remove(0, memstart_addr); } + /* + * DRTM: re-randomize the linear map from a trusted RNG. + * Coverage is bounded to present DRAM (memblock span) -- no hotplug + * reservation -- so the slack is (linear_region - span). The seed + * comes from the PSC-backed firmware TRNG, never the attacker-visible + * FDT seed; fail closed if no trusted entropy is available. + */ + if (IS_ENABLED(CONFIG_ARM64_SECURE_LAUNCH_KASLR) && slaunch_active()) { + s64 range = linear_region_size - + (memblock_end_of_DRAM() - memblock_start_of_DRAM()); + struct arm_smccc_res res; + unsigned long seed; + u64 nslots; + int tries; + + if (range < (s64)ARM64_MEMSTART_ALIGN) + panic("DRTM KASLR: linear region too small to randomize\n"); + /* + * Seed from the PSC-backed firmware TRNG (SMCCC TRNG_RND64), + * not CPU RNDR: RNDR is not exposed to NS on all DRTM + * platforms, but EL3 is (a Secure Launch arrives via SMC). + * Call the SMC directly -- the conduit predates the generic + * SMCCC probe. Retry a transiently empty pool; fail closed. + */ + for (tries = 0; tries < 10; tries++) { + arm_smccc_smc(ARM_SMCCC_TRNG_RND64, 64, 0, 0, 0, 0, 0, 0, &res); + if ((int)res.a0 == SMCCC_RET_SUCCESS) + break; + } + if ((int)res.a0 != SMCCC_RET_SUCCESS) + panic("DRTM KASLR: firmware TRNG unavailable, refusing weak KASLR\n"); + seed = res.a3; /* 64 bits requested -> entropy in X3 */ + + nslots = (u64)range / ARM64_MEMSTART_ALIGN; + memstart_addr -= ARM64_MEMSTART_ALIGN * (seed % nslots); + } + /* * If we are running with a 52-bit kernel VA config on a system that * does not support it, we have to place the available physical From 4b9a7ac96bbfd46a1b55bedd2509347573f94b36 Mon Sep 17 00:00:00 2001 From: Raghupathy Krishnamurthy Date: Wed, 15 Jul 2026 11:29:18 -0700 Subject: [PATCH 308/311] NVIDIA: SAUCE: arm64: drtm: seed kernel-image KASLR from a trusted RNG BugLink: https://bugs.launchpad.net/bugs/2161563 The kernel image KASLR offset is seeded from /chosen/kaslr-seed, which is attacker-visible under a DRTM launch. Prefer the PSC-backed firmware TRNG (SMCCC TRNG_RND64) for the image offset, falling back to the FDT seed. kaslr_early_init() runs in position-independent early code that cannot call arm_smccc_smc() or panic, so issue the SMC via inline asm and gate it on ID_AA64PFR0_EL1.EL3 != 0 (else smc would UNDEF on a no-EL3 boot). The fail-closed enforcement for a missing trusted RNG lives in the linear-map path (arm64_memblock_init). Signed-off-by: Raghupathy Krishnamurthy Signed-off-by: Ian May --- arch/arm64/kernel/pi/kaslr_early.c | 48 ++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/arch/arm64/kernel/pi/kaslr_early.c b/arch/arm64/kernel/pi/kaslr_early.c index e0e018046a46a..d252df54abf78 100644 --- a/arch/arm64/kernel/pi/kaslr_early.c +++ b/arch/arm64/kernel/pi/kaslr_early.c @@ -37,6 +37,33 @@ static u64 __init get_kaslr_seed(void *fdt, int node) return ret; } +/* SMCCC TRNG_RND64 (DEN0098) == ARM_SMCCC_TRNG_RND64; hardcoded to avoid the + * heavy arm-smccc.h include in this position-independent early code. */ +#define PI_SMCCC_TRNG_RND64 0xC4000053UL + +static bool __init __early_el3_present(void) +{ + /* ID_AA64PFR0_EL1.EL3 [15:12] != 0 => EL3 implemented => smc is legal */ + return ((read_sysreg_s(SYS_ID_AA64PFR0_EL1) >> 12) & 0xf) != 0; +} + +static bool __init __early_trng_rnd64(u64 *out) +{ + register u64 x0 asm("x0") = PI_SMCCC_TRNG_RND64; + register u64 x1 asm("x1") = 64; /* request 64 bits */ + register u64 x2 asm("x2") = 0; + register u64 x3 asm("x3") = 0; + + asm volatile("smc #0" + : "+r"(x0), "+r"(x1), "+r"(x2), "+r"(x3) + : : "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", + "x12", "x13", "x14", "x15", "x16", "x17", "memory"); + if ((long)x0 != 0) /* SMCCC_RET_SUCCESS == 0 */ + return false; + *out = x3; /* 64 bits requested -> entropy in X3 */ + return true; +} + u64 __init kaslr_early_init(void *fdt, int chosen) { u64 seed, range; @@ -45,6 +72,27 @@ u64 __init kaslr_early_init(void *fdt, int chosen) return 0; seed = get_kaslr_seed(fdt, chosen); + + /* + * DRTM: the FDT kaslr-seed is attacker-visible, so prefer the PSC- + * backed firmware TRNG (SMCCC TRNG_RND64) for the kernel image offset. + * CPU RNDR is not exposed to NS on all DRTM platforms; the SMC is. + * Call it only when EL3 is implemented (else smc is UNDEF) and fall + * back to the FDT seed on failure -- PI code cannot panic; fail-closed + * enforcement lives in arm64_memblock_init(). + */ + if (IS_ENABLED(CONFIG_ARM64_SECURE_LAUNCH_KASLR) && + __early_el3_present()) { + u64 hw; + int tries; + + for (tries = 0; tries < 10; tries++) + if (__early_trng_rnd64(&hw)) { + seed = hw; + break; + } + } + if (!seed) { if (!__early_cpu_has_rndr() || !__arm64_rndr((unsigned long *)&seed)) From 34799827483e8f0f9b161ab24d652db7f645fb1a Mon Sep 17 00:00:00 2001 From: Mohamed Sunfeer Date: Sun, 7 Jun 2026 12:29:57 -0700 Subject: [PATCH 309/311] NVIDIA: SAUCE: arm64: Add NVIDIA FIRME attestation driver for DRTM baremetal attestation BugLink: https://bugs.launchpad.net/bugs/2161563 Add nvidia-firme kernel module that provides TSM report and TSM measurement register backends for ARM FIRME (DEN0149) Platform Attestation Token retrieval on NVIDIA Grace (TH500) platforms. TSM Reports (configfs): - Invokes FIRME_ATTEST_PAT_GET SMC (0xC4000408) to retrieve platform attestation tokens from PSC via ATF. - Exposes tokens via /sys/kernel/config/tsm/report/ using the standard configfs-tsm UABI (same as ARM CCA, Intel TDX, AMD SEV). TSM Measurement Registers (sysfs): - Invokes FIRME_ATTEST_EXT_CLAIMS SMC (0xC400040B) to extend measurement registers and submit BMDR device reports to PSC. - Slot 0 (bmdr): 100-byte per-GPU device report containing identity_digest (48B), mexchange_digest (48B), gpu_device_id (1B), and reserved (3B). - Slots 1-3 (rem0-rem2): 48-byte SHA-384 extensible measurement slots. - Exposed via /sys/firmware/nvidia-firme/measurements/. Defconfig changes: - Enable CONFIG_ARM64_SECURE_LAUNCH=y for DRTM Secure Launch. - Enable CONFIG_NVIDIA_FIRME_GUEST=m for the attestation driver. - Set CONFIG_CMDLINE="drtm=on efi=noruntime nokaslr" with CONFIG_CMDLINE_EXTEND=y for default DRTM boot parameters. Tested end-to-end on GB300 hardware: DRTM launch, token retrieval (~9KB CCA token with platform + tenant + device tokens), per-GPU BMDR device reports, and REM slot extension. Signed-off-by: Mohamed Sunfeer Hyderali Signed-off-by: Ian May --- arch/arm64/configs/defconfig | 4 + drivers/virt/coco/Kconfig | 2 + drivers/virt/coco/Makefile | 1 + drivers/virt/coco/nvidia-firme/Kconfig | 16 + drivers/virt/coco/nvidia-firme/Makefile | 2 + drivers/virt/coco/nvidia-firme/nvidia-firme.c | 387 ++++++++++++++++++ 6 files changed, 412 insertions(+) create mode 100644 drivers/virt/coco/nvidia-firme/Kconfig create mode 100644 drivers/virt/coco/nvidia-firme/Makefile create mode 100644 drivers/virt/coco/nvidia-firme/nvidia-firme.c diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 269e811132f9b..31b6f5d0be67e 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1956,6 +1956,10 @@ CONFIG_CORESIGHT_SINK_ETBV10=m CONFIG_CORESIGHT_STM=m CONFIG_CORESIGHT_CPU_DEBUG=m CONFIG_CORESIGHT_CTI=m +CONFIG_ARM64_SECURE_LAUNCH=y +CONFIG_CMDLINE="drtm=on efi=noruntime nokaslr" +CONFIG_CMDLINE_EXTEND=y +CONFIG_NVIDIA_FIRME_GUEST=m CONFIG_MEMTEST=y CONFIG_NVGRACE_GPU_VFIO_PCI=m CONFIG_NVGRACE_EGM=m diff --git a/drivers/virt/coco/Kconfig b/drivers/virt/coco/Kconfig index df1cfaf26c658..ae7d390831d9d 100644 --- a/drivers/virt/coco/Kconfig +++ b/drivers/virt/coco/Kconfig @@ -14,6 +14,8 @@ source "drivers/virt/coco/tdx-guest/Kconfig" source "drivers/virt/coco/arm-cca-guest/Kconfig" +source "drivers/virt/coco/nvidia-firme/Kconfig" + source "drivers/virt/coco/guest/Kconfig" endif diff --git a/drivers/virt/coco/Makefile b/drivers/virt/coco/Makefile index cb52021912b34..16983136fbae0 100644 --- a/drivers/virt/coco/Makefile +++ b/drivers/virt/coco/Makefile @@ -7,5 +7,6 @@ obj-$(CONFIG_ARM_PKVM_GUEST) += pkvm-guest/ obj-$(CONFIG_SEV_GUEST) += sev-guest/ obj-$(CONFIG_INTEL_TDX_GUEST) += tdx-guest/ obj-$(CONFIG_ARM_CCA_GUEST) += arm-cca-guest/ +obj-$(CONFIG_NVIDIA_FIRME_GUEST) += nvidia-firme/ obj-$(CONFIG_TSM) += tsm-core.o obj-$(CONFIG_TSM_GUEST) += guest/ diff --git a/drivers/virt/coco/nvidia-firme/Kconfig b/drivers/virt/coco/nvidia-firme/Kconfig new file mode 100644 index 0000000000000..04aad6e4882af --- /dev/null +++ b/drivers/virt/coco/nvidia-firme/Kconfig @@ -0,0 +1,16 @@ +config NVIDIA_FIRME_GUEST + tristate "NVIDIA FIRME attestation driver" + depends on ARM64 + default m + select TSM_REPORTS + select TSM_MEASUREMENTS + help + TSM report and measurement register backend for ARM FIRME (DEN0149) + on NVIDIA Grace (TH500) platforms. After a DRTM Secure Launch: + + - FIRME_ATTEST_PAT_GET retrieves attestation tokens from PSC, + exposed via configfs at /sys/kernel/config/tsm/report/. + - FIRME_ATTEST_EXTEND extends measurement registers in PSC, + exposed via sysfs for GPU/device evidence binding. + + If you choose 'M' here, this module will be called nvidia-firme. diff --git a/drivers/virt/coco/nvidia-firme/Makefile b/drivers/virt/coco/nvidia-firme/Makefile new file mode 100644 index 0000000000000..235b156c4f142 --- /dev/null +++ b/drivers/virt/coco/nvidia-firme/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_NVIDIA_FIRME_GUEST) += nvidia-firme.o diff --git a/drivers/virt/coco/nvidia-firme/nvidia-firme.c b/drivers/virt/coco/nvidia-firme/nvidia-firme.c new file mode 100644 index 0000000000000..a543541580550 --- /dev/null +++ b/drivers/virt/coco/nvidia-firme/nvidia-firme.c @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * NVIDIA FIRME Attestation Driver + * + * TSM report backend and TSM measurement register backend for ARM FIRME + * (DEN0149) on NVIDIA Grace (TH500) platforms. + * + * - TSM Reports: invokes FIRME_ATTEST_PAT_GET (0xC4000408) to retrieve + * platform attestation tokens from EL3/PSC via configfs. + * - TSM MR: invokes FIRME_ATTEST_EXT_CLAIMS (0xC400040B) to extend + * measurement registers and submit BMDR device reports to PSC + * via sysfs. + * + * Copyright (c) 2025-2026, NVIDIA Corporation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* FIRME SMC Function IDs — DEN0149, SMC64, Fast call, OEN=4 (Std Svc) */ +#define FIRME_FID(fn) (0xC4000000UL | (fn)) +#define FIRME_SERVICE_VERSION FIRME_FID(0x0400) +#define FIRME_ATTEST_PAT_GET FIRME_FID(0x0408) +#define FIRME_ATTEST_EXTEND FIRME_FID(0x040B) + +/* FIRME return codes (DEN0149 Table 8.24) */ +#define FIRME_SUCCESS 0L +#define FIRME_NOT_SUPPORTED (-1L) +#define FIRME_INVALID_PARAMETERS (-2L) +#define FIRME_ABORTED (-3L) +#define FIRME_INCOMPLETE (-4L) +#define FIRME_DENIED (-5L) +#define FIRME_RETRY (-6L) + +/* SHA-384 digest size */ +#define SHA384_DIGEST_SIZE 48 + +/* + * BMDR (Baremetal Device Report) entry size for slot 0. + * Layout: [48 identity_digest][48 mexchange_digest][1 gpu_id][3 reserved] + */ +#define FIRME_BMDR_SIZE 100 + +/* + * Shared buffer sizing. ATF computes: size = (page_count + 1) * 4KB. + * page_count=3 → 16KB buffer, sufficient for most tokens. + */ +#define FIRME_BUF_PAGE_COUNT 3 +#define FIRME_MIN_SHARED_BUF_SZ SZ_4K +#define FIRME_BUF_SIZE ((FIRME_BUF_PAGE_COUNT + 1) * \ + FIRME_MIN_SHARED_BUF_SZ) +#define FIRME_MAX_TOKEN_SIZE SZ_32K +#define FIRME_MAX_RETRIES 5 + +static bool nvidia_firme_available(void) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_invoke(FIRME_SERVICE_VERSION, 0, 0, 0, 0, &res); + if ((long)res.a0 == FIRME_NOT_SUPPORTED) + return false; + + pr_info("nvidia-firme: FIRME service version %lu.%lu\n", + res.a0 >> 16, res.a0 & 0xffff); + return true; +} + +/** + * nvidia_firme_report_new - Generate an attestation report via FIRME SMC. + * @report: TSM report structure; inblob is used as the challenge nonce. + * @data: unused private data. + * + * Allocates a physically contiguous NS buffer, places the optional + * challenge at offset 0, then calls FIRME_ATTEST_PAT_GET in a loop + * to retrieve the full platform attestation token. + * + * Return: 0 on success, negative errno on failure. + */ +static int nvidia_firme_report_new(struct tsm_report *report, void *data) +{ + struct tsm_report_desc *desc = &report->desc; + struct arm_smccc_res res; + void *buf; + phys_addr_t buf_phys; + u8 *token __free(kvfree) = NULL; + u64 offset = 0; + u64 challenge_sz = 0; + size_t token_size = 0; + int retries; + + buf = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, + get_order(FIRME_BUF_SIZE)); + if (!buf) + return -ENOMEM; + + buf_phys = virt_to_phys(buf); + + /* Place challenge/nonce at offset 0 of the shared buffer */ + if (desc->inblob_len > 0) { + if (desc->inblob_len > FIRME_BUF_SIZE) { + free_pages((unsigned long)buf, get_order(FIRME_BUF_SIZE)); + return -EINVAL; + } + memcpy(buf, desc->inblob, desc->inblob_len); + challenge_sz = desc->inblob_len; + } + + token = kvzalloc(FIRME_MAX_TOKEN_SIZE, GFP_KERNEL); + if (!token) { + free_pages((unsigned long)buf, get_order(FIRME_BUF_SIZE)); + return -ENOMEM; + } + + /* Chunked retrieval loop per DEN0149 Section 8.12 */ + do { + retries = 0; +retry: + pr_info("nvidia-firme: FIRME_ATTEST_PAT_GET SMC: buf=0x%llx offset=%llu page_count=%d challenge_sz=%llu\n", + (u64)buf_phys, offset, FIRME_BUF_PAGE_COUNT, + challenge_sz); + + arm_smccc_1_1_invoke(FIRME_ATTEST_PAT_GET, + buf_phys, + offset, + FIRME_BUF_PAGE_COUNT, + challenge_sz, + &res); + + pr_info("nvidia-firme: SMC returned: status=%ld written=%lu remaining=%lu\n", + (long)res.a0, res.a1, res.a2); + + if ((long)res.a0 == FIRME_RETRY) { + pr_warn("nvidia-firme: ATF returned RETRY (%d/%d)\n", + retries + 1, FIRME_MAX_RETRIES); + if (++retries > FIRME_MAX_RETRIES) { + pr_err("nvidia-firme: too many RETRYs\n"); + free_pages((unsigned long)buf, + get_order(FIRME_BUF_SIZE)); + return -EAGAIN; + } + cond_resched(); + goto retry; + } + + if ((long)res.a0 == FIRME_ABORTED) { + pr_err("nvidia-firme: ATF returned ABORTED (-3). PSC token not available (PSC task may not be running)\n"); + free_pages((unsigned long)buf, get_order(FIRME_BUF_SIZE)); + return -ENODATA; + } + + if ((long)res.a0 != FIRME_SUCCESS && + (long)res.a0 != FIRME_INCOMPLETE) { + pr_err("nvidia-firme: SMC failed, status=%ld (NOT_SUPPORTED=%ld, INVALID_PARAMS=%ld, DENIED=%ld)\n", + (long)res.a0, FIRME_NOT_SUPPORTED, + FIRME_INVALID_PARAMETERS, (long)-5); + free_pages((unsigned long)buf, get_order(FIRME_BUF_SIZE)); + return -EIO; + } + + /* res.a1 = bytes written this call, starting at offset in buf */ + if (token_size + res.a1 > FIRME_MAX_TOKEN_SIZE) { + pr_err("nvidia-firme: token exceeds max size\n"); + free_pages((unsigned long)buf, get_order(FIRME_BUF_SIZE)); + return -ENOSPC; + } + + memcpy(&token[token_size], buf + offset, res.a1); + token_size += res.a1; + offset += res.a1; + + /* Clear challenge after first call */ + challenge_sz = 0; + } while (res.a2 > 0); /* res.a2 = remaining bytes */ + + free_pages((unsigned long)buf, get_order(FIRME_BUF_SIZE)); + + report->outblob = no_free_ptr(token); + report->outblob_len = token_size; + + pr_info("nvidia-firme: attestation token retrieved, %zu bytes\n", + token_size); + return 0; +} + +static const struct tsm_report_ops nvidia_firme_tsm_ops = { + .name = KBUILD_MODNAME, + .report_new = nvidia_firme_report_new, +}; + +/* ================================================================ + * TSM Measurement Registers (MR) — FIRME_ATTEST_EXT_CLAIMS backend + * + * Exposes PSC measurement slots via sysfs. Writing to a slot sends + * data to PSC by calling FIRME_ATTEST_EXT_CLAIMS SMC (0xC400040B). + * + * Slot 0 (bmdr): 100-byte BMDR device report per GPU + * [0-47] identity_digest (SHA-384 of device cert chain) + * [48-95] mexchange_digest (SHA-384 of SPDM measurement exchange) + * [96] gpu_device_id + * [97-99] reserved (0) + * + * Slot 1+ (rem0-rem2): 48-byte SHA-384 extensible measurement slots + * ================================================================ */ + +#define FIRME_MR_NUM_SLOTS 4 + +static u8 firme_mr_bmdr_value[FIRME_BMDR_SIZE]; +static u8 firme_mr_rem_values[FIRME_MR_NUM_SLOTS - 1][SHA384_DIGEST_SIZE]; + +/** + * firme_mr_extend - Submit data to PSC via FIRME_ATTEST_EXT_CLAIMS SMC. + * + * For slot 0 (bmdr): sends a 100-byte BMDR device report per GPU. + * For slot 1+ (rem): sends a 48-byte SHA-384 digest for REM extension. + */ +static int firme_mr_extend(const struct tsm_measurements *tm, + const struct tsm_measurement_register *mr, + const u8 *data) +{ + struct arm_smccc_res res; + unsigned int slot_index = mr - tm->mrs; + void *buf; + phys_addr_t buf_phys; + + buf = (void *)get_zeroed_page(GFP_KERNEL); + if (!buf) + return -ENOMEM; + + memcpy(buf, data, mr->mr_size); + buf_phys = virt_to_phys(buf); + + pr_info("nvidia-firme: EXT_CLAIMS SMC: slot=%u buf=0x%llx size=%u\n", + slot_index, (u64)buf_phys, mr->mr_size); + + arm_smccc_1_1_invoke(FIRME_ATTEST_EXTEND, + buf_phys, + mr->mr_size, + slot_index, + 0, + &res); + + free_page((unsigned long)buf); + + pr_info("nvidia-firme: EXT_CLAIMS returned status=%ld\n", (long)res.a0); + + if ((long)res.a0 == FIRME_SUCCESS) { + memcpy(mr->mr_value, data, mr->mr_size); + return 0; + } + + if ((long)res.a0 == FIRME_INVALID_PARAMETERS) { + pr_err("nvidia-firme: EXT_CLAIMS failed: invalid parameters (slot=%u size=%u)\n", + slot_index, mr->mr_size); + return -EINVAL; + } + + if ((long)res.a0 == FIRME_DENIED) { + pr_err("nvidia-firme: EXT_CLAIMS failed: denied (slot %u)\n", + slot_index); + return -EACCES; + } + + pr_err("nvidia-firme: EXT_CLAIMS failed: status=%ld\n", (long)res.a0); + return -EIO; +} + +static int firme_mr_refresh(const struct tsm_measurements *tm) +{ + /* MR values are cached locally after each extend call. + * PSC doesn't provide a read-back API, so refresh is a no-op. */ + return 0; +} + +static struct tsm_measurement_register firme_mrs[FIRME_MR_NUM_SLOTS] = { + { + .mr_name = "bmdr", + .mr_value = firme_mr_bmdr_value, + .mr_size = FIRME_BMDR_SIZE, + .mr_flags = TSM_MR_F_READABLE | TSM_MR_F_WRITABLE + | TSM_MR_F_NOHASH, + .mr_hash = 0, + }, + { + .mr_name = "rem0", + .mr_value = firme_mr_rem_values[0], + .mr_size = SHA384_DIGEST_SIZE, + .mr_flags = TSM_MR_F_READABLE | TSM_MR_F_WRITABLE, + .mr_hash = HASH_ALGO_SHA384, + }, + { + .mr_name = "rem1", + .mr_value = firme_mr_rem_values[1], + .mr_size = SHA384_DIGEST_SIZE, + .mr_flags = TSM_MR_F_READABLE | TSM_MR_F_WRITABLE, + .mr_hash = HASH_ALGO_SHA384, + }, + { + .mr_name = "rem2", + .mr_value = firme_mr_rem_values[2], + .mr_size = SHA384_DIGEST_SIZE, + .mr_flags = TSM_MR_F_READABLE | TSM_MR_F_WRITABLE, + .mr_hash = HASH_ALGO_SHA384, + }, +}; + +static struct tsm_measurements firme_measurements = { + .mrs = firme_mrs, + .nr_mrs = FIRME_MR_NUM_SLOTS, + .refresh = firme_mr_refresh, + .write = firme_mr_extend, +}; + +static const struct attribute_group *firme_mr_grp; +static struct kobject *firme_kobj; + +/* ================================================================ + * Module init/exit + * ================================================================ */ + +static int __init nvidia_firme_init(void) +{ + int ret; + + if (!nvidia_firme_available()) { + pr_info("nvidia-firme: FIRME service not available\n"); + return -ENODEV; + } + + ret = tsm_report_register(&nvidia_firme_tsm_ops, NULL); + if (ret < 0) { + pr_err("nvidia-firme: failed to register TSM reports (%d)\n", ret); + return ret; + } + + firme_mr_grp = tsm_mr_create_attribute_group(&firme_measurements); + if (IS_ERR(firme_mr_grp)) { + pr_err("nvidia-firme: failed to create TSM MR group (%ld)\n", + PTR_ERR(firme_mr_grp)); + firme_mr_grp = NULL; + } else { + firme_kobj = kobject_create_and_add("nvidia-firme", + firmware_kobj); + if (!firme_kobj) { + pr_err("nvidia-firme: failed to create sysfs kobject\n"); + tsm_mr_free_attribute_group(firme_mr_grp); + firme_mr_grp = NULL; + } else { + ret = sysfs_create_group(firme_kobj, firme_mr_grp); + if (ret) { + pr_err("nvidia-firme: failed to create MR sysfs group (%d)\n", ret); + kobject_put(firme_kobj); + firme_kobj = NULL; + tsm_mr_free_attribute_group(firme_mr_grp); + firme_mr_grp = NULL; + } else { + pr_info("nvidia-firme: measurement registers at /sys/firmware/nvidia-firme/\n"); + } + } + } + + pr_info("nvidia-firme: registered with TSM framework\n"); + return 0; +} +module_init(nvidia_firme_init); + +static void __exit nvidia_firme_exit(void) +{ + if (firme_mr_grp && firme_kobj) + sysfs_remove_group(firme_kobj, firme_mr_grp); + if (firme_kobj) + kobject_put(firme_kobj); + if (firme_mr_grp) + tsm_mr_free_attribute_group(firme_mr_grp); + tsm_report_unregister(&nvidia_firme_tsm_ops); + pr_info("nvidia-firme: unregistered from TSM\n"); +} +module_exit(nvidia_firme_exit); + +MODULE_AUTHOR("Hyder Ali "); +MODULE_DESCRIPTION("NVIDIA FIRME (DEN0149) Attestation and Measurement Driver"); +MODULE_LICENSE("GPL"); From 7a9e0f7a01a696df6c610b8d8036e614be86d978 Mon Sep 17 00:00:00 2001 From: Ian May Date: Thu, 23 Jul 2026 13:24:48 +0000 Subject: [PATCH 310/311] NVIDIA: SAUCE: [Config] arm64: Update annotations for DRTM BugLink: https://bugs.launchpad.net/bugs/2161563 Enable CONFIG_ARM64_SECURE_LAUNCH (selftest/fault-inject knobs off) and disable CONFIG_EFI_ZBOOT, which is incompatible with a measured launch. The remaining '-' entries (EFI_SBAT_FILE, HAVE_KERNEL_GZIP/ZSTD, KERNEL_GZIP/ZSTD) just follow from EFI_ZBOOT=n. Also enable CONFIG_NVIDIA_FIRME_GUEST=m for the FIRME attestation driver. Signed-off-by: Ian May --- debian.nvidia/config/annotations | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations index 38ff05b077b62..6801dc1e07260 100644 --- a/debian.nvidia/config/annotations +++ b/debian.nvidia/config/annotations @@ -231,3 +231,17 @@ CONFIG_BCH policy<{'amd64': 'm', 'arm64': ' CONFIG_MTD_NAND_CORE policy<{'amd64': 'm', 'arm64': 'y'}> CONFIG_PROC_CPU_RESCTRL policy<{'amd64': 'y', 'arm64': 'y'}> CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID policy<{'arm64': 'y'}> + +CONFIG_ARM64_SECURE_LAUNCH policy<{'arm64': 'y'}> +CONFIG_ARM64_SECURE_LAUNCH note<'DRTM Secure Launch support'> +CONFIG_ARM64_SECURE_LAUNCH_SELFTEST policy<{'arm64': 'n'}> +CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT policy<{'arm64': 'n'}> +CONFIG_NVIDIA_FIRME_GUEST policy<{'arm64': 'm'}> +CONFIG_NVIDIA_FIRME_GUEST note<'NVIDIA FIRME attestation driver for DRTM'> +CONFIG_EFI_ZBOOT policy<{'arm64': 'n'}> +CONFIG_EFI_ZBOOT note<'DRTM Secure Launch requires the non-zboot EFI stub path'> +CONFIG_EFI_SBAT_FILE policy<{'arm64': '-'}> +CONFIG_HAVE_KERNEL_GZIP policy<{'arm64': '-'}> +CONFIG_HAVE_KERNEL_ZSTD policy<{'arm64': '-'}> +CONFIG_KERNEL_GZIP policy<{'arm64': '-'}> +CONFIG_KERNEL_ZSTD policy<{'arm64': '-'}> From 1dab17fda017d38947ad2be72a302da58e6ba1cf Mon Sep 17 00:00:00 2001 From: Ian May Date: Tue, 23 Jun 2026 23:14:37 +0000 Subject: [PATCH 311/311] NVIDIA: SAUCE: [Packaging] arm64: Install uncompressed Image for DRTM BugLink: https://bugs.launchpad.net/bugs/2161563 DRTM Secure Launch needs the non-zboot EFI stub path (CONFIG_EFI_ZBOOT=n), so install the uncompressed, UEFI-signed Image instead of vmlinuz.efi. Signed-off-by: Ian May --- debian.nvidia/rules.d/arm64.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian.nvidia/rules.d/arm64.mk b/debian.nvidia/rules.d/arm64.mk index f086214eb37ad..af8f0f4c24b14 100644 --- a/debian.nvidia/rules.d/arm64.mk +++ b/debian.nvidia/rules.d/arm64.mk @@ -1,8 +1,8 @@ build_arch = arm64 defconfig = defconfig flavours = nvidia nvidia-64k -build_image = vmlinuz.efi -kernel_file = arch/$(build_arch)/boot/vmlinuz.efi +build_image = Image +kernel_file = arch/$(build_arch)/boot/Image install_file = vmlinuz no_dumpfile = true uefi_signed = true