diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu index 3a05604c21bf8..82d10d556cc89 100644 --- a/Documentation/ABI/testing/sysfs-devices-system-cpu +++ b/Documentation/ABI/testing/sysfs-devices-system-cpu @@ -327,6 +327,24 @@ Description: Energy performance preference This file is only present if the cppc-cpufreq driver is in use. +What: /sys/devices/system/cpu/cpuX/cpufreq/perf_limited +Date: February 2026 +Contact: linux-pm@vger.kernel.org +Description: Performance Limited + + Read to check if platform throttling (thermal/power/current + limits) caused delivered performance to fall below the + requested level. A non-zero value indicates throttling occurred. + + Write the bitmask of bits to clear: + + - 0x1 = clear bit 0 (desired performance excursion) + - 0x2 = clear bit 1 (minimum performance excursion) + - 0x3 = clear both bits + + The platform sets these bits; OSPM can only clear them. + + This file is only present if the cppc-cpufreq driver is in use. What: /sys/devices/system/cpu/cpu*/cache/index3/cache_disable_{0,1} Date: August 2008 diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index d3e7034bece52..2915345847fde 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1052,6 +1052,19 @@ Kernel parameters policy to use. This governor must be registered in the kernel before the cpufreq driver probes. + cppc_cpufreq.auto_sel_mode= + [CPU_FREQ] Enable ACPI CPPC autonomous performance + selection. When enabled, hardware automatically adjusts + CPU frequency on all CPUs based on workload demands. + In Autonomous mode, Energy Performance Preference (EPP) + hints guide hardware toward performance (0x0) or energy + efficiency (0xff). + Requires ACPI CPPC autonomous selection register support. + Format: + Default: 0 (disabled) + 0: use cpufreq governors + 1: enable if supported by hardware + cpu_init_udelay=N [X86,EARLY] Delay for N microsec between assert and de-assert of APIC INIT to start processors. This delay occurs @@ -8541,7 +8554,8 @@ Kernel parameters workqueue.default_affinity_scope= Select the default affinity scope to use for unbound workqueues. Can be one of "cpu", "smt", "cache", - "numa" and "system". Default is "cache". For more + "cache_shard", "numa" and "system". Default is + "cache_shard". For more information, see the Affinity Scopes section in Documentation/core-api/workqueue.rst. diff --git a/Documentation/arch/arm64/index.rst b/Documentation/arch/arm64/index.rst index af52edc8c0ac6..98052b4ef4a1e 100644 --- a/Documentation/arch/arm64/index.rst +++ b/Documentation/arch/arm64/index.rst @@ -23,6 +23,7 @@ ARM64 Architecture memory memory-tagging-extension mops + mpam perf pointer-authentication ptdump diff --git a/Documentation/arch/arm64/mpam.rst b/Documentation/arch/arm64/mpam.rst new file mode 100644 index 0000000000000..208ff17068c4b --- /dev/null +++ b/Documentation/arch/arm64/mpam.rst @@ -0,0 +1,89 @@ +.. SPDX-License-Identifier: GPL-2.0 + +==== +MPAM +==== + +What is MPAM +============ +MPAM (Memory Partitioning and Monitoring) is a feature in the CPUs and memory +system components such as the caches or memory controllers that allow memory +traffic to be labelled, partitioned and monitored. + +Traffic is labelled by the CPU, based on the control or monitor group the +current task is assigned to using resctrl. Partitioning policy can be set +using the schemata file in resctrl, and monitor values read via resctrl. +See Documentation/filesystems/resctrl.rst for more details. + +This allows tasks that share memory system resources, such as caches, to be +isolated from each other according to the partitioning policy (so called noisy +neighbours). + +Supported Platforms +=================== +Use of this feature requires CPU support, support in the memory system +components, and a description from firmware of where the MPAM device controls +are in the MMIO address space. (e.g. the 'MPAM' ACPI table). + +The MMIO device that provides MPAM controls/monitors for a memory system +component is called a memory system component. (MSC). + +Because the user interface to MPAM is via resctrl, only MPAM features that are +compatible with resctrl can be exposed to user-space. + +MSC are considered as a group based on the topology. MSC that correspond with +the L3 cache are considered together, it is not possible to mix MSC between L2 +and L3 to 'cover' a resctrl schema. + +The supported features are: + +* Cache portion bitmap controls (CPOR) on the L2 or L3 caches. To expose + CPOR at L2 or L3, every CPU must have a corresponding CPU cache at this + level that also supports the feature. Mismatched big/little platforms are + not supported as resctrl's controls would then also depend on task + placement. + +* Memory bandwidth maximum controls (MBW_MAX) on or after the L3 cache. + resctrl uses the L3 cache-id to identify where the memory bandwidth + control is applied. For this reason the platform must have an L3 cache + with cache-id's supplied by firmware. (It doesn't need to support MPAM.) + + To be exported as the 'MB' schema, the topology of the group of MSC chosen + must match the topology of the L3 cache so that the cache-id's can be + repainted. For example: Platforms with Memory bandwidth maximum controls + on CPU-less NUMA nodes cannot expose the 'MB' schema to resctrl as these + nodes do not have a corresponding L3 cache. If the memory bandwidth + control is on the memory rather than the L3 then there must be a single + global L3 as otherwise it is unknown which L3 the traffic came from. There + must be no caches between the L3 and the memory so that the two ends of + the path have equivalent traffic. + + When the MPAM driver finds multiple groups of MSC it can use for the 'MB' + schema, it prefers the group closest to the L3 cache. + +* Cache Storage Usage (CSU) counters can expose the 'llc_occupancy' provided + there is at least one CSU monitor on each MSC that makes up the L3 group. + Exposing CSU counters from other caches or devices is not supported. + +* Memory Bandwidth Usage (MBWU) on or after the L3 cache. resctrl uses the + L3 cache-id to identify where the memory bandwidth is measured. For this + reason the platform must have an L3 cache with cache-id's supplied by + firmware. (It doesn't need to support MPAM.) + + Memory bandwidth monitoring makes use of MBWU monitors in each MSC that + makes up the L3 group. If the memory bandwidth monitoring is on the memory + rather than the L3 then there must be a single global L3 as otherwise it + is unknown which L3 the traffic came from. + + To expose 'mbm_total_bytes', the topology of the group of MSC chosen must + match the topology of the L3 cache so that the cache-id's can be + repainted. For example: Platforms with Memory bandwidth monitors on + CPU-less NUMA nodes cannot expose 'mbm_total_bytes' as these nodes do not + have a corresponding L3 cache. 'mbm_local_bytes' is not exposed as MPAM + cannot distinguish local traffic from global traffic. + +Reporting Bugs +============== +If you are not seeing the counters or controls you expect please share the +debug messages produced when enabling dynamic debug and booting with: +dyndbg="file mpam_resctrl.c +pl" diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index 4c300caad9011..fa8b12e268712 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -128,16 +128,28 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A76 | #3324349 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A76 | #4193800 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A76AE | #4193801 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #1491015 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #1508412 | ARM64_ERRATUM_1508412 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A77 | #3324348 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A77 | #4193798 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A78 | #3324344 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A78 | #4193791 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A78AE | #4193793 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A78C | #3324346,3324347| ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A78C | #4193794 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #2119858 | ARM64_ERRATUM_2119858 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #2054223 | ARM64_ERRATUM_2054223 | @@ -146,6 +158,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #3324338 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A710 | #4193788 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A715 | #2645198 | ARM64_ERRATUM_2645198 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A715 | #3456084 | ARM64_ERRATUM_3194386 | @@ -158,20 +172,32 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X1 | #3324344 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X1 | #4193791 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X1C | #3324346 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X1C | #4193792 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #2119858 | ARM64_ERRATUM_2119858 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #2224489 | ARM64_ERRATUM_2224489 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X2 | #3324338 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X2 | #4193788 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X3 | #3324335 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X3 | #4193786 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X4 | #3194386 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X4 | #4118414 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X925 | #3324334 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-X925 | #4193781 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1188873,1418040| ARM64_ERRATUM_1418040 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #1349291 | N/A | @@ -182,6 +208,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N1 | #3324349 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-N1 | #4193800 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #2139208 | ARM64_ERRATUM_2139208 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #2067961 | ARM64_ERRATUM_2067961 | @@ -190,18 +218,32 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N2 | #3324339 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-N2 | #4193789 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-N3 | #3456111 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V1 | #1619801 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V1 | #3324341 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V1 | #4193790 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V2 | #3324336 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V2 | #4193787 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V3 | #3312417 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V3 | #4193784 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V3AE | #3312417 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Neoverse-V3AE | #4193784 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | C1-Premium | #4193780 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | C1-Ultra | #4193780 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-500 | #841119,826419 | ARM_SMMU_MMU_500_CPRE_ERRATA| | | | #562869,1047329 | | +----------------+-----------------+-----------------+-----------------------------+ @@ -214,6 +256,9 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | SI L1 | #4311569 | ARM64_ERRATUM_4311569 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | CMN-650 | #3642720 | N/A | ++----------------+-----------------+-----------------+-----------------------------+ ++----------------+-----------------+-----------------+-----------------------------+ | Broadcom | Brahma-B53 | N/A | ARM64_ERRATUM_845719 | +----------------+-----------------+-----------------+-----------------------------+ | Broadcom | Brahma-B53 | N/A | ARM64_ERRATUM_843419 | @@ -245,8 +290,16 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | Carmel Core | N/A | NVIDIA_CARMEL_CNP_ERRATUM | +----------------+-----------------+-----------------+-----------------------------+ +| NVIDIA | Olympus core | T410-OLY-1029 | ARM64_ERRATUM_4118414 | ++----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | T241 GICv3/4.x | T241-FABRIC-4 | N/A | +----------------+-----------------+-----------------+-----------------------------+ +| NVIDIA | T241 MPAM | T241-MPAM-1 | N/A | ++----------------+-----------------+-----------------+-----------------------------+ +| NVIDIA | T241 MPAM | T241-MPAM-4 | N/A | ++----------------+-----------------+-----------------+-----------------------------+ +| NVIDIA | T241 MPAM | T241-MPAM-6 | N/A | ++----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ | Freescale/NXP | LS2080A/LS1043A | A-008585 | FSL_ERRATUM_A008585 | +----------------+-----------------+-----------------+-----------------------------+ diff --git a/Documentation/core-api/workqueue.rst b/Documentation/core-api/workqueue.rst index 165ca73e83514..411e1b28b8dec 100644 --- a/Documentation/core-api/workqueue.rst +++ b/Documentation/core-api/workqueue.rst @@ -378,9 +378,9 @@ Affinity Scopes An unbound workqueue groups CPUs according to its affinity scope to improve cache locality. For example, if a workqueue is using the default affinity -scope of "cache", it will group CPUs according to last level cache -boundaries. A work item queued on the workqueue will be assigned to a worker -on one of the CPUs which share the last level cache with the issuing CPU. +scope of "cache_shard", it will group CPUs into sub-LLC shards. A work item +queued on the workqueue will be assigned to a worker on one of the CPUs +within the same shard as the issuing CPU. Once started, the worker may or may not be allowed to move outside the scope depending on the ``affinity_strict`` setting of the scope. @@ -402,7 +402,13 @@ Workqueue currently supports the following affinity scopes. ``cache`` CPUs are grouped according to cache boundaries. Which specific cache boundary is used is determined by the arch code. L3 is used in a lot of - cases. This is the default affinity scope. + cases. + +``cache_shard`` + CPUs are grouped into sub-LLC shards of at most ``wq_cache_shard_size`` + cores (default 8, tunable via the ``workqueue.cache_shard_size`` boot + parameter). Shards are always split on core (SMT group) boundaries. + This is the default affinity scope. ``numa`` CPUs are grouped according to NUMA boundaries. diff --git a/Documentation/cpu-freq/cpu-drivers.rst b/Documentation/cpu-freq/cpu-drivers.rst index c5635ac3de547..ab4f3c0f3a89b 100644 --- a/Documentation/cpu-freq/cpu-drivers.rst +++ b/Documentation/cpu-freq/cpu-drivers.rst @@ -114,8 +114,14 @@ Then, the driver must fill in the following values: |policy->cur | The current operating frequency of | | | this CPU (if appropriate) | +-----------------------------------+--------------------------------------+ -|policy->min, | | -|policy->max, | | +|policy->min | If set by the driver in ->init(), | +| | used as initial minimum frequency | +| | QoS request. | ++-----------------------------------+--------------------------------------+ +|policy->max | If set by the driver in ->init(), | +| | used as initial maximum frequency | +| | QoS request. | ++-----------------------------------+--------------------------------------+ |policy->policy and, if necessary, | | |policy->governor | must contain the "default policy" for| | | this CPU. A few moments later, | diff --git a/Documentation/filesystems/resctrl.rst b/Documentation/filesystems/resctrl.rst index ba609f8d4de57..730c6d662e040 100644 --- a/Documentation/filesystems/resctrl.rst +++ b/Documentation/filesystems/resctrl.rst @@ -215,6 +215,14 @@ related to allocation: # cat /sys/fs/resctrl/info/L3/io_alloc_cbm 0=00ff;1=000f + An ID of "*" configures all domains with the provided CBM. + + Example on a system that does not require a minimum number of consecutive bits in the mask:: + + # echo "*=0" > /sys/fs/resctrl/info/L3/io_alloc_cbm + # cat /sys/fs/resctrl/info/L3/io_alloc_cbm + 0=0;1=0 + When CDP is enabled "io_alloc_cbm" associated with the CDP_DATA and CDP_CODE resources may reflect the same values. For example, values read from and written to /sys/fs/resctrl/info/L3DATA/io_alloc_cbm may be reflected by @@ -228,12 +236,11 @@ with respect to allocation: user can request. "bandwidth_gran": - The granularity in which the memory bandwidth + The approximate granularity in which the memory bandwidth percentage is allocated. The allocated b/w percentage is rounded off to the next - control step available on the hardware. The - available bandwidth control steps are: - min_bandwidth + N * bandwidth_gran. + control step available on the hardware. The available + steps are at least as small as this value. "delay_linear": Indicates if the delay scale is linear or @@ -419,9 +426,9 @@ with the following files: Two MBM events are supported by default: mbm_local_bytes and mbm_total_bytes. Each MBM event's sub-directory contains a file named "event_filter" that is - used to view and modify which memory transactions the MBM event is configured - with. The file is accessible only when "mbm_event" counter assignment mode is - enabled. + used to view and (if writable) modify which memory transactions the MBM event + is configured with. The file is accessible only when "mbm_event" counter + assignment mode is enabled. List of memory transaction types supported: @@ -446,9 +453,8 @@ with the following files: # cat /sys/fs/resctrl/info/L3_MON/event_configs/mbm_local_bytes/event_filter local_reads,local_non_temporal_writes,local_reads_slow_memory - Modify the event configuration by writing to the "event_filter" file within - the "event_configs" directory. The read/write "event_filter" file contains the - configuration of the event that reflects which memory transactions are counted by it. + The memory transactions the MBM event is configured with can be changed + if "event_filter" is writable. For example:: @@ -472,6 +478,12 @@ with the following files: "1": Auto assignment is enabled. + Automatic counter assignment is done with best effort. If auto + assignment is enabled but there are not enough available counters then + monitor group creation could succeed while one or more events belonging + to the group may not have a counter assigned in all domains. Consult + mbm_L3_assignments for counter assignment states of the new groups. + Example:: # echo 0 > /sys/fs/resctrl/info/L3_MON/mbm_assign_on_mkdir @@ -562,6 +574,11 @@ All groups contain the following files: then the task must already belong to the CTRL_MON parent of this group. The task is removed from any previous MON group. + When writing to this file, a task id of 0 is interpreted as the + task id of the currently running task. On reading the file, a task + id of 0 will never be shown and there is no representation of the + idle tasks. Instead, a CPU's idle task is always considered as a + member of the group owning the CPU. "cpus": Reading this file shows a bitmask of the logical CPUs owned by diff --git a/MAINTAINERS b/MAINTAINERS index 85bbf2d242458..b9028c49b421e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -22198,11 +22198,13 @@ F: tools/testing/selftests/net/rds/ RDT - RESOURCE ALLOCATION M: Tony Luck M: Reinette Chatre +M: x86@kernel.org R: Dave Martin R: James Morse R: Babu Moger L: linux-kernel@vger.kernel.org S: Supported +P: Documentation/process/maintainer-tip.rst F: Documentation/filesystems/resctrl.rst F: arch/x86/include/asm/resctrl.h F: arch/x86/kernel/cpu/resctrl/ diff --git a/Ubuntu.md b/Ubuntu.md index a49309c1d8c9d..85afdc3463f40 100644 --- a/Ubuntu.md +++ b/Ubuntu.md @@ -1,8 +1,8 @@ -Name: linux +Name: linux-nvidia Version: 7.0.0 Series: 26.04 (resolute) Description: - This is the source code for the Ubuntu linux kernel for the 26.04 series. This - source tree is used to produce the flavours: generic, generic-64k. + This is the source code for the NVIDIA linux kernel for the 26.04 series. This + source tree is used to produce the flavours: nvidia, nvidia-64k. This kernel is configured to support the widest range of desktop, laptop and server configurations. diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index d4b97eebf9965..e6e2b37f8988d 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1176,6 +1176,43 @@ config ARM64_ERRATUM_4311569 If unsure, say Y. +config ARM64_ERRATUM_4118414 + bool "Various: Completion of affected memory accesses might not be guaranteed by completion of a TLBI" + default y + select ARM64_WORKAROUND_REPEAT_TLBI + help + This option adds a workaround for the following errata: + + * ARM C1-Premium erratum 4193780 + * ARM C1-Ultra erratum 4193780 + * ARM Cortex-A76 erratum 4193800 + * ARM Cortex-A76AE erratum 4193801 + * ARM Cortex-A77 erratum 4193798 + * ARM Cortex-A78 erratum 4193791 + * ARM Cortex-A78AE erratum 4193793 + * ARM Cortex-A78C erratum 4193794 + * ARM Cortex-A710 erratum 4193788 + * ARM Cortex-X1 erratum 4193791 + * ARM Cortex-X1C erratum 4193792 + * ARM Cortex-X2 erratum 4193788 + * ARM Cortex-X3 erratum 4193786 + * ARM Cortex-X4 erratum 4118414 + * ARM Cortex-X925 erratum 4193781 + * ARM Neoverse-N1 erratum 4193800 + * ARM Neoverse-N2 erratum 4193789 + * ARM Neoverse-V1 erratum 4193790 + * ARM Neoverse-V2 erratum 4193787 + * ARM Neoverse-V3 erratum 4193784 + * ARM Neoverse-V3AE erratum 4193784 + * NVIDIA Olympus erratum T410-OLY-1029 + + On affected cores, some memory accesses might not be completed by + broadcast TLB invalidation. + + This issue is also known as CVE-2025-10263. + + If unsure, say Y. + config CAVIUM_ERRATUM_22375 bool "Cavium erratum 22375, 24313" default y @@ -2017,8 +2054,8 @@ config ARM64_TLB_RANGE config ARM64_MPAM bool "Enable support for MPAM" - select ARM64_MPAM_DRIVER if EXPERT # does nothing yet - select ACPI_MPAM if ACPI + select ARM64_MPAM_DRIVER + select ARCH_HAS_CPU_RESCTRL help Memory System Resource Partitioning and Monitoring (MPAM) is an optional extension to the Arm architecture that allows each @@ -2040,6 +2077,8 @@ config ARM64_MPAM MPAM is exposed to user-space via the resctrl pseudo filesystem. + This option enables the extra context switch code. + endmenu # "ARMv8.4 architectural features" menu "ARMv8.5 architectural features" @@ -2424,6 +2463,82 @@ config EFI allow the kernel to be booted as an EFI application. This is only useful on systems that have UEFI firmware. +config ARM64_SECURE_LAUNCH + bool "ARM64 DRTM Secure Launch support" + depends on ARM64 && EFI + default y + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 + help + Enable support for ARM DRTM (Dynamic Root of Trust for Measurement) + Secure Launch. When enabled, the kernel can act as a DLME, receiving + control after a DRTM dynamic launch with MMU off at EL2. + + The EFI stub triggers DRTM_DYNAMIC_LAUNCH after ExitBootServices + when "drtm=on" is on the kernel command line. D-CRTM measures the + kernel and returns control. The kernel then parses DLME data and + manages DRTM SMC calls during boot. Booting with drtm=on requires + firmware with an EL3 DRTM monitor; without drtm=on the kernel + boots normally and issues no DRTM calls. + + If unsure, say N. + +config ARM64_SECURE_LAUNCH_SELFTEST + bool "ARM64 DRTM Secure Launch validation self-tests" + depends on ARM64_SECURE_LAUNCH + default n + help + Build slaunch_selftest() into the kernel. At the end of + slaunch_measure_post_efi() the self-test calls the address-map + validation helpers (dcrtm_range_in_normal, + dcrtm_range_overlaps_non_normal, efi_regions_overlap) with + crafted inputs that previously bypassed validation due to + integer overflow. The helpers must reject the bad inputs and + accept the good ones; mismatch panics with selftest:* message. + + Useful for proving the overflow / wrap guards work as designed. + + If unsure, say N. + +config ARM64_SECURE_LAUNCH_FAULT_INJECT + bool "ARM64 DRTM Secure Launch fault injection" + depends on ARM64_SECURE_LAUNCH + default n + help + Build slaunch_inject_fault() into the kernel. When the kernel + command line contains "slaunch_inject=", mutate the EFI + memory map (or other tainted inputs) in-place before + slaunch_validate_efi_early() runs the real validation. The new + guard is expected to panic with a specific message; the harness + treats that as a passing negative test. + + Tokens implemented: mmap_wrap, mmap_pages_overflow, + mmap_size_huge, systab_nr_tables_huge, sha384_force, + sha_algo_bad. + + Useful for proving end-to-end that the validation pipeline + catches the canonical exploits. Intended for development testing only; leave disabled in production builds. + + If unsure, say N. + +config ARM64_SECURE_LAUNCH_KASLR + bool "ARM64 DRTM trusted linear-map KASLR" + depends on ARM64_SECURE_LAUNCH && RANDOMIZE_BASE + depends on !MEMORY_HOTPLUG + default n + help + After a DRTM Secure Launch, re-randomize the kernel linear + (direct) map from the CPU's FEAT_RNG instead of leaving it at a + fixed offset. The randomization slack is measured against present + DRAM only, with no reservation for memory hotplug, so it works + even when the CPU PArange equals the virtual address size. + + Fail-closed: if the CPU lacks FEAT_RNG (RNDR) the kernel panics + rather than booting with a predictable linear map. Requires memory + hotplug to be disabled. + + If unsure, say N. + config COMPRESSED_INSTALL bool "Install compressed image by default" help diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index b67d5b1fc45b0..31b6f5d0be67e 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1956,4 +1956,15 @@ CONFIG_CORESIGHT_SINK_ETBV10=m CONFIG_CORESIGHT_STM=m CONFIG_CORESIGHT_CPU_DEBUG=m CONFIG_CORESIGHT_CTI=m +CONFIG_ARM64_SECURE_LAUNCH=y +CONFIG_CMDLINE="drtm=on efi=noruntime nokaslr" +CONFIG_CMDLINE_EXTEND=y +CONFIG_NVIDIA_FIRME_GUEST=m CONFIG_MEMTEST=y +CONFIG_NVGRACE_GPU_VFIO_PCI=m +CONFIG_NVGRACE_EGM=m +CONFIG_VFIO_DEVICE_CDEV=y +CONFIG_FAULT_INJECTION=y +CONFIG_IOMMUFD_DRIVER=y +CONFIG_IOMMUFD=y +CONFIG_IOMMUFD_TEST=y diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 08860d482e600..45f96ca01cea5 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -97,7 +97,9 @@ #define ARM_CPU_PART_CORTEX_X925 0xD85 #define ARM_CPU_PART_CORTEX_A725 0xD87 #define ARM_CPU_PART_CORTEX_A720AE 0xD89 +#define ARM_CPU_PART_C1_ULTRA 0xD8C #define ARM_CPU_PART_NEOVERSE_N3 0xD8E +#define ARM_CPU_PART_C1_PREMIUM 0xD90 #define APM_CPU_PART_XGENE 0x000 #define APM_CPU_VAR_POTENZA 0x00 @@ -188,7 +190,9 @@ #define MIDR_CORTEX_X925 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X925) #define MIDR_CORTEX_A725 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A725) #define MIDR_CORTEX_A720AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A720AE) +#define MIDR_C1_ULTRA MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_ULTRA) #define MIDR_NEOVERSE_N3 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N3) +#define MIDR_C1_PREMIUM MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PREMIUM) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) diff --git a/arch/arm64/include/asm/drtm.h b/arch/arm64/include/asm/drtm.h new file mode 100644 index 0000000000000..8aeab02e24da2 --- /dev/null +++ b/arch/arm64/include/asm/drtm.h @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * ARM64 DRTM (Dynamic Root of Trust for Measurement) definitions. + * Based on DEN0113 v1.2 — Arm DRTM Architecture Specification. + */ +#ifndef _ASM_ARM64_DRTM_H +#define _ASM_ARM64_DRTM_H + +#ifndef __ASSEMBLY__ +#include +#endif + +/* DRTM SMC Function IDs (DEN0113 v1.2 §3.2-3.10) */ +#define DRTM_SMC_FN_BASE 0xC4000110UL +#define DRTM_SMC_VERSION (DRTM_SMC_FN_BASE + 0x00) +#define DRTM_SMC_FEATURES (DRTM_SMC_FN_BASE + 0x01) +#define DRTM_SMC_UNPROTECT_MEMORY (DRTM_SMC_FN_BASE + 0x03) +#define DRTM_SMC_DYNAMIC_LAUNCH (DRTM_SMC_FN_BASE + 0x04) +#define DRTM_SMC_CLOSE_LOCALITY (DRTM_SMC_FN_BASE + 0x05) +#define DRTM_SMC_GET_ERROR (DRTM_SMC_FN_BASE + 0x06) +#define DRTM_SMC_SET_ERROR (DRTM_SMC_FN_BASE + 0x07) +#define DRTM_SMC_SET_TCB_HASH (DRTM_SMC_FN_BASE + 0x08) +#define DRTM_SMC_LOCK_TCB_HASH (DRTM_SMC_FN_BASE + 0x09) +#define DRTM_SMC_ENABLE_SECURE_INTERRUPTS (DRTM_SMC_FN_BASE + 0x0A) /* 0xC400011A */ + +/* DRTM Return Codes (DEN0113 v1.2 §3.18, Table 20) */ +#define DRTM_SUCCESS 0 +#define DRTM_NOT_SUPPORTED (-1) +#define DRTM_INVALID_PARAMETERS (-2) +#define DRTM_DENIED (-3) +#define DRTM_INTERNAL_ERROR (-5) + +/* DEN0113 v1.2 Table 9: DRTM_PARAMETERS revision is 2 */ +#define DRTM_PARAMS_REVISION 2 + +/* Launch features */ +#define DRTM_LAUNCH_FEAT_MEM_PROT_ALL (0x0 << 3) +#define DRTM_LAUNCH_FEAT_SEC_INT_DISABLE (0x1 << 7) /* DEN0113 Table 9 */ + +/* DRTM page size */ +#define DRTM_PAGE_SIZE 0x1000 + +/* + * Preamble->DLME DTB-PA handoff slot: DTB PA written 8 bytes below the + * DLME data region; sl_entry reads it via X0+X1-8 after D-CRTM ERET. + * Private contract (not DEN0113); DTB PA validated before FDT is parsed. + */ +#define SL_DLME_DTB_SLOT_OFFSET (-8) + +/* + * Full-range DMA protection sentinel (DEN0113 v1.2 §3.14 Table 11 + + * §4.6.2): type = NORMAL, start = 0, page count = 2^52 - 1. + */ +#define DRTM_MEM_PROT_FULL_RANGE \ + ((0x0ULL << 55) | (0x0ULL << 52) | ((1ULL << 52) - 1ULL)) + +#ifndef __ASSEMBLY__ +/* + * Memory Region Descriptor Table (DEN0113 v1.2 §3.14, Table 11): header + * followed by num_regions descriptors, consumed in place from the DLME + * data region (no fixed-size copy, no region-count cap). + */ +struct drtm_mem_region_hdr { + __le16 revision; + __le16 reserved; + __le32 num_regions; +} __packed; + +struct drtm_mem_region { + __le64 start_address; + __le64 size_and_type; +} __packed; + +/* + * Address map region types in size_and_type bits [54:52] (DEN0113 v1.2 + * §3.14 Table 11): 0 normal, 1 normal+cacheability, 2 device/MMIO, + * 3 non-volatile, 4 reserved. + */ +#define DRTM_REGION_TYPE_NORMAL 0 +#define DRTM_REGION_TYPE_NORMAL_CACHED 1 +#define DRTM_REGION_TYPE_DEVICE 2 +#define DRTM_REGION_TYPE_NV 3 +#define DRTM_REGION_TYPE_RSVD 4 + +/* + * size_and_type field helpers (DEN0113 v1.2 §3.14 Table 11): page count + * [51:0], region type [54:52], cacheability [56:55] (valid for + * NORMAL_CACHED), reserved [63:57]. + */ +#define DRTM_MEM_REGION_PAGE_COUNT(x) ((x) & ((1ULL << 52) - 1)) +#define DRTM_MEM_REGION_TYPE(x) (((x) >> 52) & 0x7) +#define DRTM_MEM_REGION_CACHEABILITY(x) (((x) >> 55) & 0x3) + +/* DLME Data Header (DEN0113 v1.2 §3.15, Table 14) — populated by D-CRTM */ +struct dlme_data_header { + __le16 version; + __le16 this_hdr_size; + __le32 reserved; + __le64 dlme_data_size; + __le64 protected_regions_size; + __le64 address_map_size; + __le64 drtm_event_log_size; + __le64 tcb_hash_table_size; + __le64 acpi_table_region_size; + __le64 impl_defined_region_size; +}; + +#ifdef CONFIG_ARM64_SECURE_LAUNCH +extern unsigned long sl_dlme_region_pa; +extern unsigned long sl_dlme_data_offset; + +void slaunch_early_init(void); +void slaunch_setup(void); +void slaunch_validate_initrd(void); +void slaunch_reserve_dlme_data(void); +void slaunch_measure_post_efi(void); +bool slaunch_phys_is_protected_ram(phys_addr_t pa); +bool slaunch_phys_range_overlaps_protected_ram(phys_addr_t pa, size_t size); + +/* True iff a DRTM Secure Launch occurred (stub-planted DLME handoff). */ +static inline bool slaunch_active(void) { return sl_dlme_region_pa != 0; } +#else +static inline void slaunch_early_init(void) { } +static inline void slaunch_setup(void) { } +static inline void slaunch_validate_initrd(void) { } +static inline void slaunch_reserve_dlme_data(void) { } +static inline void slaunch_measure_post_efi(void) { } +static inline bool slaunch_phys_is_protected_ram(phys_addr_t pa) { return false; } +static inline bool slaunch_phys_range_overlaps_protected_ram(phys_addr_t pa, size_t size) { return false; } +static inline bool slaunch_active(void) { return false; } +#endif + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_ARM64_DRTM_H */ diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index 85f4c1615472d..4d15071a4f3fc 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -513,7 +513,8 @@ check_override id_aa64pfr0, ID_AA64PFR0_EL1_MPAM_SHIFT, .Linit_mpam_\@, .Lskip_mpam_\@, x1, x2 .Linit_mpam_\@: - msr_s SYS_MPAM2_EL2, xzr // use the default partition + mov x0, #MPAM2_EL2_EnMPAMSM_MASK + msr_s SYS_MPAM2_EL2, x0 // use the default partition, // and disable lower traps mrs_s x0, SYS_MPAMIDR_EL1 tbz x0, #MPAMIDR_EL1_HAS_HCR_SHIFT, .Lskip_mpam_\@ // skip if no MPAMHCR reg diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h index 74a4f738c5f52..aba887c561c0a 100644 --- a/arch/arm64/include/asm/kernel-pgtable.h +++ b/arch/arm64/include/asm/kernel-pgtable.h @@ -68,7 +68,13 @@ #define KERNEL_SEGMENT_COUNT 5 #if SWAPPER_BLOCK_SIZE > SEGMENT_ALIGN -#define EARLY_SEGMENT_EXTRA_PAGES (KERNEL_SEGMENT_COUNT + 1) +/* + * The EFI stub's writable statics (.efistub) are mapped as a separate + * segment so they can be unmapped after init; each of its two page-aligned + * boundaries may force one extra page-table page where a swapper-block + * mapping would otherwise be used. + */ +#define EARLY_SEGMENT_EXTRA_PAGES (KERNEL_SEGMENT_COUNT + 1 + 2 * __is_defined(CONFIG_EFI)) /* * The initial ID map consists of the kernel image, mapped as two separate * segments, and may appear misaligned wrt the swapper block size. This means diff --git a/arch/arm64/include/asm/mpam.h b/arch/arm64/include/asm/mpam.h new file mode 100644 index 0000000000000..70d396e7b6da8 --- /dev/null +++ b/arch/arm64/include/asm/mpam.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2025 Arm Ltd. */ + +#ifndef __ASM__MPAM_H +#define __ASM__MPAM_H + +#include +#include +#include +#include +#include + +#include + +DECLARE_STATIC_KEY_FALSE(mpam_enabled); +DECLARE_PER_CPU(u64, arm64_mpam_default); +DECLARE_PER_CPU(u64, arm64_mpam_current); + +/* + * The value of the MPAM0_EL1 sysreg when a task is in resctrl's default group. + * This is used by the context switch code to use the resctrl CPU property + * instead. The value is modified when CDP is enabled/disabled by mounting + * the resctrl filesystem. + */ +extern u64 arm64_mpam_global_default; + +#ifdef CONFIG_ARM64_MPAM +static inline u64 __mpam_regval(u16 partid_d, u16 partid_i, u8 pmg_d, u8 pmg_i) +{ + return FIELD_PREP(MPAM0_EL1_PARTID_D, partid_d) | + FIELD_PREP(MPAM0_EL1_PARTID_I, partid_i) | + FIELD_PREP(MPAM0_EL1_PMG_D, pmg_d) | + FIELD_PREP(MPAM0_EL1_PMG_I, pmg_i); +} + +static inline void mpam_set_cpu_defaults(int cpu, u16 partid_d, u16 partid_i, + u8 pmg_d, u8 pmg_i) +{ + u64 default_val = __mpam_regval(partid_d, partid_i, pmg_d, pmg_i); + + WRITE_ONCE(per_cpu(arm64_mpam_default, cpu), default_val); +} + +/* + * The resctrl filesystem writes to the partid/pmg values for threads and CPUs, + * which may race with reads in mpam_thread_switch(). Ensure only one of the old + * or new values are used. Particular care should be taken with the pmg field as + * mpam_thread_switch() may read a partid and pmg that don't match, causing this + * value to be stored with cache allocations, despite being considered 'free' by + * resctrl. + */ +static inline u64 mpam_get_regval(struct task_struct *tsk) +{ + return READ_ONCE(task_thread_info(tsk)->mpam_partid_pmg); +} + +static inline void mpam_set_task_partid_pmg(struct task_struct *tsk, + u16 partid_d, u16 partid_i, + u8 pmg_d, u8 pmg_i) +{ + u64 regval = __mpam_regval(partid_d, partid_i, pmg_d, pmg_i); + + WRITE_ONCE(task_thread_info(tsk)->mpam_partid_pmg, regval); +} + +static inline void mpam_thread_switch(struct task_struct *tsk) +{ + u64 oldregval; + int cpu = smp_processor_id(); + u64 regval = mpam_get_regval(tsk); + + if (!static_branch_likely(&mpam_enabled)) + return; + + if (regval == READ_ONCE(arm64_mpam_global_default)) + regval = READ_ONCE(per_cpu(arm64_mpam_default, cpu)); + + oldregval = READ_ONCE(per_cpu(arm64_mpam_current, cpu)); + if (oldregval == regval) + return; + + write_sysreg_s(regval | MPAM1_EL1_MPAMEN, SYS_MPAM1_EL1); + if (system_supports_sme()) + write_sysreg_s(regval & (MPAMSM_EL1_PARTID_D | MPAMSM_EL1_PMG_D), SYS_MPAMSM_EL1); + isb(); + + /* Synchronising the EL0 write is left until the ERET to EL0 */ + write_sysreg_s(regval, SYS_MPAM0_EL1); + + WRITE_ONCE(per_cpu(arm64_mpam_current, cpu), regval); +} +#else +static inline void mpam_thread_switch(struct task_struct *tsk) {} +#endif /* CONFIG_ARM64_MPAM */ + +#endif /* __ASM__MPAM_H */ diff --git a/arch/arm64/include/asm/resctrl.h b/arch/arm64/include/asm/resctrl.h new file mode 100644 index 0000000000000..b506e95cf6e37 --- /dev/null +++ b/arch/arm64/include/asm/resctrl.h @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include diff --git a/arch/arm64/include/asm/sections.h b/arch/arm64/include/asm/sections.h index 51b0d594239eb..ad402c0ba605e 100644 --- a/arch/arm64/include/asm/sections.h +++ b/arch/arm64/include/asm/sections.h @@ -23,6 +23,7 @@ extern char __irqentry_text_start[], __irqentry_text_end[]; extern char __mmuoff_data_start[], __mmuoff_data_end[]; extern char __entry_tramp_text_start[], __entry_tramp_text_end[]; extern char __relocate_new_kernel_start[], __relocate_new_kernel_end[]; +extern char __efistub_start[], __efistub_end[]; static inline size_t entry_tramp_text_size(void) { diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index 7942478e40658..5d7fe3e153c85 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -41,6 +41,9 @@ struct thread_info { #ifdef CONFIG_SHADOW_CALL_STACK void *scs_base; void *scs_sp; +#endif +#ifdef CONFIG_ARM64_MPAM + u64 mpam_partid_pmg; #endif u32 cpu; }; diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index fe627100d1990..411e47910a1ee 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -50,6 +50,7 @@ obj-$(CONFIG_HAVE_STATIC_CALL) += static_call.o obj-$(CONFIG_CPU_PM) += sleep.o suspend.o obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_EFI) += efi.o efi-rt-wrapper.o +obj-$(CONFIG_ARM64_SECURE_LAUNCH) += sl_stub.o slaunch.o obj-$(CONFIG_PCI) += pci.o obj-$(CONFIG_ARMV8_DEPRECATED) += armv8_deprecated.o obj-$(CONFIG_ACPI) += acpi.o @@ -68,6 +69,7 @@ obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_VMCORE_INFO) += vmcore_info.o obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o +obj-$(CONFIG_ARM64_MPAM) += mpam.o obj-$(CONFIG_ARM64_MTE) += mte.o obj-y += vdso-wrap.o obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o diff --git a/arch/arm64/kernel/acpi.c b/arch/arm64/kernel/acpi.c index a9d884fd1d001..d7a56c8fe2e5e 100644 --- a/arch/arm64/kernel/acpi.c +++ b/arch/arm64/kernel/acpi.c @@ -29,6 +29,8 @@ #include #include +#include + #include #include #include @@ -282,6 +284,11 @@ pgprot_t __acpi_get_mem_attribute(phys_addr_t addr) u64 attr; + /* Never let the untrusted EFI attribute device-map kernel RAM or + * the measured DLME; force cacheable for those. */ + if (slaunch_phys_is_protected_ram(addr)) + return PAGE_KERNEL; + attr = efi_mem_attributes(addr); if (attr & EFI_MEMORY_WB) return PAGE_KERNEL; @@ -369,6 +376,14 @@ void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size) fallthrough; default: + /* Refuse a non-RAM descriptor whose span reaches kernel + * RAM or the measured DLME: a device alias of normal + * memory is unsafe. Check the whole range — a span may + * start in MMIO and extend into protected memory. */ + if (slaunch_phys_range_overlaps_protected_ram(phys, size)) { + pr_warn(FW_BUG "DRTM: refusing device-map of kernel RAM/DLME @ %pa\n", &phys); + return NULL; + } if (region->attribute & EFI_MEMORY_WB) prot = PAGE_KERNEL; else if (region->attribute & EFI_MEMORY_WC) @@ -376,6 +391,11 @@ void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size) else if (region->attribute & EFI_MEMORY_WT) prot = __acpi_get_writethrough_mem_attribute(); } + } else if (slaunch_phys_range_overlaps_protected_ram(phys, size)) { + /* Span absent from the EFI map but reaching kernel RAM or the + * DLME: refuse rather than device-map it. */ + pr_warn(FW_BUG "DRTM: refusing device-map of unmapped kernel RAM/DLME @ %pa\n", &phys); + return NULL; } return __ioremap_prot(phys, size, prot); } diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 5c0ab6bfd44a6..0b39556c88349 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -339,7 +339,36 @@ static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = { ERRATA_MIDR_RANGE(MIDR_CORTEX_A510, 0, 0, 1, 1), }, #endif - {}, +#ifdef CONFIG_ARM64_ERRATUM_4118414 + { + ERRATA_MIDR_RANGE_LIST(((const struct midr_range[]) { + MIDR_ALL_VERSIONS(MIDR_C1_PREMIUM), + MIDR_ALL_VERSIONS(MIDR_C1_ULTRA), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A76), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A76AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A77), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X2), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X3), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X4), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X925), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3AE), + MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS), + {} + })), + }, +#endif + {} }; #endif @@ -675,7 +704,7 @@ const struct arm64_cpu_capabilities arm64_errata[] = { #endif #ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI { - .desc = "Qualcomm erratum 1009, or ARM erratum 1286807, 2441009", + .desc = "Broken broadcast TLBI completion", .capability = ARM64_WORKAROUND_REPEAT_TLBI, .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, .matches = cpucap_multi_entry_cap_matches, diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 32c2dbcc0c641..f57c2ff98326b 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -86,6 +86,7 @@ #include #include #include +#include #include #include #include @@ -2501,13 +2502,19 @@ test_has_mpam(const struct arm64_cpu_capabilities *entry, int scope) static void cpu_enable_mpam(const struct arm64_cpu_capabilities *entry) { - /* - * Access by the kernel (at EL1) should use the reserved PARTID - * which is configured unrestricted. This avoids priority-inversion - * where latency sensitive tasks have to wait for a task that has - * been throttled to release the lock. - */ - write_sysreg_s(0, SYS_MPAM1_EL1); + int cpu = smp_processor_id(); + u64 regval = 0; + + if (IS_ENABLED(CONFIG_ARM64_MPAM) && static_branch_likely(&mpam_enabled)) + regval = READ_ONCE(per_cpu(arm64_mpam_current, cpu)); + + write_sysreg_s(regval | MPAM1_EL1_MPAMEN, SYS_MPAM1_EL1); + if (cpus_have_cap(ARM64_SME)) + write_sysreg_s(regval & (MPAMSM_EL1_PARTID_D | MPAMSM_EL1_PMG_D), SYS_MPAMSM_EL1); + isb(); + + /* Synchronising the EL0 write is left until the ERET to EL0 */ + write_sysreg_s(regval, SYS_MPAM0_EL1); } static bool diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index d7b0d12b10155..a37e1bf22917b 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -42,6 +42,20 @@ PROVIDE(__efistub_sysfb_primary_display = sysfb_primary_display); #endif PROVIDE(__efistub__ctype = _ctype); +#ifdef CONFIG_ARM64_SECURE_LAUNCH +PROVIDE(__efistub_sl_entry = sl_entry); +#endif + +/* + * PI code maps the page-aligned .efistub section as its own segment. + * Plain PROVIDE (not PI_EXPORT_SYM): PI_EXPORT_SYM asserts sym < + * __bss_start, which a boundary symbol equal to __bss_start + * (page-aligned __efistub_end) would fail; these are address bounds, + * not storage. + */ +PROVIDE(__pi___efistub_start = __efistub_start); +PROVIDE(__pi___efistub_end = __efistub_end); + PROVIDE(__pi___memcpy = __pi_memcpy); PROVIDE(__pi___memmove = __pi_memmove); PROVIDE(__pi___memset = __pi_memset); diff --git a/arch/arm64/kernel/mpam.c b/arch/arm64/kernel/mpam.c new file mode 100644 index 0000000000000..3a490de4fa125 --- /dev/null +++ b/arch/arm64/kernel/mpam.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2025 Arm Ltd. */ + +#include + +#include +#include +#include +#include + +DEFINE_STATIC_KEY_FALSE(mpam_enabled); +DEFINE_PER_CPU(u64, arm64_mpam_default); +DEFINE_PER_CPU(u64, arm64_mpam_current); + +u64 arm64_mpam_global_default; + +static int mpam_pm_notifier(struct notifier_block *self, + unsigned long cmd, void *v) +{ + u64 regval; + int cpu = smp_processor_id(); + + switch (cmd) { + case CPU_PM_EXIT: + /* + * Don't use mpam_thread_switch() as the system register + * value has changed under our feet. + */ + regval = READ_ONCE(per_cpu(arm64_mpam_current, cpu)); + write_sysreg_s(regval | MPAM1_EL1_MPAMEN, SYS_MPAM1_EL1); + if (system_supports_sme()) { + write_sysreg_s(regval & (MPAMSM_EL1_PARTID_D | MPAMSM_EL1_PMG_D), + SYS_MPAMSM_EL1); + } + isb(); + + write_sysreg_s(regval, SYS_MPAM0_EL1); + + return NOTIFY_OK; + default: + return NOTIFY_DONE; + } +} + +static struct notifier_block mpam_pm_nb = { + .notifier_call = mpam_pm_notifier, +}; + +static int __init arm64_mpam_register_cpus(void) +{ + u64 mpamidr = read_sanitised_ftr_reg(SYS_MPAMIDR_EL1); + u16 partid_max = FIELD_GET(MPAMIDR_EL1_PARTID_MAX, mpamidr); + u8 pmg_max = FIELD_GET(MPAMIDR_EL1_PMG_MAX, mpamidr); + + if (!system_supports_mpam()) + return 0; + + cpu_pm_register_notifier(&mpam_pm_nb); + return mpam_register_requestor(partid_max, pmg_max); +} +/* Must occur before mpam_msc_driver_init() from subsys_initcall() */ +arch_initcall(arm64_mpam_register_cpus) diff --git a/arch/arm64/kernel/pi/kaslr_early.c b/arch/arm64/kernel/pi/kaslr_early.c index e0e018046a46a..d252df54abf78 100644 --- a/arch/arm64/kernel/pi/kaslr_early.c +++ b/arch/arm64/kernel/pi/kaslr_early.c @@ -37,6 +37,33 @@ static u64 __init get_kaslr_seed(void *fdt, int node) return ret; } +/* SMCCC TRNG_RND64 (DEN0098) == ARM_SMCCC_TRNG_RND64; hardcoded to avoid the + * heavy arm-smccc.h include in this position-independent early code. */ +#define PI_SMCCC_TRNG_RND64 0xC4000053UL + +static bool __init __early_el3_present(void) +{ + /* ID_AA64PFR0_EL1.EL3 [15:12] != 0 => EL3 implemented => smc is legal */ + return ((read_sysreg_s(SYS_ID_AA64PFR0_EL1) >> 12) & 0xf) != 0; +} + +static bool __init __early_trng_rnd64(u64 *out) +{ + register u64 x0 asm("x0") = PI_SMCCC_TRNG_RND64; + register u64 x1 asm("x1") = 64; /* request 64 bits */ + register u64 x2 asm("x2") = 0; + register u64 x3 asm("x3") = 0; + + asm volatile("smc #0" + : "+r"(x0), "+r"(x1), "+r"(x2), "+r"(x3) + : : "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", + "x12", "x13", "x14", "x15", "x16", "x17", "memory"); + if ((long)x0 != 0) /* SMCCC_RET_SUCCESS == 0 */ + return false; + *out = x3; /* 64 bits requested -> entropy in X3 */ + return true; +} + u64 __init kaslr_early_init(void *fdt, int chosen) { u64 seed, range; @@ -45,6 +72,27 @@ u64 __init kaslr_early_init(void *fdt, int chosen) return 0; seed = get_kaslr_seed(fdt, chosen); + + /* + * DRTM: the FDT kaslr-seed is attacker-visible, so prefer the PSC- + * backed firmware TRNG (SMCCC TRNG_RND64) for the kernel image offset. + * CPU RNDR is not exposed to NS on all DRTM platforms; the SMC is. + * Call it only when EL3 is implemented (else smc is UNDEF) and fall + * back to the FDT seed on failure -- PI code cannot panic; fail-closed + * enforcement lives in arm64_memblock_init(). + */ + if (IS_ENABLED(CONFIG_ARM64_SECURE_LAUNCH_KASLR) && + __early_el3_present()) { + u64 hw; + int tries; + + for (tries = 0; tries < 10; tries++) + if (__early_trng_rnd64(&hw)) { + seed = hw; + break; + } + } + if (!seed) { if (!__early_cpu_has_rndr() || !__arm64_rndr((unsigned long *)&seed)) diff --git a/arch/arm64/kernel/pi/map_kernel.c b/arch/arm64/kernel/pi/map_kernel.c index a852264958c36..96aaf51714cd8 100644 --- a/arch/arm64/kernel/pi/map_kernel.c +++ b/arch/arm64/kernel/pi/map_kernel.c @@ -43,6 +43,10 @@ static void __init map_kernel(u64 kaslr_offset, u64 va_offset, int root_level) phys_addr_t pgdp = (phys_addr_t)init_pg_dir + PAGE_SIZE; pgprot_t text_prot = PAGE_KERNEL_ROX; pgprot_t data_prot = PAGE_KERNEL; + void *efistub_start = IS_ENABLED(CONFIG_EFI) ? (void *)__efistub_start + : (void *)_end; + void *efistub_end = IS_ENABLED(CONFIG_EFI) ? (void *)__efistub_end + : (void *)_end; pgprot_t prot; /* @@ -92,8 +96,19 @@ static void __init map_kernel(u64 kaslr_offset, u64 va_offset, int root_level) __inittext_end, prot, false, root_level); map_segment(init_pg_dir, &pgdp, va_offset, __initdata_begin, __initdata_end, data_prot, false, root_level); - map_segment(init_pg_dir, &pgdp, va_offset, _data, _end, data_prot, - true, root_level); + /* + * Map the EFI stub's writable statics (the page-aligned .efistub + * section) as their own segment, without contiguous PTEs, so that no + * block or contiguous mapping straddles a reclaim boundary and + * free_initmem() can unmap [__efistub_start, __efistub_end) at page + * granularity. + */ + map_segment(init_pg_dir, &pgdp, va_offset, _data, efistub_start, + data_prot, true, root_level); + map_segment(init_pg_dir, &pgdp, va_offset, efistub_start, efistub_end, + data_prot, false, root_level); + map_segment(init_pg_dir, &pgdp, va_offset, efistub_end, _end, + data_prot, true, root_level); dsb(ishst); idmap_cpu_replace_ttbr1((phys_addr_t)init_pg_dir); diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 489554931231e..47698955fa1e4 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -738,6 +739,12 @@ struct task_struct *__switch_to(struct task_struct *prev, if (prev->thread.sctlr_user != next->thread.sctlr_user) update_sctlr_el1(next->thread.sctlr_user); + /* + * MPAM thread switch happens after the DSB to ensure prev's accesses + * use prev's MPAM settings. + */ + mpam_thread_switch(next); + /* the actual thread switch */ last = cpu_switch_to(prev, next); diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 23c05dc7a8f2a..35b92094cf365 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -289,6 +290,8 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) early_fixmap_init(); early_ioremap_init(); + slaunch_early_init(); + setup_machine_fdt(__fdt_pointer); /* @@ -319,8 +322,12 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) */ cpu_uninstall_idmap(); + slaunch_setup(); + slaunch_validate_initrd(); + xen_early_init(); efi_init(); + slaunch_reserve_dlme_data(); if (!efi_enabled(EFI_BOOT)) { if ((u64)_text % MIN_KIMG_ALIGN) @@ -333,11 +340,25 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) paging_init(); + /* + * Measure ACPI tables before acpi_boot_table_init() consumes them; + * after paging_init() so memblock_alloc() returns linear-mapped ptrs. + */ + slaunch_measure_post_efi(); + acpi_table_upgrade(); /* Parse the ACPI tables for possible boot-time configuration */ acpi_boot_table_init(); + /* + * DEN0113 defines the DRTM measurement/attestation chain over ACPI. + * A launched system that fell back to device tree runs on unmeasured + * topology with no valid attestation, so fail closed. + */ + if (sl_dlme_region_pa && acpi_disabled) + panic("slaunch: DRTM launch requires ACPI mode, but ACPI is disabled\n"); + if (acpi_disabled) unflatten_device_tree(); diff --git a/arch/arm64/kernel/sl_stub.S b/arch/arm64/kernel/sl_stub.S new file mode 100644 index 0000000000000..da2f7c46b4397 --- /dev/null +++ b/arch/arm64/kernel/sl_stub.S @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * ARM64 DRTM Secure Launch entry stub — DLME entry point. D-CRTM (TF-A) + * ERETs here per DEN0113 v1.2 §4.6.1 Table 33 with X0 = DLME region PA, + * X1 = data offset, MMU/D-cache off; recovers the DTB PA, branches to + * primary_entry for normal boot. + */ + +#include +#include +#include + + .section ".idmap.text","a" + +/* + * sl_entry — DRTM dynamic launch entry point. Only reached via DRTM + * ERET (DRTM_PARAMETERS entry_point_offset); never during normal boot. + */ +SYM_CODE_START(sl_entry) + /* + * Save DLME region PA for slaunch_setup(). MMU/D-cache off + * (DEN0113 v1.2 §4.6.1 Table 33), so the str hits DRAM; dc ivac + * (not cvac) drops the stale cache line so the kernel re-reads + * the fresh value once caches are on. + */ + adr_l x9, sl_dlme_region_pa + str x0, [x9] + dc ivac, x9 + + /* Same for DLME data offset. */ + adr_l x9, sl_dlme_data_offset + str x1, [x9] + dc ivac, x9 + dsb sy + + /* + * Load DTB PA from the Preamble->DLME slot at + * (region_base + data_offset + SL_DLME_DTB_SLOT_OFFSET). + */ + add x9, x0, x1 + ldr x0, [x9, #SL_DLME_DTB_SLOT_OFFSET] /* X0 = DTB PA */ + + /* Clear X1-X3 per arm64 boot protocol */ + mov x1, xzr + mov x2, xzr + mov x3, xzr + + /* Enter normal kernel boot with X0 = DTB PA */ + b primary_entry +SYM_CODE_END(sl_entry) + + .section .data + .align 3 +SYM_DATA(sl_dlme_region_pa, .quad 0) +SYM_DATA(sl_dlme_data_offset, .quad 0) diff --git a/arch/arm64/kernel/slaunch.c b/arch/arm64/kernel/slaunch.c new file mode 100644 index 0000000000000..48e139a8fc22e --- /dev/null +++ b/arch/arm64/kernel/slaunch.c @@ -0,0 +1,2341 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * ARM64 DRTM Secure Launch support. Processes DRTM state when the kernel + * is launched as a DLME via DRTM dynamic launch; called early in + * setup_arch(). Copyright (c) 2025-2026, NVIDIA Corporation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/* + * Hash algorithm dispatch. The DLME must match the D-CRTM firmware hash + * algo (DRTM_FEATURES feature 0x1) or the event-log chain cannot replay. + * Supports SHA-256 (0x000B) and SHA-384 (0x000C) via the kernel library + * API, whose arch SIMD paths are static-key gated until subsys_initcall. + */ +#define SL_TPM_ALG_SHA256 0x000B +#define SL_TPM_ALG_SHA384 0x000C + +enum sl_hash_algo { + SL_HASH_SHA256 = 0, + SL_HASH_SHA384 = 1, +}; + +#define SL_HASH_MAX_DIGEST_SIZE SHA512_DIGEST_SIZE /* 64 */ + +struct sl_hash_alg_info { + const char *name; + u16 tpm_alg_id; + u8 digest_size; +}; + +static const struct sl_hash_alg_info sl_hash_algs[] = { + [SL_HASH_SHA256] = { + .name = "SHA-256", + .tpm_alg_id = SL_TPM_ALG_SHA256, + .digest_size = SHA256_DIGEST_SIZE, + }, + [SL_HASH_SHA384] = { + .name = "SHA-384", + .tpm_alg_id = SL_TPM_ALG_SHA384, + .digest_size = SHA384_DIGEST_SIZE, + }, +}; + +/* Active algo for the lifetime of this boot. Default SHA-256 until + * slaunch_verify_hash_algo() updates it from DRTM_FEATURES. + */ +static enum sl_hash_algo sl_active_algo __initdata = SL_HASH_SHA256; + +static inline const struct sl_hash_alg_info *sl_active_alg_info(void) +{ + return &sl_hash_algs[sl_active_algo]; +} + +/* + * Hash one contiguous in-RAM buffer with the active algorithm; out must + * hold sl_active_alg_info()->digest_size bytes. Uses sha256()/sha384() + * (safe in setup_arch(); see the dispatch comment above). + */ +static void __init sl_hash_data(const void *data, size_t size, u8 *out) +{ + if (sl_active_algo == SL_HASH_SHA384) + sha384(data, size, out); + else + sha256(data, size, out); +} + +/* FDT magic number (big-endian 0xd00dfeed at offset 0) */ +#define FDT_HEADER_MAGIC 0xd00dfeed + +/* + * D-CRTM address map in the DLME data region (populated by D-CRTM, + * memblock_reserve'd by slaunch_setup()). Mapped once in early_init; + * pointer kept for all validation. No copy, no region-count cap. + */ +static struct drtm_mem_region *dcrtm_regions; +static u32 dcrtm_num_regions; + +/* + * DLME data extent saved at slaunch_setup() time so that + * slaunch_measure_post_efi() can re-reserve the region after efi_init()'s + * memblock_remove(0, PHYS_ADDR_MAX) wipes our earlier reservation. + */ +static phys_addr_t sl_dlme_data_pa; +static u64 sl_dlme_data_size; + +/* + * Validated UEFI SRTM TPM event log location, captured in + * slaunch_validate_raw_systab() so the main-vs-final-events "last writer + * wins" selection can prefer the primary log. The SRTM log is a pre-DRTM + * (untrusted) artifact, so it is validated for in-bounds extent only, not + * measured. Zero when firmware published no GUID (e.g. FVP without Tcg2Dxe). + */ +static phys_addr_t sl_srtm_log_pa; +static size_t sl_srtm_log_size; /* includes sizeof(linux_efi_tpm_eventlog) */ + +/* + * Raw EFI memory map extent saved by slaunch_validate_raw_mmap() and + * read by slaunch_validate_srtm_log() so the SRTM log extent can be + * rejected if it overlaps the raw mmap buffer. + * + * sl_efi_mmap_desc_size and sl_efi_nr_tables are additionally consumed + * by slaunch_extend_drtm_event_log() to compute a runtime upper bound + * for the kernel-side event-log buffer allocation. The bound scales with + * the firmware-reported UEFI input counts (one event per memory-map + * descriptor / ConfigurationTable entry in the upper bound), plus a small + * fixed-event budget for ACPI/CMDLINE/initrd/SRTM. + * + * These values come from pre-DRTM untrusted UEFI inputs; an attacker + * inflating them can only waste memory, not escalate (the kernel image + * hash covers these statics' writes via the past-_edata placement of + * EFI-stub writable globals). + */ +static phys_addr_t sl_efi_mmap_pa; +static u64 sl_efi_mmap_size; +static u32 sl_efi_mmap_desc_size; +static unsigned long sl_efi_nr_tables; + +/* + * Kernel-side DRTM event log buffer (memblock_alloc'd in + * slaunch_measure_post_efi(), not __initdata so securityfs can read it + * at runtime). DCE bytes are copied in first, then DLME TCG_PCR_EVENT2 + * records are appended here; the DCE buffer is never written back. + */ +static u8 *sl_kernel_evlog; +static size_t sl_kernel_evlog_size; +static size_t sl_kernel_evlog_capacity; + +/* + * Close TPM locality 2 — the DLME's locality, per DEN0113 v1.2 §4.6.1. + * Locality 3 is the DCE's; it's closed by the DCE, not the DLME. + */ +static void __init slaunch_tpm_setup(void) +{ + struct arm_smccc_res res; + + arm_smccc_smc(DRTM_SMC_CLOSE_LOCALITY, 2, 0, 0, 0, 0, 0, 0, &res); + if ((long)res.a0 == DRTM_NOT_SUPPORTED) + pr_warn("slaunch: CLOSE_LOCALITY not supported (no TPM backend)\n"); + else if ((long)res.a0 != DRTM_SUCCESS) + pr_err("slaunch: CLOSE_LOCALITY failed: %ld\n", (long)res.a0); + else + pr_info("slaunch: TPM locality 2 closed\n"); +} + +/* + * Re-enable Secure interrupts requested disabled via launch_features bit 7 + * (DEN0113 §3.11). Runs as the first post-launch step so no failable init + * path can leave them stranded disabled. NOT_SUPPORTED/DENIED mean they were + * never disabled; any other failure after an explicit request is ambiguous. + */ +static void __init slaunch_enable_secure_ints(void) +{ + struct arm_smccc_res res; + + arm_smccc_smc(DRTM_SMC_ENABLE_SECURE_INTERRUPTS, 0, 0, 0, 0, 0, 0, 0, &res); + if ((long)res.a0 == DRTM_NOT_SUPPORTED || + (long)res.a0 == DRTM_DENIED) + pr_info("slaunch: Secure interrupts not disabled by platform (%ld)\n", + (long)res.a0); + else if ((long)res.a0 != DRTM_SUCCESS) + panic("slaunch: ENABLE_SECURE_INTERRUPTS failed: %ld\n", + (long)res.a0); + else + pr_info("slaunch: Secure interrupts re-enabled\n"); +} + +/* Defined after the address-map parser below. */ +static const char *dcrtm_type_name(int type); + +/* + * Parse the D-CRTM address map (at header_size + protected_regions_size + * within DLME data). A trusted EL3-populated input describing physical + * memory layout; stored for validating untrusted data (DTB, EFI mmap). + * Returns true on success. + */ +static bool __init slaunch_parse_address_map(phys_addr_t dlme_data_pa, + struct dlme_data_header *hdr) +{ + phys_addr_t map_pa; + u64 map_size; + u64 hdr_size, prot_size; + struct drtm_mem_region_hdr *map_hdr; + struct drtm_mem_region *regions; + u32 num_regions, i; + + /* + * All offset arithmetic below assumes the v1 dlme_data_header + * layout; a later revision would target wrong bytes. Reject != v1. + */ + if (le16_to_cpu(hdr->version) != 1) + panic("slaunch: DLME data header version %u; only v1 supported\n", + le16_to_cpu(hdr->version)); + + hdr_size = le16_to_cpu(hdr->this_hdr_size); + prot_size = le64_to_cpu(hdr->protected_regions_size); + map_size = le64_to_cpu(hdr->address_map_size); + + if (map_size == 0) { + pr_err("slaunch: D-CRTM address map is empty\n"); + return false; + } + + if (map_size < sizeof(struct drtm_mem_region_hdr)) { + pr_err("slaunch: address map too small (%llu bytes)\n", + map_size); + return false; + } + + map_pa = dlme_data_pa + hdr_size + prot_size; + + /* Map temporarily to validate and log */ + map_hdr = early_memremap(map_pa, (size_t)map_size); + if (!map_hdr) { + pr_err("slaunch: failed to map address map at 0x%llx\n", + (u64)map_pa); + return false; + } + + num_regions = le32_to_cpu(map_hdr->num_regions); + pr_info("slaunch: D-CRTM address map: revision=%u, %u regions\n", + le16_to_cpu(map_hdr->revision), num_regions); + + if (sizeof(struct drtm_mem_region_hdr) + + (u64)num_regions * sizeof(struct drtm_mem_region) > map_size) { + pr_err("slaunch: address map regions overflow map size\n"); + early_memunmap(map_hdr, (size_t)map_size); + return false; + } + + /* Log regions for debug */ + regions = (struct drtm_mem_region *)((u8 *)map_hdr + + sizeof(struct drtm_mem_region_hdr)); + for (i = 0; i < num_regions; i++) { + u64 addr = le64_to_cpu(regions[i].start_address); + u64 st = le64_to_cpu(regions[i].size_and_type); + u64 pages = DRTM_MEM_REGION_PAGE_COUNT(st); + u32 type = DRTM_MEM_REGION_TYPE(st); + + pr_info("slaunch: [%u] 0x%012llx - 0x%012llx %s (%llu pages)\n", + i, addr, addr + pages * DRTM_PAGE_SIZE, + dcrtm_type_name(type), pages); + } + + early_memunmap(map_hdr, (size_t)map_size); + + /* + * Map regions and keep the pointer (physical memory is in DLME + * data, memblock_reserve'd later); validators use it directly. + */ + dcrtm_regions = early_memremap_ro(map_pa + sizeof(struct drtm_mem_region_hdr), + num_regions * sizeof(struct drtm_mem_region)); + if (!dcrtm_regions) { + pr_err("slaunch: failed to map address map regions\n"); + return false; + } + dcrtm_num_regions = num_regions; + return true; +} + +/* + * Verify D-CRTM published full-range DMA protection (DEN0113 v1.2 + * §4.6.2). Walks the protected_regions sub-region and requires the + * spec-conformant "single entry, start=0, full-range" encoding; a partial + * map breaks the measure-then-parse soundness model. + */ +static void __init slaunch_assert_dma_protection(phys_addr_t dlme_data_pa, + u64 hdr_size, u64 prot_size) +{ + const struct drtm_mem_region_hdr *phdr; + const struct drtm_mem_region *regs; + phys_addr_t prot_pa; + u32 num; + u64 start, st; + + if (prot_size == 0) + panic("slaunch: DCE published empty protected_regions; cannot verify SMMU DMA protection\n"); + if (prot_size < sizeof(*phdr) + sizeof(*regs)) + panic("slaunch: protected_regions size %llu too small for 1 entry\n", + prot_size); + + prot_pa = dlme_data_pa + hdr_size; + phdr = early_memremap_ro(prot_pa, (size_t)prot_size); + if (!phdr) + panic("slaunch: cannot map protected_regions at 0x%llx\n", + (u64)prot_pa); + + num = le32_to_cpu(phdr->num_regions); + regs = (const struct drtm_mem_region *)((const u8 *)phdr + sizeof(*phdr)); + start = le64_to_cpu(regs[0].start_address); + st = le64_to_cpu(regs[0].size_and_type); + + /* + * Strict spec match (DEN0113 v1.2 §3.15 R314110 + §4.6.2): single + * entry, start=0, size_and_type = DRTM_MEM_PROT_FULL_RANGE. Anything + * else is partial DMA protection or non-conformant — fatal. + */ + if (num != 1 || start != 0 || st != DRTM_MEM_PROT_FULL_RANGE) + panic("slaunch: SMMU DMA protection not full-range (num=%u start=0x%llx st=0x%llx; want 1/0/0x%llx); DRTM secure launch requires full DRAM coverage\n", + num, start, st, (u64)DRTM_MEM_PROT_FULL_RANGE); + + early_memunmap((void *)phdr, (size_t)prot_size); + pr_info("slaunch: SMMU DMA protection verified: full NS-DRAM coverage\n"); +} + +/* + * True if [start, start+size) falls entirely within normal memory of + * the D-CRTM address map. Normal = type 0 or type 1 (cacheability); + * DEVICE/NV/RSVD are rejected (DEN0113 v1.2 §3.14 Table 11 + R314100). + * Supports ranges spanning multiple adjacent/overlapping regions. + */ +static bool __init dcrtm_range_in_normal(u64 start, u64 size) +{ + u64 pos = start; + u64 end; + + if (!size || !dcrtm_regions) + return false; + /* Reject ranges that wrap u64 (attacker-controlled inputs). */ + if (check_add_overflow(start, size, &end)) + return false; + + while (pos < end) { + bool advanced = false; + u32 i; + + for (i = 0; i < dcrtm_num_regions; i++) { + u64 st = le64_to_cpu(dcrtm_regions[i].size_and_type); + u32 type = DRTM_MEM_REGION_TYPE(st); + u64 pages = DRTM_MEM_REGION_PAGE_COUNT(st); + u64 rstart = le64_to_cpu(dcrtm_regions[i].start_address); + u64 rsize, rend; + + if (type != DRTM_REGION_TYPE_NORMAL && + type != DRTM_REGION_TYPE_NORMAL_CACHED) + continue; + + /* Skip malformed regions whose size or end wraps. */ + if (check_mul_overflow(pages, (u64)DRTM_PAGE_SIZE, &rsize)) + continue; + if (check_add_overflow(rstart, rsize, &rend)) + continue; + + if (pos >= rstart && pos < rend) { + pos = (rend < end) ? rend : end; + advanced = true; + break; + } + } + + if (!advanced) + return false; + } + return true; +} + +/* + * Check if a range overlaps any non-normal region (DEVICE, NV, RSVD); + * type 0/1 normal memory is skipped (DEN0113 v1.2 §3.14 Table 11). + * Returns the region type if overlap found, -1 otherwise. + */ +static int __init dcrtm_range_overlaps_non_normal(u64 start, u64 size) +{ + u32 i; + u64 end; + + if (!dcrtm_regions) + return -1; + /* Wrapping range — fail closed: pretend it overlaps an RSVD region. */ + if (check_add_overflow(start, size, &end)) + return DRTM_REGION_TYPE_RSVD; + + for (i = 0; i < dcrtm_num_regions; i++) { + u64 st = le64_to_cpu(dcrtm_regions[i].size_and_type); + u32 type = DRTM_MEM_REGION_TYPE(st); + u64 pages = DRTM_MEM_REGION_PAGE_COUNT(st); + u64 rstart = le64_to_cpu(dcrtm_regions[i].start_address); + u64 rsize, rend; + + if (type == DRTM_REGION_TYPE_NORMAL || + type == DRTM_REGION_TYPE_NORMAL_CACHED) + continue; + + /* Malformed region whose size or end wraps: fail closed + * (assume it overlaps). + */ + if (check_mul_overflow(pages, (u64)DRTM_PAGE_SIZE, &rsize)) + return type; + if (check_add_overflow(rstart, rsize, &rend)) + return type; + + if (start < rend && end > rstart) + return type; + } + return -1; +} + +static const char * __init dcrtm_type_name(int type) +{ + switch (type) { + case DRTM_REGION_TYPE_NORMAL: return "NORMAL"; + case DRTM_REGION_TYPE_NORMAL_CACHED: return "NORMAL_CACHED"; + case DRTM_REGION_TYPE_DEVICE: return "DEVICE"; + case DRTM_REGION_TYPE_NV: return "NV"; + case DRTM_REGION_TYPE_RSVD: return "RSVD"; + default: return "UNKNOWN"; + } +} + +/* + * Check if two EFI memory regions overlap. + */ +static bool __init efi_regions_overlap(u64 s1, u64 sz1, u64 s2, u64 sz2) +{ + u64 e1, e2; + + /* Either range wraps u64 -> fail closed (report overlap). */ + if (check_add_overflow(s1, sz1, &e1) || + check_add_overflow(s2, sz2, &e2)) + return true; + return (s1 < e2) && (s2 < e1); +} + +/* True when @pa is kernel RAM or in the measured DLME extent; such a PA + * must never be device-mapped from the untrusted EFI map. False when DRTM + * is inactive; runtime-callable since the DLME handoff values persist. */ +bool slaunch_phys_is_protected_ram(phys_addr_t pa) +{ + if (!sl_dlme_region_pa) + return false; + if (memblock_is_map_memory(pa)) + return true; + return sl_dlme_data_size && + pa >= sl_dlme_region_pa && + pa < sl_dlme_data_pa + sl_dlme_data_size; +} + +/* True if [pa, pa+size) touches kernel RAM or the measured DLME extent. + * A range that wraps is refused. Used by the ACPI ioremap path, where a + * span may start outside protected memory and extend into it. */ +bool slaunch_phys_range_overlaps_protected_ram(phys_addr_t pa, size_t size) +{ + phys_addr_t p, end; + + if (!sl_dlme_region_pa) + return false; + if (check_add_overflow(pa, (phys_addr_t)size, &end)) + return true; + + if (sl_dlme_data_size && + pa < sl_dlme_data_pa + sl_dlme_data_size && + sl_dlme_region_pa < end) + return true; + + for (p = ALIGN_DOWN(pa, PAGE_SIZE); p < end; p += PAGE_SIZE) + if (memblock_is_map_memory(p)) + return true; + return false; +} + +/* + * slaunch_early_init() -- earliest DRTM validation, called BEFORE + * setup_machine_fdt() (after early_ioremap_init). Parses the D-CRTM + * address map, then validates the DTB PA/magic/size against it. Panics + * on any failure: the DTB is untrusted until validated. + */ +void __init slaunch_early_init(void) +{ + struct dlme_data_header *hdr; + phys_addr_t dlme_data_pa; + phys_addr_t dtb_pa; + __be32 *dtb_hdr; + u32 fdt_magic, fdt_size; + + if (!sl_dlme_region_pa) + return; + + /* First post-launch action: nothing failable may run before this. */ + slaunch_enable_secure_ints(); + + pr_info("slaunch: DRTM early init -- validating DTB before consumption\n"); + pr_info("slaunch: DLME region PA: 0x%lx, data offset: 0x%lx\n", + sl_dlme_region_pa, sl_dlme_data_offset); + + /* Step 1: Map DLME data header and parse address map */ + dlme_data_pa = sl_dlme_region_pa + sl_dlme_data_offset; + hdr = early_memremap(dlme_data_pa, sizeof(*hdr)); + if (!hdr) + panic("slaunch: failed to map DLME data header at 0x%llx\n", + (u64)dlme_data_pa); + + if (!slaunch_parse_address_map(dlme_data_pa, hdr)) + panic("slaunch: failed to parse D-CRTM address map -- cannot validate untrusted data\n"); + + early_memunmap(hdr, sizeof(*hdr)); + + /* Step 2: Validate DTB PA is in a NORMAL region */ + dtb_pa = __fdt_pointer; + if (!dtb_pa) + panic("slaunch: no DTB physical address available for validation\n"); + + pr_info("slaunch: DTB PA: 0x%llx\n", (u64)dtb_pa); + + /* + * Check the FDT header (magic + totalsize) is in a NORMAL region; + * the full-size check follows once fdt_totalsize is read. + */ + if (!dcrtm_range_in_normal(dtb_pa, sizeof(u32) * 2)) + panic("slaunch: DTB PA 0x%llx is NOT in a D-CRTM NORMAL region\n", + (u64)dtb_pa); + + /* Step 3: Map DTB header and verify FDT magic + size */ + dtb_hdr = early_memremap(dtb_pa, sizeof(u32) * 2); + if (!dtb_hdr) + panic("slaunch: failed to map DTB header at 0x%llx\n", + (u64)dtb_pa); + + fdt_magic = be32_to_cpu(dtb_hdr[0]); + fdt_size = be32_to_cpu(dtb_hdr[1]); + + early_memunmap(dtb_hdr, sizeof(u32) * 2); + + if (fdt_magic != FDT_HEADER_MAGIC) + panic("slaunch: DTB at 0x%llx has invalid FDT magic: 0x%08x (expected 0x%08x)\n", + (u64)dtb_pa, fdt_magic, FDT_HEADER_MAGIC); + + if (fdt_size == 0) + panic("slaunch: DTB at 0x%llx reports zero totalsize\n", + (u64)dtb_pa); + + /* + * Validate the full DTB range is in NORMAL memory; containment also + * caps fdt_size, so an over-large totalsize fails. + */ + if (!dcrtm_range_in_normal(dtb_pa, fdt_size)) + panic("slaunch: DTB range [0x%llx - 0x%llx] extends outside D-CRTM NORMAL region\n", + (u64)dtb_pa, (u64)(dtb_pa + fdt_size)); + + pr_info("slaunch: DTB validated: magic=0x%08x, size=%u bytes, in NORMAL region\n", + fdt_magic, fdt_size); +} + +/* + * Validate all untrusted EFI inputs BEFORE efi_init() consumes them + * (it reads systab tables and walks the EFI mmap into memblock). The + * early validators below operate on the raw firmware buffers, not + * efi.memmap, which does not exist yet at this stage. + */ +struct sl_efi_info { + bool present; + u64 systab_pa; + u64 mmap_pa; + u64 mmap_size; + u32 desc_size; + u32 desc_ver; +}; + +/* Forward decls — bodies defined later (or stubbed out by #ifdef + * gating below). + */ +#ifdef CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT +static void __init slaunch_inject_fault(struct sl_efi_info *info); +#else +static inline void slaunch_inject_fault(struct sl_efi_info *info) { } +#endif +#ifdef CONFIG_ARM64_SECURE_LAUNCH_SELFTEST +static void __init slaunch_selftest(void); +#else +static inline void slaunch_selftest(void) { } +#endif + +static void __init slaunch_read_chosen_efi(struct sl_efi_info *info) +{ + const void *fdt = initial_boot_params; + const __be64 *p64; + const __be32 *p32; + int node, len; + + memset(info, 0, sizeof(*info)); + if (!fdt) + return; + node = fdt_path_offset(fdt, "/chosen"); + if (node < 0) + return; + +#define _GET64(name, field) \ + do { \ + p64 = fdt_getprop(fdt, node, name, &len); \ + if (!p64 || len < (int)sizeof(__be64)) \ + return; \ + info->field = be64_to_cpu(*p64); \ + } while (0) +#define _GET32(name, field) \ + do { \ + p32 = fdt_getprop(fdt, node, name, &len); \ + if (!p32 || len < (int)sizeof(__be32)) \ + return; \ + info->field = be32_to_cpu(*p32); \ + } while (0) + _GET64("linux,uefi-system-table", systab_pa); + _GET64("linux,uefi-mmap-start", mmap_pa); + _GET32("linux,uefi-mmap-size", mmap_size); + _GET32("linux,uefi-mmap-desc-size", desc_size); + _GET32("linux,uefi-mmap-desc-ver", desc_ver); +#undef _GET64 +#undef _GET32 + info->present = true; +} + +/* + * Per-GUID minimum sizes for known EFI ConfigurationTable entries (spec + * entry-point minimums, e.g. RSDP v2.0+ = 36 B); the validator checks + * NORMAL containment of this size, not a flat 4 KB. Unknown GUIDs use + * SL_CFGTBL_UNKNOWN_BOUND. + */ +struct sl_cfgtbl_size_entry { + efi_guid_t guid; + u32 min_size; +}; + +static const struct sl_cfgtbl_size_entry sl_cfgtbl_sizes[] __initconst = { + { ACPI_20_TABLE_GUID, 36 }, /* RSDP v2.0+ */ + { SMBIOS3_TABLE_GUID, 24 }, /* SMBIOS3 entry point */ + { EFI_RT_PROPERTIES_TABLE_GUID, 8 }, /* version + flags */ + { LINUX_EFI_MEMRESERVE_TABLE_GUID, 32 }, /* header struct */ + { LINUX_EFI_RANDOM_SEED_TABLE_GUID, 32 }, /* header struct */ +}; + +#define SL_CFGTBL_UNKNOWN_BOUND EFI_PAGE_SIZE + +static u32 __init sl_cfgtbl_min_size(const efi_guid_t *guid) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(sl_cfgtbl_sizes); i++) { + if (efi_guidcmp(*guid, sl_cfgtbl_sizes[i].guid) == 0) + return sl_cfgtbl_sizes[i].min_size; + } + return SL_CFGTBL_UNKNOWN_BOUND; +} + +/* Forward decl — definition lives near slaunch_measure_initrd. */ +static bool __init slaunch_ranges_overlap(u64 a_start, u64 a_size, + u64 b_start, u64 b_size); + +/* + * Validate a UEFI SRTM TPM event log Configuration Table entry against + * the D-CRTM address map: cover the full header+body extent and reject + * overlap with DLME/DTB/mmap so firmware cannot alias an input. The log + * is a pre-DRTM artifact, so this validates its extent only (it is not + * measured); on success captures sl_srtm_log_pa/size. + */ +static void __init slaunch_validate_srtm_log(u64 log_pa, + const efi_guid_t *guid) +{ + efi_guid_t main_guid = LINUX_EFI_TPM_EVENT_LOG_GUID; + efi_guid_t final_guid = EFI_TCG2_FINAL_EVENTS_TABLE_GUID; + size_t hdr_size; + u64 body_size; + u64 total_size; + u64 dlme_size; + u64 fdt_size = 0; + phys_addr_t dtb_pa = __fdt_pointer; + bool is_main = (efi_guidcmp(*guid, main_guid) == 0); + bool is_final = (efi_guidcmp(*guid, final_guid) == 0); + + if (!is_main && !is_final) + return; + + if (!log_pa) + panic("slaunch: SRTM log GUID entry has NULL vendor_table\n"); + + hdr_size = is_main ? sizeof(struct linux_efi_tpm_eventlog) + : sizeof(struct efi_tcg2_final_events_table); + + /* Header extent must be in NORMAL before we can read the size + * field. dcrtm_range_in_normal rejects size==0 and u64 wrap. */ + if (!dcrtm_range_in_normal(log_pa, hdr_size)) + panic("slaunch: SRTM log PA 0x%llx (header %zu B) NOT in NORMAL region\n", + log_pa, hdr_size); + + /* Header extent must not alias the DLME region, DTB, or raw EFI + * mmap buffer (an attacker could otherwise coerce the validator + * into mis-interpreting attestation-critical bytes). */ + dlme_size = (sl_dlme_data_pa + sl_dlme_data_size) - sl_dlme_region_pa; + if (slaunch_ranges_overlap(log_pa, hdr_size, + sl_dlme_region_pa, dlme_size)) + panic("slaunch: SRTM log header [0x%llx+%zu] overlaps DLME region [0x%llx+%llu]\n", + log_pa, hdr_size, (u64)sl_dlme_region_pa, dlme_size); + + if (dtb_pa) { + /* Read fdt_totalsize from the (already-validated) DTB. */ + __be32 *p = early_memremap(dtb_pa, sizeof(u32) * 2); + + if (p) { + fdt_size = be32_to_cpu(p[1]); + early_memunmap(p, sizeof(u32) * 2); + } + } + if (fdt_size && slaunch_ranges_overlap(log_pa, hdr_size, + dtb_pa, fdt_size)) + panic("slaunch: SRTM log header [0x%llx+%zu] overlaps DTB [0x%llx+%llu]\n", + log_pa, hdr_size, (u64)dtb_pa, fdt_size); + + if (sl_efi_mmap_size && + slaunch_ranges_overlap(log_pa, hdr_size, + sl_efi_mmap_pa, sl_efi_mmap_size)) + panic("slaunch: SRTM log header [0x%llx+%zu] overlaps raw EFI mmap [0x%llx+%llu]\n", + log_pa, hdr_size, (u64)sl_efi_mmap_pa, + sl_efi_mmap_size); + + /* Read the firmware-published body size from the header. */ + if (is_main) { + struct linux_efi_tpm_eventlog *log_tbl; + + log_tbl = early_memremap(log_pa, hdr_size); + if (!log_tbl) + panic("slaunch: SRTM log header remap failed at 0x%llx\n", + log_pa); + body_size = log_tbl->size; + early_memunmap(log_tbl, hdr_size); + } else { + struct efi_tcg2_final_events_table *final_tbl; + + final_tbl = early_memremap(log_pa, hdr_size); + if (!final_tbl) + panic("slaunch: SRTM final-events header remap failed at 0x%llx\n", + log_pa); + /* + * Final-events per-event encoding needs the primary log's + * algorithm-set descriptor we may not have seen, so treat + * events[] as opaque. version must be 1 per TCG2 ACPI spec; + * reject otherwise before nr_events. + */ + if (final_tbl->version != 1) + panic("slaunch: SRTM final-events table version %llu != 1\n", + final_tbl->version); + body_size = 0; + (void)final_tbl->nr_events; + early_memunmap(final_tbl, hdr_size); + } + + if (check_add_overflow(log_pa, (u64)hdr_size + body_size, + &total_size)) + panic("slaunch: SRTM log PA + size wraps u64 (pa 0x%llx + %zu + %llu)\n", + log_pa, hdr_size, body_size); + + total_size = (u64)hdr_size + body_size; + + if (!dcrtm_range_in_normal(log_pa, total_size)) + panic("slaunch: SRTM log [0x%llx+%llu] NOT in NORMAL region (full extent)\n", + log_pa, total_size); + + if (slaunch_ranges_overlap(log_pa, total_size, + sl_dlme_region_pa, dlme_size)) + panic("slaunch: SRTM log [0x%llx+%llu] overlaps DLME region [0x%llx+%llu]\n", + log_pa, total_size, (u64)sl_dlme_region_pa, dlme_size); + if (fdt_size && + slaunch_ranges_overlap(log_pa, total_size, dtb_pa, fdt_size)) + panic("slaunch: SRTM log [0x%llx+%llu] overlaps DTB [0x%llx+%llu]\n", + log_pa, total_size, (u64)dtb_pa, fdt_size); + if (sl_efi_mmap_size && + slaunch_ranges_overlap(log_pa, total_size, + sl_efi_mmap_pa, sl_efi_mmap_size)) + panic("slaunch: SRTM log [0x%llx+%llu] overlaps raw EFI mmap [0x%llx+%llu]\n", + log_pa, total_size, (u64)sl_efi_mmap_pa, + sl_efi_mmap_size); + + /* Last writer wins when both GUIDs are present — both are + * structurally validated above; the main TPM event log is the + * preferred measurement target. */ + if (is_main || !sl_srtm_log_pa) { + sl_srtm_log_pa = log_pa; + sl_srtm_log_size = (size_t)total_size; + } + pr_info("slaunch: SRTM TPM event log validated: PA 0x%llx, %llu B (%s)\n", + log_pa, total_size, + is_main ? "LINUX_EFI_TPM_EVENT_LOG" : "EFI_TCG2_FINAL_EVENTS"); +} + +/* Validate System Table + ConfigurationTable pointers against D-CRTM + * map. Runs pre-efi_init, panics on bad input. + */ +static void __init slaunch_validate_raw_systab(u64 systab_pa) +{ + efi_system_table_t *systab; + efi_config_table_t *cfgtbl; + unsigned long tables_pa; + unsigned long nr_tables; + size_t tbl_size; + unsigned long j; + + if (!dcrtm_range_in_normal(systab_pa, sizeof(efi_system_table_t))) + panic("slaunch: EFI System Table PA 0x%llx NOT in NORMAL region\n", + systab_pa); + + systab = early_memremap_ro(systab_pa, sizeof(efi_system_table_t)); + if (!systab) + panic("slaunch: failed to map EFI System Table at 0x%llx\n", + systab_pa); + + nr_tables = systab->nr_tables; + tables_pa = (unsigned long)systab->tables; + early_memunmap(systab, sizeof(efi_system_table_t)); + + if (nr_tables == 0 || !tables_pa) { + pr_info("slaunch: EFI System Table has no ConfigurationTable entries\n"); + return; + } + /* + * Reject nr_tables that overflow when multiplied by the per-entry + * size; NORMAL containment then caps any structurally too-large value. + */ + if (check_mul_overflow(nr_tables, + (unsigned long)sizeof(efi_config_table_t), + &tbl_size)) + panic("slaunch: EFI System Table nr_tables=%lu overflows tbl_size\n", + nr_tables); + + if (!dcrtm_range_in_normal(tables_pa, tbl_size)) + panic("slaunch: EFI ConfigurationTable array at 0x%lx NOT in NORMAL region\n", + tables_pa); + + cfgtbl = early_memremap_ro(tables_pa, tbl_size); + if (!cfgtbl) + panic("slaunch: failed to map ConfigurationTable at 0x%lx\n", + tables_pa); + + for (j = 0; j < nr_tables; j++) { + unsigned long tbl_ptr = (unsigned long)cfgtbl[j].table; + u32 size; + + if (!tbl_ptr) + continue; + size = sl_cfgtbl_min_size(&cfgtbl[j].guid); + if (!dcrtm_range_in_normal(tbl_ptr, size)) + panic("slaunch: EFI ConfigurationTable[%lu] 0x%lx [size %u] NOT in NORMAL region\n", + j, tbl_ptr, size); + /* If the entry advertises an SRTM TPM event log (header + * + variable-length body), extend the structural check to + * cover the full body extent (validate-only; the log is a + * pre-DRTM artifact and is not measured). */ + slaunch_validate_srtm_log((u64)tbl_ptr, &cfgtbl[j].guid); + } + early_memunmap(cfgtbl, tbl_size); + /* + * Save validated nr_tables so slaunch_extend_drtm_event_log() can + * scale the kernel event-log buffer allocation by the count of + * ConfigurationTable entries (upper bound on per-entry events). + */ + sl_efi_nr_tables = nr_tables; + pr_info("slaunch: early EFI System Table validation PASSED (%lu entries)\n", + nr_tables); +} + +/* + * Validate the raw EFI memory map at /chosen/linux,uefi-mmap-start. + * Walks descriptors at desc-size stride (NOT sizeof(efi_memory_desc_t), + * which can differ for forward compatibility), pre-efi_init. + */ +static void __init slaunch_validate_raw_mmap(const struct sl_efi_info *info) +{ + void *mmap; + u64 offset; + u32 ndesc, nchecked = 0; + + /* Validate desc-size / desc-ver / mmap-size sanity. */ + if (info->desc_ver != 1) + panic("slaunch: linux,uefi-mmap-desc-ver=%u (expected 1)\n", + info->desc_ver); + if (info->desc_size < sizeof(efi_memory_desc_t) || info->desc_size > 128) + panic("slaunch: linux,uefi-mmap-desc-size=%u out of sane range\n", + info->desc_size); + if (info->mmap_size == 0 || + info->mmap_size % info->desc_size != 0) + panic("slaunch: linux,uefi-mmap-size=%llu not a multiple of desc-size=%u\n", + info->mmap_size, info->desc_size); + + /* + * mmap_pa + mmap_size must not wrap u64 and must lie in a NORMAL + * region; containment caps mmap_size, rejecting over-large values. + */ + { + u64 mmap_end; + + if (check_add_overflow(info->mmap_pa, info->mmap_size, &mmap_end)) + panic("slaunch: linux,uefi-mmap [0x%llx + %llu] wraps u64\n", + info->mmap_pa, info->mmap_size); + } + + /* The full mmap buffer must be in NORMAL memory. */ + if (!dcrtm_range_in_normal(info->mmap_pa, info->mmap_size)) + panic("slaunch: linux,uefi-mmap [0x%llx+0x%llx] NOT entirely in NORMAL\n", + info->mmap_pa, info->mmap_size); + + mmap = early_memremap_ro(info->mmap_pa, info->mmap_size); + if (!mmap) + panic("slaunch: failed to map raw EFI mmap at 0x%llx (size %llu)\n", + info->mmap_pa, info->mmap_size); + + ndesc = (u32)(info->mmap_size / info->desc_size); + for (offset = 0; offset < info->mmap_size; offset += info->desc_size) { + efi_memory_desc_t *md = (efi_memory_desc_t *)((u8 *)mmap + offset); + u64 phys = md->phys_addr; + u64 region_size, phys_end; + int otype; + + /* Skip MMIO and EfiReservedMemoryType: secure carveouts are + * reserved in the UEFI map but excluded from the NS-DRAM D-CRTM + * map and never ingested as usable RAM, so need not be NORMAL. */ + if (md->type == EFI_MEMORY_MAPPED_IO || + md->type == EFI_MEMORY_MAPPED_IO_PORT_SPACE || + md->type == EFI_RESERVED_TYPE) + continue; + + /* num_pages * page_size overflow guard. */ + if (check_mul_overflow(md->num_pages, (u64)EFI_PAGE_SIZE, + ®ion_size)) + panic("slaunch: raw EFI mmap[%llu]: type=%u num_pages=%llu overflows u64\n", + offset / info->desc_size, md->type, md->num_pages); + /* phys_addr + region_size address-wrap guard. */ + if (region_size && + check_add_overflow(phys, region_size, &phys_end)) + panic("slaunch: raw EFI mmap[%llu]: [0x%012llx + 0x%llx] type=%u wraps u64\n", + offset / info->desc_size, phys, region_size, + md->type); + + if (!dcrtm_range_in_normal(phys, region_size)) + panic("slaunch: raw EFI mmap[%llu]: region [0x%012llx-0x%012llx] type=%u NOT in NORMAL\n", + offset / info->desc_size, phys, + phys + region_size, md->type); + + otype = dcrtm_range_overlaps_non_normal(phys, region_size); + if (otype >= 0) + panic("slaunch: raw EFI mmap[%llu]: region [0x%012llx-0x%012llx] type=%u OVERLAPS %s\n", + offset / info->desc_size, phys, + phys + region_size, md->type, + dcrtm_type_name(otype)); + nchecked++; + } + + /* Defense-in-depth: warn on any overlapping descriptor pair. + * First-match attribute lookups would let a planted MMIO descriptor + * order-pick the type for a measured PA; the acpi_os_ioremap consumer + * guard blocks the RAM/DLME case regardless, so this is non-fatal. */ + for (offset = 0; offset < info->mmap_size; offset += info->desc_size) { + efi_memory_desc_t *a = (efi_memory_desc_t *)((u8 *)mmap + offset); + u64 asz, off2; + + if (check_mul_overflow(a->num_pages, (u64)EFI_PAGE_SIZE, &asz)) + continue; + for (off2 = offset + info->desc_size; off2 < info->mmap_size; + off2 += info->desc_size) { + efi_memory_desc_t *b = + (efi_memory_desc_t *)((u8 *)mmap + off2); + u64 bsz; + + if (check_mul_overflow(b->num_pages, + (u64)EFI_PAGE_SIZE, &bsz)) + continue; + if (efi_regions_overlap(a->phys_addr, asz, + b->phys_addr, bsz)) + pr_warn("slaunch: raw EFI mmap descriptors [%llu]/[%llu] OVERLAP @ 0x%012llx (types %u/%u)\n", + offset / info->desc_size, + off2 / info->desc_size, + a->phys_addr, a->type, b->type); + } + } + + early_memunmap(mmap, info->mmap_size); + pr_info("slaunch: early raw EFI mmap validation PASSED (%u of %u descriptors checked)\n", + nchecked, ndesc); + + /* Remember raw mmap extent for later overlap checks (e.g. the SRTM + * TPM event log validator must reject a published log PA that aliases + * the firmware-provided memory map buffer). desc_size is also saved + * so slaunch_extend_drtm_event_log() can compute the descriptor count + * as part of the kernel event-log buffer sizing formula. */ + sl_efi_mmap_pa = info->mmap_pa; + sl_efi_mmap_size = info->mmap_size; + sl_efi_mmap_desc_size = info->desc_size; +} + +static void __init slaunch_validate_efi_early(const struct sl_efi_info *info) +{ + if (!info->present) { + pr_info("slaunch: /chosen does not have all linux,uefi-* properties — skipping early EFI validation\n"); + return; + } + /* + * Stage the raw EFI mmap extent before the systab validator so + * slaunch_validate_srtm_log() can reject an SRTM log PA aliasing it; + * validate_raw_mmap() re-publishes these after its own checks. + * desc_size is pre-staged for the event-log buffer sizing formula. + */ + sl_efi_mmap_pa = info->mmap_pa; + sl_efi_mmap_size = info->mmap_size; + sl_efi_mmap_desc_size = info->desc_size; + slaunch_validate_raw_systab(info->systab_pa); + slaunch_validate_raw_mmap(info); +} + +/* + * Process DRTM state early in setup_arch() (address map already parsed + * by slaunch_early_init): reserve DLME data in memblock, CLOSE_LOCALITY, + * disable EFI runtime services, and validate the DTB /chosen EFI pointers. + */ +void __init slaunch_setup(void) +{ + struct dlme_data_header *hdr; + phys_addr_t dlme_data_pa; + + if (!sl_dlme_region_pa) + return; + + pr_info("slaunch: DRTM Secure Launch detected\n"); + + /* Map DLME data header for reservation */ + dlme_data_pa = sl_dlme_region_pa + sl_dlme_data_offset; + hdr = early_memremap(dlme_data_pa, sizeof(*hdr)); + /* The header locates the protected-regions table and event log that + * the DMA-protection assertion and locality close below need; fail + * closed if it cannot be mapped rather than boot on with them skipped. */ + if (!hdr) + panic("slaunch: failed to map DLME data header at 0x%llx\n", + (u64)dlme_data_pa); + + pr_info("slaunch: DLME data version: %u\n", + le16_to_cpu(hdr->version)); + pr_info("slaunch: DLME data size: %llu\n", + le64_to_cpu(hdr->dlme_data_size)); + pr_info("slaunch: Event log size: %llu\n", + le64_to_cpu(hdr->drtm_event_log_size)); + + /* Enforce full-range DMA-protection assumption per DEN0113 v1.2 §4.6.2. + * Called from slaunch_setup (not slaunch_early_init) so panic + * prints — earlycon is registered by this point. + */ + slaunch_assert_dma_protection(dlme_data_pa, + le16_to_cpu(hdr->this_hdr_size), + le64_to_cpu(hdr->protected_regions_size)); + + /* Stash for slaunch_reserve_dlme_data() to reserve post-efi_init. */ + sl_dlme_data_pa = dlme_data_pa; + sl_dlme_data_size = le64_to_cpu(hdr->dlme_data_size); + + early_memunmap(hdr, sizeof(*hdr)); + + slaunch_tpm_setup(); + + /* + * A detected DRTM launch forces EFI runtime services off: their + * pointers come from untrusted pre-DRTM firmware. efi_disable_runtime() + * sets disable_runtime so arm_enable_runtime_services() never re-enables + * them; the bit/mask clear is defense in depth. + */ + efi_disable_runtime(); + clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); + efi.runtime_supported_mask = 0; + pr_info("slaunch: EFI runtime services disabled (DRTM launch)\n"); + + /* + * Validate all untrusted EFI inputs before efi_init() consumes them: + * read /chosen, optionally inject a fault, then run the validators. + */ + { + struct sl_efi_info efi_info; + + slaunch_read_chosen_efi(&efi_info); + slaunch_inject_fault(&efi_info); + slaunch_validate_efi_early(&efi_info); + } +} + +/* Reserve DLME data after efi_init's memblock_remove(0, PHYS_ADDR_MAX). */ +void __init slaunch_reserve_dlme_data(void) +{ + if (!sl_dlme_data_pa || !sl_dlme_data_size) + return; + memblock_reserve(sl_dlme_data_pa, sl_dlme_data_size); +} + +/* + * Record an in-memory SHA-256 measurement into slaunch_measurements[] + * for later replay into the DRTM event log on PCR 18. The table is + * memblock_alloc-backed and grows geometrically, so platforms with many + * ACPI tables are not capped (seeded by slaunch_measurements_init()). + */ +/* + * hash is sized for the largest digest (SHA-512, 64 B) so one struct + * serves SHA-256 and SHA-384; digest_size marks the live bytes and + * tpm_alg_id records the algorithm at measurement time. + */ +struct slaunch_measurement { + char desc[16]; + u8 hash[SL_HASH_MAX_DIGEST_SIZE]; + u8 digest_size; + u16 tpm_alg_id; +}; + +static struct slaunch_measurement *slaunch_measurements __initdata; +static unsigned int slaunch_measurement_count __initdata; +static unsigned int slaunch_measurement_capacity __initdata; + +static void __init slaunch_measurements_init(void) +{ + size_t bytes; + + if (slaunch_measurements) + return; + + slaunch_measurement_capacity = 256; + bytes = (size_t)slaunch_measurement_capacity * + sizeof(*slaunch_measurements); + slaunch_measurements = memblock_alloc(bytes, SMP_CACHE_BYTES); + if (!slaunch_measurements) + panic("slaunch: memblock_alloc for measurement table failed (%zu bytes)\n", + bytes); +} + +static void __init slaunch_measurements_reserve(unsigned int needed) +{ + struct slaunch_measurement *new_arr; + unsigned int new_cap; + size_t new_bytes, old_bytes; + + if (needed <= slaunch_measurement_capacity) + return; + + new_cap = slaunch_measurement_capacity ? + slaunch_measurement_capacity : 1; + while (new_cap < needed) + new_cap *= 2; + + new_bytes = (size_t)new_cap * sizeof(*slaunch_measurements); + old_bytes = (size_t)slaunch_measurement_capacity * + sizeof(*slaunch_measurements); + + new_arr = memblock_alloc(new_bytes, SMP_CACHE_BYTES); + if (!new_arr) + panic("slaunch: measurement table grow failed at %u entries\n", + slaunch_measurement_count); + + if (slaunch_measurements && old_bytes) { + memcpy(new_arr, slaunch_measurements, + slaunch_measurement_count * + sizeof(*slaunch_measurements)); + memblock_free(slaunch_measurements, old_bytes); + } + + slaunch_measurements = new_arr; + slaunch_measurement_capacity = new_cap; +} + +static void __init slaunch_measure(const char *desc, const void *data, + size_t size) +{ + const struct sl_hash_alg_info *ainfo = sl_active_alg_info(); + u8 hash[SL_HASH_MAX_DIGEST_SIZE]; + struct slaunch_measurement *m; + + /* + * Hash with the active algorithm via sl_hash_data() (sha256()/ + * sha384()), safe in setup_arch() since the arch SIMD fast-paths + * are static-key gated until subsys_initcall. + */ + sl_hash_data(data, size, hash); + + pr_info("slaunch: measured %s (%zu bytes) %s: %*phN\n", + desc, size, ainfo->name, + (int)ainfo->digest_size, hash); + + /* Future revision: also extend this hash into a hardware measurement + * engine (TPM HASH_START or platform-specific equivalent) so the + * measurement is anchored beyond the in-memory event log. + */ + + slaunch_measurements_reserve(slaunch_measurement_count + 1); + + m = &slaunch_measurements[slaunch_measurement_count++]; + strscpy(m->desc, desc, sizeof(m->desc)); + memcpy(m->hash, hash, ainfo->digest_size); + if (ainfo->digest_size < SL_HASH_MAX_DIGEST_SIZE) + memset(m->hash + ainfo->digest_size, 0, + SL_HASH_MAX_DIGEST_SIZE - ainfo->digest_size); + m->digest_size = ainfo->digest_size; + m->tpm_alg_id = ainfo->tpm_alg_id; +} + +/* + * Measure ACPI tables (RSDP, XSDT, and each XSDT-referenced table) while + * DMA protection is still active (before the slaunch_unprotect_memory + * late_initcall), so no device can tamper with them. The hashes are + * attestation evidence a remote verifier compares to known-good values. + */ +/* + * RSDP layout (ACPI 2.0+): we only need xsdt_physical_address at offset 24 + * and length at offset 20. Avoid depending on for the struct. + */ +/* Minimal ACPI table header — avoid full dependency. + * Full header is 36 bytes; XSDT entries follow after this. + */ +struct slaunch_acpi_hdr { + char signature[4]; + u32 length; + u8 revision; + u8 checksum; + char oem_id[6]; + char oem_table_id[8]; + u32 oem_revision; + u32 creator_id; + u32 creator_revision; +} __packed; + +#define RSDP_SIZE_V1 20 +#define RSDP_OFF_LEN 20 /* u32 length (ACPI 2.0+) */ +#define RSDP_OFF_XSDT 24 /* u64 xsdt_physical_address */ +#define RSDP_MIN_MAP 36 /* enough to read through xsdt_physical_address */ + +/* FADT field offsets (ACPI 6.x §5.2.9) — used to follow indirection + * to DSDT and FACS. Local copies to avoid pulling in . + */ +#define FADT_FIRMWARE_CTRL_OFF 36 /* u32 */ +#define FADT_DSDT_OFF 40 /* u32 */ +#define FADT_X_FIRMWARE_CTRL_OFF 132 /* u64, ACPI 2.0+ */ +#define FADT_X_DSDT_OFF 140 /* u64, ACPI 2.0+ */ + +/* Validate PA + length against D-CRTM map, then measure. Every + * failure is fatal — silent skip breaks attestation soundness. + */ +static void __init slaunch_measure_one_acpi(phys_addr_t pa, const char *desc) +{ + struct slaunch_acpi_hdr *tbl; + u32 tbl_len; + + if (!pa) + panic("slaunch: %s PA is 0 — cannot measure\n", desc); + if (!dcrtm_range_in_normal(pa, sizeof(*tbl))) + panic("slaunch: %s PA 0x%llx (header) NOT in NORMAL region\n", + desc, (u64)pa); + + tbl = early_memremap(pa, sizeof(*tbl)); + if (!tbl) + panic("slaunch: %s header remap failed at 0x%llx\n", + desc, (u64)pa); + tbl_len = tbl->length; + early_memunmap(tbl, sizeof(*tbl)); + + if (tbl_len < sizeof(*tbl)) + panic("slaunch: %s length %u < header size %zu\n", + desc, tbl_len, sizeof(*tbl)); + if (!dcrtm_range_in_normal(pa, tbl_len)) + panic("slaunch: %s [0x%llx+%u] NOT in NORMAL region\n", + desc, (u64)pa, tbl_len); + + tbl = early_memremap(pa, tbl_len); + if (!tbl) + panic("slaunch: %s full remap failed at 0x%llx (size %u)\n", + desc, (u64)pa, tbl_len); + slaunch_measure(desc, tbl, tbl_len); + early_memunmap(tbl, tbl_len); +} + +/* + * Query the D-CRTM TPM hash algorithm (DRTM_FEATURES feature 0x1, + * DEN0113 v1.2 §3.3), setting sl_active_algo so later measurements use + * the matching path; refuse to proceed on mismatch. Supports 0xB SHA-256 + * / 0xC SHA-384. FAULT_INJECT adds sha384_force / sha_algo_bad tokens. + */ +#define SL_DRTM_FW_HASH_MASK 0xFFFFULL + +#ifdef CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT +static bool __init sl_cmdline_has(const char *tok); +#endif + +static void __init slaunch_verify_hash_algo(void) +{ + struct arm_smccc_res res; + u64 features; + u32 algo; + bool fw_features_supported = true; + + /* + * Feature 0x1 = TPM features (bit 63 set per spec). TF-A returns + * a0 = 1 or DRTM_NOT_SUPPORTED, a1 = the tpm_features bitfield. + */ + arm_smccc_smc(DRTM_SMC_FEATURES, (1ULL << 63) | 0x1, + 0, 0, 0, 0, 0, 0, &res); + if ((long)res.a0 == DRTM_NOT_SUPPORTED) { + pr_warn("slaunch: DRTM_FEATURES(TPM) not supported; assuming SHA-256\n"); + fw_features_supported = false; + algo = SL_TPM_ALG_SHA256; + } else { + features = res.a1; + algo = features & SL_DRTM_FW_HASH_MASK; + } + + pr_info("slaunch: DCE firmware hash algorithm: 0x%x\n", algo); + +#ifdef CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT + /* + * Fault-injection override (FAULT_INJECT only): coerce a SHA-256 + * firmware (FVP) into the SHA-384 path to regression-test it without + * a separate TF-A build, plus a "bad algo" token to check the + * dispatch panics on an unsupported algorithm. + */ + if (sl_cmdline_has("slaunch_inject=sha384_force")) { + pr_warn("slaunch: INJECT: overriding hash algo -> SHA-384\n"); + algo = SL_TPM_ALG_SHA384; + } else if (sl_cmdline_has("slaunch_inject=sha_algo_bad")) { + pr_warn("slaunch: INJECT: overriding hash algo -> 0xDEAD (unsupported)\n"); + algo = 0xDEAD; + } +#endif + + switch (algo) { + case SL_TPM_ALG_SHA256: + sl_active_algo = SL_HASH_SHA256; + break; + case SL_TPM_ALG_SHA384: + sl_active_algo = SL_HASH_SHA384; + break; + default: + panic("slaunch: DCE reports hash algo 0x%x; kernel only implements SHA-256 (0xB) and SHA-384 (0xC). Add new algo to sl_hash_algs[] or use a supported DCE.\n", + algo); + } + + pr_info("slaunch: DLME will use %s for the measurement chain%s\n", + sl_active_alg_info()->name, + fw_features_supported ? "" : " (DRTM_FEATURES unavailable)"); +} + +static void __init slaunch_measure_acpi(void) +{ + struct slaunch_acpi_hdr *xsdt; + phys_addr_t rsdp_pa, xsdt_pa; + phys_addr_t dsdt_pa = 0, facs_pa = 0; + u32 rsdp_len, xsdt_len, num_entries, i; + u64 *entry_ptrs; + void *rsdp; + + /* Every failure below is fatal. The "kernel acts on unmeasured + * bytes" case breaks the attestation-based trust model. + */ + rsdp_pa = efi.acpi20; + if (rsdp_pa == EFI_INVALID_TABLE_ADDR || !rsdp_pa) + panic("slaunch: no ACPI RSDP in EFI System Table (DRTM requires ACPI)\n"); + + if (!dcrtm_range_in_normal(rsdp_pa, RSDP_MIN_MAP)) + panic("slaunch: RSDP PA 0x%llx NOT in NORMAL region\n", + (u64)rsdp_pa); + + rsdp = early_memremap(rsdp_pa, RSDP_MIN_MAP); + if (!rsdp) + panic("slaunch: RSDP header remap failed at 0x%llx\n", + (u64)rsdp_pa); + + /* ACPI 2.0+ RSDP must be >= 36 bytes so xsdt_pa (offset 24-31) is measured. */ + rsdp_len = *(u32 *)((u8 *)rsdp + RSDP_OFF_LEN); + if (rsdp_len < RSDP_MIN_MAP) + panic("slaunch: RSDP length %u < %u\n", + rsdp_len, (u32)RSDP_MIN_MAP); + xsdt_pa = *(u64 *)((u8 *)rsdp + RSDP_OFF_XSDT); + early_memunmap(rsdp, RSDP_MIN_MAP); + + if (!dcrtm_range_in_normal(rsdp_pa, rsdp_len)) + panic("slaunch: RSDP [0x%llx+%u] NOT in NORMAL region\n", + (u64)rsdp_pa, rsdp_len); + rsdp = early_memremap(rsdp_pa, rsdp_len); + if (!rsdp) + panic("slaunch: RSDP full remap failed at 0x%llx (size %u)\n", + (u64)rsdp_pa, rsdp_len); + slaunch_measure("RSDP", rsdp, rsdp_len); + early_memunmap(rsdp, rsdp_len); + + if (!dcrtm_range_in_normal(xsdt_pa, sizeof(*xsdt))) + panic("slaunch: XSDT PA 0x%llx NOT in NORMAL region\n", xsdt_pa); + xsdt = early_memremap(xsdt_pa, sizeof(*xsdt)); + if (!xsdt) + panic("slaunch: XSDT header remap failed at 0x%llx\n", xsdt_pa); + xsdt_len = xsdt->length; + early_memunmap(xsdt, sizeof(*xsdt)); + + if (xsdt_len < sizeof(*xsdt)) + panic("slaunch: XSDT length %u < header size %zu\n", + xsdt_len, sizeof(*xsdt)); + if (!dcrtm_range_in_normal(xsdt_pa, xsdt_len)) + panic("slaunch: XSDT [0x%llx+%u] NOT in NORMAL region\n", + xsdt_pa, xsdt_len); + xsdt = early_memremap(xsdt_pa, xsdt_len); + if (!xsdt) + panic("slaunch: XSDT full remap failed at 0x%llx (size %u)\n", + xsdt_pa, xsdt_len); + slaunch_measure("XSDT", xsdt, xsdt_len); + + /* Walk XSDT entries. Each is a 64-bit PA to a top-level ACPI table. */ + num_entries = (xsdt_len - sizeof(*xsdt)) / sizeof(u64); + entry_ptrs = (u64 *)((u8 *)xsdt + sizeof(*xsdt)); + + for (i = 0; i < num_entries; i++) { + struct slaunch_acpi_hdr *tbl; + u64 tbl_pa = entry_ptrs[i]; + u32 tbl_len; + char desc[32]; + + if (!dcrtm_range_in_normal(tbl_pa, sizeof(*tbl))) + panic("slaunch: XSDT entry[%u] PA 0x%llx (hdr) NOT in NORMAL\n", + i, tbl_pa); + tbl = early_memremap(tbl_pa, sizeof(*tbl)); + if (!tbl) + panic("slaunch: XSDT entry[%u] header remap failed at 0x%llx\n", + i, tbl_pa); + tbl_len = tbl->length; + early_memunmap(tbl, sizeof(*tbl)); + + if (tbl_len < sizeof(*tbl)) + panic("slaunch: XSDT entry[%u] length %u < header size %zu\n", + i, tbl_len, sizeof(*tbl)); + if (!dcrtm_range_in_normal(tbl_pa, tbl_len)) + panic("slaunch: XSDT entry[%u] [0x%llx+%u] NOT in NORMAL\n", + i, tbl_pa, tbl_len); + tbl = early_memremap(tbl_pa, tbl_len); + if (!tbl) + panic("slaunch: XSDT entry[%u] full remap failed at 0x%llx (size %u)\n", + i, tbl_pa, tbl_len); + + snprintf(desc, sizeof(desc), "ACPI:%.4s", tbl->signature); + slaunch_measure(desc, tbl, tbl_len); + + /* Capture FADT indirection while FADT is mapped. Prefer + * 64-bit X_* fields (ACPI 2.0+); fall back to 32-bit + * fields if FADT length is too short to carry them. + */ + if (!memcmp(tbl->signature, "FACP", 4)) { + if (tbl_len >= FADT_X_DSDT_OFF + sizeof(u64)) + memcpy(&dsdt_pa, + (u8 *)tbl + FADT_X_DSDT_OFF, + sizeof(u64)); + if (!dsdt_pa && + tbl_len >= FADT_DSDT_OFF + sizeof(u32)) { + u32 d32; + + memcpy(&d32, + (u8 *)tbl + FADT_DSDT_OFF, + sizeof(u32)); + dsdt_pa = d32; + } + if (tbl_len >= FADT_X_FIRMWARE_CTRL_OFF + sizeof(u64)) + memcpy(&facs_pa, + (u8 *)tbl + FADT_X_FIRMWARE_CTRL_OFF, + sizeof(u64)); + if (!facs_pa && + tbl_len >= FADT_FIRMWARE_CTRL_OFF + sizeof(u32)) { + u32 f32; + + memcpy(&f32, + (u8 *)tbl + FADT_FIRMWARE_CTRL_OFF, + sizeof(u32)); + facs_pa = f32; + } + } + early_memunmap(tbl, tbl_len); + } + + early_memunmap(xsdt, xsdt_len); + + /* + * Follow FADT indirections (not in XSDT): DSDT carries the AML + * executed by ACPICA — mandatory; FACS is optional on HW-reduced + * ACPI. Other indirect tables (BERT/ERST/HEST/EINJ/PCCT) are data, + * not boot-TCB, so they are deliberately not followed. + */ + if (!dsdt_pa) + panic("slaunch: FADT present but X_Dsdt/Dsdt both zero — DSDT cannot be measured\n"); + slaunch_measure_one_acpi(dsdt_pa, "DSDT"); + if (facs_pa) + slaunch_measure_one_acpi(facs_pa, "FACS"); + else + pr_info("slaunch: FACS absent (HW-reduced ACPI) — no measurement needed\n"); +} + +/* Half-open range overlap. Fails closed (returns true) on u64 wrap of + * either range — refuse to consume rather than silently miscompute. */ +static bool __init slaunch_ranges_overlap(u64 a_start, u64 a_size, + u64 b_start, u64 b_size) +{ + u64 a_end, b_end; + + if (check_add_overflow(a_start, a_size, &a_end)) + return true; + if (check_add_overflow(b_start, b_size, &b_end)) + return true; + return a_start < b_end && b_start < a_end; +} + +#ifdef CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT +static bool __init sl_cmdline_has(const char *tok); +static void __init slaunch_inject_initrd(u64 *start, u64 *size); +#endif + +/* + * Measure the bootloader-supplied initramfs. The buffer sits in + * untrusted DRAM not covered by D-CRTM, so its extent is validated + * before any read and failures are fatal. Absence is legitimate and + * logged (no marker hash); the verifier infers coverage from the event. + */ +/* Returns validated (post-inject) start/size, or false if no initrd. */ +static bool __init slaunch_validate_initrd_extents(u64 *out_start, u64 *out_size) +{ + u64 start = phys_initrd_start; + u64 size = phys_initrd_size; + u64 end, dlme_size; + + if (!start || !size) + return false; + +#ifdef CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT + slaunch_inject_initrd(&start, &size); +#endif + + if (!IS_ALIGNED(start, PAGE_SIZE)) + panic("slaunch: initrd start 0x%llx not page-aligned\n", + start); + + if (check_add_overflow(start, size, &end)) + panic("slaunch: initrd [0x%llx + %llu] wraps u64\n", + start, size); + + if (!dcrtm_range_in_normal(start, size)) + panic("slaunch: initrd [0x%llx+%llu] NOT in NORMAL region\n", + start, size); + + dlme_size = (sl_dlme_data_pa + sl_dlme_data_size) - sl_dlme_region_pa; + if (slaunch_ranges_overlap(start, size, + sl_dlme_region_pa, dlme_size)) + panic("slaunch: initrd [0x%llx+%llu] overlaps DLME region [0x%llx+%llu]\n", + start, size, (u64)sl_dlme_region_pa, dlme_size); + + *out_start = start; + *out_size = size; + return true; +} + +/* Called from slaunch_setup, before arm64_memblock_init consumes the extents. */ +void __init slaunch_validate_initrd(void) +{ + u64 start, size; + + if (!sl_dlme_region_pa) + return; + if (!slaunch_validate_initrd_extents(&start, &size)) + pr_info("slaunch: no initrd present, skipping measurement\n"); +} + +static void __init slaunch_measure_initrd(void) +{ + const struct sl_hash_alg_info *ainfo = sl_active_alg_info(); + u64 start, size; + u64 off = 0, remaining; + struct sha256_ctx sctx256; + struct sha384_ctx sctx384; + u8 hash[SL_HASH_MAX_DIGEST_SIZE]; + struct slaunch_measurement *m; + void *p; + + if (!slaunch_validate_initrd_extents(&start, &size)) + return; + + /* + * Chunked early_memremap + streaming hash with the active algorithm: + * map one page at a time, feed into the streaming context, finalise. + * Both APIs (sha256/sha384_init/update/final) are safe in setup_arch() + * (see the sl_hash_data() comment). + */ + if (sl_active_algo == SL_HASH_SHA384) + sha384_init(&sctx384); + else + sha256_init(&sctx256); + + remaining = size; + while (remaining > 0) { + size_t chunk = min_t(u64, remaining, (u64)PAGE_SIZE); + + p = early_memremap(start + off, chunk); + if (!p) + panic("slaunch: initrd chunk remap failed at 0x%llx (chunk %zu)\n", + start + off, chunk); + if (sl_active_algo == SL_HASH_SHA384) + sha384_update(&sctx384, p, chunk); + else + sha256_update(&sctx256, p, chunk); + early_memunmap(p, chunk); + off += chunk; + remaining -= chunk; + } + if (sl_active_algo == SL_HASH_SHA384) + sha384_final(&sctx384, hash); + else + sha256_final(&sctx256, hash); + + pr_info("slaunch: measured initrd (%llu bytes) %s: %*phN\n", + size, ainfo->name, (int)ainfo->digest_size, hash); + + slaunch_measurements_reserve(slaunch_measurement_count + 1); + m = &slaunch_measurements[slaunch_measurement_count++]; + strscpy(m->desc, "initrd", sizeof(m->desc)); + memcpy(m->hash, hash, ainfo->digest_size); + if (ainfo->digest_size < SL_HASH_MAX_DIGEST_SIZE) + memset(m->hash + ainfo->digest_size, 0, + SL_HASH_MAX_DIGEST_SIZE - ainfo->digest_size); + m->digest_size = ainfo->digest_size; + m->tpm_alg_id = ainfo->tpm_alg_id; +} + +/* + * Extend the DRTM event log with DLME-side measurements. The kernel + * allocates its OWN buffer (sl_kernel_evlog), copies the DCE-published + * bytes in (read-only), and appends one bounds-checked TCG_PCR_EVENT2 + * per measurement; the combined log is exposed via securityfs. + */ +/* + * TODO: no Arm event type (DEN0113 v1.2 §3.17.2 Table 19, base 0x9000) + * matches a DLME-side ACPI measurement, so generic TCG + * EV_PLATFORM_CONFIG_FLAGS (0x0A) is used; switch to a dedicated + * EVTYPE_ARM_* once one is registered. + */ +#define SL_EV_PLATFORM_CONFIG_FLAGS 0x0000000A +#define SL_DRTM_PCR_INDEX 18 /* DEN0113 v1.2: PCR[18] DLME schema, §4.8.4 Table 40 */ + +/* + * Per-event upper bound (TCG PFP §10.2.2): PCRIndex/EventType/count + * (12) + alg_id (2) + max digest (64) + EventSize (4) + desc (16) = 98, + * rounded to 128. Describes one event's shape, not a total-buffer cap. + */ +#define SL_TCG_EVENT2_MAX_BYTES 128 + +/* + * Fixed-event budget: events not scaled by efi_nr_tables/efi_mmap_descs + * (CMDLINE, initrd, SRTM_TPM_LOG) plus headroom. 16 covers the existing + * 4 with margin. + */ +#define SL_DLME_FIXED_EVENTS 16 + +/* + * Constant headroom for small overhead beyond pure TCG_PCR_EVENT2 + * records (future header prefix or padding); 1 KiB, not a buffer cap. + */ +#define SL_KEVLOG_HEADROOM 1024 + +/* + * Sanity cap on the DCE-published event log size the kernel copies: an + * absolute structural bound (1 MiB, well above any realistic log) to + * refuse pathological firmware sizes arithmetic could miss. + */ +#define SL_DCE_EVLOG_MAX (1ULL * 1024 * 1024) + +/* + * Write one TCG_PCR_EVENT2 at *off in buf (advances *off); -ENOSPC if no + * room within max. Packed LE: u32 PCRIndex, EventType, digest_count; + * per digest u16 hashAlg + hash[digest_size]; u32 EventSize + Event[]. + * digest_size/alg are variable (SHA-256 32 B, SHA-384 48 B) per record. + */ +static int __init sl_evlog_append_event2(u8 *buf, size_t *off, size_t max, + u32 pcr, u32 type, + u16 tpm_alg_id, + const u8 *hash, u8 digest_size, + const void *event_data, + u32 event_size) +{ + size_t need; + u8 *p; + + if (digest_size == 0 || digest_size > SL_HASH_MAX_DIGEST_SIZE) + return -EINVAL; + + need = 4 + 4 + 4 + 2 + digest_size + 4 + event_size; + + if (*off + need > max) + return -ENOSPC; + + p = buf + *off; + *(__le32 *)(p + 0) = cpu_to_le32(pcr); + *(__le32 *)(p + 4) = cpu_to_le32(type); + *(__le32 *)(p + 8) = cpu_to_le32(1); + *(__le16 *)(p + 12) = cpu_to_le16(tpm_alg_id); + memcpy(p + 14, hash, digest_size); + *(__le32 *)(p + 14 + digest_size) = cpu_to_le32(event_size); + if (event_size) + memcpy(p + 14 + digest_size + 4, event_data, event_size); + *off += need; + return 0; +} + +static void __init slaunch_extend_drtm_event_log(void) +{ + phys_addr_t dlme_data_pa, evlog_pa; + struct dlme_data_header *hdr; + u64 hdr_size, prot_size, map_size, dlme_data_size; + u64 evlog_size_initial; + size_t evlog_max, evlog_off; + u8 *evlog_va; + unsigned int i; + u64 efi_mmap_descs; + size_t kbuf_cap; + u64 per_event_budget; + + if (!sl_dlme_region_pa) + return; + + dlme_data_pa = sl_dlme_region_pa + sl_dlme_data_offset; + + /* Read offsets from the DLME data header. Unmap before we touch + * the event log to avoid fixmap-slot overlap on the same page. + */ + hdr = early_memremap(dlme_data_pa, sizeof(*hdr)); + if (!hdr) + panic("slaunch: event log: cannot map DLME data header\n"); + hdr_size = le16_to_cpu(hdr->this_hdr_size); + prot_size = le64_to_cpu(hdr->protected_regions_size); + map_size = le64_to_cpu(hdr->address_map_size); + dlme_data_size = le64_to_cpu(hdr->dlme_data_size); + evlog_size_initial = le64_to_cpu(hdr->drtm_event_log_size); + early_memunmap(hdr, sizeof(*hdr)); + + if (evlog_size_initial == 0) + panic("slaunch: DCE published empty DRTM event log\n"); + + /* + * DCE event log runs from after header+protected_regions+address_map + * to the end of the DLME data region; guard the sub-region + * arithmetic against u64 wrap/underflow before subtracting. + */ + { + u64 sub_total; + + if (check_add_overflow(hdr_size, prot_size, &sub_total) || + check_add_overflow(sub_total, map_size, &sub_total)) + panic("slaunch: DLME header sub-region sizes wrap u64 (hdr=%llu prot=%llu map=%llu)\n", + hdr_size, prot_size, map_size); + if (sub_total > sl_dlme_data_size) + panic("slaunch: DLME header sub-region total %llu exceeds dlme_data_size %llu\n", + sub_total, sl_dlme_data_size); + + evlog_pa = dlme_data_pa + sub_total; + evlog_max = (size_t)(sl_dlme_data_size - sub_total); + } + + if (evlog_size_initial > SL_DCE_EVLOG_MAX) + panic("slaunch: DCE-published event log size %llu exceeds 1 MiB sanity cap\n", + evlog_size_initial); + if (evlog_size_initial > (u64)evlog_max) + panic("slaunch: DCE-published event log size %llu exceeds DLME-data event_log capacity %zu\n", + evlog_size_initial, evlog_max); + + /* + * Allocate the kernel event-log buffer: dce_evlog_size + + * (efi_nr_tables + efi_mmap_descs + SL_DLME_FIXED_EVENTS) * + * SL_TCG_EVENT2_MAX_BYTES + SL_KEVLOG_HEADROOM. Untrusted UEFI counts + * only bound the size; appended bytes are all validated. + */ + efi_mmap_descs = sl_efi_mmap_desc_size ? + sl_efi_mmap_size / sl_efi_mmap_desc_size : 0; + + per_event_budget = (u64)sl_efi_nr_tables + efi_mmap_descs + + SL_DLME_FIXED_EVENTS; + /* + * Multiplication guard for per_event_budget * SL_TCG_EVENT2_MAX_BYTES; + * bounded in practice, but check explicitly before use. + */ + if (per_event_budget > SIZE_MAX / SL_TCG_EVENT2_MAX_BYTES) + panic("slaunch: event log capacity formula overflow (nr_tables=%lu mmap_descs=%llu)\n", + sl_efi_nr_tables, efi_mmap_descs); + + kbuf_cap = (size_t)evlog_size_initial + + (size_t)per_event_budget * SL_TCG_EVENT2_MAX_BYTES + + SL_KEVLOG_HEADROOM; + + sl_kernel_evlog = memblock_alloc(kbuf_cap, SMP_CACHE_BYTES); + if (!sl_kernel_evlog) + panic("slaunch: memblock_alloc for kernel event log buffer failed (%zu bytes)\n", + kbuf_cap); + sl_kernel_evlog_capacity = kbuf_cap; + sl_kernel_evlog_size = 0; + + pr_info("slaunch: kernel event log: VA %p, capacity %zu B (DCE=%llu + (tables=%lu + mmap=%llu + fixed=%u)*%u + headroom=%u)\n", + sl_kernel_evlog, kbuf_cap, evlog_size_initial, + sl_efi_nr_tables, efi_mmap_descs, + SL_DLME_FIXED_EVENTS, SL_TCG_EVENT2_MAX_BYTES, + SL_KEVLOG_HEADROOM); + + pr_info("slaunch: DCE event log: PA 0x%llx, DLME-data sub-region capacity %zu B, DCE-published %llu B\n", + (u64)evlog_pa, evlog_max, evlog_size_initial); + + /* + * Copy DCE's published event log into the kernel buffer (the only + * read of the DCE buffer; later appends target sl_kernel_evlog). + * Done while DLME-data is still under SMMU DMA protection + * (unprotect is a late_initcall), so no DMA TOCTOU is possible. + */ + evlog_va = early_memremap(evlog_pa, (size_t)evlog_size_initial); + if (!evlog_va) + panic("slaunch: cannot map DCE event log at 0x%llx (%llu B)\n", + (u64)evlog_pa, evlog_size_initial); + memcpy(sl_kernel_evlog, evlog_va, (size_t)evlog_size_initial); + early_memunmap(evlog_va, (size_t)evlog_size_initial); + sl_kernel_evlog_size = (size_t)evlog_size_initial; + + /* + * Append DLME-side measurements into the kernel buffer (NOT the + * DCE buffer). Every sl_evlog_append_event2() call bounds-checks + * against sl_kernel_evlog_capacity. + */ + evlog_off = sl_kernel_evlog_size; + for (i = 0; i < slaunch_measurement_count; i++) { + const struct slaunch_measurement *m = &slaunch_measurements[i]; + size_t dlen = strnlen(m->desc, sizeof(m->desc)); + + if (sl_evlog_append_event2(sl_kernel_evlog, &evlog_off, + sl_kernel_evlog_capacity, + SL_DRTM_PCR_INDEX, + SL_EV_PLATFORM_CONFIG_FLAGS, + m->tpm_alg_id, + m->hash, m->digest_size, + m->desc, (u32)dlen)) + panic("slaunch: kernel event log overflow at DLME entry %u (%s) — buffer cap=%zu, off=%zu\n", + i, m->desc, sl_kernel_evlog_capacity, evlog_off); + } + sl_kernel_evlog_size = evlog_off; + + pr_info("slaunch: kernel event log: copied %llu B from DCE, appended %u DLME entries (total %zu B, slack %zu B)\n", + evlog_size_initial, slaunch_measurement_count, + sl_kernel_evlog_size, + sl_kernel_evlog_capacity - sl_kernel_evlog_size); + + /* Canonical event-log bytes (DCE + DLME) are what a verifier + * replays; exposed via securityfs rather than dumped to the log. + */ + pr_info("slaunch: DRTM event log canonical size (DCE + DLME): %zu B\n", + sl_kernel_evlog_size); + + pr_info("slaunch: DLME event log entries (PCR %u, EV_PLATFORM_CONFIG_FLAGS):\n", + SL_DRTM_PCR_INDEX); + for (i = 0; i < slaunch_measurement_count; i++) { + const struct slaunch_measurement *m = &slaunch_measurements[i]; + const char *algo_name = + (m->tpm_alg_id == SL_TPM_ALG_SHA384) ? "SHA-384" : + (m->tpm_alg_id == SL_TPM_ALG_SHA256) ? "SHA-256" : + "UNKNOWN"; + + pr_info("slaunch: [%2u] %-12s %s: %*phN\n", + i, m->desc, algo_name, + (int)m->digest_size, m->hash); + } +} + +#ifdef CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT +/* + * Negative-test fault injection (slaunch_inject=): mutates the + * raw EFI mmap/systab/SRTM-log before slaunch_validate_efi_early so the + * validator panics. Tokens: mmap_*, systab_nr_tables_huge, srtm_log_*, + * sha384_force, sha_algo_bad. Not for production. + */ +static bool __init sl_cmdline_has(const char *tok) +{ + return strstr(boot_command_line, tok); +} + +/* Mutate one descriptor in the raw EFI mmap buffer. Returns true if + * we found a CONVENTIONAL_MEMORY descriptor and applied the mutation. + */ +typedef void (*sl_md_mutator_t)(efi_memory_desc_t *md); + +static bool __init slaunch_inject_raw_mmap(const struct sl_efi_info *info, + sl_md_mutator_t mutate) +{ + void *mmap; + u64 offset; + bool applied = false; + + mmap = early_memremap(info->mmap_pa, info->mmap_size); + if (!mmap) { + pr_warn("slaunch: INJECT: failed to map raw mmap\n"); + return false; + } + for (offset = 0; offset < info->mmap_size; offset += info->desc_size) { + efi_memory_desc_t *md = + (efi_memory_desc_t *)((u8 *)mmap + offset); + + if (md->type != EFI_CONVENTIONAL_MEMORY) + continue; + mutate(md); + applied = true; + break; + } + early_memunmap(mmap, info->mmap_size); + return applied; +} + +static void __init sl_mutate_mmap_wrap(efi_memory_desc_t *md) +{ + /* phys_addr + num_pages*4096 wraps past U64_MAX */ + md->num_pages = (~md->phys_addr / EFI_PAGE_SIZE) + 2; + pr_warn("slaunch: INJECT mmap_wrap on phys=0x%llx -> num_pages=%llu (expect panic 'wraps u64')\n", + md->phys_addr, md->num_pages); +} + +static void __init sl_mutate_mmap_pages_overflow(efi_memory_desc_t *md) +{ + /* num_pages * EFI_PAGE_SIZE itself overflows */ + md->num_pages = (U64_MAX / EFI_PAGE_SIZE) + 2; + pr_warn("slaunch: INJECT mmap_pages_overflow on phys=0x%llx -> num_pages=%llu (expect panic 'overflows u64')\n", + md->phys_addr, md->num_pages); +} + +/* Force two CONVENTIONAL descriptors to overlap so the overlap + * detector in slaunch_validate_raw_mmap fires its WARN (non-fatal). */ +static void __init slaunch_inject_mmap_overlap(const struct sl_efi_info *info) +{ + void *mmap; + u64 off; + efi_memory_desc_t *first = NULL; + + mmap = early_memremap(info->mmap_pa, info->mmap_size); + if (!mmap) { + pr_warn("slaunch: INJECT mmap_overlap: map failed\n"); + return; + } + for (off = 0; off < info->mmap_size; off += info->desc_size) { + efi_memory_desc_t *md = + (efi_memory_desc_t *)((u8 *)mmap + off); + + if (md->type != EFI_CONVENTIONAL_MEMORY) + continue; + if (!first) { + first = md; + continue; + } + md->phys_addr = first->phys_addr; + if (!md->num_pages) + md->num_pages = 1; + pr_warn("slaunch: INJECT mmap_overlap: 2nd CONV desc -> phys=0x%llx (expect 'OVERLAP')\n", + md->phys_addr); + early_memunmap(mmap, info->mmap_size); + return; + } + pr_warn("slaunch: INJECT mmap_overlap: <2 CONVENTIONAL descriptors\n"); + early_memunmap(mmap, info->mmap_size); +} + +/* Overwrite the first non-null EFI ConfigurationTable entry in the raw + * systab's cfgtbl array with a synthetic SRTM-log entry. Returns true + * if we managed to plant the entry. Used by srtm_log_* injectors only; + * not safe for production. */ +static bool __init slaunch_inject_srtm_cfgtbl(const struct sl_efi_info *info, + u64 vendor_pa) +{ + efi_guid_t srtm_guid = LINUX_EFI_TPM_EVENT_LOG_GUID; + efi_system_table_t *systab; + efi_config_table_t *cfgtbl; + unsigned long tables_pa; + unsigned long nr_tables; + size_t tbl_size; + unsigned long j; + bool applied = false; + + systab = early_memremap(info->systab_pa, sizeof(*systab)); + if (!systab) { + pr_warn("slaunch: INJECT srtm_log: systab remap failed\n"); + return false; + } + nr_tables = systab->nr_tables; + tables_pa = (unsigned long)systab->tables; + early_memunmap(systab, sizeof(*systab)); + + if (!nr_tables || !tables_pa) { + pr_warn("slaunch: INJECT srtm_log: systab has no cfgtbl entries\n"); + return false; + } + tbl_size = nr_tables * sizeof(*cfgtbl); + cfgtbl = early_memremap(tables_pa, tbl_size); + if (!cfgtbl) { + pr_warn("slaunch: INJECT srtm_log: cfgtbl remap failed\n"); + return false; + } + for (j = 0; j < nr_tables; j++) { + if (!cfgtbl[j].table) + continue; + cfgtbl[j].guid = srtm_guid; + cfgtbl[j].table = (void *)(unsigned long)vendor_pa; + applied = true; + break; + } + early_memunmap(cfgtbl, tbl_size); + return applied; +} + +static void __init slaunch_inject_srtm_log(const struct sl_efi_info *info) +{ + u64 vendor_pa; + const char *tag = NULL; + + if (sl_cmdline_has("slaunch_inject=srtm_log_overlap_dlme")) { + /* Aim the entry at the start of the DLME region. The header + * extent overlap check in slaunch_validate_srtm_log fires + * before any size field is read. */ + vendor_pa = sl_dlme_region_pa; + tag = "srtm_log_overlap_dlme"; + } else if (sl_cmdline_has("slaunch_inject=srtm_log_outside_normal")) { + /* FVP PL011 UART base — a DEVICE region per DEN0113 v1.2 + * Table 11. dcrtm_range_in_normal must reject. */ + vendor_pa = 0x1c090000ULL; + tag = "srtm_log_outside_normal"; + } else if (sl_cmdline_has("slaunch_inject=srtm_log_overlap_mmap")) { + /* + * Aim the entry at the raw EFI mmap PA: it is NORMAL so the + * range check passes and only the overlap-with-mmap guard in + * slaunch_validate_srtm_log rejects it. + */ + vendor_pa = info->mmap_pa; + tag = "srtm_log_overlap_mmap"; + } else { + return; + } + + if (!slaunch_inject_srtm_cfgtbl(info, vendor_pa)) + pr_warn("slaunch: INJECT %s: could not plant cfgtbl entry\n", + tag); + else + pr_warn("slaunch: INJECT %s: planted SRTM-log entry PA=0x%llx (expect panic)\n", + tag, vendor_pa); +} + +static void __init slaunch_inject_fault(struct sl_efi_info *info) +{ + if (!info->present) { + /* Nothing to inject into. */ + return; + } + + slaunch_inject_srtm_log(info); + + if (sl_cmdline_has("slaunch_inject=mmap_wrap")) { + if (!slaunch_inject_raw_mmap(info, sl_mutate_mmap_wrap)) + pr_warn("slaunch: INJECT mmap_wrap: no EFI_CONVENTIONAL_MEMORY descriptor found\n"); + return; + } + if (sl_cmdline_has("slaunch_inject=mmap_pages_overflow")) { + if (!slaunch_inject_raw_mmap(info, sl_mutate_mmap_pages_overflow)) + pr_warn("slaunch: INJECT mmap_pages_overflow: no EFI_CONVENTIONAL_MEMORY descriptor found\n"); + return; + } + if (sl_cmdline_has("slaunch_inject=mmap_overlap")) { + slaunch_inject_mmap_overlap(info); + return; + } + if (sl_cmdline_has("slaunch_inject=mmap_size_huge")) { + /* + * Inflate the local mmap size (a 48-multiple, so the + * multiple-of-desc guard is not what trips) past any NORMAL + * region so the containment check in validate_raw_mmap fires. + */ + info->mmap_size = (0x10000000000ULL / 48ULL) * 48ULL; + pr_warn("slaunch: INJECT mmap_size_huge: info->mmap_size=%llu (expect panic 'NOT entirely in NORMAL')\n", + info->mmap_size); + return; + } + if (sl_cmdline_has("slaunch_inject=systab_nr_tables_huge")) { + /* + * Inflate nr_tables in-place so nr_tables * sizeof(entry) + * overflows and check_mul_overflow in validate_raw_systab panics. + */ + efi_system_table_t *systab; + + systab = early_memremap(info->systab_pa, + sizeof(efi_system_table_t)); + if (!systab) { + pr_warn("slaunch: INJECT systab_nr_tables_huge: remap failed\n"); + return; + } + systab->nr_tables = + (ULONG_MAX / sizeof(efi_config_table_t)) + 1UL; + pr_warn("slaunch: INJECT systab_nr_tables_huge: nr_tables=%lu (expect panic 'overflows tbl_size')\n", + (unsigned long)systab->nr_tables); + early_memunmap(systab, sizeof(efi_system_table_t)); + return; + } +} + +/* + * Negative-test fault injection for initrd measurement (cmdline + * slaunch_inject=). Mutates the local start/size copies before + * validation, exercising the same path an attacker-controlled DTB would. + * Tokens: initrd_wrap, initrd_size_huge, initrd_outside_normal, _overlap_dlme. + */ +static void __init slaunch_inject_initrd(u64 *start, u64 *size) +{ + if (sl_cmdline_has("slaunch_inject=initrd_wrap")) { + *size = (~(*start)) + 2; + pr_warn("slaunch: INJECT initrd_wrap: start=0x%llx size=%llu (expect panic 'wraps u64')\n", + *start, *size); + return; + } + if (sl_cmdline_has("slaunch_inject=initrd_size_huge")) { + /* + * Force the initrd end past its NORMAL region with a size just + * below the u64 wrap, so check_add_overflow passes and the + * NORMAL-containment check is what panics. + */ + if (*start && U64_MAX - *start > 1ULL) + *size = U64_MAX - *start - 1ULL; + else + *size = (1ULL << 62); + pr_warn("slaunch: INJECT initrd_size_huge: size=%llu (expect panic 'NOT in NORMAL region')\n", + *size); + return; + } + if (sl_cmdline_has("slaunch_inject=initrd_outside_normal")) { + *start = 0x1c090000ULL; + *size = PAGE_SIZE; + pr_warn("slaunch: INJECT initrd_outside_normal: start=0x%llx (expect panic 'NOT in NORMAL region')\n", + *start); + return; + } + if (sl_cmdline_has("slaunch_inject=initrd_overlap_dlme")) { + *start = sl_dlme_region_pa; + *size = PAGE_SIZE; + pr_warn("slaunch: INJECT initrd_overlap_dlme: start=0x%llx (expect panic 'overlaps DLME')\n", + *start); + return; + } +} +#endif /* CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT */ + +#ifdef CONFIG_ARM64_SECURE_LAUNCH_SELFTEST +/* + * Self-test of the validation guards (end of slaunch_measure_post_efi). + * Calls the helpers with crafted wrapping inputs plus valid ones; the + * helpers must reject the bad and accept the good, else panic with a + * "selftest:" prefix that the harness flags as a regression. + */ +static void __init slaunch_selftest(void) +{ + /* T1: dcrtm_range_in_normal MUST reject wrapping start+size */ + if (dcrtm_range_in_normal(0xFFFFFFFFFFFFE000ULL, 0x10000ULL)) + panic("selftest: dcrtm_range_in_normal accepted wrapping range\n"); + + /* T2: dcrtm_range_in_normal MUST reject size==0 */ + if (dcrtm_range_in_normal(0x80000000ULL, 0)) + panic("selftest: dcrtm_range_in_normal accepted size=0\n"); + + /* T3: dcrtm_range_in_normal MUST accept a known-good NORMAL range + * (one page in NS DRAM). Regression check. + */ + if (!dcrtm_range_in_normal(0x80000000ULL, EFI_PAGE_SIZE)) + panic("selftest: dcrtm_range_in_normal rejected known NORMAL range (regression)\n"); + + /* T4: dcrtm_range_overlaps_non_normal MUST fail closed on wrap + * (returns RSVD instead of -1). + */ + if (dcrtm_range_overlaps_non_normal(0xFFFFFFFFFFFFE000ULL, + 0x10000ULL) != DRTM_REGION_TYPE_RSVD) + panic("selftest: dcrtm_range_overlaps_non_normal did not fail closed on wrap\n"); + + /* T5: efi_regions_overlap MUST fail closed (true) on either wrap */ + if (!efi_regions_overlap(0xFFFFFFFFFFFFE000ULL, 0x10000ULL, + 0x80000000ULL, EFI_PAGE_SIZE)) + panic("selftest: efi_regions_overlap did not fail closed on wrap\n"); + + /* T6: efi_regions_overlap on disjoint ranges MUST return false */ + if (efi_regions_overlap(0x80000000ULL, 0x1000ULL, + 0x90000000ULL, 0x1000ULL)) + panic("selftest: efi_regions_overlap reported false overlap on disjoint ranges\n"); + + /* T7: efi_regions_overlap on truly-overlapping ranges MUST return true */ + if (!efi_regions_overlap(0x80000000ULL, 0x10000ULL, + 0x80008000ULL, 0x10000ULL)) + panic("selftest: efi_regions_overlap missed real overlap\n"); + + /* T8: check_mul_overflow catches num_pages * EFI_PAGE_SIZE wrap + * (the macro behind the slaunch_validate_efi guard). + */ + { + u64 out; + + if (!check_mul_overflow((u64)0x10000000000000ULL, + (u64)EFI_PAGE_SIZE, &out)) + panic("selftest: check_mul_overflow missed num_pages*PAGE_SIZE wrap\n"); + } + + /* T9: slaunch_phys_is_protected_ram is the exact predicate + * acpi_os_ioremap/__acpi_get_mem_attribute gate on. It MUST flag the + * measured DLME region and MUST NOT flag a non-RAM PA. */ + if (sl_dlme_region_pa) { + if (!slaunch_phys_is_protected_ram(sl_dlme_region_pa)) + panic("selftest: slaunch_phys_is_protected_ram missed DLME region\n"); + if (slaunch_phys_is_protected_ram(0)) + panic("selftest: slaunch_phys_is_protected_ram flagged non-RAM PA\n"); + } + + pr_info("slaunch: ALL SELFTESTS PASSED (9/9)\n"); +} +#endif /* CONFIG_ARM64_SECURE_LAUNCH_SELFTEST */ + +/* + * Post-efi_init jobs that need efi_init's outputs: re-reserve DLME data + * (efi_init's memblock_remove wiped the slaunch_setup reservation), + * measure ACPI tables via efi.acpi20, and run the validation-helper + * self-test. Untrusted EFI inputs were already validated in slaunch_setup(). + */ +void __init slaunch_measure_post_efi(void) +{ + if (!sl_dlme_region_pa) + return; + + if (sl_dlme_data_size) + memblock_reserve(sl_dlme_data_pa, sl_dlme_data_size); + + slaunch_measurements_init(); + slaunch_selftest(); + slaunch_verify_hash_algo(); + slaunch_measure_acpi(); + + /* + * Measure the effective kernel command line (boot_command_line, the + * post-/chosen cmdline the kernel actually used). Per-deployment + * variance makes the hash vary, so verifier policy must allow for it + * (or pin it via CONFIG_CMDLINE_FORCE). + */ + slaunch_measure("CMDLINE", boot_command_line, strlen(boot_command_line)); + + slaunch_measure_initrd(); + slaunch_extend_drtm_event_log(); + + /* + * Release the dcrtm_regions early_memremap_ro slot; no further + * validators consume it. NULLing the pointer makes any stray + * post-init caller fail-fast in dcrtm_range_in_normal's guard. + */ + if (dcrtm_regions) { + early_memunmap(dcrtm_regions, + dcrtm_num_regions * sizeof(*dcrtm_regions)); + dcrtm_regions = NULL; + } +} + +/* + * Securityfs exposure for the kernel-side DRTM event log, mirroring the + * TPM "binary_bios_measurements" file. Same information-disclosure + * surface: measurement hashes are not secrets but the attestation + * primitive a remote verifier fetches. + */ +static ssize_t slaunch_evlog_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + if (!sl_kernel_evlog || !sl_kernel_evlog_size) + return 0; + return simple_read_from_buffer(buf, count, ppos, + sl_kernel_evlog, + sl_kernel_evlog_size); +} + +static const struct file_operations slaunch_evlog_fops = { + .owner = THIS_MODULE, + .read = slaunch_evlog_read, + .llseek = default_llseek, +}; + +static struct dentry *sl_securityfs_dir; +static struct dentry *sl_securityfs_evlog; + +static int __init slaunch_securityfs_init(void) +{ + if (!sl_kernel_evlog || !sl_kernel_evlog_size) + return 0; + + sl_securityfs_dir = securityfs_create_dir("slaunch", NULL); + if (IS_ERR(sl_securityfs_dir)) { + pr_warn("slaunch: securityfs_create_dir failed: %ld\n", + PTR_ERR(sl_securityfs_dir)); + sl_securityfs_dir = NULL; + return 0; + } + + sl_securityfs_evlog = securityfs_create_file("drtm_event_log", + 0440, + sl_securityfs_dir, + NULL, + &slaunch_evlog_fops); + if (IS_ERR(sl_securityfs_evlog)) { + pr_warn("slaunch: securityfs_create_file failed: %ld\n", + PTR_ERR(sl_securityfs_evlog)); + securityfs_remove(sl_securityfs_dir); + sl_securityfs_dir = NULL; + sl_securityfs_evlog = NULL; + return 0; + } + + pr_info("slaunch: securityfs/slaunch/drtm_event_log exposed (%zu B)\n", + sl_kernel_evlog_size); + return 0; +} +late_initcall(slaunch_securityfs_init); + +/* + * Release DRTM DMA protection after IOMMU/SMMU drivers have + * established their own DMA isolation. + */ +static int __init slaunch_unprotect_memory(void) +{ + struct arm_smccc_res res; + + if (!sl_dlme_region_pa) + return 0; + + pr_info("slaunch: Calling DRTM_UNPROTECT_MEMORY\n"); + arm_smccc_smc(DRTM_SMC_UNPROTECT_MEMORY, 0, 0, 0, 0, 0, 0, 0, &res); + if ((long)res.a0 != DRTM_SUCCESS) { + pr_err("slaunch: UNPROTECT_MEMORY failed: %ld\n", + (long)res.a0); + return -EIO; + } + + pr_info("slaunch: DMA protection released\n"); + return 0; +} +late_initcall(slaunch_unprotect_memory); diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 2d1e75263f033..1a7a0008de790 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -239,7 +239,20 @@ SECTIONS __init_begin = .; __inittext_begin = .; - INIT_TEXT_SECTION(8) + /* + * Inlined INIT_TEXT_SECTION(8) with extra .efistub.text/.rodata + * catchers so the renamed EFI-stub text/rodata stays inside + * [_sinittext, _einittext) and the PE/COFF .text range; otherwise + * the stub entry point lands in PE .data and the loader rejects it. + */ + . = ALIGN(8); + .init.text : { + _sinittext = .; + INIT_TEXT + *(.efistub.text .efistub.text.*) + *(.efistub.rodata .efistub.rodata.*) + _einittext = .; + } __exittext_begin = .; .exit.text : { @@ -270,7 +283,9 @@ SECTIONS INIT_CALLS CON_INITCALL INIT_RAM_FS - *(.init.altinstructions .init.bss) /* from the EFI stub */ + *(.init.altinstructions .init.bss) /* from PI / EFI stub */ + /* .efistub.data / .efistub.bss are placed past _edata + * in their own .efistub PROGBITS block, see below. */ } .exit.data : { EXIT_DATA @@ -327,9 +342,29 @@ SECTIONS } PECOFF_EDATA_PADDING - __pecoff_data_rawsize = ABSOLUTE(. - __initdata_begin); _edata = .; + /* + * EFI-stub writable statics (.efistub.data/.bss) placed AFTER _edata + * so the DRTM measurement bounds [_text, _edata) never cover + * stub-mutable data; __pecoff_data_rawsize is computed past + * __efistub_end. Stub text/rodata stay in PE .text (buckets above). + * The section is padded to a page so [__efistub_start, __efistub_end) + * owns its tail page exclusively and free_initmem() can free and + * unmap it on any page size; PAGE_SIZE is a multiple of the PE + * FileAlignment (0x200), so no separate PE padding is needed, and + * the zero fill is PROGBITS so `objcopy -O binary` writes the bytes. + */ + . = ALIGN(PAGE_SIZE); + __efistub_start = .; + .efistub : ALIGN(PAGE_SIZE) { + *(.efistub.data .efistub.data.*) + *(.efistub.bss .efistub.bss.*) + . = ALIGN(PAGE_SIZE); + } + __efistub_end = .; + __pecoff_data_rawsize = ABSOLUTE(. - __initdata_begin); + /* start of zero-init region */ BSS_SECTION(SBSS_ALIGN, 0, 0) __pi___bss_start = __bss_start; @@ -410,3 +445,12 @@ ASSERT(KEXEC_CONTROL_PAGE_SIZE >= SZ_4K, "KEXEC_CONTROL_PAGE_SIZE is broken") ASSERT(__relocate_new_kernel_start == arm64_relocate_new_kernel, "kexec control page does not start with arm64_relocate_new_kernel") #endif + +/* + * The EFI stub's writable statics are freed and unmapped after init; both + * boundaries must be page-aligned so the reclaim covers whole pages only. + */ +ASSERT((__efistub_start & (PAGE_SIZE - 1)) == 0, + "__efistub_start is not page aligned") +ASSERT((__efistub_end & (PAGE_SIZE - 1)) == 0, + "__efistub_end is not page aligned") diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 2597e8bda8672..0b50ddd530f3e 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -267,7 +267,8 @@ static inline void __deactivate_traps_hfgxtr(struct kvm_vcpu *vcpu) static inline void __activate_traps_mpam(struct kvm_vcpu *vcpu) { - u64 r = MPAM2_EL2_TRAPMPAM0EL1 | MPAM2_EL2_TRAPMPAM1EL1; + u64 clr = MPAM2_EL2_EnMPAMSM; + u64 set = MPAM2_EL2_TRAPMPAM0EL1 | MPAM2_EL2_TRAPMPAM1EL1; if (!system_supports_mpam()) return; @@ -277,18 +278,21 @@ static inline void __activate_traps_mpam(struct kvm_vcpu *vcpu) write_sysreg_s(MPAMHCR_EL2_TRAP_MPAMIDR_EL1, SYS_MPAMHCR_EL2); } else { /* From v1.1 TIDR can trap MPAMIDR, set it unconditionally */ - r |= MPAM2_EL2_TIDR; + set |= MPAM2_EL2_TIDR; } - write_sysreg_s(r, SYS_MPAM2_EL2); + sysreg_clear_set_s(SYS_MPAM2_EL2, clr, set); } static inline void __deactivate_traps_mpam(void) { + u64 clr = MPAM2_EL2_TRAPMPAM0EL1 | MPAM2_EL2_TRAPMPAM1EL1 | MPAM2_EL2_TIDR; + u64 set = MPAM2_EL2_EnMPAMSM; + if (!system_supports_mpam()) return; - write_sysreg_s(0, SYS_MPAM2_EL2); + sysreg_clear_set_s(SYS_MPAM2_EL2, clr, set); if (system_supports_mpam_hcr()) write_sysreg_s(MPAMHCR_HOST_FLAGS, SYS_MPAMHCR_EL2); diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 1b4cacb6e918a..0edd655934a97 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -3376,6 +3376,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { { SYS_DESC(SYS_MPAM1_EL1), undef_access }, { SYS_DESC(SYS_MPAM0_EL1), undef_access }, + { SYS_DESC(SYS_MPAMSM_EL1), undef_access }, + { SYS_DESC(SYS_VBAR_EL1), access_rw, reset_val, VBAR_EL1, 0 }, { SYS_DESC(SYS_DISR_EL1), NULL, reset_val, DISR_EL1, 0 }, diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 2ea9f1c7ebcd0..963d2699cd9a3 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -597,8 +597,10 @@ static void vgic_its_invalidate_cache(struct vgic_its *its) unsigned long idx; xa_for_each(&its->translation_cache, idx, irq) { - xa_erase(&its->translation_cache, idx); - vgic_put_irq(kvm, irq); + /* Only the context that erases the entry drops its cache ref. */ + irq = xa_erase(&its->translation_cache, idx); + if (irq) + vgic_put_irq(kvm, irq); } } diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index b9b248d24fd10..c256876b77da4 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -40,6 +40,8 @@ #include #include #include +#include +#include #include #include #include @@ -234,6 +236,43 @@ void __init arm64_memblock_init(void) memblock_remove(0, memstart_addr); } + /* + * DRTM: re-randomize the linear map from a trusted RNG. + * Coverage is bounded to present DRAM (memblock span) -- no hotplug + * reservation -- so the slack is (linear_region - span). The seed + * comes from the PSC-backed firmware TRNG, never the attacker-visible + * FDT seed; fail closed if no trusted entropy is available. + */ + if (IS_ENABLED(CONFIG_ARM64_SECURE_LAUNCH_KASLR) && slaunch_active()) { + s64 range = linear_region_size - + (memblock_end_of_DRAM() - memblock_start_of_DRAM()); + struct arm_smccc_res res; + unsigned long seed; + u64 nslots; + int tries; + + if (range < (s64)ARM64_MEMSTART_ALIGN) + panic("DRTM KASLR: linear region too small to randomize\n"); + /* + * Seed from the PSC-backed firmware TRNG (SMCCC TRNG_RND64), + * not CPU RNDR: RNDR is not exposed to NS on all DRTM + * platforms, but EL3 is (a Secure Launch arrives via SMC). + * Call the SMC directly -- the conduit predates the generic + * SMCCC probe. Retry a transiently empty pool; fail closed. + */ + for (tries = 0; tries < 10; tries++) { + arm_smccc_smc(ARM_SMCCC_TRNG_RND64, 64, 0, 0, 0, 0, 0, 0, &res); + if ((int)res.a0 == SMCCC_RET_SUCCESS) + break; + } + if ((int)res.a0 != SMCCC_RET_SUCCESS) + panic("DRTM KASLR: firmware TRNG unavailable, refusing weak KASLR\n"); + seed = res.a3; /* 64 bits requested -> entropy in X3 */ + + nslots = (u64)range / ARM64_MEMSTART_ALIGN; + memstart_addr -= ARM64_MEMSTART_ALIGN * (seed % nslots); + } + /* * If we are running with a 52-bit kernel VA config on a system that * does not support it, we have to place the available physical @@ -403,6 +442,21 @@ void free_initmem(void) * is not supported by kallsyms. */ vunmap_range((u64)__init_begin, (u64)__init_end); + + if (__efistub_start != __efistub_end) { + void *lm_begin = lm_alias(__efistub_start); + void *lm_end = lm_alias(__efistub_end); + + /* + * EFI-stub writable statics: dead after boot; freed and + * unmapped like initmem (the .efistub layout exists for all + * arm64 EFI builds). + */ + memblock_free(lm_begin, lm_end - lm_begin); + free_reserved_area(lm_begin, lm_end, + POISON_FREE_INITMEM, "unused efistub"); + vunmap_range((u64)__efistub_start, (u64)__efistub_end); + } } void dump_mem_limit(void) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 9d1c211080571..1287cb1de6f3c 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -5172,6 +5172,14 @@ Field 31:16 PARTID_D Field 15:0 PARTID_I EndSysreg +Sysreg MPAMSM_EL1 3 0 10 5 3 +Res0 63:48 +Field 47:40 PMG_D +Res0 39:32 +Field 31:16 PARTID_D +Res0 15:0 +EndSysreg + Sysreg ISR_EL1 3 0 12 1 0 Res0 63:11 Field 10 IS diff --git a/arch/x86/include/asm/resctrl.h b/arch/x86/include/asm/resctrl.h index 575f8408a9e7c..40a74a0617345 100644 --- a/arch/x86/include/asm/resctrl.h +++ b/arch/x86/include/asm/resctrl.h @@ -191,6 +191,11 @@ static inline void resctrl_arch_mon_ctx_free(struct rdt_resource *r, enum resctrl_event_id evtid, void *ctx) { } +static inline bool resctrl_arch_mon_can_overflow(void) +{ + return true; +} + void resctrl_cpu_detect(struct cpuinfo_x86 *c); #else diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index 7667cf7c4e945..00bc2e3ed0ff1 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -88,7 +88,6 @@ struct rdt_hw_resource rdt_resources_all[RDT_NUM_RESOURCES] = { .name = "MB", .ctrl_scope = RESCTRL_L3_CACHE, .ctrl_domains = ctrl_domain_init(RDT_RESOURCE_MBA), - .schema_fmt = RESCTRL_SCHEMA_RANGE, }, }, [RDT_RESOURCE_SMBA] = @@ -97,7 +96,6 @@ struct rdt_hw_resource rdt_resources_all[RDT_NUM_RESOURCES] = { .name = "SMBA", .ctrl_scope = RESCTRL_L3_CACHE, .ctrl_domains = ctrl_domain_init(RDT_RESOURCE_SMBA), - .schema_fmt = RESCTRL_SCHEMA_RANGE, }, }, [RDT_RESOURCE_PERF_PKG] = @@ -211,22 +209,23 @@ static __init bool __get_mem_config_intel(struct rdt_resource *r) cpuid_count(0x00000010, 3, &eax.full, &ebx, &ecx, &edx.full); hw_res->num_closid = edx.split.cos_max + 1; max_delay = eax.split.max_delay + 1; + r->schema_fmt = RESCTRL_SCHEMA_PERCENT; r->membw.max_bw = MAX_MBA_BW; - r->membw.arch_needs_linear = true; + r->mba.arch_needs_linear = true; if (ecx & MBA_IS_LINEAR) { - r->membw.delay_linear = true; + r->mba.delay_linear = true; r->membw.min_bw = MAX_MBA_BW - max_delay; r->membw.bw_gran = MAX_MBA_BW - max_delay; } else { if (!rdt_get_mb_table(r)) return false; - r->membw.arch_needs_linear = false; + r->mba.arch_needs_linear = false; } if (boot_cpu_has(X86_FEATURE_PER_THREAD_MBA)) - r->membw.throttle_mode = THREAD_THROTTLE_PER_THREAD; + r->mba.throttle_mode = THREAD_THROTTLE_PER_THREAD; else - r->membw.throttle_mode = THREAD_THROTTLE_MAX; + r->mba.throttle_mode = THREAD_THROTTLE_MAX; r->alloc_capable = true; @@ -246,17 +245,18 @@ static __init bool __rdt_get_mem_config_amd(struct rdt_resource *r) cpuid_count(0x80000020, subleaf, &eax, &ebx, &ecx, &edx); hw_res->num_closid = edx + 1; + r->schema_fmt = RESCTRL_SCHEMA__AMD_MBA; r->membw.max_bw = 1 << eax; /* AMD does not use delay */ - r->membw.delay_linear = false; - r->membw.arch_needs_linear = false; + r->mba.delay_linear = false; + r->mba.arch_needs_linear = false; /* * AMD does not use memory delay throttle model to control * the allocation like Intel does. */ - r->membw.throttle_mode = THREAD_THROTTLE_UNDEFINED; + r->mba.throttle_mode = THREAD_THROTTLE_UNDEFINED; r->membw.min_bw = 0; r->membw.bw_gran = 1; @@ -325,7 +325,7 @@ static void mba_wrmsr_amd(struct msr_param *m) */ static u32 delay_bw_map(unsigned long bw, struct rdt_resource *r) { - if (r->membw.delay_linear) + if (r->mba.delay_linear) return MAX_MBA_BW - bw; pr_warn_once("Non Linear delay-bw map not supported but queried\n"); @@ -378,7 +378,7 @@ static void setup_default_ctrlval(struct rdt_resource *r, u32 *dc) * For Memory Allocation: Set b/w requested to 100% */ for (i = 0; i < hw_res->num_closid; i++, dc++) - *dc = resctrl_get_default_ctrl(r); + *dc = resctrl_get_resource_default_ctrl(r); } static void ctrl_domain_free(struct rdt_hw_ctrl_domain *hw_dom) diff --git a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c index b20e705606b8f..91ce05256a004 100644 --- a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c +++ b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c @@ -16,9 +16,15 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include "internal.h" +u32 resctrl_arch_round_bw(u32 val, const struct resctrl_schema *s) +{ + return roundup(val, (unsigned long)s->membw.bw_gran); +} + int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d, u32 closid, enum resctrl_conf_type t, u32 cfg_val) { diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index 9bd87bae49834..794a6fb175e4e 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -454,6 +454,7 @@ int __init rdt_get_l3_mon_config(struct rdt_resource *r) (rdt_cpu_has(X86_FEATURE_CQM_MBM_TOTAL) || rdt_cpu_has(X86_FEATURE_CQM_MBM_LOCAL))) { r->mon.mbm_cntr_assignable = true; + r->mon.mbm_cntr_configurable = true; cpuid_count(0x80000020, 5, &eax, &ebx, &ecx, &edx); r->mon.num_mbm_cntrs = (ebx & GENMASK(15, 0)) + 1; hw_res->mbm_cntr_assign_enabled = true; diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index 8850264684405..8a017f1111028 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -253,7 +253,7 @@ void resctrl_arch_reset_all_ctrls(struct rdt_resource *r) hw_dom = resctrl_to_arch_ctrl_dom(d); for (i = 0; i < hw_res->num_closid; i++) - hw_dom->ctrl_val[i] = resctrl_get_default_ctrl(r); + hw_dom->ctrl_val[i] = resctrl_get_resource_default_ctrl(r); msr_param.dom = d; smp_call_function_any(&d->hdr.cpu_mask, rdt_ctrl_update, &msr_param, 1); } diff --git a/block/blk.h b/block/blk.h index a55e2e4fcda4f..a7abf3be34ef8 100644 --- a/block/blk.h +++ b/block/blk.h @@ -132,6 +132,8 @@ static inline bool biovec_phys_mergeable(struct request_queue *q, if (addr1 + vec1->bv_len != addr2) return false; + if (!zone_device_pages_have_same_pgmap(vec1->bv_page, vec2->bv_page)) + return false; if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2->bv_page)) return false; if ((addr1 | mask) != ((addr2 + vec2->bv_len - 1) | mask)) diff --git a/debian.master/changelog b/debian.master/changelog index a323b50416a91..6bc92ed5dac8c 100644 --- a/debian.master/changelog +++ b/debian.master/changelog @@ -1,3 +1,51 @@ +linux (7.0.0-27.27) resolute; urgency=medium + + * resolute/linux: 7.0.0-27.27 -proposed tracker (LP: #2157114) + + * Packaging resync (LP: #1786013) + - [Packaging] update annotations scripts + + * Ubuntu 26.04 linux kernel has non-functional nova-core GPU driver enabled, + conflicting with nouveau (LP: #2150845) + - [Config] Disable NOVA_CORE + + * CVE-2026-46316 + - KVM: arm64: vgic-its: Drop the translation cache reference only for the + erased entry + + * CVE-2026-46244 + - netfilter: nft_inner: Fix IPv6 inner_thoff desync + + * CVE-2026-46137 + - mptcp: pm: ADD_ADDR rtx: allow ID 0 + - mptcp: pm: ADD_ADDR rtx: fix potential data-race + + * CVE-2026-46185 + - smb/client: fix out-of-bounds read in symlink_data() + + * CVE-2026-46195 + - smb: client: validate dacloffset before building DACL pointers + + * CVE-2026-46289 + - lib/scatterlist: fix length calculations in extract_kvec_to_sg + + * CVE-2026-46119 + - libceph: Fix slab-out-of-bounds access in auth message processing + + * CVE-2026-46135 + - nvmet-tcp: fix race between ICReq handling and queue teardown + + * CVE-2026-46155 + - smb/client: fix out-of-bounds read in smb2_compound_op() + + * CVE-2026-46115 + - block: add pgmap check to biovec_phys_mergeable + + * CVE-2026-46243 + - smb: client: reject userspace cifs.spnego descriptions + + -- Manuel Diewald Thu, 18 Jun 2026 18:54:56 +0200 + linux (7.0.0-26.26) resolute; urgency=medium * resolute/linux: 7.0.0-26.26 -proposed tracker (LP: #2154530) diff --git a/debian.master/config/annotations b/debian.master/config/annotations index 0a6dfbb7a3063..dde1e2948bdea 100644 --- a/debian.master/config/annotations +++ b/debian.master/config/annotations @@ -459,6 +459,9 @@ CONFIG_NLS note<'dependancy of boot essenti CONFIG_NOP_USB_XCEIV policy<{'amd64': 'm', 'arm64': 'm', 'armhf': 'y', 'ppc64el': 'm', 'riscv64': 'm', 's390x': '-'}> CONFIG_NOP_USB_XCEIV note<'boot essential on omap/highbank'> +CONFIG_NOVA_CORE policy<{'amd64': 'n', 'arm64': 'n'}> +CONFIG_NOVA_CORE note<'LP: #2150845'> + CONFIG_NO_HZ_FULL policy<{'amd64': 'y', 'arm64': 'y', 'armhf': 'n', 'ppc64el': 'n', 'riscv64': 'n'}> CONFIG_NO_HZ_FULL note<'LP: #2051342'> @@ -9857,7 +9860,6 @@ CONFIG_NOUVEAU_DEBUG_DEFAULT policy<{'amd64': '3', 'arm64': ' CONFIG_NOUVEAU_DEBUG_MMU policy<{'amd64': 'n', 'arm64': 'n', 'armhf': 'n', 'ppc64el': 'n', 'riscv64': 'n', 's390x': '-'}> CONFIG_NOUVEAU_DEBUG_PUSH policy<{'amd64': 'n', 'arm64': 'n', 'armhf': 'n', 'ppc64el': 'n', 'riscv64': 'n', 's390x': '-'}> CONFIG_NOUVEAU_PLATFORM_DRIVER policy<{'arm64': 'y', 'armhf': 'y'}> -CONFIG_NOVA_CORE policy<{'amd64': 'm', 'arm64': 'm'}> CONFIG_NOZOMI policy<{'amd64': 'm', 'arm64': 'm', 'armhf': 'm', 'ppc64el': 'm', 'riscv64': 'm', 's390x': 'n'}> CONFIG_NO_HZ policy<{'amd64': 'y', 'arm64': 'y', 'armhf': 'y', 'ppc64el': 'y', 'riscv64': 'y', 's390x': 'y'}> CONFIG_NO_HZ_COMMON policy<{'amd64': 'y', 'arm64': 'y', 'armhf': 'y', 'ppc64el': 'y', 'riscv64': 'y', 's390x': 'y'}> diff --git a/debian.master/tracking-bug b/debian.master/tracking-bug index afdbf0d4c9bc1..57fff0be3022b 100644 --- a/debian.master/tracking-bug +++ b/debian.master/tracking-bug @@ -1 +1 @@ -2154530 2026.05.18-6 +2157114 s2026.05.18-1 diff --git a/debian.nvidia/changelog b/debian.nvidia/changelog new file mode 100644 index 0000000000000..05ece2617a2c8 --- /dev/null +++ b/debian.nvidia/changelog @@ -0,0 +1,1843 @@ +linux-nvidia (7.0.0-1015.15) resolute; urgency=medium + + * resolute/linux-nvidia: 7.0.0-1015.15 -proposed tracker (LP: #2158924) + + * Packaging resync (LP: #1786013) + - [Packaging] debian.nvidia/dkms-versions -- update from kernel-versions + (adhoc/d2026.06.30) + + * Backport mana support for PF device 0x00C1 (LP: #2156821) + - net: mana: Add support for PF device 0x00C1 + + * Backport the arm-smmu-v3 kdump adoption series (LP: #2156531) + - iommu/arm-smmu-v3: Add arm_smmu_kdump_adopt_strtab() for kdump + - iommu/arm-smmu-v3: Implement is_attach_deferred() for kdump + - iommu/arm-smmu-v3: Do not enable EVTQ/PRIQ interrupts in kdump kernel + - iommu/arm-smmu-v3: Skip EVTQ/PRIQ setup in kdump kernel + - iommu/arm-smmu-v3: Retain CR0_SMMUEN during kdump device reset + - iommu/arm-smmu-v3: Skip RMR bypass for kdump adoption + - iommu/arm-smmu-v3: Detect ARM_SMMU_OPT_KDUMP_ADOPT in probe() + + * Backport: fuse: back uncached readdir buffers with pages (LP: #2156632) + - fuse: back uncached readdir buffers with pages + + * Backport: Mitigate TLBI errata on various Arm CPUs (LP: #2156557) // CVE- + Enable ARM64_ERRATUM_4118414 to mitigate 2025-10263 on NVIDIA platforms. + - NVIDIA: [Config] Enable ARM64_ERRATUM_4118414 + + * Backport: Mitigate TLBI errata on various Arm CPUs (LP: #2156557) // + CVE-2025-10263. The existing ARM64_ERRATUM_4118414 handling already uses + - arm64: errata: Mitigate TLBI errata on NVIDIA Olympus CPU + + * Backport: Mitigate TLBI errata on various Arm CPUs (LP: #2156557) + - arm64: cputype: Add C1-Ultra definitions + - arm64: cputype: Add C1-Premium definitions + - arm64: errata: Mitigate TLBI errata on various Arm CPUs + + * linux-nvidia: Port MPAM Functionality into Kernel (LP: #2154527) + - arm_mpam: Ensure in_reset_state is false after applying configuration + - arm_mpam: Reset when feature configuration bit unset + - arm64/sysreg: Add MPAMSM_EL1 register + - KVM: arm64: Preserve host MPAM configuration when changing traps + - KVM: arm64: Make MPAMSM_EL1 accesses UNDEF + - arm64: mpam: Context switch the MPAM registers + - arm64: mpam: Re-initialise MPAM regs when CPU comes online + - arm64: mpam: Drop the CONFIG_EXPERT restriction + - arm64: mpam: Advertise the CPUs MPAM limits to the driver + - arm64: mpam: Add cpu_pm notifier to restore MPAM sysregs + - arm64: mpam: Initialise and context switch the MPAMSM_EL1 register + - arm64: mpam: Add helpers to change a task or cpu's MPAM PARTID/PMG + values + - arm_mpam: resctrl: Add boilerplate cpuhp and domain allocation + - arm_mpam: resctrl: Pick the caches we will use as resctrl resources + - arm_mpam: resctrl: Implement resctrl_arch_reset_all_ctrls() + - arm_mpam: resctrl: Add resctrl_arch_get_config() + - arm_mpam: resctrl: Implement helpers to update configuration + - arm_mpam: resctrl: Add plumbing against arm64 task and cpu hooks + - arm_mpam: resctrl: Add CDP emulation + - arm_mpam: resctrl: Hide CDP emulation behind CONFIG_EXPERT + - arm_mpam: resctrl: Convert to/from MPAMs fixed-point formats + - arm_mpam: resctrl: Add rmid index helpers + - arm_mpam: resctrl: Wait for cacheinfo to be ready + - arm_mpam: resctrl: Add support for 'MB' resource + - arm_mpam: resctrl: Add kunit test for control format conversions + - arm_mpam: resctrl: Add monitor initialisation and domain boilerplate + - arm_mpam: resctrl: Add support for csu counters + - arm_mpam: resctrl: Allow resctrl to allocate monitors + - arm_mpam: resctrl: Add resctrl_arch_rmid_read() + - arm_mpam: resctrl: Update the rmid reallocation limit + - arm_mpam: resctrl: Add empty definitions for assorted resctrl functions + - ALSA: usb-audio: Replace hard-coded number with MAX_CHANNELS + - arm64: mpam: Select ARCH_HAS_CPU_RESCTRL + - arm_mpam: resctrl: Call resctrl_init() on platforms that can support + resctrl + - arm_mpam: Add quirk framework + - arm_mpam: Add workaround for T241-MPAM-1 + - arm_mpam: Add workaround for T241-MPAM-4 + - arm_mpam: Add workaround for T241-MPAM-6 + - arm_mpam: Quirk CMN-650's CSU NRDY behaviour + - arm64: mpam: Add initial MPAM documentation + - fs/resctrl: Report invalid domain ID when parsing io_alloc_cbm + - fs/resctrl: Add "*" shorthand to set io_alloc CBM for all domains + - MAINTAINERS: Update resctrl entry + - fs/resctrl: Add missing return value descriptions + - arm_mpam: resctrl: Fix MBA CDP alloc_capable handling on unmount + - arm_mpam: resctrl: Fix the check for no monitor components found + - arm_mpam: resctrl: Make resctrl_mon_ctx_waiters static + - NVIDIA: SAUCE: Update annotations to set CONFIG_RESCTRL_FS + - NVIDIA: SAUCE: fs/resctrl: Tidy up the error path in + resctrl_mkdir_event_configs() + - NVIDIA: SAUCE: x86,fs/resctrl: Create 'event_filter' files read only if + they're not configurable + - NVIDIA: SAUCE: fs/resctrl: Disallow the software controller when MBM + counters are assignable + - NVIDIA: SAUCE: fs/resctrl: Add monitor property 'mbm_cntr_assign_fixed' + - NVIDIA: SAUCE: fs/resctrl: Continue counter allocation after failure + - NVIDIA: SAUCE: fs/resctrl: Document that automatic counter assignment is + best effort + - NVIDIA: SAUCE: fs/resctrl: Document tasks file behaviour for task id 0 + and idle tasks + - NVIDIA: SAUCE: arm_mpam: resctrl: Pick classes for use as MBM counters + - NVIDIA: SAUCE: arm_mpam: resctrl: Pre-allocate assignable monitors + - NVIDIA: SAUCE: arm_mpam: resctrl: Add resctrl_arch_config_cntr() for + ABMC use + - NVIDIA: SAUCE: arm_mpam: resctrl: Add resctrl_arch_cntr_read() & + resctrl_arch_reset_cntr() + - NVIDIA: SAUCE: arm64: mpam: Add memory bandwidth usage (MBWU) + documentation + - NVIDIA: SAUCE: arm_mpam: Add resctrl_arch_round_bw() + - NVIDIA: SAUCE: fs/resctrl,x86/resctrl: Factor mba rounding to be per- + arch + - NVIDIA: SAUCE: x86/resctrl: Add stub to allow other architecture to + disable monitor overflow + - NVIDIA: SAUCE: arm_mpam: resctrl: Determine if any exposed counter can + overflow + - NVIDIA: SAUCE: fs/restrl: Allow the overflow handler to be disabled + - NVIDIA: SAUCE: arm_mpam: Allow cmax/cmin to be configured + - NVIDIA: SAUCE: arm_mpam: Rename mbw conversion to 'fract16' for code re- + use + - NVIDIA: SAUCE: fs/resctrl: Group all the MBA specific properties in a + separate struct + - NVIDIA: SAUCE: fs/resctrl: Abstract duplicate domain test to a helper + - NVIDIA: SAUCE: fs/resctrl: Move MBA supported check to parse_line() + instead of parse_bw() + - NVIDIA: SAUCE: fs/resctrl: Rename resctrl_get_default_ctrl() to include + resource + - NVIDIA: SAUCE: fs/resctrl: Add a schema format to the schema, allowing + it to be different + - NVIDIA: SAUCE: fs/resctrl: Add specific schema types for 'range' + - NVIDIA: SAUCE: x86/resctrl: Move over to specifying MBA control formats + - NVIDIA: SAUCE: fs/resctrl: Add additional files for percentage and + bitmap controls + - NVIDIA: SAUCE: fs/resctrl: Add fflags_from_schema() for files based on + schema format + - NVIDIA: SAUCE: fs/resctrl: Expose the schema format to user-space + - NVIDIA: SAUCE: fs/resctrl: Add L2 and L3 'MAX' resource schema + - NVIDIA: SAUCE: arm_mpam: resctrl: Add the glue code to convert to/from + cmax + - NVIDIA: SAUCE: resctrl/mpam: reset RIS by applying explicit default + config + - NVIDIA: SAUCE: arm_mpam: Fix MPAMCFG_MBW_PBM register setting + + * fs/ntfs3: fix mount failure on 64K page-size kernels (LP: #2155467) + - fs/ntfs3: fix mount failure on 64K page-size kernels + + * linux-nvidia: backport FF-A partition info descriptor size fix + (LP: #2154045) + - firmware: arm_ffa: Bound PARTITION_INFO_GET_REGS copies + - firmware: arm_ffa: Honor partition info descriptor size + + -- Jacob Martin Wed, 01 Jul 2026 14:01:27 -0500 + +linux-nvidia (7.0.0-1014.14) resolute; urgency=medium + + * resolute/linux-nvidia: 7.0.0-1014.14 -proposed tracker (LP: #2153496) + + * Packaging resync (LP: #1786013) + - [Packaging] update variants + - [Packaging] debian.nvidia/dkms-versions -- update from kernel-versions + (adhoc/d2026.05.20) + + * Installer fails internally with a RSync error due to page fault + (LP: #2150640) + - NVIDIA: SAUCE: ovl: keep err zero after successful ovl_cache_get() + + * Pull CPPC mailing list patches for Spark (LP: #2131705) + - ACPI: CPPC: Add cppc_get_perf() API to read performance controls + - ACPI: CPPC: Warn on missing mandatory DESIRED_PERF register + - ACPI: CPPC: Extend cppc_set_epp_perf() for FFH/SystemMemory + - cpufreq: CPPC: Update cached perf_ctrls on sysfs write + - cpufreq: cppc: Update MIN_PERF/MAX_PERF in target callbacks + - ACPI: CPPC: add APIs and sysfs interface for perf_limited + - cpufreq: CPPC: Add sysfs documentation for perf_limited + - ACPI: CPPC: Move reference performance to capabilities + - ACPI: CPPC: Fix uninitialized ref variable in cppc_get_perf_caps() + - ACPI: CPPC: Check cpc_read() return values consistently + - cpufreq: Remove max_freq_req update for pre-existing policy + - cpufreq: Add boost_freq_req QoS request + - cpufreq: Allocate QoS freq_req objects with policy + - cpufreq/amd-pstate: Cache the max frequency in cpudata + - NVIDIA: SAUCE: cpufreq: Extract cpufreq_policy_init_qos() function + - NVIDIA: SAUCE: cpufreq: Set default policy->min/max values for all + drivers + - NVIDIA: SAUCE: cpufreq: Remove driver default policy->min/max init + - NVIDIA: SAUCE: cpufreq: Use policy->min/max init as QoS request + - NVIDIA: SAUCE: cpufreq: CPPC: add autonomous mode boot parameter support + + * Backport lan743x driver patches (LP: #2152064) + - net: microchip: lan743x: add ethtool nway_reset support + - net: lan743x: fix SGMII detection on PCI1xxxx B0+ during warm reset + - net: lan743x: rename chip_rev to fpga_rev + + * Backport SMT-aware asymmetric CPU capacity idle selection (LP: #2150671) + - NVIDIA: VR: SAUCE: sched/fair: Attach sched_domain_shared to + sd_asym_cpucapacity + - NVIDIA: VR: SAUCE: sched/fair: Prefer fully-idle SMT cores in asym- + capacity idle selection + - NVIDIA: VR: SAUCE: sched/fair: Reject misfit pulls onto busy SMT + siblings on asym-capacity + - NVIDIA: VR: SAUCE: sched/fair: Add SIS_UTIL support to + select_idle_capacity() + + * Introduce a sharded cache affinity scope (LP: #2150467) + - workqueue: fix parse_affn_scope() prefix matching bug + - workqueue: fix typo in WQ_AFFN_SMT comment + - workqueue: add WQ_AFFN_CACHE_SHARD affinity scope + - workqueue: set WQ_AFFN_CACHE_SHARD as the default affinity scope + - tools/workqueue: add CACHE_SHARD support to wq_dump.py + - workqueue: add test_workqueue benchmark module + - docs: workqueue: document WQ_AFFN_CACHE_SHARD affinity scope + - workqueue: avoid unguarded 64-bit division + - workqueue: validate cpumask_first() result in + llc_populate_cpu_shard_id() + - [Config] nvidia: Defaults for CONFIG_TEST_WORKQUEUE + + * UBUNTU: [Config] nvidia: Disable default CMA reservation (LP: #2150898) + - [Config] nvidia: Disable default CMA reservation + + * Backport Use device ID range for DGX Spark iGPU (LP: #2150487) + - NVIDIA: SAUCE: iommu/arm-smmu-v3: Use device ID range for DGX Spark iGPU + iommu quirk + + * Backport NVIDIA: SAUCE: iommu/arm-smmu-v3: Use identity domain for ASPEED + BMC devices (LP: #2150470) + - NVIDIA: SAUCE: iommu/arm-smmu-v3: Use identity domain for ASPEED BMC + devices + + * Update GDS/NVMe SAUCE for v6.17 (LP: #2134960) // [linux-nvidia-7.0]: + Forward-port GDS/NVFS content (LP: #2150289) + - NVIDIA: SAUCE: Patch NVMe/NVMeoF driver to support GDS on Linux 7.0 + Kernel + + * serial: 8250_mtk: Update ACPI support to add NVDA0240 device ID + (LP: #2148607) + - Revert "NVIDIA: SAUCE: serial: 8250_mtk: Add ACPI support" + - NVIDIA: SAUCE: MEDIATEK: serial: 8250_mtk: Add ACPI support + + * fix r8169 vs r8127 contention for Spark (LP: #2144345) + - NVIDIA: SAUCE: r8169: remove PCI IDs claimed by r8127 driver + + [ Ubuntu: 7.0.0-27.27 ] + + * resolute/linux: 7.0.0-27.27 -proposed tracker (LP: #2157114) + * Packaging resync (LP: #1786013) + - [Packaging] update annotations scripts + * Ubuntu 26.04 linux kernel has non-functional nova-core GPU driver enabled, + conflicting with nouveau (LP: #2150845) + - [Config] Disable NOVA_CORE + * CVE-2026-46316 + - KVM: arm64: vgic-its: Drop the translation cache reference only for the + erased entry + * CVE-2026-46244 + - netfilter: nft_inner: Fix IPv6 inner_thoff desync + * CVE-2026-46137 + - mptcp: pm: ADD_ADDR rtx: allow ID 0 + - mptcp: pm: ADD_ADDR rtx: fix potential data-race + * CVE-2026-46185 + - smb/client: fix out-of-bounds read in symlink_data() + * CVE-2026-46195 + - smb: client: validate dacloffset before building DACL pointers + * CVE-2026-46289 + - lib/scatterlist: fix length calculations in extract_kvec_to_sg + * CVE-2026-46119 + - libceph: Fix slab-out-of-bounds access in auth message processing + * CVE-2026-46135 + - nvmet-tcp: fix race between ICReq handling and queue teardown + * CVE-2026-46155 + - smb/client: fix out-of-bounds read in smb2_compound_op() + * CVE-2026-46115 + - block: add pgmap check to biovec_phys_mergeable + * CVE-2026-46243 + - smb: client: reject userspace cifs.spnego descriptions + + [ Ubuntu: 7.0.0-26.26 ] + + * resolute/linux: 7.0.0-26.26 -proposed tracker (LP: #2154530) + * Packaging resync (LP: #1786013) + - Revert "UBUNTU: SAUCE: import Huawei ES3000_V2 (2.1.0.23)" + - [Packaging] debian.master/dkms-versions -- remove dkms-versions + (main/2026.05.18) + * Fix mic mute led on a HP EliteBook 6 G2a platform (LP: #2150065) + - ALSA: hda/realtek: Add LED fixup for HP EliteBook 6 G2a Laptops + * ov08x40 module mounted upside down on a certain DELL platforms + (LP: #2146517) + - SAUCE: media: ipu-bridge: Add DMI quirk for new Dell XPS laptops with + upside down sensors + - SAUCE: media: ipu-bridge: Add DMI quirk for Dell 14 laptops with upside + down sensors + * Support additional 2888x1808@30fps 900MHz for OVTI05C1 camera sensor + (LP: #2147409) + - SAUCE: media: ipu-bridge: Add 900MHz for OV05C10 + - SAUCE: platform/x86: int3472: increase handshake delay to 50ms for + OV05C10 + * Support Samsung S5K3J1 sensor for Intel MIPI camera (LP: #2121852) + - SAUCE: media: ipu-bridge: Support s5k3j1 sensor + * [SRU] ASoC: enable rt1320 speaker amp and DMIC on PTL SoundWire platforms + (LP: #2150196) + - ASoC: Intel: soc-acpi-intel-ptl-match: drop rt722 monolithic match + tables + - ASoC: SOF: Intel: Add a is_amp flag to fix the wrong name prefix + - ASoC: sdw_utils: add rt1320 and rt1321 dmic dai in codec_info_list + * powerpc-build in ubuntu_kernel_selftests fails to build due to + uninitialized value (LP: #2129844) + - selftests/powerpc: Suppress -Wmaybe-uninitialized with GCC 15 + * Ubuntu 26.04 linux kernel has non-functional nova-core GPU driver enabled, + conflicting with nouveau (LP: #2150845) + - [Config] Disable DRM_NOVA + * Resolute update: v7.0.6 upstream stable release (LP: #2152558) + - Linux 7.0.6 + - Upstream stable to v7.0.6 + * Resolute update: v7.0.5 upstream stable release (LP: #2152556) + - Linux 7.0.5 + - Upstream stable to v7.0.5 + * Resolute update: v7.0.4 upstream stable release (LP: #2152552) + - ALSA: usb-audio: stop parsing UAC2 rates at MAX_NR_RATES + - ALSA: usb-audio: Avoid false E-MU sample-rate notifications + - ALSA: usb-audio: Fix Audio Advantage Micro II SPDIF switch + - usb: xhci: Make usb_host_endpoint.hcpriv survive endpoint_disable() + - usb: chipidea: otg: not wait vbus drop if use role_switch + - usb: chipidea: core: allow ci_irq_handler() handle both ID and VBUS + change + - ALSA: usb-audio: Evaluate packsize caps at the right place + - LoongArch: Add spectre boundry for syscall dispatch table + - drm/nouveau: fix u32 overflow in pushbuf reloc bounds check + - leds: qcom-lpg: Check for array overflow when selecting the high + resolution + - greybus: gb-beagleplay: bound bootloader receive buffering + - greybus: gb-beagleplay: fix sleep in atomic context in hdlc_tx_frames() + - misc: ibmasm: fix OOB MMIO read in ibmasm_handle_mouse_interrupt() + - ibmasm: fix OOB reads in command_file_write due to missing size checks + - ibmasm: fix heap over-read in ibmasm_send_i2o_message() + - sysfs: attribute_group: Respect is_visible_const() when changing owner + - driver core: Don't let a device probe until it's ready + - device property: Make modifications of fwnode "flags" thread safe + - drm/nouveau: fix nvkm_device leak on aperture removal failure + - rust: dma: remove DMA_ATTR_NO_KERNEL_MAPPING from public attrs + - kbuild: rust: allow `clippy::uninlined_format_args` + - fs: afs: revert mmap_prepare() change + - firmware: google: framebuffer: Do not mark framebuffer as busy + - lib: test_hmm: evict device pages on file close to avoid use-after-free + - arm64/mm: Enable batched TLB flush in unmap_hotplug_range() + - arm64: mm: Fix rodata=full block mapping support for realm guests + - mm: migrate: requeue destination folio on deferred split queue + - mm: prevent droppable mappings from being locked + - mm: fix deferred split queue races during migration + - ocfs2: split transactions in dio completion to avoid credit exhaustion + - Input: edt-ft5x06 - fix use-after-free in debugfs teardown + - zram: do not forget to endio for partial discard requests + - wifi: rtw88: check for PCI upstream bridge existence + - wifi: mwifiex: fix use-after-free in mwifiex_adapter_cleanup() + - vfio: selftests: Fix VLA initialisation in vfio_pci_irq_set() + - vfio/xe: Add a missing vfio_pci_core_release_dev() + - vfio/virtio: Convert list_lock from spinlock to mutex + - vfio/cdx: Serialize VFIO_DEVICE_SET_IRQS with a per-device mutex + - vfio/cdx: Fix NULL pointer dereference in interrupt trigger path + - um: drivers: call kernel_strrchr() explicitly in cow_user.c + - thermal: core: Fix thermal zone governor cleanup issues + - spi: imx: fix use-after-free on unbind + - spi: ch341: fix memory leaks on probe failures + - crypto: algif_aead - snapshot IV for async AEAD requests + - crypto: pcrypt - Fix handling of MAY_BACKLOG requests + - dt-bindings: display: ti, am65x-dss: Fix AM62L DSS reg and clock + constraints + - of: unittest: fix use-after-free in of_unittest_changeset() + - of: unittest: fix use-after-free in testdrv_probe() + - hwmon: (powerz) Fix missing usb_kill_urb() on signal interrupt + - EDAC/versalnet: Fix device_node leak in mc_probe() + - PCI: imx6: Skip waiting for L2/L3 Ready on i.MX6SX + - media: amphion: Fix race between m2m job_abort and device_run + - ALSA: control: Validate buf_len before strnlen() in + snd_ctl_elem_init_enum_names() + - net: caif: clear client service pointer on teardown + - net: strparser: fix skb_head leak in strp_abort_strp() + - media: mtk-jpeg: fix use-after-free in release path due to uncancelled + work + - crypto: atmel-sha204a - Fix OTP sysfs read and error handling + - PCI: endpoint: pci-epf-ntb: Remove duplicate resource teardown + - Revert "ALSA: usb: Increase volume range that triggers a warning" + - phy: qcom: m31-eusb2: clear PLL_EN during init + - PCI: epf-mhi: Return 0, not remaining timeout, when eDMA ops complete + - lib/ts_kmp: fix integer overflow in pattern length calculation + - media: i2c: imx219: Check return value of devm_gpiod_get_optional() in + imx219_probe() + - net: qrtr: ns: Fix use-after-free in driver remove() + - ext2: reject inodes with zero i_nlink and valid mode in ext2_iget() + - mm/zsmalloc: copy KMSAN metadata in zs_page_migrate() + - ALSA: aoa: i2sbus: clear stale prepared state + - ALSA: aoa: i2sbus: fix OF node lifetime handling + - ALSA: aoa: Skip devices with no codecs in i2sbus_resume() + - ALSA: ctxfi: Add fallback to default RSR for S/PDIF + - ALSA: seq_oss: return full count for successful SEQ_FULLSIZE writes + - erofs: fix the out-of-bounds nameoff handling for trailing dirents + - ipmi:ssif: Clean up kthread on errors + - jbd2: fix deadlock in jbd2_journal_cancel_revoke() + - KVM: selftests: Fix reserved value WRMSR testcase for multi-feature MSRs + - md/raid10: fix deadlock with check operation and nowait requests + - media: rc: igorplugusb: heed coherency rules + - media: rockchip: rkcif: fix off by one bugs + - media: rockchip: rkcif: comply with minimum number of buffers + requirement + - mfd: stpmic1: Attempt system shutdown twice in case PMIC is confused + - mm/alloc_tag: clear codetag for pages allocated before page_ext + initialization + - mm/damon/core: fix damon_call() vs kdamond_fn() exit race + - mm/damon/core: fix damos_walk() vs kdamond_fn() exit race + - mm/hugetlb: fix early boot crash on parameters without '=' separator + - mtd: docg3: fix use-after-free in docg3_release() + - nvme-pci: add NVME_QUIRK_DISABLE_WRITE_ZEROES for Kingston OM3SGP4 + - nvme: respect NVME_QUIRK_DISABLE_WRITE_ZEROES when wzsl is set + - parisc: _llseek syscall is only available for 32-bit userspace + - parisc: Drop ip_fast_csum() inline assembly implementation + - PCI: cadence: Use cdns_pcie_read_sz() for byte or word read access + - PCI: imx6: Fix reference clock source selection for i.MX95 + - perf annotate: Use jump__delete when freeing LoongArch jumps + - RDMA/mana_ib: Disable RX steering on RSS QP destroy + - remoteproc: xlnx: Only access buffer information if IPI is buffered + - reset: rzv2h-usb2phy: Keep PHY clock enabled for entire device lifetime + - sched: Use u64 for bandwidth ratio calculations + - selftests/mqueue: Fix incorrectly named file + - landlock: Fix LOG_SUBDOMAINS_OFF inheritance across fork() + - landlock: Allow TSYNC with LOG_SUBDOMAINS_OFF and fd=-1 + - selftests/landlock: Drain stale audit records on init + - selftests/landlock: Fix format warning for __u64 in net_test + - selftests/landlock: Fix snprintf truncation checks in audit helpers + - selftests/landlock: Skip stale records in audit_match_record() + - rbd: fix null-ptr-deref when device_add_disk() fails + - mm/zone_device: do not touch device folio after calling ->folio_free() + - block: fix zone write plugs refcount handling in + disk_zone_wplug_schedule_bio_work() + - io_uring/zcrx: return back two step unregistration + - io_uring/timeout: check unused sqe fields + - block: relax pgmap check in bio_add_page for compatible zone device + pages + - iio: adc: ti-ads7950: use iio_push_to_buffers_with_ts_unaligned() + - io_uring/register: fix ring resizing with mixed/large SQEs/CQEs + - io_uring/zcrx: fix user_struct uaf + - io_uring/poll: fix signed comparison in io_poll_get_ownership() + - io_uring/poll: ensure EPOLL_ONESHOT is propagated for EPOLL_URING_WAKE + - module.lds,codetag: force 0 sh_addr for sections + - module.lds.S: Fix modules on 32-bit parisc architecture + - ALSA: core: Fix potential data race at fasync handling + - ALSA: caiaq: Fix control_put() result and cache rollback + - ALSA: caiaq: Handle probe errors properly + - ALSA: 6fire: Fix input volume change detection + - ALSA: hda/realtek - Add mute LED support for HP Victus 15-fa2xxx + - ALSA: pcmtest: fix reference leak on failed device registration + - ALSA: pcmtest: Fix resource leaks in module init error paths + - iio: adc: ad7768-1: fix one-shot mode data acquisition + - iio: adc: ad7768-1: remove switch to one-shot mode + - rxrpc: Fix memory leaks in rxkad_verify_response() + - rxrpc: Fix rxkad crypto unalignment handling + - rxrpc: Fix error handling in rxgk_extract_token() + - rxrpc: Fix re-decryption of RESPONSE packets + - EDAC/versalnet: Fix memory leak in remove and probe error paths + - tools/accounting: handle truncated taskstats netlink messages + - net: txgbe: fix RTNL assertion warning when remove module + - arm64: dts: marvell: uDPU: add ethernet aliases + - net: qrtr: ns: Limit the maximum server registration per node + - net: qrtr: ns: Limit the maximum number of lookups + - net: qrtr: ns: Free the node during ctrl_cmd_bye() + - net: qrtr: ns: Limit the total number of nodes + - net: rds: fix MR cleanup on copy error + - net: txgbe: fix firmware version check + - net/smc: avoid early lgr access in smc_clc_wait_msg + - net: ks8851: Reinstate disabling of BHs around IRQ handler + - net: bridge: use a stable FDB dst snapshot in RCU readers + - netconsole: avoid out-of-bounds access on empty string in trim_newline() + - net: mctp: fix don't require received header reserved bits to be zero + - net: ks8851: Avoid excess softirq scheduling + - drm/arcpgu: fix device node leak + - slub: fix data loss and overflow in krealloc() + - tracing/fprobe: Reject registration of a registered fprobe before init + - RDMA/rxe: Validate pad and ICRC before payload_size() in rxe_rcv + - printf: Compile the kunit test with DISABLE_BRANCH_PROFILING + DISABLE_BRANCH_PROFILING + - ipv4: icmp: validate reply type before using icmp_pointers + - libceph: Prevent potential null-ptr-deref in ceph_handle_auth_reply() + - spi: fix resource leaks on device setup failure + - extract-cert: Wrap key_pass with '#ifdef USE_PKCS11_ENGINE' + - tpm: avoid -Wunused-but-set-variable + - LoongArch: Make arch_irq_work_has_interrupt() true only if IPI HW exist + - LoongArch: Show CPU vulnerabilites correctly + - fbdev: defio: Disconnect deferred I/O from the lifetime of struct + fb_info + - power: supply: axp288_charger: Do not cancel work before initializing it + - hwmon: (isl28022) Fix integer overflow in power calculation on 32-bit + - hwmon: (powerz) Avoid cacheline sharing for DMA buffer + - media: rzv2h-ivc: Revise default VBLANK formula + - media: rzv2h-ivc: Fix AXIRX_VBLANK register write + - fs: prepare for adding LSM blob to backing_file + - lsm: add backing_file LSM hooks + - selinux: fix overlayfs mmap() and mprotect() access checks + - hwmon: (pt5161l) Fix bugs in pt5161l_read_block_data() + - randomize_kstack: Maintain kstack_offset per task + - mmc: block: use single block write in retry + - mmc: sdhci-of-dwcmshc: Disable clock before DLL configuration + - arm64: dts: ti: am62-verdin: Enable pullup for eMMC data pins + - crypto: qat - fix IRQ cleanup on 6xxx probe failure + - xfs: start gc on zonegc_low_space attribute updates + - xfs: fix a resource leak in xfs_alloc_buftarg() + - firmware: google: framebuffer: Do not unregister platform device + - firmware: exynos-acpm: Drop fake 'const' on handle pointer + - crypto: talitos - fix SEC1 32k ahash request limitation + - crypto: talitos - rename first/last to first_desc/last_desc + - pwm: imx-tpm: Count the number of enabled channels in probe + - tpm2-sessions: Fix missing tpm_buf_destroy() in tpm2_read_public() + - tpm: Fix auth session leak in tpm2_get_random() error path + - tpm: Use kfree_sensitive() to free auth session in tpm_dev_release() + - tpm: tpm_tis: add error logging for data transfer + - tpm: tpm_tis: stop transmit if retries are exhausted + - rtc: ntxec: fix OF node reference imbalance + - mm/vmalloc: take vmap_purge_lock in shrinker + - mm/memfd_luo: fix physical address conversion in put_folios cleanup + - mm/mempolicy: fix memory leaks in weighted_interleave_auto_store() + - mm/damon/stat: fix memory leak on damon_start() failure in + damon_stat_start() + - mm/damon/core: validate damos_quota_goal->nid for + node_mem_{used,free}_bp + - mm/damon/core: validate damos_quota_goal->nid for + node_memcg_{used,free}_bp + - mm/damon/core: use time_in_range_open() for damos quota window start + - mm/damon/core: disallow time-quota setting zero esz + - mm/damon/core: disallow non-power of two min_region_sz on damon_start() + - userfaultfd: allow registration of ranges below mmap_min_addr + - LoongArch: KVM: Use CSR_CRMD_PLV in kvm_arch_vcpu_in_kernel() + - KVM: x86: Defer non-architectural deliver of exception payload to + userspace read + - KVM: nSVM: Mark all of vmcb02 dirty when restoring nested state + - KVM: nSVM: Sync NextRIP to cached vmcb12 after VMRUN of L2 + - KVM: nSVM: Sync interrupt shadow to cached vmcb12 after VMRUN of L2 + - KVM: SVM: Inject #UD for INVLPGA if EFER.SVME=0 + - KVM: SVM: Explicitly mark vmcb01 dirty after modifying VMCB intercepts + - KVM: nSVM: Ensure AVIC is inhibited when restoring a vCPU to guest mode + - KVM: nSVM: Always use NextRIP as vmcb02's NextRIP after first L2 VMRUN + - KVM: nSVM: Delay stuffing L2's current RIP into NextRIP until vCPU run + - KVM: nSVM: Use vcpu->arch.cr2 when updating vmcb12 on nested #VMEXIT + - KVM: arm64: Account for RESx bits in __compute_fgt() + - KVM: nSVM: Avoid clearing VMCB_LBR in vmcb12 + - KVM: nSVM: Delay setting soft IRQ RIP tracking fields until vCPU run + - KVM: SVM: Switch svm_copy_lbrs() to a macro + - KVM: SVM: Add missing save/restore handling of LBR MSRs + - KVM: nSVM: Always inject a #GP if mapping VMCB12 fails on nested VMRUN + - KVM: nSVM: Refactor checking LBRV enablement in vmcb12 into a helper + - KVM: nSVM: Refactor writing vmcb12 on nested #VMEXIT as a helper + - KVM: nSVM: Triple fault if restore host CR3 fails on nested #VMEXIT + - KVM: nSVM: Triple fault if mapping VMCB12 fails on nested #VMEXIT + - KVM: nSVM: Clear GIF on nested #VMEXIT(INVALID) + - KVM: nSVM: Clear EVENTINJ fields in vmcb12 on nested #VMEXIT + - KVM: nSVM: Clear tracking of L1->L2 NMI and soft IRQ on nested #VMEXIT + - KVM: nSVM: Add missing consistency check for EFER, CR0, CR4, and CS + - KVM: nSVM: Drop the non-architectural consistency check for NP_ENABLE + - KVM: nSVM: Add missing consistency check for nCR3 validity + - KVM: nSVM: Raise #UD if unhandled VMMCALL isn't intercepted by L1 + - KVM: nSVM: Always intercept VMMCALL when L2 is active + - ARM: 9472/1: fix race condition on PG_dcache_clean in + __sync_icache_dcache() + - ring-buffer: Do not double count the reader_page + - ext4: fix bounds check in check_xattrs() to prevent out-of-bounds access + - ext4: fix missing brelse() in ext4_xattr_inode_dec_ref_all() + - udf: fix partition descriptor append bookkeeping + - mtd: spi-nor: sst: Fix write enable before AAI sequence + - mtd: spinand: winbond: Declare the QE bit on W25NxxJW + - amdgpu/jpeg: fix deepsleep register for jpeg 5_0_0 and 5_0_2 + - md/md-llbitmap: skip reading rdevs that are not in_sync + - md/md-llbitmap: raise barrier before state machine transition + - md/raid5: fix soft lockup in retry_aligned_read() + - md/raid5: validate payload size before accessing journal metadata + - check-uapi: link into shared objects + - mm, swap: speed up hibernation allocation and writeout + - HID: apple: ensure the keyboard backlight is off if suspending + - inotify: fix watch count leak when fsnotify_add_inode_mark_locked() + fails + - x86/cpu: Disable FRED when PTI is forced on + - x86/shstk: Prevent deadlock during shstk sigreturn + - wifi: rtl8xxxu: fix potential use of uninitialized value + - tcp: call sk_data_ready() after listener migration + - taskstats: set version in TGID exit notifications + - mptcp: sync the msk->sndbuf at accept() time + - mfd: core: Preserve OF node when ACPI handle is present + - 9p: fix access mode flags being ORed instead of replaced + - Bluetooth: hci_event: fix potential UAF in SSP passkey handlers + - bus: mhi: host: pci_generic: Switch to async power up to avoid boot + delays + - can: ucan: fix devres lifetime + - crypto: acomp - fix wrong pointer stored by acomp_save_req() + - crypto: arm64/aes - Fix 32-bit aes_mac_update() arg treated as 64-bit + - crypto: atmel-aes - Fix 3-page memory leak in atmel_aes_buff_cleanup + - crypto: atmel-ecc - Release client on allocation failure + - crypto: hisilicon - Fix dma_unmap_single() direction + - crypto: ccree - fix a memory leak in cc_mac_digest() + - crypto: atmel-tdes - fix DMA sync direction + - crypto: atmel-sha204a - Fix error codes in OTP reads + - crypto: atmel-sha204a - Fix potential UAF and memory leak in remove path + - crypto: atmel-sha204a - Fix uninitialized data access on OTP read error + - crypto: nx - fix bounce buffer leaks in nx842_crypto_{alloc,free}_ctx + - crypto: nx - fix context leak in nx842_crypto_free_ctx + - crypto: nx - Fix packed layout in struct nx842_crypto_header + - dm mirror: fix integer overflow in create_dirty_log() + - erofs: fix unsigned underflow in z_erofs_lz4_handle_overlap() + - ceph: fix num_ops off-by-one when crypto allocation fails + - ceph: only d_add() negative dentries when they are unhashed + - gtp: disable BH before calling udp_tunnel_xmit_skb() + - IB/core: Fix zero dmac race in neighbor resolution + - ktest: Fix the month in the name of the failure directory + - NFSv4.1: Apply session size limits on clone path + - ntfs3: add buffer boundary checks to run_unpack() + - ntfs3: fix integer overflow in run_unpack() volume boundary check + - rtmutex: Use waiter::task instead of current in remove_waiter() + - rxgk: Fix potential integer overflow in length check + - sched_ext: Documentation: Clarify ops.dispatch() role in task lifecycle + - scsi: sd: fix missing put_disk() when device_add(&disk_dev) fails + - seg6: fix seg6 lwtunnel output redirect for L2 reduced encap mode + - perf loongarch: Fix build failure with CONFIG_LIBDW_DWARF_UNWIND + - iio: frequency: admv1013: add dev variable + - iio: frequency: admv1013: fix NULL pointer dereference on str + - wifi: mt76: mt792x: describe USB WFSYS reset with a descriptor + - wifi: mt76: mt792x: fix mt7925u USB WFSYS reset handling + - mm: various small mmap_prepare cleanups + - mm: avoid deadlock when holding rmap on mmap_prepare error + - mei: me: use PCI_DEVICE_DATA macro + - mei: me: add nova lake point H DID + - crypto: authencesn - reject short ahash digests during instance creation + - driver core: Add kernel-doc for DEV_FLAG_COUNT enum value + - ALSA: caiaq: Fix potentially leftover ep1_in_urb at error path + - ALSA: caiaq: Don't abort when no input device is available + - ipv6: rpl: reserve mac_len headroom when recompressed SRH grows + - drm/amdgpu: fix zero-size GDS range init on RDNA4 + - drm/imagination: Fix segfault when updating ftrace mask + - ALSA: caiaq: fix usb_dev refcount leak on probe failure + - ALSA: aloop: Fix peer runtime UAF during format-change stop + - vmalloc: fix buffer overflow in vrealloc_node_align() + - mm/page_alloc: return NULL early from alloc_frozen_pages_nolock() in NMI + on UP + - mm/slab: return NULL early from kmalloc_nolock() in NMI on UP + - net: ipv6: fix NOREF dst use in seg6 and rpl lwtunnels + - netfilter: reject zero shift in nft_bitwise + - ipmi:ssif: Remove unnecessary indention + - ipmi:ssif: NULL thread on error + - Linux 7.0.4 + - Upstream stable to v7.0.4 + * Resolute update: v7.0.3 upstream stable release (LP: #2152550) + - Buffer overflow in drivers/xen/sys-hypervisor.c + - xen/privcmd: fix double free via VMA splitting + - Linux 7.0.3 + - Upstream stable to v7.0.3 + * Resolute update: v7.0.2 upstream stable release (LP: #2150553) + - crypto: authencesn - Fix src offset when decrypting in-place + - pwm: th1520: fix `CLIPPY=1` warning + - drm/amdgpu: replace PASID IDR with XArray + - crypto: krb5enc - fix sleepable flag handling in encrypt dispatch + - crypto: krb5enc - fix async decrypt skipping hash verification + - ksmbd: fix use-after-free in __ksmbd_close_fd() via durable scavenger + - ksmbd: validate owner of durable handle on reconnect + - scripts: generate_rust_analyzer.py: define scripts + - scripts/dtc: Remove unused dts_version in dtc-lexer.l + - fs/ntfs3: validate rec->used in journal-replay file record check + - f2fs: fix to do sanity check on dcc->discard_cmd_cnt conditionally + - f2fs: fix UAF caused by decrementing sbi->nr_pages[] in + f2fs_write_end_io() + - f2fs: fix to avoid memory leak in f2fs_rename() + - f2fs: fix to avoid uninit-value access in f2fs_sanity_check_node_footer + - fuse: reject oversized dirents in page cache + - fuse: abort on fatal signal during sync init + - fuse: Check for large folio with SPLICE_F_MOVE + - fuse: quiet down complaints in fuse_conn_limit_write + - fuse: fuse_dev_ioctl_clone() should wait for device file to be + initialized + - ksmbd: require minimum ACE size in smb_check_perm_dacl() + - smb: server: fix active_num_conn leak on transport allocation failure + - smb: client: fix dir separator in SMB1 UNIX mounts + - smb: server: fix max_connections off-by-one in tcp accept path + - smb: client: require a full NFS mode SID before reading mode bits + - smb: client: validate the whole DACL before rewriting it in cifsacl + - smb: client: fix OOB read in smb2_ioctl_query_info QUERY_INFO path + - ksmbd: validate response sizes in ipc_validate_msg() + - ksmbd: validate num_aces and harden ACE walk in smb_inherit_dacl() + - ksmbd: fix out-of-bounds write in smb2_get_ea() EA alignment + - ksmbd: use check_add_overflow() to prevent u16 DACL size overflow + - ksmbd: reset rcount per connection in ksmbd_conn_wait_idle_sess_id() + - writeback: Fix use after free in inode_switch_wbs_work_fn() + - f2fs: fix use-after-free of sbi in f2fs_compress_write_end_io() + - ALSA: usb-audio: apply quirk for MOONDROP JU Jiu + - ALSA: hda/realtek: Add quirk for Legion S7 15IMH + - ALSA: caiaq: take a reference on the USB device in create_card() + - net/packet: fix TOCTOU race on mmap'd vnet_hdr in tpacket_snd() + - crypto: ccp: Don't attempt to copy CSR to userspace if PSP command + failed + - crypto: ccp: Don't attempt to copy PDH cert to userspace if PSP command + failed + - crypto: ccp: Don't attempt to copy ID to userspace if PSP command failed + - rxrpc: Fix missing validation of ticket length in non-XDR key preparsing + - mshv_vtl: Fix vmemmap_shift exceeding MAX_FOLIO_ORDER + - Linux 7.0.2 + * Resolute update: v7.0.1 upstream stable release (LP: #2150547) + - Revert "UBUNTU: SAUCE: cdc-acm: Exclude Exar USB serial ports" + - nfc: llcp: add missing return after LLCP_CLOSED checks + - x86/CPU: Fix FPDSS on Zen1 + - can: raw: fix ro->uniq use-after-free in raw_rcv() + - i2c: s3c24xx: check the size of the SMBUS message before using it + - staging: rtl8723bs: initialize le_tmp64 in rtw_BIP_verify() + - HID: alps: fix NULL pointer dereference in alps_raw_event() + - HID: core: clamp report_size in s32ton() to avoid undefined shift + - net: usb: cdc-phonet: fix skb frags[] overflow in rx_complete() + - NFC: digital: Bounds check NFC-A cascade depth in SDD response handler + - drm/vc4: platform_get_irq_byname() returns an int + - bnge: return after auxiliary_device_uninit() in error path + - ALSA: usx2y: us144mkii: fix NULL deref on missing interface 0 + - ALSA: fireworks: bound device-supplied status before string array lookup + - fbdev: tdfxfb: avoid divide-by-zero on FBIOPUT_VSCREENINFO + - usb: gadget: f_ncm: validate minimum block_len in ncm_unwrap_ntb() + - usb: gadget: f_phonet: fix skb frags[] overflow in pn_rx_complete() + - usb: gadget: renesas_usb3: validate endpoint index in standard request + handlers + - smb: client: fix off-by-8 bounds check in check_wsl_eas() + - smb: client: fix OOB reads parsing symlink error response + - ksmbd: validate EaNameLength in smb2_get_ea() + - ksmbd: require 3 sub-authorities before reading sub_auth[2] + - ksmbd: fix mechToken leak when SPNEGO decode fails after token alloc + - smb: client: avoid double-free in smbd_free_send_io() after + smbd_send_batch_flush() + - smb: server: avoid double-free in smb_direct_free_sendmsg after + smb_direct_flush_send_list() + - usbip: validate number_of_packets in usbip_pack_ret_submit() + - usb: typec: fusb302: Switch to threaded IRQ handler + - usb: storage: Expand range of matched versions for VL817 quirks entry + - USB: cdc-acm: Add quirks for Yoga Book 9 14IAH10 INGENIC touchscreen + - usb: gadget: f_hid: don't call cdev_init while cdev in use + - usb: port: add delay after usb_hub_set_port_power() + - fbdev: udlfb: avoid divide-by-zero on FBIOPUT_VSCREENINFO + - scripts/gdb/symbols: handle module path parameters + - scripts: generate_rust_analyzer.py: avoid FD leak + - wifi: rtw88: fix device leak on probe failure + - staging: sm750fb: fix division by zero in ps_to_hz() + - selftests/mm: hmm-tests: don't hardcode THP size to 2MB + - USB: serial: option: add Telit Cinterion FN990A MBIM composition + - Docs/admin-guide/mm/damon/reclaim: warn commit_inputs vs param updates + race + - Docs/admin-guide/mm/damon/lru_sort: warn commit_inputs vs param updates + race + - ALSA: ctxfi: Limit PTP to a single page + - dcache: Limit the minimal number of bucket to two + - vfio/xe: Reorganize the init to decouple migration from reset + - arm64: mm: Handle invalid large leaf mappings correctly + - media: vidtv: fix NULL pointer dereference in + vidtv_channel_pmt_match_sections + - ocfs2: fix possible deadlock between unlink and dio_end_io_write + - ocfs2: fix use-after-free in ocfs2_fault() when VM_FAULT_RETRY + - ocfs2: handle invalid dinode in ocfs2_group_extend + - PCI: endpoint: pci-epf-vntb: Stop cmd_handler work in + epf_ntb_epc_cleanup + - PCI: endpoint: pci-epf-vntb: Remove duplicate resource teardown + - KVM: selftests: Remove duplicate LAUNCH_UPDATE_VMSA call in SEV-ES + migrate test + - KVM: SEV: Reject attempts to sync VMSA of an already-launched/encrypted + vCPU + - KVM: SEV: Protect *all* of sev_mem_enc_register_region() with kvm->lock + - KVM: SEV: Disallow LAUNCH_FINISH if vCPUs are actively being created + - KVM: SEV: Lock all vCPUs when synchronzing VMSAs for SNP launch finish + - KVM: SEV: Drop WARN on large size for KVM_MEMORY_ENCRYPT_REG_REGION + - mm: call ->free_folio() directly in folio_unmap_invalidate() + - checkpatch: add support for Assisted-by tag + - x86-64: rename misleadingly named '__copy_user_nocache()' function + - x86: rename and clean up __copy_from_user_inatomic_nocache() + - x86-64/arm64/powerpc: clean up and rename __copy_from_user_flushcache + - KVM: x86: Use scratch field in MMIO fragment to hold small write values + - ASoC: qcom: q6apm: move component registration to unmanaged version + - mm/kasan: fix double free for kasan pXds + - mm: blk-cgroup: fix use-after-free in cgwb_release_workfn() + - media: vidtv: fix nfeeds state corruption on start_streaming failure + - media: mediatek: vcodec: fix use-after-free in encoder release path + - media: em28xx: fix use-after-free in em28xx_v4l2_open() + - hwmon: (powerz) Fix use-after-free on USB disconnect + - ALSA: 6fire: fix use-after-free on disconnect + - bcache: fix cached_dev.sb_bio use-after-free and crash + - wireguard: device: use exit_rtnl callback instead of manual rtnl_lock in + pre_exit + - media: as102: fix to not free memory after the device is registered in + as102_usb_probe() + - nilfs2: fix NULL i_assoc_inode dereference in + nilfs_mdt_save_to_shadow_map + - media: vidtv: fix pass-by-value structs causing MSAN warnings + - media: hackrf: fix to not free memory after the device is registered in + hackrf_probe() + - mm/userfaultfd: fix hugetlb fault mutex hash calculation + - clockevents: Add missing resets of the next_event_forced flag + - Linux 7.0.1 + * GRO managed-frag use-after-free leading to local privilege escalation + (LP: #2154172) + - net: gro: don't merge zcopy skbs + * AppArmor Vulnerabilities (LP: #2151747) + - SAUCE: apparmor: pass big_resp to handler + - SAUCE: apparmor: remove redundant kref_init for listener->count + - SAUCE: apparmor: fix NULL pointer dereference in unpack_pdb + * AppArmor Vulnerabilities (LP: #2151747) // CVE-2026-47337 + - SAUCE: apparmor: fix NULL pointer dereference in bind_map_addr + * AppArmor Vulnerabilities (LP: #2151747) // CVE-2026-47334 + - SAUCE: apparmor: fix sleep prone memory allocation under a spin_lock + * AppArmor Vulnerabilities (LP: #2151747) // CVE-2026-47333 + - SAUCE: apparmor: fix dfa unpacking size of the notification filter + * AppArmor Vulnerabilities (LP: #2151747) // CVE-2026-47332 + - SAUCE: apparmor: fix size check against type instead of pointer + * apparmor: LLVM/clang build failure due to uninitialized variable in + notify.c (LP: #2148809) // CVE-2026-47330 + - SAUCE: apparmor: initialize variable used in uninitialized context + * AppArmor Vulnerabilities (LP: #2151747) // CVE-2026-47329 + - SAUCE: apparmor: fix name validation bypass on notification + * AppArmor Vulnerabilities (LP: #2151747) // CVE-2026-47327 // + CVE-2026-47328 + - SAUCE: apparmor: fix glob memory leak after kstrdup + * AppArmor Vulnerabilities (LP: #2151747) // CVE-2026-47326 + - SAUCE: apparmor: fix inverted NULL check after aa_get_buffer + * CVE-2026-46300 + - net: skbuff: preserve shared-frag marker during coalescing + - net: skbuff: propagate shared-frag marker through frag-transfer helpers + * net/rds: reset op_nents when zerocopy page pin fails (LP: #2153962) + - net/rds: reset op_nents when zerocopy page pin fails + * CVE-2026-46333 + - ptrace: slightly saner 'get_dumpable()' logic + * CVE-2026-43500 + - rxrpc: Fix conn-level packet handling to unshare RESPONSE packets + - rxrpc: Fix potential UAF after skb_unshare() failure + - rxrpc: Fix rxrpc_input_call_event() to only unshare DATA packets + - rxrpc: Also unshare DATA/RESPONSE packets when paged frags are present + * CVE-2026-43284 + - xfrm: esp: avoid in-place decrypt on shared skb frags + + [ Ubuntu: 7.0.0-15.15 ] + + * resolute/linux: 7.0.0-15.15 -proposed tracker (LP: #2148866) + * Qualcomm X1E: Speaker overdrive causes hardware protection shutdown + (LP: #2149808) + - SAUCE: ASoC: qcom: x1e80100: limit speaker volumes + * intel-ipu7 / intel-ipu7-isys modules are shipped unsigned in latest + Resolute kernels, breaking Secure Boot systems (LP: #2148718) + - [packaging] add intel-ipu7 to signature inclusion list + + -- Jacob Martin Thu, 18 Jun 2026 18:23:46 -0500 + +linux-nvidia (7.0.0-1006.6) resolute; urgency=medium + + * resolute/linux-nvidia: 7.0.0-1006.6 -proposed tracker (LP: #2148214) + + [ Ubuntu: 7.0.0-14.14 ] + + * resolute/linux: 7.0.0-14.14 -proposed tracker (LP: #2148159) + * support vflip/hflip for Sony IMX471 camera sensor (LP: #2138841) + - SAUCE: media: ipu-bridge: add TBE20A0 ACPI id for Sony IMX471 + * AA: disable SECURITY_APPARMOR_PACKET_MEDIATION_ENABLED (LP: #2147533) + - [Config] disable SECURITY_APPARMOR_PACKET_MEDIATION_ENABLED + * System doesn't response with mt76 call trace (LP: #2137448) + - wifi: mt76: mt792x: Fix a potential deadlock in high-load situations + * The second tbt storage plugged on the dock will not be recognized + (LP: #2139572) + - SAUCE: thunderbolt: Fix PCIe device enumeration with delayed rescan + * dma-buf filesystem flags fix (LP: #2139656) + - SAUCE: dma-buf: set SB_I_NOEXEC and SB_I_NODEV on dmabuf filesystem + * Bluetooth device (MT7925) not detected on USB bus with linux-oem-6.17 + (LP: #2145164) + - SAUCE: USB: hub: call ACPI _PRR reset during port power-cycle on + enumeration failure + * drm/i915/lnl+/tc: Fix false disconnect of active DP-alt TC port during + long HPD pulse (LP: #2143879) + - SAUCE: drm/i915/lnl+/tc: Fix false disconnect of active DP-alt TC port + during long HPD pulse + * i915 WARN_ON call trace during CB/WB on MTL/ARL platforms (LP: #2144537) + - SAUCE: drm/i915/xelpdp/tc: Convert TCSS power check WARN to a debug + message + * Miscellaneous Ubuntu changes + - [Packaging] Add support for per-flavour depends + - [Packaging] Don't hard-code lmm zfs dependency + - [Config] updateconfigs following v7.0 release + + [ Ubuntu: 7.0.0-13.13 ] + + * resolute/linux: 7.0.0-13.13 -proposed tracker (LP: #2147403) + * ubuntu_kselftests:_net/net:gre_gso.sh failing (LP: #2136820) + - SAUCE increase socat timeout in gre_gso.sh + * Canonical Kmod 2025 key rotation (LP: #2147447) + - [Packaging] ubuntu-compatible-signing -- make Ubuntu-Compatible-Signing + extensible + - [Packaging] ubuntu-compatible-signing -- allow consumption of positive + certs + - [Packaging] ubuntu-compatible-signing -- report the livepatch:2025 key + - [Config] prepare for Canonical Kmod key rotation + - [Packaging] ubuntu-compatible-signing -- report the kmod:2025 key + - [Packaging] ensure our cert rollups are always fresh + * On Dell system, the internal OLED display drops to a visibly low FPS after + suspend/resume (LP: #2144712) + - drm/i915/psr: Disable Panel Replay on Dell XPS 14 DA14260 as a quirk + - drm/i915/psr: Fixes for Dell XPS DA14260 quirk + * Realtek RTL8116AF SFP option module fails to get connected (LP: #2116144) + - SAUCE: r8169: add quirk for RTL8116af SerDes + * Miscellaneous Ubuntu changes + - [Config] updateconfigs following v7.0-rc7 rebase + + -- Jacob Martin Tue, 14 Apr 2026 16:19:48 -0500 + +linux-nvidia (7.0.0-1005.5) resolute; urgency=medium + + * resolute/linux-nvidia: 7.0.0-1005.5 -proposed tracker (LP: #2145971) + + * Packaging resync (LP: #1786013) + - [Packaging] update variants + + * Please make dracut the default initrd generator (LP: #2142775) + - [Packaging] recommends dracut instead of initramfs-tools + + [ Ubuntu: 7.0.0-12.12 ] + + * resolute/linux: 7.0.0-12.12 -proposed tracker (LP: #2146778) + * Packaging resync (LP: #1786013) + - [Packaging] update variants + * linux-generic does not run scripts in /usr/share/kernel/*.d (LP: #2147005) + - [Packaging] templates: Use consistent indentation + - [Packaging] templates: Run scripts in /usr/share/kernel/*.d too + * RISC-V kernel config is out of sync with other archs (LP: #1981437) + - [Config] riscv64: Enable COUNTER=m + - [Config] riscv64: Use GENDWARFKSYMS like other architectures + * unconfined profile denies userns_create for chromium based processes + (LP: #1990064) + - [Config] disable CONFIG_SECURITY_APPARMOR_RESTRICT_USERNS + * FFe: add network interface mediation to 26.04 (LP: #2144679) + - SAUCE: apparmor5.0.0 [57/57]: apparmor: add the ability to use interface + in network mediation. + * Jellyfin Desktop Flatpak doesn't work with the current AppArmor profile + (LP: #2142956) + - SAUCE: apparmor5.0.0 [29/57]: apparmor: fix fine grained inet mediation + sock_file_perm + - SAUCE: apparmor5.0.0 [30/57]: apparmor-next 7.1: aapparmor: use target + task's context in apparmor_getprocattr() + - SAUCE: apparmor5.0.0 [31/57]: apparmor-next 7.1: apparmor: return error + on namespace mismatch in verify_header + - SAUCE: apparmor5.0.0 [32/57]: apparmor-next 7.1: apparmor: enable + differential encoding + - SAUCE: apparmor5.0.0 [33/57]: apparmor-next 7.1: apparmor: propagate + -ENOMEM correctly in unpack_table + - SAUCE: apparmor5.0.0 [34/57]: apparmor-next 7.1: apparmor: Replace + memcpy + NUL termination with kmemdup_nul in do_setattr + - SAUCE: apparmor5.0.0 [35/57]: apparmor-next 7.1: apparmor: Remove + redundant if check in sk_peer_get_label + - SAUCE: apparmor5.0.0 [36/57]: apparmor-next 7.1: apparmor: use + __label_make_stale in __aa_proxy_redirect + - SAUCE: apparmor5.0.0 [37/57]: apparmor-next 7.1: apparmor: fix net.h and + policy.h circular include pattern + - SAUCE: apparmor5.0.0 [39/57]: apparmor-next 7.1: apparmor: make include + headers self-contained + - SAUCE: apparmor5.0.0 [40/57]: apparmor-next 7.1: apparmor: Use + sysfs_emit in param_get_{audit,mode} + - SAUCE: apparmor5.0.0 [41/57]: apparmor-next 7.1: apparmor: fix + rawdata_f_data implicit flex array + - SAUCE: apparmor5.0.0 [42/57]: apparmor-next 7.1: apparmor: free rawdata + as soon as possible + - SAUCE: apparmor5.0.0 [43/57]: apparmor-next 7.1: apparmor: Initial + support for compressed policies + - SAUCE: apparmor5.0.0 [44/57]: apparmor-next 7.1: apparmor: fix potential + UAF in aa_replace_profiles + - SAUCE: apparmor5.0.0 [45/57]: apparmor-next 7.1: apparmor: hide unused + get_loaddata_common_ref() function + - SAUCE: apparmor5.0.0 [46/57]: apparmor-next 7.1: apparmor: Fix string + overrun due to missing termination + - SAUCE: apparmor5.0.0 [47/57]: apparmor: fix packed tag on v5 header + struct + - SAUCE: apparmor5.0.0 [48/57]: apparmor: add temporal caching to audit + responses. + - SAUCE: apparmor5.0.0 [49/57]: apparmor: change fn_label_build() call to + not return NULL + - SAUCE: apparmor5.0.0 [50/57]: apparmor: make fn_label_build() capable of + handling not supported + - SAUCE: apparmor5.0.0 [51/57]: apparmor: move netfilter functions next to + the LSM network operations + - SAUCE: apparmor5.0.0 [52/57]: apparmor: move sock_rvc_skb() next to + inet_conn_request + - SAUCE: apparmor5.0.0 [53/57]: apparmor: fix af_unix local addr mediation + binding + - SAUCE: apparmor5.0.0 [54/57]: cleanups of apparmor af_unix mediation + - SAUCE: apparmor5.0.0 [55/57]: apparmor: fix apparmor_secmark_check() + when !inet and secmark defined. + - SAUCE: apparmor5.0.0 [56/57]: apparmor: fix auditing of non-mediation + falures + * snap service cannot change apparmor hat (LP: #2139664) // Jellyfin Desktop + Flatpak doesn't work with the current AppArmor profile (LP: #2142956) + - SAUCE: apparmor5.0.0 [38/57]: apparmor-next 7.1: apparmor: grab ns lock + and refresh when looking up changehat child profiles + * AppArmor blocks write(2) to network sockets with Linux 6.19 (LP: #2141298) + - SAUCE: apparmor5.0.0 [28/57]: apparmor: fix aa_label_sk_perm to check + for RULE_MEDIATES_NET + * update apparmor and LSM stacking patch set (LP: #2028253) + - SAUCE: apparmor5.0.0 [1/57]: Stacking: LSM: Single calls in secid hooks + - SAUCE: apparmor5.0.0 [2/57]: Stacking: LSM: Exclusive secmark usage + - SAUCE: apparmor5.0.0 [3/57]: Stacking: AppArmor: Remove the exclusive + flag + - SAUCE: apparmor5.0.0 [4/57]: Revert "apparmor: fix dbus permission + queries to v9 ABI" + - SAUCE: apparmor5.0.0 [5/57]: Revert "apparmor: gate make fine grained + unix mediation behind v9 abi" + - SAUCE: apparmor5.0.0 [6/57]: apparmor: net: patch to provide + compatibility with v2.x net rules + - SAUCE: apparmor5.0.0 [7/57]: apparmor: net: add fine grained ipv4/ipv6 + mediation + - SAUCE: apparmor5.0.0 [8/57]: apparmor: lift compatibility check out of + profile_af_perm + - SAUCE: apparmor5.0.0 [9/57]: apparmor: userns: add unprivileged user ns + mediation + - SAUCE: apparmor5.0.0 [10/57]: apparmor: userns: Add sysctls for + additional controls of unpriv userns restrictions + - SAUCE: apparmor5.0.0 [12/57]: apparmor: userns: open userns related + sysctl so lxc can check if restriction are in place + - SAUCE: apparmor5.0.0 [13/57]: apparmor: userns: allow profile to be + transitioned when a userns is created + - SAUCE: apparmor5.0.0 [14/57]: apparmor: mqueue: call + security_inode_init_security on inode creation + - SAUCE: apparmor5.0.0 [15/57]: apparmor: mqueue: add fine grained + mediation of posix mqueues + - SAUCE: apparmor5.0.0 [16/57]: apparmor: uring: add io_uring mediation + - SAUCE: apparmor5.0.0 [19/57]: apparmor: prompt: setup slab cache for + audit data + - SAUCE: apparmor5.0.0 [20/57]: apparmor: prompt: add the ability for + profiles to have a learning cache + - SAUCE: apparmor5.0.0 [21/57]: apparmor: prompt: enable userspace upcall + for mediation + - SAUCE: apparmor5.0.0 [22/57]: apparmor: prompt: pass prompt boolean + through into path_name as well + - SAUCE: apparmor5.0.0 [23/57]: apparmor: check for supported version in + notification messages. + - SAUCE: apparmor5.0.0 [24/57]: apparmor: refactor building notice so it + is easier to extend + - SAUCE: apparmor5.0.0 [25/57]: apparmor: switch from ENOTSUPP to + EPROTONOSUPPORT + - SAUCE: apparmor5.0.0 [26/57]: apparmor: add support for meta data tags + - SAUCE: apparmor5.0.0 [27/57]: apparmor: prevent profile->disconnected + double free in aa_free_profile + * update apparmor and LSM stacking patch set (LP: #2028253) // Installation + of AppArmor on a 6.14 kernel produces error message "Illegal number: yes" + (LP: #2102680) + - SAUCE: apparmor5.0.0 [17/57]: apparmor: create an + AA_SFS_TYPE_BOOLEAN_INTPRINT sysctl variant + - SAUCE: apparmor5.0.0 [18/57]: apparmor: Use AA_SFS_FILE_BOOLEAN_INTPRINT + for userns and io_uring sysctls + * update apparmor and LSM stacking patch set (LP: #2028253) // [FFe] + apparmor-4.0.0-alpha2 for unprivileged user namespace restrictions in + mantic (LP: #2032602) + - SAUCE: apparmor5.0.0 [11/57]: apparmor: userns - make it so special + unconfined profiles can mediate user namespaces + * Enable new Intel WCL soundwire support (LP: #2143301) + - ASoC: sdw_utils: Add CS42L43B codec info + - ASoC: dt-bindings: cirrus, cs42l43: Add CS42L43B variant + - mfd: cs42l43: Add support for the B variant + - ASoC: cs42l43: Add support for the B variant + * Enable audio functions on Dell Huracan/Renegade platforms w/o built-in + microphone (LP: #2143902) + - ASoC: SDCA: Add default value for mipi-sdca-function-reset-max-delay + - ASoC: SDCA: Update counting of SU/GE DAPM routes + - ASoC: SDCA: Improve mapping of Q7.8 SDCA volumes + - ASoC: SDCA: Pull the Q7.8 volume helpers out of soc-ops + - ASoC: add snd_soc_lookup_component_by_name helper + - ASoC: soc_sdw_utils: partial match the codec name + - ASoC: soc_sdw_utils: remove index from sdca codec name + * [SRU] MIPI camera is not working after upgrading to 6.17-oem + (LP: #2145171) + - SAUCE: ACPI: respect items already in honor_dep before skipping + * linux-tools: consider linking perf against LLVM (LP: #2138328) + - [Packaging] Actually enable llvm for perf + * Pull patch in qla2xxx to Resolute (LP: #2144856) + - scsi: qla2xxx: Add support to report MPI FW state + * Ubuntu Resolute Desktop image arm64 - Boot on SC8280XP stalls with gpi-dma + errors (LP: #2142403) + - Revert "arm64: dts: qcom: sc8280xp: Enable GPI DMA" + * 26.04 Snapdragon X Elite: Sync concept kernel changes (LP: #2144643) + - SAUCE: arm64: dts: add missing denali-oled.dtb to Makefile + - SAUCE: dt-bindings: phy: qcom: Add CSI2 C-PHY/DPHY schema + - SAUCE: phy: qcom-mipi-csi2: Add a CSI2 MIPI DPHY driver + - SAUCE: dt-bindings: media: qcom,x1e80100-camss: Add simple-mfd + compatible + - SAUCE: dt-bindings: media: qcom,x1e80100-camss: Add optional PHY handle + definitions + - SAUCE: dt-bindings: media: qcom,x1e80100-camss: Add support for combo- + mode endpoints + - SAUCE: dt-bindings: media: qcom,x1e80100-camss: Describe iommu entries + - SAUCE: media: qcom: camss: Add legacy_phy flag to SoC definition + structures + - SAUCE: media: qcom: camss: Add support for PHY API devices + - SAUCE: media: qcom: camss: Drop legacy PHY descriptions from x1e + - SAUCE: arm64: dts: qcom: x1e80100: Add CAMCC block definition + - SAUCE: arm64: dts: qcom: x1e80100: Add CCI definitions + - SAUCE: arm64: dts: qcom: x1e80100: Add CAMSS block definition + - SAUCE: arm64: dts: qcom: x1e80100-crd: Add pm8010 CRD pmic,id=m + regulators + - SAUCE: arm64: dts: qcom: x1e80100-crd: Add ov08x40 RGB sensor on CSIPHY4 + - SAUCE: arm64: dts: qcom: x1e80100-t14s: Add pm8010 camera PMIC with + voltage levels for IR and RGB camera + - SAUCE: arm64: dts: qcom: x1e80100-t14s: Add on ov02c10 RGB sensor on + CSIPHY4 + - SAUCE: arm64: dts: qcom: x1e80100-lenovo-yoga-slim7x: Add pm8010 camera + PMIC with voltage levels for IR and RGB camera + - SAUCE: arm64: dts: qcom: x1e80100-lenovo-yoga-slim7x: Add l7b_2p8 + voltage regulator for RGB camera + - SAUCE: arm64: dts: qcom: x1e80100-lenovo-yoga-slim7x: Add ov02c10 RGB + sensor on CSIPHY4 + - SAUCE: arm64: dts: qcom: x1e80100-dell-inspiron14-7441: Switch on CAMSS + RGB sensor + - SAUCE: arm64: dts: qcom: x1-asus-zenbook-a14: Add on OV02C10 RGB sensor + on CSIPHY4 + - SAUCE: arm64: dts: qcom: x1e80100-dell-xps13-9345: add camera support + - SAUCE: arm64: dts: qcom: x1e78100-t14s: enable camera privacy indicator + - SAUCE: arm64: dts: qcom: x1e80100-lenovo-yoga-slim7x: enable camera + privacy indicator + - SAUCE: arm64: dts: qcom: x1e80100-dell-xps13-9345: enable camera privacy + indicator + - SAUCE: dt-bindings: arm: qcom: Add ASUS Vivobook X1P42100 variant + - SAUCE: arm64: dts: qcom: x1-vivobook-s15: create a common dtsi for Hamoa + and Purwa variants + - SAUCE: arm64: dts: qcom: x1-vivobook-s15: add Purwa-compatible device + tree + - SAUCE: firmware: qcom: scm: allow QSEECOM on ASUS Vivobook X1P42100 + variant + - SAUCE: arm64: dts: qcom: hamoa: Move PCIe PERST and Wake GPIOs to port + nodes + - SAUCE: arm64: dts: qcom: x1e-acer-swift-14: Move PCIe PERST and Wake + GPIOs to port nodes + * 25.10 Snapdragon X Elite: Sync concept kernel changes (LP: #2121477) + - SAUCE: wip: arm64: dts: qcom: x1e78100-t14s: enable bluetooth + * Miscellaneous Ubuntu changes + - SAUCE: dt-bindings: arm: qcom: Document HP EliteBook 6 G1q + - SAUCE: firmware: qcom: scm: Allow QSEECOM for HP EliteBook 6 G1q + - SAUCE: arm64: dts: qcom: x1p42100-hp-elitebook-6-g1q: DT for HP + EliteBook 6 G1q + - [Config] PHY_QCOM_MIPI_CSI2=m + - SAUCE: arm64: dts: x1e80100-lenovo-yoga-slim7x: Fix RGB camera supplies + - [Config] toolchain version update + - Update Changes.md after v7.0-rc5 rebase + - [Packaging] update Ubuntu.md + - [Config] enable SECURITY_APPARMOR_PACKET_MEDIATION_ENABLED + - [Packaging] Add linux-main-modules-zfs to linux-modules depends + * Miscellaneous upstream changes + - Revert "UBUNTU: SAUCE: Add Bluetooth support for the Lenovo Yoga Slim + 7x" + + [ Ubuntu: 7.0.0-10.10 ] + + * resolute/linux: 7.0.0-10.10 -proposed tracker (LP: #2144865) + * Miscellaneous upstream changes + - Revert "powerpc: fix KUAP warning in VMX usercopy path" + + [ Ubuntu: 7.0.0-9.9 ] + + * resolute/linux: 7.0.0-9.9 -proposed tracker (LP: #2144735) + * Please make dracut the default initrd generator (LP: #2142775) + - [Packaging] recommends dracut instead of initramfs-tools + * Miscellaneous Ubuntu changes + - SAUCE: Change RISC-V target to RVA23 (riscv64a23-unknown-linux-gnu) + + [ Ubuntu: 7.0.0-8.8 ] + + * resolute/linux: 7.0.0-8.8 -proposed tracker (LP: #2144652) + * UBUNTU: SAUCE: igc: Increase Thunderbolt MAC passthrough delay to 1000ms + (LP: #2143197) + - SAUCE: igc: Increase Thunderbolt MAC passthrough delay to 1000ms + * [usrmerge] evaluate kernel owned packages for DEP17 compliance + (LP: #2139276) + - [Packaging] Install modules in /usr/lib/modules + * Miscellaneous Ubuntu changes + - [Config] hardening: enable LIST_HARDENED + - [Config] hardening: disable LDISC_AUTOLOAD + - [Config] hardening: disable LEGACY_PTYS + - [Config] updateconfigs following v7.0-rc4 rebase + + [ Ubuntu: 7.0.0-7.7 ] + + * resolute/linux: 7.0.0-7.7 -proposed tracker (LP: #2143974) + * unconfined profile denies userns_create for chromium based processes + (LP: #1990064) + - [Config] disable CONFIG_SECURITY_APPARMOR_RESTRICT_USERNS + * Jellyfin Desktop Flatpak doesn't work with the current AppArmor profile + (LP: #2142956) + - SAUCE: apparmor5.0.0 [29/29]: apparmor: fix fine grained inet mediation + sock_file_perm + * AppArmor blocks write(2) to network sockets with Linux 6.19 (LP: #2141298) + - SAUCE: apparmor5.0.0 [28/29]: apparmor: fix aa_label_sk_perm to check + for RULE_MEDIATES_NET + * update apparmor and LSM stacking patch set (LP: #2028253) + - SAUCE: apparmor5.0.0 [1/29]: Stacking: LSM: Single calls in secid hooks + - SAUCE: apparmor5.0.0 [2/29]: Stacking: LSM: Exclusive secmark usage + - SAUCE: apparmor5.0.0 [3/29]: Stacking: AppArmor: Remove the exclusive + flag + - SAUCE: apparmor5.0.0 [4/29]: Revert "apparmor: fix dbus permission + queries to v9 ABI" + - SAUCE: apparmor5.0.0 [5/29]: Revert "apparmor: gate make fine grained + unix mediation behind v9 abi" + - SAUCE: apparmor5.0.0 [6/29]: apparmor: net: patch to provide + compatibility with v2.x net rules + - SAUCE: apparmor5.0.0 [7/29]: apparmor: net: add fine grained ipv4/ipv6 + mediation + - SAUCE: apparmor5.0.0 [8/29]: apparmor: lift compatibility check out of + profile_af_perm + - SAUCE: apparmor5.0.0 [9/29]: apparmor: userns: add unprivileged user ns + mediation + - SAUCE: apparmor5.0.0 [10/29]: apparmor: userns: Add sysctls for + additional controls of unpriv userns restrictions + - SAUCE: apparmor5.0.0 [12/29]: apparmor: userns: open userns related + sysctl so lxc can check if restriction are in place + - SAUCE: apparmor5.0.0 [13/29]: apparmor: userns: allow profile to be + transitioned when a userns is created + - SAUCE: apparmor5.0.0 [14/29]: apparmor: mqueue: call + security_inode_init_security on inode creation + - SAUCE: apparmor5.0.0 [15/29]: apparmor: mqueue: add fine grained + mediation of posix mqueues + - SAUCE: apparmor5.0.0 [16/29]: apparmor: uring: add io_uring mediation + - SAUCE: apparmor5.0.0 [19/29]: apparmor: prompt: setup slab cache for + audit data + - SAUCE: apparmor5.0.0 [20/29]: apparmor: prompt: add the ability for + profiles to have a learning cache + - SAUCE: apparmor5.0.0 [21/29]: apparmor: prompt: enable userspace upcall + for mediation + - SAUCE: apparmor5.0.0 [22/29]: apparmor: prompt: pass prompt boolean + through into path_name as well + - SAUCE: apparmor5.0.0 [23/29]: apparmor: check for supported version in + notification messages. + - SAUCE: apparmor5.0.0 [24/29]: apparmor: refactor building notice so it + is easier to extend + - SAUCE: apparmor5.0.0 [25/29]: apparmor: switch from ENOTSUPP to + EPROTONOSUPPORT + - SAUCE: apparmor5.0.0 [26/29]: apparmor: add support for meta data tags + - SAUCE: apparmor5.0.0 [27/29]: apparmor: prevent profile->disconnected + double free in aa_free_profile + * update apparmor and LSM stacking patch set (LP: #2028253) // Installation + of AppArmor on a 6.14 kernel produces error message "Illegal number: yes" + (LP: #2102680) + - SAUCE: apparmor5.0.0 [17/29]: apparmor: create an + AA_SFS_TYPE_BOOLEAN_INTPRINT sysctl variant + - SAUCE: apparmor5.0.0 [18/29]: apparmor: Use AA_SFS_FILE_BOOLEAN_INTPRINT + for userns and io_uring sysctls + * update apparmor and LSM stacking patch set (LP: #2028253) // [FFe] + apparmor-4.0.0-alpha2 for unprivileged user namespace restrictions in + mantic (LP: #2032602) + - SAUCE: apparmor5.0.0 [11/29]: apparmor: userns - make it so special + unconfined profiles can mediate user namespaces + * NPU utilization on amdxdna is missing (LP: #2143243) + - SAUCE: accel/amdxdna: Add IOCTL to retrieve realtime NPU power estimate + - SAUCE: accel/amdxdna: Support sensors for column utilization + - SAUCE: accel/amdxdna: Import AMD_PMF namespace + * Adopting dark mode by default for OLED panel (LP: #2143203) + - SAUCE: drm/connector: Add a new 'panel_type' property + - SAUCE: drm/amd/display: Attach OLED property to eDP panels + * Support AMD Image Signal Processing (ISP) unit V4.0 (LP: #2110092) + - SAUCE: media: platform: amd: Introduce amd isp4 capture driver + - SAUCE: media: platform: amd: low level support for isp4 firmware + - SAUCE: media: platform: amd: Add isp4 fw and hw interface + - SAUCE: media: platform: amd: isp4 subdev and firmware loading handling + added + - SAUCE: media: platform: amd: isp4 video node and buffers handling added + - SAUCE: Documentation: add documentation of AMD isp 4 driver + - SAUCE: media: platform: amd: isp4 debug fs logging and more descriptive + errors + - [Config] Enable VIDEO_AMD_ISP4_CAPTURE + * Miscellaneous Ubuntu changes + - [Config] temporarily disable OBJTOOL_WERROR + + -- Jacob Martin Thu, 02 Apr 2026 10:48:01 -0500 + +linux-nvidia (7.0.0-1003.3) resolute; urgency=medium + + * Packaging resync (LP: #1786013) + - [Packaging] debian.nvidia/dkms-versions -- update from kernel-versions + (adhoc/d2026.02.16) + + * Backport NVIDIA: SAUCE: vfio/nvgrace-egm: split zapping EGM into 1GB + chunks (LP: #2142160) + - NVIDIA: SAUCE: vfio/nvgrace-egm: split zapping EGM into 1GB chunks + + * r8127 module unload triggers NAPI WARN in netif_napi_del_locked() + (LP: #2141780) + - NVIDIA: SAUCE: r8127: fix NAPI warning on module removal + + * Enable Coresight in Perf (LP: #2093957) + - [Packaging] Enable coresight in Perf if arm64 + - [Packaging] Add libopencsd-dev as a build dependency + + * Backport nvgrace-gpu hugepfnmap, ecc patches and miscellaneous cleanups + (LP: #2138892) + - NVIDIA: SAUCE: vfio/nvgrace-egm: register EGM PFNMAP range with + memory_failure + - NVIDIA: SAUCE: vfio: Remove vfio_device_from_file() declaration + + * missing prototype for vfio_device_from_file() (LP: #2138132) + - NVIDIA: SAUCE: vfio: Fix missing prototype warning + + * r8127: Downgrade GPL claim to info (LP: #2137588) + - NVIDIA: SAUCE: r8127: print GPL_CLAIM with KERN_INFO + + * mt7925: Incorrect MLO mode in firmware control (LP: #2138755) + - NVIDIA: SAUCE: wifi: mt76: mt7925: Fix incorrect MLO mode in firmware + control + + * Enable GDS in the 6.8 based linux-nvidia kernel (LP: #2059814) + - NVIDIA: [Packaging] Add nvidia-fs build dependencies + + * Add PCIe Hotplug Driver for CX7 on DGX Spark (LP: #2138269) + - NVIDIA: SAUCE: MEDIATEK: platform: Add PCIe Hotplug Driver for CX7 on + DGX Spark + + * Backport support for Grace MPAM (LP: #2122432) + - NVIDIA: SAUCE: DT: cacheinfo: Expose the code to generate a cache-id + from a device_node + - NVIDIA: SAUCE: DT: dt-bindings: arm: Add MPAM MSC binding + - NVIDIA: SAUCE: arm64: mpam: Context switch the MPAM registers + - NVIDIA: SAUCE: arm64: mpam: Re-initialise MPAM regs when CPU comes + online + - NVIDIA: SAUCE: arm64: mpam: Advertise the CPUs MPAM limits to the driver + - NVIDIA: SAUCE: arm64: mpam: Add cpu_pm notifier to restore MPAM sysregs + - NVIDIA: SAUCE: arm64: mpam: Add helpers to change a tasks and cpu mpam + partid/pmg values + - NVIDIA: SAUCE: cacheinfo: Add helper to find the cache size from + cpu+level + - NVIDIA: SAUCE: arm_mpam: resctrl: Add boilerplate cpuhp and domain + allocation + - NVIDIA: SAUCE: arm_mpam: resctrl: Pick the caches we will use as resctrl + resources + - NVIDIA: SAUCE: arm_mpam: resctrl: Implement + resctrl_arch_reset_all_ctrls() + - NVIDIA: SAUCE: arm_mpam: resctrl: Add resctrl_arch_get_config() + - NVIDIA: SAUCE: arm_mpam: resctrl: Implement helpers to update + configuration + - NVIDIA: SAUCE: arm_mpam: resctrl: Add plumbing against arm64 task and + cpu hooks + - NVIDIA: SAUCE: arm_mpam: resctrl: Add CDP emulation + - NVIDIA: SAUCE: arm_mpam: resctrl: Add rmid index helpers + - NVIDIA: SAUCE: arm_mpam: resctrl: Convert to/from MPAMs bitmaps and + fixed-point formats + - NVIDIA: SAUCE: arm_mpam: resctrl: Add support for 'MB' resource + - NVIDIA: SAUCE: arm_mpam: resctrl: Reject oversized memory bandwidth + portion bitmaps + - NVIDIA: SAUCE: arm_mpam: resctrl: Fix MB min_bandwidth value exposed to + userspace + - NVIDIA: SAUCE: arm_mpam: resctrl: Add kunit test for control format + conversions + - NVIDIA: SAUCE: arm_mpam: resctrl: Add support for csu counters + - NVIDIA: SAUCE: arm_mpam: resctrl: Pre-allocate free running monitors + - NVIDIA: SAUCE: arm_mpam: resctrl: Pre-allocate assignable monitors + - NVIDIA: SAUCE: arm_mpam: resctrl: Add kunit test for ABMC/CDP + interactions + - NVIDIA: SAUCE: arm_mpam: resctrl: Add resctrl_arch_config_cntr() for + ABMC use + - NVIDIA: SAUCE: arm_mpam: resctrl: Allow resctrl to allocate monitors + - NVIDIA: SAUCE: arm_mpam: resctrl: Add resctrl_arch_rmid_read() and + resctrl_arch_reset_rmid() + - NVIDIA: SAUCE: arm_mpam: resctrl: Add resctrl_arch_cntr_read() & + resctrl_arch_reset_cntr() + - NVIDIA: SAUCE: untested: arm_mpam: resctrl: Allow monitors to be + configured with filters + - NVIDIA: SAUCE: arm_mpam: resctrl: Add empty definitions for fine-grained + enables + - NVIDIA: SAUCE: arm64: mpam: Select ARCH_HAS_CPU_RESCTRL + - NVIDIA: SAUCE: fs/resctrl: Don't touch rmid_ptrs[] in free_rmid() when + there are no monitors + - NVIDIA: SAUCE: fs/resctrl: Avoid a race with dom_data_exit() and + closid_num_dirty_rmid[] + - NVIDIA: SAUCE: fs/resctrl: Avoid a race with dom_data_exit() and + rmid_ptrs[] + - NVIDIA: SAUCE: perf/arm-cmn: Stop claiming all the resources + - NVIDIA: SAUCE: arm_mpam: resctrl: Call resctrl_init() on platforms that + can support resctrl + - NVIDIA: SAUCE: arm_mpam: resctrl: Call resctrl_exit() in the event of + errors + - NVIDIA: SAUCE: arm_mpam: resctrl: Update the rmid reallocation limit + - NVIDIA: SAUCE: arm_mpam: resctrl: Sort the order of the domain lists + - NVIDIA: SAUCE: arm_mpam: Generate a configuration for min controls + - NVIDIA: SAUCE: arm_mpam: Add quirk framework + - NVIDIA: SAUCE: arm_mpam: Add workaround for T241-MPAM-1 + - NVIDIA: SAUCE: arm_mpam: Add workaround for T241-MPAM-4 + - NVIDIA: SAUCE: arm_mpam: Add workaround for T241-MPAM-6 + - NVIDIA: SAUCE: arm_mpam: Quirk CMN-650's CSU NRDY behaviour + - NVIDIA: SAUCE: debugfs: Add helpers for creating cpumask entries in + debugfs + - NVIDIA: SAUCE: arm_mpam: Add debugfs entries to show the MSC/RIS the + driver discovered + - NVIDIA: SAUCE: arm_mpam: Add force-disable debugfs trigger + - NVIDIA: SAUCE: arm_mpam: Expose the number of NRDY retries in debugfs + - NVIDIA: SAUCE: arm_mpam: Add resctrl_arch_round_bw() + - NVIDIA: SAUCE: fs/resctrl,x86/resctrl: Factor mba rounding to be per- + arch + - NVIDIA: SAUCE: arm_mpam: Relax num_rmids parameter advertised to + userspace + - NVIDIA: SAUCE: arm_mpam: Split the locking around the mon_sel registers + - NVIDIA: SAUCE: arm_mpam: Allow the maximum partid to be overridden from + the command line + - NVIDIA: SAUCE: arm_mpam: Allow MSC to be forced to have an unknown + location + - NVIDIA: SAUCE: fs/resctrl: Add this_is_not_abi mount option + - NVIDIA: SAUCE: iommu/arm-smmu-v3: Register SMMU capabilities with MPAM + - NVIDIA: SAUCE: iommu/arm-smmu-v3: Add mpam helpers to query and set + state + - NVIDIA: SAUCE: iommu: Add helpers to get and set the QoS state + - NVIDIA: SAUCE: iommu: Add helpers to retrieve iommu_groups by id or + kobject + - NVIDIA: SAUCE: iommu: Add helper to retrieve iommu kset + - NVIDIA: SAUCE: kobject: Add kset_get_next_obj() to allow a kset to be + walked + - NVIDIA: SAUCE: arm_mpam: resctrl: Add iommu helpers to get/set the + partid and pmg + - NVIDIA: SAUCE: fs/resctrl: Add support for assigning iommu_groups to + resctrl groups + - NVIDIA: SAUCE: firmware: arm_scmi: add MPAM-FB SCMI protocol stub + - NVIDIA: SAUCE: arm_mpam: add MPAM-FB MSC firmware access support + - NVIDIA: SAUCE: arm_mpam: Allow duplicate PCC subspace_ids + - NVIDIA: SAUCE: untested: mpam: Convert pcc_channels list to XArray and + cleanup + - NVIDIA: SAUCE: x86/resctrl: Add stub to allow other architecture to + disable monitor overflow + - NVIDIA: SAUCE: arm_mpam: resctrl: Determine if any exposed counter can + overflow + - NVIDIA: SAUCE: fs/restrl: Allow the overflow handler to be disabled + - NVIDIA: SAUCE: fs/resctrl: Uniform data type of + component_id/domid/id/cache_id + - NVIDIA: SAUCE: arm_mpam: Allow cmax/cmin to be configured + - NVIDIA: SAUCE: arm_mpam: Rename mbw conversion to 'fract16' for code re- + use + - NVIDIA: SAUCE: fs/resctrl: Group all the MBA specific properties in a + separate struct + - NVIDIA: SAUCE: fs/resctrl: Abstract duplicate domain test to a helper + - NVIDIA: SAUCE: fs/resctrl: Move MBA supported check to parse_line() + instead of parse_bw() + - NVIDIA: SAUCE: fs/resctrl: Rename resctrl_get_default_ctrl() to include + resource + - NVIDIA: SAUCE: fs/resctrl: Add a schema format to the schema, allowing + it to be different + - NVIDIA: SAUCE: fs/resctrl: Use schema format to check the resource is a + bitmap + - NVIDIA: SAUCE: fs/resctrl: Add specific schema types for 'range' + - NVIDIA: SAUCE: x86/resctrl: Move over to specifying MBA control formats + - NVIDIA: SAUCE: arm_mpam: resctrl: Convert MB resource to use percentage + - NVIDIA: SAUCE: fs/resctrl: Remove 'range' schema format + - NVIDIA: SAUCE: fs/resctrl: Add additional files for percentage and + bitmap controls + - NVIDIA: SAUCE: fs/resctrl: Add fflags_from_schema() for files based on + schema format + - NVIDIA: SAUCE: fs/resctrl: Expose the schema format to user-space + - NVIDIA: SAUCE: fs/resctrl: Add L2 and L3 'MAX' resource schema + - NVIDIA: SAUCE: arm_mpam: resctrl: Add the glue code to convert to/from + cmax + - NVIDIA: SAUCE: mm,memory_hotplug: Add lockdep assertion helper + - NVIDIA: SAUCE: fs/resctrl: Take memory hotplug lock whenever taking CPU + hotplug lock + - NVIDIA: SAUCE: fs/resctrl: Add mount option for mb_uses_numa_nid and + arch stubs + - NVIDIA: SAUCE: Fix unused variable warning + - NVIDIA: SAUCE: arm_mpam: resctrl: Pick whether MB can use NUMA nid + instead of cache-id + - NVIDIA: SAUCE: arm_mpam: resctrl: Change domain_hdr online/offline to + work with a set of CPUs + - NVIDIA: SAUCE: untested: arm_mpam: resctrl: Split + mpam_resctrl_alloc_domain() to have CPU and node + - NVIDIA: SAUCE: arm_mpam: resctrl: Add NUMA node notifier for domain + online/offline + - NVIDIA: SAUCE: untested: arm_mpam: resctrl: Allow resctrl to enable NUMA + nid as MB domain-id + - NVIDIA: SAUCE: [Config] RESCTRL configs added to annotations + - NVIDIA: SAUCE: arm_mpam: Fix missing SHIFT definitions + - NVIDIA: SAUCE: arm_mpam: resctrl: Fix MPAM kunit + - NVIDIA: SAUCE: resctrl/mpam: Align packed mpam_props to fix arm64 KUnit + alignment fault + - NVIDIA: SAUCE: resctrl/tests: mpam_devices: compare only meaningful + bytes of mpam_props + + * r8127: fix for LTS test panic (LP: #2134991) + - NVIDIA: SAUCE: r8127: Remove registers2 proc entry + + * Add two more Spark iGPU IDs for the existing iommu quirk (LP: #2132033) + - NVIDIA: SAUCE: iommu/arm-smmu-v3: Add two more DGX Spark iGPU IDs for + existing iommu quirk + + * Pull CPPC mailing list patches for Spark (LP: #2131705) + - NVIDIA: SAUCE: ACPI: CPPC: Add cppc_get_perf() API to read performance + controls + - NVIDIA: SAUCE: ACPI: CPPC: extend APIs to support auto_sel and epp + - NVIDIA: SAUCE: ACPI: CPPC: add APIs and sysfs interface for min/max_perf + - NVIDIA: SAUCE: ACPI: CPPC: add APIs and sysfs interface for perf_limited + register + - NVIDIA: SAUCE: cpufreq: CPPC: Add sysfs for min/max_perf and + perf_limited + - NVIDIA: SAUCE: cpufreq: CPPC: update policy min/max when toggling + auto_select + - NVIDIA: SAUCE: cpufreq: CPPC: add autonomous mode boot parameter support + + * r8127: fix kernel panic when dump all registers (LP: #2130445) + - NVIDIA: SAUCE: r8127: fix a kernel panic when dump all registers + - NVIDIA: SAUCE: r8127: add support for RTL8127 cable diagnostic test + + * Set CONFIG_IOMMU_DEFAULT_PASSTHROUGH as default for Nvidia CPUs + (LP: #2129776) + - NVIDIA: SAUCE: iommu/arm-smmu-v3: Set DGX Spark iGPU default domain type + to DMA + - [Config] nvidia: Update annotations to set + CONFIG_IOMMU_DEFAULT_PASSTHROUGH + + * mt7925: Introduce CSA support in non-MLO mode (LP: #2129209) + - NVIDIA: SAUCE: wifi: mt76: mt7925: introduce CSA support in non-MLO mode + + * IOMMU: Support contiguous bit in translation tables (LP: #2112600) + - NVIDIA: SAUCE: iommu/io-pgtable-arm: backport contiguous bit support + + * NVIDIA: SAUCE: MEDIATEK: usb: host: xhci-hub: fix MT89xx SoCs return + PORTLI value (LP: #2125126) + - NVIDIA: SAUCE: MEDIATEK: usb: host: xhci-hub: fix MT89xx SoCs return + PORTLI value + + * NVIDIA: SAUCE: ffa notification count initialization fix (LP: #2123861) + - NVIDIA: SAUCE: Fix FFA notification count initialization + + * Pull-request for setting CPU frequency gov to performance (LP: #2028576) + - [Config] nvidia: Use performance CPU frequency governor on amd64 + + * Set CONFIG_IOMMU_DEFAULT_DMA_LAZY as default for Nvidia CPUs + (LP: #2119661) + - [Config] nvidia: Update annotations to set CONFIG_IOMMU_DEFAULT_DMA_LAZY + + * Backport support for Grace virtualization features: vEVENTQ, HW QUEUE, and + vEGM (LP: #2119656) + - NVIDIA: SAUCE: arm64: configs: Build NVGRACE_GPU_VFIO_PCI as LKM + - NVIDIA: SAUCE: arm64: configs: Enable IOMMUFD and VFIO_DEVICE_CDEV + - NVIDIA: SAUCE: vfio/nvgrace-egm: Introduce module to manage EGM + - NVIDIA: SAUCE: vfio/nvgrace-egm: Handle pages with ECC errors on the EGM + - NVIDIA: SAUCE: arm64: configs: Build CONFIG_NVGRACE_EGM as LKM + - NVIDIA: SAUCE: vfio/nvgrace-egm: Move the egm header file to include + - NVIDIA: SAUCE: vfio/nvgrace-egm: Free region memory during + unregistration + - NVIDIA: SAUCE: vfio/nvgrace-egm: Move region hash initialization + - NVIDIA: SAUCE: vfio/nvgrace-egm: Handle and convey EGM registration + errors + - NVIDIA: SAUCE: vfio/nvgrace-gpu: Handle EGM registration failure + - NVIDIA: SAUCE: vfio/nvgrace-egm: Address sparse errors + - NVIDIA: SAUCE: vfio/nvgrace-gpu: Address smatch errors + - NVIDIA: SAUCE: vfio/nvgrace-egm: Ensure ACPI value reads are successful + - NVIDIA: SAUCE: vfio/nvgrace-egm: Avoid invalid retired pages base + - NVIDIA: SAUCE: vfio/nvgrace-egm: Update EGM unregistration API + - NVIDIA: SAUCE: vfio/nvgrace-egm: track GPUs associated with the EGM + regions + - NVIDIA: SAUCE: vfio/nvgrace-egm: list gpus through sysfs + - NVIDIA: SAUCE: vfio/nvgrace-egm: expose the egm size through sysfs + - NVIDIA: SAUCE: arm64: configs: enable NVGRACE_EGM as module + + * Backport support for arm64 BRBE and a future NVIDIA CPU ID (LP: #2118663) + - [Config] nvidia: Enable BRBE + + * nvidia-ffa-ec: Fix FFH data response length (LP: #2118357) + - NVIDIA: SAUCE: Fix FFH data response length + + * Add pincontrol driver for MT8901 chip (LP: #2117784) + - NVIDIA: SAUCE: MEDIATEK: pinctrl: mediatek: Add gpio-range record in + pinctrl driver + - NVIDIA: SAUCE: MEDIATEK: pinctrl: mediatek: Add acpi support + - NVIDIA: SAUCE: MEDIATEK: pinctrl: mt8901: Add pinctrl driver + - [Config] nvidia: Update annotations to enable CONFIG_PINCTRL_MT8901 + + * NVIDIA: SAUCE: Add FFA and EC Secure Service Driver to -nvidia kernel + (LP: #2114230) + - NVIDIA: SAUCE: Add support for custom ARM FFH offset handler + - NVIDIA: SAUCE: Add nvidia ffa driver for EC communication + - NVIDIA: SAUCE: Add ffa driver for each secure EC service + - NVIDIA: SAUCE: Add support for EC secure service communication + - NVIDIA: SAUCE: Rescan acpi devices that uses secure EC communication + - NVIDIA: SAUCE: irqchip/gic-v3: Allow unused SGIs for drivers/modules + - NVIDIA: SAUCE: Add support for notifications from secure EC services + - [Config] nvidia: Update annotations to enable NVIDIA FFA EC driver + + * Backport: TPM Service Command Response Buffer Interface Over FF-A + (LP: #2111511) + - [Config] nvidia-6.14: Update annotations to enable TPM over FFA + + * Backport: ALSA: hda - Add new driver for HDA controllers listed via ACPI + (LP: #2111447) + - NVIDIA: SAUCE: [Config] nvidia: CONFIG_SND_HDA_ACPI=m on arm64 + + * Pull request to enable GPU passthrough for CUDA (LP: #2095028) + - NVIDIA: SAUCE: WAR: iommufd/pages: Bypass PFNMAP + - NVIDIA: SAUCE: [Config] nvidia: Update annotations for Grace I/O + virtualization + - [Config] nvidia-6.14: Drop CONFIG_TEGRA241_CMDQV from annotations + + * Add Realtek r8127 ethernet driver (LP: #2109730) + - NVIDIA: SAUCE: r8127: Add Realtek r8127 ethernet driver + - NVIDIA: SAUCE: r8127: Remove Realtek r8127 non required files + - NVIDIA: SAUCE: r8127: Moved files from r8127/src to r8127 folder + - NVIDIA: SAUCE: Add r8127 in kernel build + - [Config] nvidia-6.11: Update annotations to enable realtek R8127 module + + * Pull request: Add quirk and disable SBR on Gen5 ports (LP: #2107509) + - NVIDIA: SAUCE: MEDIATEK: usb: host: xhci-plat: support usb3 bulks stream + low power + + * Apply backport of upstream commit to enable Realtek Bluetooth module + (LP: #2096882) + - NVIDIA: SAUCE: Adds MT7925 BT devices + + * Apply SAUCE patch to enable 8250 serial device (LP: #2096888) + - NVIDIA: SAUCE: serial: 8250_mtk: Add ACPI support + + * Backport: "Add support for AArch64 AMUv1-based average freq" Series + (LP: #2100032) + - NVIDIA: [Config] set CONFIG_CPUFREQ_ARCH_CUR_FREQ=y for x86 + + * MANA: include driver fixes and enable module on ARM64 (LP: #2084598) + - [Config] nvidia-6.17: Enable MANA configs on x86 and arm64 + + * Apply patch to set CONFIG_EFI_CAPSULE_LOADER=y for arm64 (LP: #2067111) + - NVIDIA: [Config] EFI: set CAPSULE_LOADER=y for arm64 + + * linux-nvidia-6.5_6.5.0-1014.14 breaks with earlier BIOS release, and + modeset/resolutions are wrong (LP: #2061930) // Blacklist coresight_etm4x + (LP: #2067106) + - [Packaging] blacklist coresight_etm4x + + * backport arm64 THP improvements from 6.9 (LP: #2059316) + - NVIDIA: [Config] arm64: ARM64_CONTPTE=y + + * Reapply the linux-nvidia kernel config options from the 5.15 and 6.5 + kernels (LP: #2060327) + - NVIDIA: [Config]: Disable the NOUVEAU driver which is not used with + -nvidia kernels + - NVIDIA: [Config]: Adding CORESIGHT and ARM64_ERRATUM configs to + annotations + + [ Ubuntu: 7.0.0-6.6 ] + + * resolute/linux: 7.0.0-6.6 -proposed tracker (LP: #2143745) + * Miscellaneous Ubuntu changes + - [Packaging] drop unstable suffix + + [ Ubuntu: 7.0.0-5.5 ] + + * resolute/linux-unstable: 7.0.0-5.5 -proposed tracker (LP: #2143700) + * Resolute real-time patchset: 7.0-rc1-rt1 (LP: #2143181) + - SAUCE: Reapply "serial: 8250: Switch to nbcon console" + - SAUCE: Reapply "serial: 8250: Revert "drop lockdep annotation from + serial8250_clear_IER()"" + - SAUCE: drm/i915: Use preempt_disable/enable_rt() where recommended + - SAUCE: drm/i915: Don't disable interrupts on PREEMPT_RT during atomic + updates + - SAUCE: drm/i915: Disable tracing points on PREEMPT_RT + - SAUCE: drm/i915/gt: Use spin_lock_irq() instead of local_irq_disable() + + spin_lock() + - SAUCE: drm/i915: Drop the irqs_disabled() check + - SAUCE: drm/i915/guc: Consider also RCU depth in busy loop. + - SAUCE: drm/i915: Consider RCU read section as atomic. + - SAUCE: Revert "drm/i915: Depend on !PREEMPT_RT." + - SAUCE: sysfs: Add /sys/kernel/realtime entry + - Real-time patchset 7.0-rc1-rt1 + * Miscellaneous Ubuntu changes + - [Config] rust toolchain version update + + [ Ubuntu-unstable: 7.0.0-4.4 ] + + * resolute/linux-unstable: 7.0.0-4.4 -proposed tracker (LP: #2143123) + * efi: Fix swapped arguments to bsearch() in efi_status_to_*() SAUCE patch + (LP: #2141276) + - SAUCE efi: Fix swapped arguments to bsearch() in efi_status_to_*() + * Plucky preinstalled server fails to boot on rb3gen2 (LP: #2106681) // + Questing preinstalled server fails to boot on sa8775p boards + (LP: #2121347) + - [Config] move more qcom interconnect/pinctrl/gcc options to builtin + * linux-tools: consider linking perf against LLVM (LP: #2138328) + - [Packaging] Add llvm-21-dev to build-depends for perf + * Miscellaneous Ubuntu changes + - [Packaging] Add intel-speed-select to linux-tools + - [Packaging] remove stale debian/dkms-versions + - [Packaging] remove stale debian/dkms-versions scripting + + [ Ubuntu-unstable: 7.0.0-3.3 ] + + * resolute/linux-unstable: 7.0.0-3.3 -proposed tracker (LP: #2143020) + * Miscellaneous Ubuntu changes + - [Config] updateconfig after rebase to v7.0-rc2 + - [Config] switch to PREEMPT_LAZY + + [ Ubuntu-unstable: 7.0.0-2.2 ] + + * resolute/linux-unstable: 7.0.0-2.2 -proposed tracker (LP: #2142764) + + [ Ubuntu-unstable: 7.0.0-1.1 ] + + * resolute/linux-unstable: 7.0.0-1.1 -proposed tracker (LP: #2142402) + * Miscellaneous Ubuntu changes + - [packaging] rename to linux-unstable + - [Config] updateconfig after rebase to v7.0-rc1 + - Update Changes.md + - [Packaging] add libbpf-dev to Build-Depends + - [Config] disable AMD_ISP4, FTBFS + - [Packaging] debian.master/dkms-versions -- temporarily remove zfs FTBFS + - [Packaging] debian.master/dkms-versions -- temporarily remove evdi FTBFS + - [Config] updateconfig after rebase to v7.0-rc1 + - [Config] update toolchain version + + -- Jacob Martin Thu, 26 Mar 2026 11:14:51 -0500 + +linux-nvidia (6.19.0-1001.1) resolute; urgency=medium + + * linux-tools: consider linking perf against LLVM (LP: #2138328) + - [Packaging] Add llvm-21-dev to build-depends for perf + + [ Ubuntu: 6.19.0-6.6 ] + + * resolute/linux: 6.19.0-6.6 -proposed tracker (LP: #2142114) + * Resolute update: v6.19.2 upstream stable release (LP: #2142112) + - Revert "driver core: enforce device_lock for driver_match_device()" + - Linux 6.19.2 + * Resolute update: v6.19.1 upstream stable release (LP: #2142111) + - io_uring/io-wq: add exit-on-idle state + - io_uring: allow io-wq workers to exit when unused + - smb: client: split cached_fid bitfields to avoid shared-byte RMW races + - ksmbd: fix infinite loop caused by next_smb2_rcv_hdr_off reset in error + paths + - ksmbd: add chann_lock to protect ksmbd_chann_list xarray + - smb: server: fix leak of active_num_conn in ksmbd_tcp_new_connection() + - smb: smbdirect: introduce smbdirect_socket.recv_io.credits.available + - smb: smbdirect: introduce smbdirect_socket.send_io.bcredits.* + - smb: server: make use of smbdirect_socket.recv_io.credits.available + - smb: server: let recv_done() queue a refill when the peer is low on + credits + - smb: server: make use of smbdirect_socket.send_io.bcredits + - smb: server: fix last send credit problem causing disconnects + - smb: server: let send_done handle a completion without IB_SEND_SIGNALED + - smb: client: make use of smbdirect_socket.recv_io.credits.available + - smb: client: let recv_done() queue a refill when the peer is low on + credits + - smb: client: let smbd_post_send() make use of request->wr + - smb: client: remove pointless sc->recv_io.credits.count rollback + - smb: client: remove pointless sc->send_io.pending handling in + smbd_post_send_iter() + - smb: client: port and use the wait_for_credits logic used by server + - smb: client: split out smbd_ib_post_send() + - smb: client: introduce and use smbd_{alloc, free}_send_io() + - smb: client: use smbdirect_send_batch processing + - smb: client: make use of smbdirect_socket.send_io.bcredits + - smb: client: fix last send credit problem causing disconnects + - smb: client: let smbd_post_send_negotiate_req() use smbd_post_send() + - smb: client: let send_done handle a completion without IB_SEND_SIGNALED + - driver core: enforce device_lock for driver_match_device() + - Bluetooth: btusb: Add USB ID 7392:e611 for Edimax EW-7611UXB + - ALSA: hda/conexant: Add quirk for HP ZBook Studio G4 + - crypto: iaa - Fix out-of-bounds index in find_empty_iaa_compression_mode + - crypto: octeontx - Fix length check to avoid truncation in + ucode_load_store + - crypto: omap - Allocate OMAP_CRYPTO_FORCE_COPY scatterlists correctly + - crypto: virtio - Add spinlock protection with virtqueue notification + - crypto: virtio - Remove duplicated virtqueue_kick in + virtio_crypto_skcipher_crypt_req + - nilfs2: Fix potential block overflow that cause system hang + - hfs: ensure sb->s_fs_info is always cleaned up + - wifi: rtw88: Fix alignment fault in rtw_core_enable_beacon() + - scsi: qla2xxx: Validate sp before freeing associated memory + - scsi: qla2xxx: Allow recovery for tape devices + - scsi: qla2xxx: Delay module unload while fabric scan in progress + - scsi: qla2xxx: Free sp in error path to fix system crash + - scsi: qla2xxx: Query FW again before proceeding with login + - sched/mmcid: Don't assume CID is CPU owned on mode switch + - bus: fsl-mc: fix use-after-free in driver_override_show() + - erofs: fix UAF issue for file-backed mounts w/ directio option + - xfs: fix UAF in xchk_btree_check_block_owner + - drm/exynos: vidi: use ctx->lock to protect struct vidi_context member + variables related to memory alloc/free + - PCI: endpoint: Avoid creating sub-groups asynchronously + - wifi: rtl8xxxu: fix slab-out-of-bounds in rtl8xxxu_sta_add + - Linux 6.19.1 + * AppArmor blocks write(2) to network sockets with Linux 6.19 (LP: #2141298) + - SAUCE: apparmor: fix aa_label_sk_perm to check for RULE_MEDIATES_NET + + -- Jacob Martin Tue, 03 Mar 2026 11:01:18 -0600 + +linux-nvidia (6.19.0-1000.0) resolute; urgency=medium + + * Initial changelog entry. + + -- Jacob Martin Fri, 27 Feb 2026 14:49:42 -0600 + +linux (6.19.0-6.6) resolute; urgency=medium + + * resolute/linux: 6.19.0-6.6 -proposed tracker (LP: #2142114) + + * Resolute update: v6.19.2 upstream stable release (LP: #2142112) + - Revert "driver core: enforce device_lock for driver_match_device()" + - Linux 6.19.2 + + * Resolute update: v6.19.1 upstream stable release (LP: #2142111) + - io_uring/io-wq: add exit-on-idle state + - io_uring: allow io-wq workers to exit when unused + - smb: client: split cached_fid bitfields to avoid shared-byte RMW races + - ksmbd: fix infinite loop caused by next_smb2_rcv_hdr_off reset in error + paths + - ksmbd: add chann_lock to protect ksmbd_chann_list xarray + - smb: server: fix leak of active_num_conn in ksmbd_tcp_new_connection() + - smb: smbdirect: introduce smbdirect_socket.recv_io.credits.available + - smb: smbdirect: introduce smbdirect_socket.send_io.bcredits.* + - smb: server: make use of smbdirect_socket.recv_io.credits.available + - smb: server: let recv_done() queue a refill when the peer is low on + credits + - smb: server: make use of smbdirect_socket.send_io.bcredits + - smb: server: fix last send credit problem causing disconnects + - smb: server: let send_done handle a completion without IB_SEND_SIGNALED + - smb: client: make use of smbdirect_socket.recv_io.credits.available + - smb: client: let recv_done() queue a refill when the peer is low on + credits + - smb: client: let smbd_post_send() make use of request->wr + - smb: client: remove pointless sc->recv_io.credits.count rollback + - smb: client: remove pointless sc->send_io.pending handling in + smbd_post_send_iter() + - smb: client: port and use the wait_for_credits logic used by server + - smb: client: split out smbd_ib_post_send() + - smb: client: introduce and use smbd_{alloc, free}_send_io() + - smb: client: use smbdirect_send_batch processing + - smb: client: make use of smbdirect_socket.send_io.bcredits + - smb: client: fix last send credit problem causing disconnects + - smb: client: let smbd_post_send_negotiate_req() use smbd_post_send() + - smb: client: let send_done handle a completion without IB_SEND_SIGNALED + - driver core: enforce device_lock for driver_match_device() + - Bluetooth: btusb: Add USB ID 7392:e611 for Edimax EW-7611UXB + - ALSA: hda/conexant: Add quirk for HP ZBook Studio G4 + - crypto: iaa - Fix out-of-bounds index in find_empty_iaa_compression_mode + - crypto: octeontx - Fix length check to avoid truncation in + ucode_load_store + - crypto: omap - Allocate OMAP_CRYPTO_FORCE_COPY scatterlists correctly + - crypto: virtio - Add spinlock protection with virtqueue notification + - crypto: virtio - Remove duplicated virtqueue_kick in + virtio_crypto_skcipher_crypt_req + - nilfs2: Fix potential block overflow that cause system hang + - hfs: ensure sb->s_fs_info is always cleaned up + - wifi: rtw88: Fix alignment fault in rtw_core_enable_beacon() + - scsi: qla2xxx: Validate sp before freeing associated memory + - scsi: qla2xxx: Allow recovery for tape devices + - scsi: qla2xxx: Delay module unload while fabric scan in progress + - scsi: qla2xxx: Free sp in error path to fix system crash + - scsi: qla2xxx: Query FW again before proceeding with login + - sched/mmcid: Don't assume CID is CPU owned on mode switch + - bus: fsl-mc: fix use-after-free in driver_override_show() + - erofs: fix UAF issue for file-backed mounts w/ directio option + - xfs: fix UAF in xchk_btree_check_block_owner + - drm/exynos: vidi: use ctx->lock to protect struct vidi_context member + variables related to memory alloc/free + - PCI: endpoint: Avoid creating sub-groups asynchronously + - wifi: rtl8xxxu: fix slab-out-of-bounds in rtl8xxxu_sta_add + - Linux 6.19.1 + + * AppArmor blocks write(2) to network sockets with Linux 6.19 (LP: #2141298) + - SAUCE: apparmor: fix aa_label_sk_perm to check for RULE_MEDIATES_NET + + -- Timo Aaltonen Wed, 18 Feb 2026 14:31:48 +0200 diff --git a/debian.nvidia/config/README.rst b/debian.nvidia/config/README.rst new file mode 100644 index 0000000000000..751ce7f3b284d --- /dev/null +++ b/debian.nvidia/config/README.rst @@ -0,0 +1,185 @@ +================== +Config Annotations +================== + +:Author: Andrea Righi + +Overview +======== + +Each Ubuntu kernel needs to maintain its own .config for each supported +architecture and each flavour. + +Every time a new patch is applied or a kernel is rebased on top of a new +one, we need to update the .config's accordingly (config options can be +added, removed and also renamed). + +So, we need to make sure that some critical config options are always +matching the desired value in order to have a functional kernel. + +State of the art +================ + +At the moment configs are maintained as a set of Kconfig chunks (inside +`debian./config/`): a global one, plus per-arch / per-flavour +chunks. + +In addition to that, we need to maintain also a file called +'annotations'; the purpose of this file is to make sure that some +critical config options are not silently removed or changed when the +real .config is re-generated (for example after a rebase or after +applying a new set of patches). + +The main problem with this approach is that, often, we have duplicate +information that is stored both in the Kconfig chunks *and* in the +annotations files and, at the same time, the whole .config's information +is distributed between Kconfig chunks and annotations, making it hard to +maintain, review and manage in general. + +Proposed solution +================= + +The proposed solution is to store all the config information into the +"annotations" format and get rid of the config chunks (basically the +real .config's can be produced "compiling" annotations). + +Implementation +============== + +To help the management of the annotations an helper script is provided +(`debian/scripts/misc/annotations`): + +``` +usage: annotations [-h] [--version] [--file FILE] [--arch ARCH] [--flavour FLAVOUR] [--config CONFIG] + (--query | --export | --import FILE | --update FILE | --check FILE) + +Manage Ubuntu kernel .config and annotations + +options: + -h, --help show this help message and exit + --version, -v show program's version number and exit + --file FILE, -f FILE Pass annotations or .config file to be parsed + --arch ARCH, -a ARCH Select architecture + --flavour FLAVOUR, -l FLAVOUR + Select flavour (default is "generic") + --config CONFIG, -c CONFIG + Select a specific config option + +Action: + --query, -q Query annotations + --export, -e Convert annotations to .config format + --import FILE, -i FILE + Import a full .config for a specific arch and flavour into annotations + --update FILE, -u FILE + Import a partial .config into annotations (only resync configs specified in FILE) + --check FILE, -k FILE + Validate kernel .config with annotations +``` + +This script allows to query config settings (per arch/flavour/config), +export them into the Kconfig format (generating the real .config files) +and check if the final .config matches the rules defined in the +annotations. + +Examples (annotations is defined as an alias to `debian/scripts/annotations`): + + - Show settings for `CONFIG_DEBUG_INFO_BTF` for master kernel across all the + supported architectures and flavours: + +``` +$ annotations --query --config CONFIG_DEBUG_INFO_BTF +{ + "policy": { + "amd64": "y", + "arm64": "y", + "armhf": "n", + "ppc64el": "y", + "riscv64": "y", + "s390x": "y" + }, + "note": "'Needs newer pahole for armhf'" +} +``` + + - Dump kernel .config for arm64 and flavour generic-64k: + +``` +$ annotations --arch arm64 --flavour generic-64k --export +CONFIG_DEBUG_FS=y +CONFIG_DEBUG_KERNEL=y +CONFIG_COMPAT=y +... +``` + + - Update annotations file with a new kernel .config for amd64 flavour + generic: + +``` +$ annotations --arch amd64 --flavour generic --import build/.config +``` + +Moreover, an additional kernelconfig commands are provided +(via debian/rules targets): + - `migrateconfigs`: automatically merge all the previous configs into + annotations (local changes still need to be committed) + +Annotations headers +=================== + +The main annotations file should contain a header to define the architectures +and flavours that are supported. + +Here is the format of the header for the generic kernel: +``` +# Menu: HEADER +# FORMAT: 4 +# ARCH: amd64 arm64 armhf ppc64el riscv64 s390x +# FLAVOUR: amd64-generic arm64-generic arm64-generic-64k armhf-generic armhf-generic-lpae ppc64el-generic riscv64-generic s390x-generic + +``` + +Example header of a derivative (linux-aws): +``` +# Menu: HEADER +# FORMAT: 4 +# ARCH: amd64 arm64 +# FLAVOUR: amd64-aws arm64-aws +# FLAVOUR_DEP: {'amd64-aws': 'amd64-generic', 'arm64-aws': 'arm64-generic'} + +include "../../debian.master/config/annotations" + +# Below you can define only the specific linux-aws configs that differ from linux generic + +``` + +Pros and Cons +============= + + Pros: + - avoid duplicate information in .config's and annotations + - allow to easily define groups of config settings (for a specific + environment or feature, such as annotations.clouds, annotations.ubuntu, + annotations.snapd, etc.) + - config options are more accessible, easy to change and review + - we can easily document how config options are managed (and external + contributors won't be discouraged anymore when they need to to change a + config option) + + Cons: + - potential regressions: the new tool/scripts can have potential bugs, + so we could experience regressions due to some missed config changes + - kernel team need to understand the new process (even if everything + is transparent, kernel cranking process is the same, there might be + corner cases that need to be addressed and resolved manually) + +TODO +==== + + - Migrate all flavour and arch definitions into annotations (rather + than having this information defined in multiple places inside + debian/scripts); right now this information is "partially" migrated, + meaning that we need to define arches and flavours in the headers + section of annotations (so that the annotations tool can figure out + the list of supported arches and flavours), but arches and flavours + are still defined elsewhere, ideally we would like to have arches and + flavours defined only in one place: annotations. diff --git a/debian.nvidia/config/annotations b/debian.nvidia/config/annotations new file mode 100644 index 0000000000000..6801dc1e07260 --- /dev/null +++ b/debian.nvidia/config/annotations @@ -0,0 +1,247 @@ +# Menu: HEADER +# FORMAT: 4 +# ARCH: amd64 arm64 +# FLAVOUR: amd64-nvidia arm64-nvidia arm64-nvidia-64k +# FLAVOUR_DEP: {'amd64-nvidia': 'amd64-generic', 'arm64-nvidia': 'arm64-generic', 'arm64-nvidia-64k': 'arm64-generic-64k'} + +include "../../debian.master/config/annotations" + +CONFIG_ARM64_ERRATUM_1902691 policy<{'arm64': 'y'}> +CONFIG_ARM64_ERRATUM_1902691 note<'Required for Grace enablement'> + +CONFIG_ARM64_ERRATUM_2038923 policy<{'arm64': 'y'}> +CONFIG_ARM64_ERRATUM_2038923 note<'Required for Grace enablement'> + +CONFIG_ARM64_ERRATUM_2064142 policy<{'arm64': 'y'}> +CONFIG_ARM64_ERRATUM_2064142 note<'Required for Grace enablement'> + +CONFIG_ARM64_ERRATUM_2119858 policy<{'arm64': 'y'}> +CONFIG_ARM64_ERRATUM_2119858 note<'Required for Grace enablement'> + +CONFIG_ARM64_ERRATUM_2139208 policy<{'arm64': 'y'}> +CONFIG_ARM64_ERRATUM_2139208 note<'Required for Grace enablement'> + +CONFIG_ARM64_ERRATUM_2224489 policy<{'arm64': 'y'}> +CONFIG_ARM64_ERRATUM_2224489 note<'Required for Grace enablement'> + +CONFIG_ARM64_ERRATUM_2253138 policy<{'arm64': 'y'}> +CONFIG_ARM64_ERRATUM_2253138 note<'Required for Grace enablement'> + +CONFIG_ARM64_ERRATUM_4118414 policy<{'arm64': 'y'}> +CONFIG_ARM64_ERRATUM_4118414 note<'Required for Grace and Vera enablement'> + +CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE policy<{'arm64': 'y'}> +CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE note<'Required for Grace enablement'> + +CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE policy<{'arm64': 'y'}> +CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE note<'Required for Grace enablement'> + +CONFIG_ARM_FFA_TRANSPORT policy<{'arm64': 'y'}> +CONFIG_ARM_FFA_TRANSPORT note<'LP: #2111511'> + +CONFIG_ARM_SMMU_V3_IOMMUFD policy<{'arm64': 'y'}> +CONFIG_ARM_SMMU_V3_IOMMUFD note<'LP: #2095028'> + +CONFIG_CMA_SIZE_MBYTES policy<{'amd64': '0', 'arm64': '0'}> +CONFIG_CMA_SIZE_MBYTES note<'LP: #2150898'> + +CONFIG_CORESIGHT policy<{'arm64': 'm'}> +CONFIG_CORESIGHT note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_CATU policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_CATU note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_CPU_DEBUG policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_CPU_DEBUG note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_CPU_DEBUG_DEFAULT_ON policy<{'arm64': 'n'}> +CONFIG_CORESIGHT_CPU_DEBUG_DEFAULT_ON note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_CTCU policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_CTCU note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_CTI policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_CTI note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_CTI_INTEGRATION_REGS policy<{'arm64': 'n'}> +CONFIG_CORESIGHT_CTI_INTEGRATION_REGS note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_DUMMY policy<{'arm64': 'n'}> +CONFIG_CORESIGHT_DUMMY note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_LINKS_AND_SINKS policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_LINKS_AND_SINKS note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_LINK_AND_SINK_TMC policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_LINK_AND_SINK_TMC note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_SINK_ETBV10 policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_SINK_ETBV10 note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_SINK_TPIU policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_SINK_TPIU note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_SOURCE_ETM4X policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_SOURCE_ETM4X note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_STM policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_STM note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_TNOC policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_TNOC note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_TPDA policy<{'arm64': 'n'}> +CONFIG_CORESIGHT_TPDA note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_TPDM policy<{'arm64': 'n'}> +CONFIG_CORESIGHT_TPDM note<'Required for Grace enablement'> + +CONFIG_CORESIGHT_TRBE policy<{'arm64': 'm'}> +CONFIG_CORESIGHT_TRBE note<'Required for Grace enablement'> + +CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND policy<{'arm64': 'n'}> +CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND note<'LP: #2028576: Perf governor required for NVIDIA workloads'> + +CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE policy<{'amd64': 'y', 'arm64': 'y'}> +CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE note<'LP: #2028576: Perf governor required for NVIDIA workloads'> + +CONFIG_CPU_FREQ_DEFAULT_GOV_SCHEDUTIL policy<{'amd64': 'n', 'arm64': 'n'}> +CONFIG_CPU_FREQ_DEFAULT_GOV_SCHEDUTIL note<'LP: #2028576: Perf governor required for NVIDIA workloads'> + +CONFIG_DRM_NOUVEAU policy<{'amd64': 'n', 'arm64': 'n'}> +CONFIG_DRM_NOUVEAU note<'Disable nouveau for NVIDIA kernels'> + +CONFIG_DRM_NOUVEAU_BACKLIGHT policy<{'amd64': '-', 'arm64': '-'}> +CONFIG_DRM_NOUVEAU_BACKLIGHT note<'Disable nouveau for NVIDIA kernels'> + +CONFIG_DRM_NOUVEAU_CH7006 policy<{'amd64': '-', 'arm64': '-'}> +CONFIG_DRM_NOUVEAU_CH7006 note<'Disable nouveau for NVIDIA kernels'> + +CONFIG_DRM_NOUVEAU_SIL164 policy<{'amd64': '-', 'arm64': '-'}> +CONFIG_DRM_NOUVEAU_SIL164 note<'Disable nouveau for NVIDIA kernels'> + +CONFIG_DRM_NOUVEAU_SVM policy<{'amd64': '-', 'arm64': '-'}> +CONFIG_DRM_NOUVEAU_SVM note<'Disable nouveau for NVIDIA kernels'> + +CONFIG_EFI_CAPSULE_LOADER policy<{'amd64': 'm', 'arm64': 'y'}> +CONFIG_EFI_CAPSULE_LOADER note<'LP: #2067111'> + +CONFIG_ETM4X_IMPDEF_FEATURE policy<{'arm64': 'n'}> +CONFIG_ETM4X_IMPDEF_FEATURE note<'Required for Grace enablement'> + +CONFIG_GPIO_AAEON policy<{'amd64': '-'}> +CONFIG_GPIO_AAEON note<'Disable all Ubuntu ODM drivers'> + +CONFIG_IOMMUFD_VFIO_CONTAINER policy<{'arm64': 'y'}> +CONFIG_IOMMUFD_VFIO_CONTAINER note<'LP: #2095028'> + +CONFIG_IOMMU_DEFAULT_DMA_LAZY policy<{'amd64': 'y', 'arm64': 'n'}> +CONFIG_IOMMU_DEFAULT_DMA_LAZY note<'On Nvidia CPU passthrough mode is recommend so set passthrough mode as default for better performance'> + +CONFIG_IOMMU_DEFAULT_DMA_STRICT policy<{'amd64': 'n', 'arm64': 'n'}> +CONFIG_IOMMU_DEFAULT_DMA_STRICT note<'On Nvidia CPU passthrough mode is recommend so set passthrough mode as default for better performance'> + +CONFIG_IOMMU_DEFAULT_PASSTHROUGH policy<{'amd64': 'n', 'arm64': 'y'}> +CONFIG_IOMMU_DEFAULT_PASSTHROUGH note<'On Nvidia CPU passthrough mode is recommend so set passthrough mode as default for better performance'> + +CONFIG_LEDS_AAEON policy<{'amd64': '-'}> +CONFIG_LEDS_AAEON note<'Disable all Ubuntu ODM drivers'> + +CONFIG_MFD_AAEON policy<{'amd64': '-'}> +CONFIG_MFD_AAEON note<'Disable all Ubuntu ODM drivers'> + +CONFIG_MTD policy<{'amd64': 'm', 'arm64': 'y'}> +CONFIG_MTD note<'Essential for boot on ARM64'> + +CONFIG_MTK_PCIE_HOTPLUG policy<{'arm64': 'm'}> +CONFIG_MTK_PCIE_HOTPLUG note<'CX7 PCIe hotplug driver for NVIDIA DGX Spark systems with GB10 SoC.'> + +CONFIG_NOUVEAU_DEBUG policy<{'amd64': '-', 'arm64': '-'}> +CONFIG_NOUVEAU_DEBUG note<'Disable nouveau for NVIDIA kernels'> + +CONFIG_NOUVEAU_DEBUG_DEFAULT policy<{'amd64': '-', 'arm64': '-'}> +CONFIG_NOUVEAU_DEBUG_DEFAULT note<'Disable nouveau for NVIDIA kernels'> + +CONFIG_NOUVEAU_DEBUG_MMU policy<{'amd64': '-', 'arm64': '-'}> +CONFIG_NOUVEAU_DEBUG_MMU note<'Disable nouveau for NVIDIA kernels'> + +CONFIG_NOUVEAU_DEBUG_PUSH policy<{'amd64': '-', 'arm64': '-'}> +CONFIG_NOUVEAU_DEBUG_PUSH note<'Disable nouveau for NVIDIA kernels'> + +CONFIG_NOUVEAU_PLATFORM_DRIVER policy<{'arm64': '-'}> +CONFIG_NOUVEAU_PLATFORM_DRIVER note<'Disable nouveau for NVIDIA kernels'> + +CONFIG_NR_CPUS policy<{'amd64': '8192', 'arm64': '512'}> +CONFIG_NR_CPUS note<'LP: #1864198'> + +CONFIG_NVGRACE_EGM policy<{'arm64': 'm'}> +CONFIG_NVGRACE_EGM note<'LP: #2119656'> + +CONFIG_NVIDIA_FFA_EC policy<{'arm64': 'y'}> +CONFIG_NVIDIA_FFA_EC note<'LP: #2114230'> + +CONFIG_PID_IN_CONTEXTIDR policy<{'arm64': 'y'}> +CONFIG_PID_IN_CONTEXTIDR note<'Required for Grace enablement'> + +CONFIG_PINCTRL_MT8901 policy<{'arm64': 'y'}> +CONFIG_PINCTRL_MT8901 note<'LP: #2117784'> + +CONFIG_R8127 policy<{'amd64': 'n', 'arm64': 'm'}> +CONFIG_R8127 note<'LP: #2109730'> + +CONFIG_RESCTRL_FS policy<{'amd64': 'y', 'arm64': 'y'}> +CONFIG_RESCTRL_FS note<'LP: #2122432'> + +CONFIG_SAMPLE_CORESIGHT_SYSCFG policy<{'arm64': 'n'}> +CONFIG_SAMPLE_CORESIGHT_SYSCFG note<'Required for Grace enablement'> + +CONFIG_SENSORS_AAEON policy<{'amd64': '-'}> +CONFIG_SENSORS_AAEON note<'Disable all Ubuntu ODM drivers'> + +CONFIG_SPI_TEGRA210_QUAD policy<{'arm64': 'y'}> +CONFIG_SPI_TEGRA210_QUAD note<'Ensures the TPM is available before the IMA driver initializes'> + +CONFIG_TCG_ARM_CRB_FFA policy<{'arm64': 'y'}> +CONFIG_TCG_ARM_CRB_FFA note<'LP: #2111511'> + +CONFIG_TCG_TIS_SPI policy<{'amd64': 'm', 'arm64': 'y'}> +CONFIG_TCG_TIS_SPI note<'Ensures the TPM is available before the IMA driver initializes'> + +CONFIG_TEST_WORKQUEUE policy<{'amd64': 'n', 'arm64': 'n'}> +CONFIG_TEST_WORKQUEUE note<'LP: #2150467'> + +CONFIG_UBUNTU_ODM_DRIVERS policy<{'amd64': 'n', 'arm64': 'n'}> +CONFIG_UBUNTU_ODM_DRIVERS note<'Disable all Ubuntu ODM drivers'> + +CONFIG_ULTRASOC_SMB policy<{'arm64': 'n'}> +CONFIG_ULTRASOC_SMB note<'Required for Grace enablement'> + +CONFIG_VFIO_CONTAINER policy<{'amd64': 'y', 'arm64': 'n'}> +CONFIG_VFIO_CONTAINER note<'LP: #2095028'> + +CONFIG_VFIO_IOMMU_TYPE1 policy<{'amd64': 'm', 'arm64': '-'}> +CONFIG_VFIO_IOMMU_TYPE1 note<'LP: #2095028'> + + +# ---- Annotations without notes ---- + +CONFIG_ARCH_HAS_CPU_RESCTRL policy<{'amd64': 'y', 'arm64': 'y'}> +CONFIG_ARM64_MPAM_RESCTRL_FS policy<{'arm64': 'y'}> +CONFIG_BCH policy<{'amd64': 'm', 'arm64': 'y'}> +CONFIG_MTD_NAND_CORE policy<{'amd64': 'm', 'arm64': 'y'}> +CONFIG_PROC_CPU_RESCTRL policy<{'amd64': 'y', 'arm64': 'y'}> +CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID policy<{'arm64': 'y'}> + +CONFIG_ARM64_SECURE_LAUNCH policy<{'arm64': 'y'}> +CONFIG_ARM64_SECURE_LAUNCH note<'DRTM Secure Launch support'> +CONFIG_ARM64_SECURE_LAUNCH_SELFTEST policy<{'arm64': 'n'}> +CONFIG_ARM64_SECURE_LAUNCH_FAULT_INJECT policy<{'arm64': 'n'}> +CONFIG_NVIDIA_FIRME_GUEST policy<{'arm64': 'm'}> +CONFIG_NVIDIA_FIRME_GUEST note<'NVIDIA FIRME attestation driver for DRTM'> +CONFIG_EFI_ZBOOT policy<{'arm64': 'n'}> +CONFIG_EFI_ZBOOT note<'DRTM Secure Launch requires the non-zboot EFI stub path'> +CONFIG_EFI_SBAT_FILE policy<{'arm64': '-'}> +CONFIG_HAVE_KERNEL_GZIP policy<{'arm64': '-'}> +CONFIG_HAVE_KERNEL_ZSTD policy<{'arm64': '-'}> +CONFIG_KERNEL_GZIP policy<{'arm64': '-'}> +CONFIG_KERNEL_ZSTD policy<{'arm64': '-'}> diff --git a/debian.nvidia/control.d/flavour-control.stub b/debian.nvidia/control.d/flavour-control.stub new file mode 100644 index 0000000000000..4055c769adb64 --- /dev/null +++ b/debian.nvidia/control.d/flavour-control.stub @@ -0,0 +1,87 @@ +# Items that get replaced: +# FLAVOUR +# ARCH +# SUPPORTED +# TARGET +# BOOTLOADER +# =PROVIDES= +# +# Items marked with =FOO= are optional +# +# This file describes the template for packages that are created for each flavour +# in debian/control.d/vars.* +# +# This file gets edited in a couple of places. See the debian/control.stub rule in +# debian/rules. PGGVER, ABINUM, and SRCPKGNAME are all converted in the +# process of creating debian/control. +# +# The flavour specific strings (ARCH, etc) are converted using values from the various +# flavour files in debian/control.d/vars.* +# +# XXX: Leave the blank line before the first package!! + +Package: linux-modules-PKGVER-ABINUM-FLAVOUR +Build-Profiles: +Architecture: ARCH +Section: kernel +Priority: optional +Depends: ${misc:Depends}, ${shlibs:Depends}, wireless-regdb +Built-Using: ${linux:BuiltUsing} +Description: Linux kernel modules for version PKGVER + Contains the corresponding System.map file, the modules built by the + packager, and scripts that try to ensure that the system is not left in an + unbootable state after an update. + . + Supports SUPPORTED processors. + . + TARGET + . + You likely do not want to install this package directly. Instead, install + the linux-FLAVOUR meta-package, which will ensure that upgrades work + correctly, and that supporting packages are also installed. + +Package: linux-headers-PKGVER-ABINUM-FLAVOUR +Build-Profiles: +Architecture: ARCH +Section: devel +Priority: optional +Depends: ${misc:Depends}, SRCPKGNAME-headers-PKGVER-ABINUM, ${shlibs:Depends} +Provides: linux-headers, linux-headers-3.0 +Description: Linux kernel headers for version PKGVER + This package provides kernel header files for version PKGVER. + . + This is for sites that want the latest kernel headers. Please read + /usr/share/doc/linux-headers-PKGVER-ABINUM/debian.README.gz for details. + +Package: linux-lib-rust-PKGVER-ABINUM-FLAVOUR +Build-Profiles: +Architecture: amd64 +Multi-Arch: foreign +Section: devel +Priority: optional +Depends: ${misc:Depends}, coreutils +Description: Rust library files related to Linux kernel version PKGVER + This package provides kernel library files for version PKGVER, that allow to + compile out-of-tree kernel modules written in Rust. + +Package: linux-tools-PKGVER-ABINUM-FLAVOUR +Build-Profiles: +Architecture: ARCH +Section: devel +Priority: optional +Depends: ${misc:Depends}, SRCPKGNAME-tools-PKGVER-ABINUM +Description: Linux kernel version specific tools for version PKGVER-ABINUM + This package provides the architecture dependant parts for kernel + version locked tools (such as x86_energy_perf_policy) for + version PKGVER-ABINUM. + +Package: linux-cloud-tools-PKGVER-ABINUM-FLAVOUR +Build-Profiles: +Architecture: ARCH +Section: devel +Priority: optional +Depends: ${misc:Depends}, SRCPKGNAME-cloud-tools-PKGVER-ABINUM +Description: Linux kernel version specific cloud tools for version PKGVER-ABINUM + This package provides the architecture dependant parts for kernel + version locked tools for cloud for version PKGVER-ABINUM. + diff --git a/debian.nvidia/control.d/flavour-signed-control.stub b/debian.nvidia/control.d/flavour-signed-control.stub new file mode 100644 index 0000000000000..b059c7aeceedd --- /dev/null +++ b/debian.nvidia/control.d/flavour-signed-control.stub @@ -0,0 +1,38 @@ +Package: linux-image=SIGN-ME-PKG=-PKGVER-ABINUM-FLAVOUR +Build-Profiles: +Architecture: ARCH +Section: kernel +Priority: optional +Provides: linux-image, fuse-module, =PROVIDES=${linux:rprovides} +Depends: ${misc:Depends}, ${shlibs:Depends}, kmod, linux-base (>= 4.5ubuntu1~16.04.1), linux-modules-PKGVER-ABINUM-FLAVOUR +Recommends: BOOTLOADER, dracut | linux-initramfs-tool +Breaks: flash-kernel (<< 3.90ubuntu2) [arm64 armhf], s390-tools (<< 2.3.0-0ubuntu3) [s390x] +Conflicts: linux-image=SIGN-PEER-PKG=-PKGVER-ABINUM-FLAVOUR +Suggests: bpftool, linux-perf, SRCPKGNAME-tools, linux-headers-PKGVER-ABINUM-FLAVOUR +Description: Linux kernel image for version PKGVER + This package contains the=SIGN-ME-TXT= Linux kernel image for version PKGVER. + . + Supports SUPPORTED processors. + . + TARGET + . + You likely do not want to install this package directly. Instead, install + the linux-FLAVOUR meta-package, which will ensure that upgrades work + correctly, and that supporting packages are also installed. + +Package: linux-image=SIGN-ME-PKG=-PKGVER-ABINUM-FLAVOUR-dbgsym +Build-Profiles: +Architecture: ARCH +Section: devel +Priority: optional +Depends: ${misc:Depends} +Provides: linux-debug +Description: Linux kernel debug image for version PKGVER + This package provides the=SIGN-ME-TXT= kernel debug image for version PKGVER. + . + This is for sites that wish to debug the kernel. + . + The kernel image contained in this package is NOT meant to boot from. It + is uncompressed, and unstripped. This package also includes the + unstripped modules. + diff --git a/debian.nvidia/control.d/vars.nvidia b/debian.nvidia/control.d/vars.nvidia new file mode 100644 index 0000000000000..1444699fc2bf2 --- /dev/null +++ b/debian.nvidia/control.d/vars.nvidia @@ -0,0 +1,5 @@ +arch="amd64 arm64" +supported="NVIDIA" +target="Intended for NVIDIA platforms" +bootloader="grub-pc [amd64] | grub-efi-amd64 [amd64] | grub-efi-ia32 [amd64] | grub [amd64] | lilo [amd64] | flash-kernel [armhf arm64] | grub-efi-arm64 [arm64] | grub-efi-arm [armhf] | grub-ieee1275 [ppc64el]" +provides="kvm-api-4, redhat-cluster-modules, ivtv-modules, virtualbox-guest-modules [amd64]" diff --git a/debian.nvidia/control.d/vars.nvidia-64k b/debian.nvidia/control.d/vars.nvidia-64k new file mode 100644 index 0000000000000..072ec63f21074 --- /dev/null +++ b/debian.nvidia/control.d/vars.nvidia-64k @@ -0,0 +1,5 @@ +arch="arm64" +supported="NVIDIA 64K pages" +target="Intended for NVIDIA systems" +bootloader="grub-efi-arm64 [arm64] | flash-kernel [arm64]" +provides="kvm-api-4, redhat-cluster-modules, ivtv-modules" diff --git a/debian.nvidia/control.stub.in b/debian.nvidia/control.stub.in new file mode 100644 index 0000000000000..ad3894f188532 --- /dev/null +++ b/debian.nvidia/control.stub.in @@ -0,0 +1,105 @@ +Source: SRCPKGNAME +Section: devel +Priority: optional +Maintainer: Ubuntu Kernel Team +Rules-Requires-Root: no +Standards-Version: 3.9.4.0 +Build-Depends: + autoconf , + automake , + bc , + bindgen:native [amd64 arm64], + bison , + clang-21:native [amd64 arm64], + cpio, + curl , + debhelper-compat (= 10), + default-jdk-headless:native , + dkms , + flex , + gawk , + java-common , + kmod , + libaudit-dev , + libbpf-dev , + libcap-dev , + libdebuginfod-dev [amd64 arm64] , + libdw-dev , + libelf-dev , + libiberty-dev , + liblzma-dev , + libnewt-dev , + libnl-3-dev, + libnl-genl-3-dev, + libnuma-dev [amd64 arm64] , + libpci-dev , + libssl-dev , + libstdc++-dev, + libtool , + libtraceevent-dev [amd64 arm64] , + libtracefs-dev [amd64 arm64] , + libudev-dev , + libunwind8-dev [amd64 arm64] , + llvm-21-dev, + makedumpfile:native [amd64] , + openssl , + pahole (>= 1.29-2ubuntu2) [amd64 arm64] | dwarves (>= 1.21) [amd64 arm64] , + pkg-config , + python3:native , + python3-dev:native , + libpython3-dev , + python3-setuptools, + rsync [!i386] , + rust-src:native [amd64 arm64], + rustc:native (>= 1.82) [amd64 arm64], + rustfmt:native [amd64 arm64], + uuid-dev , + zstd , + bpftool:native [amd64 arm64] , + nvidia-dkms-580-open [amd64 arm64] , + nvidia-kernel-source-580-open [amd64 arm64] , + libopencsd-dev [arm64] , +Build-Depends-Indep: + asciidoc , + bzip2 , + python3-docutils , + sharutils , + xmlto , +Vcs-Git: git://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-nvidia/+git/=SERIES= +XS-Testsuite: autopkgtest +#XS-Testsuite-Depends: gcc-4.7 binutils + +Package: SRCPKGNAME-headers-PKGVER-ABINUM +Build-Profiles: +Architecture: all +Multi-Arch: foreign +Section: devel +Priority: optional +Depends: ${misc:Depends}, coreutils +Description: Header files related to Linux kernel version PKGVER + This package provides kernel header files for version PKGVER, for sites + that want the latest kernel headers. Please read + /usr/share/doc/SRCPKGNAME-headers-PKGVER-ABINUM/debian.README.gz for details + +Package: SRCPKGNAME-tools-PKGVER-ABINUM +Build-Profiles: +Architecture: amd64 arm64 +Section: devel +Priority: optional +Depends: ${misc:Depends}, ${shlibs:Depends}, linux-tools-common +Description: Linux kernel version specific tools for version PKGVER-ABINUM + This package provides the architecture dependant parts for kernel + version locked tools (such as perf and x86_energy_perf_policy) for + version PKGVER-ABINUM. + You probably want to install linux-tools-PKGVER-ABINUM-. + +Package: SRCPKGNAME-cloud-tools-PKGVER-ABINUM +Build-Profiles: +Architecture: amd64 +Section: devel +Priority: optional +Depends: ${misc:Depends}, ${shlibs:Depends}, linux-cloud-tools-common +Description: Linux kernel version specific cloud tools for version PKGVER-ABINUM + This package provides the architecture dependant parts for kernel + version locked tools for cloud tools for version PKGVER-ABINUM. + You probably want to install linux-cloud-tools-PKGVER-ABINUM-. diff --git a/debian.nvidia/dkms-versions b/debian.nvidia/dkms-versions new file mode 100644 index 0000000000000..4a941b2e62257 --- /dev/null +++ b/debian.nvidia/dkms-versions @@ -0,0 +1,4 @@ +zfs-linux 2.4.1-1ubuntu5 modulename=zfs debpath=pool/universe/z/%package%/zfs-dkms_%version%_all.deb arch=amd64 arch=arm64 arch=ppc64el arch=riscv64 arch=s390x rprovides=spl-modules rprovides=spl-dkms rprovides=zfs-modules rprovides=zfs-dkms off_series=true +v4l2loopback 0.15.3-1ubuntu2 modulename=v4l2loopback debpath=pool/universe/v/%package%/v4l2loopback-dkms_%version%_all.deb arch=amd64 rprovides=v4l2loopback-modules rprovides=v4l2loopback-dkms off_series=true +nvidia-fs 2.29.4-1 modulename=nvidia-fs debpath=pool/universe/n/%package%/nvidia-fs-dkms_%version%_amd64.deb arch=amd64 arch=arm64 rprovides=nvidia-fs-modules rprovides=nvidia-fs-dkms type=standalone +mstflint 4.33.0+1-1.1 modulename=mstflint_access debpath=pool/universe/m/%package%/mstflint-dkms_%version%_all.deb arch=amd64 arch=arm64 rprovides=mstflint-modules rprovides=mstflint-dkms diff --git a/debian.nvidia/etc/update.conf b/debian.nvidia/etc/update.conf new file mode 100644 index 0000000000000..3917a390a39f5 --- /dev/null +++ b/debian.nvidia/etc/update.conf @@ -0,0 +1,7 @@ +# WARNING: we do not create update.conf when we are not a +# derivative. Various cranky components make use of this. +# If we start unconditionally creating update.conf we need +# to fix at least cranky close and cranky rebase. +RELEASE_REPO=git://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/resolute +SOURCE_RELEASE_BRANCH=master-next +DEBIAN_MASTER=debian.master diff --git a/debian.nvidia/modprobe.d/common.conf b/debian.nvidia/modprobe.d/common.conf new file mode 100644 index 0000000000000..619c9a23fe210 --- /dev/null +++ b/debian.nvidia/modprobe.d/common.conf @@ -0,0 +1,4 @@ +# LP:1434842 -- disable OSS drivers by default to allow pulseaudio to emulate +blacklist snd-mixer-oss +blacklist snd-pcm-oss +blacklist coresight_etm4x diff --git a/debian.nvidia/reconstruct b/debian.nvidia/reconstruct new file mode 100644 index 0000000000000..c74b364c08a46 --- /dev/null +++ b/debian.nvidia/reconstruct @@ -0,0 +1,47 @@ +# Recreate any symlinks created since the orig. +[ ! -L 'ubuntu/igh-ecat/master/rtdm-ioctl.c' ] && ln -sf 'ioctl.c' 'ubuntu/igh-ecat/master/rtdm-ioctl.c' +chmod +x 'debian/cloud-tools/hv_get_dhcp_info' +chmod +x 'debian/cloud-tools/hv_get_dns_info' +chmod +x 'debian/cloud-tools/hv_set_ifconfig' +chmod +x 'debian/rules' +chmod +x 'debian/scripts/checks/final-checks' +chmod +x 'debian/scripts/checks/module-signature-check' +chmod +x 'debian/scripts/control-create' +chmod +x 'debian/scripts/dkms-build' +chmod +x 'debian/scripts/dkms-build--nvidia-N' +chmod +x 'debian/scripts/dkms-build-configure--zfs' +chmod +x 'debian/scripts/file-downloader' +chmod +x 'debian/scripts/link-headers' +chmod +x 'debian/scripts/link-lib-rust' +chmod +x 'debian/scripts/misc/annotations' +chmod +x 'debian/scripts/misc/find-missing-sauce.sh' +chmod +x 'debian/scripts/misc/gen-auto-reconstruct' +chmod +x 'debian/scripts/misc/git-ubuntu-log' +chmod +x 'debian/scripts/misc/insert-changes' +chmod +x 'debian/scripts/misc/insert-ubuntu-changes' +chmod +x 'debian/scripts/misc/kernelconfig' +chmod +x 'debian/scripts/sign-module' +chmod +x 'debian/templates/extra.postinst.in' +chmod +x 'debian/templates/extra.postrm.in' +chmod +x 'debian/templates/headers.postinst.in' +chmod +x 'debian/templates/image.postinst.in' +chmod +x 'debian/templates/image.postrm.in' +chmod +x 'debian/templates/image.preinst.in' +chmod +x 'debian/templates/image.prerm.in' +chmod +x 'debian/tests/rebuild' +chmod +x 'debian/tests/ubuntu-regression-suite' +chmod +x 'drivers/net/ethernet/realtek/r8127/Makefile' +chmod +x 'drivers/net/ethernet/realtek/r8127/r8127.h' +chmod +x 'drivers/net/ethernet/realtek/r8127/r8127_dash.h' +chmod +x 'drivers/net/ethernet/realtek/r8127/r8127_firmware.h' +chmod +x 'drivers/net/ethernet/realtek/r8127/r8127_n.c' +chmod +x 'drivers/net/ethernet/realtek/r8127/r8127_realwow.h' +chmod +x 'drivers/net/ethernet/realtek/r8127/r8127_rss.h' +chmod +x 'drivers/net/ethernet/realtek/r8127/rtl_eeprom.c' +chmod +x 'drivers/net/ethernet/realtek/r8127/rtl_eeprom.h' +chmod +x 'drivers/net/ethernet/realtek/r8127/rtltool.c' +chmod +x 'drivers/net/ethernet/realtek/r8127/rtltool.h' +# Remove any files deleted from the orig. +rm -f 'arch/parisc/lib/checksum.c' +rm -f 'tools/testing/selftests/mqueue/setting' +exit 0 diff --git a/debian.nvidia/rules.d/amd64.mk b/debian.nvidia/rules.d/amd64.mk new file mode 100644 index 0000000000000..8aa96b3e758cc --- /dev/null +++ b/debian.nvidia/rules.d/amd64.mk @@ -0,0 +1,20 @@ +build_arch = x86 +defconfig = defconfig +flavours = nvidia +build_image = bzImage +kernel_file = arch/$(build_arch)/boot/bzImage +install_file = vmlinuz +vdso = vdso_install +no_dumpfile = true +uefi_signed = true +do_tools_usbip = true +do_tools_cpupower = true +do_tools_perf = true +do_tools_perf_jvmti = true +do_tools_perf_python = true +do_tools_bpftool = true +do_tools_x86 = true +do_tools_hyperv = false +do_tools_rtla = true +do_tools_acpidbg = true +do_lib_rust = false diff --git a/debian.nvidia/rules.d/arm64.mk b/debian.nvidia/rules.d/arm64.mk new file mode 100644 index 0000000000000..af8f0f4c24b14 --- /dev/null +++ b/debian.nvidia/rules.d/arm64.mk @@ -0,0 +1,20 @@ +build_arch = arm64 +defconfig = defconfig +flavours = nvidia nvidia-64k +build_image = Image +kernel_file = arch/$(build_arch)/boot/Image +install_file = vmlinuz +no_dumpfile = true +uefi_signed = true + +vdso = vdso_install + +do_tools_usbip = true +do_tools_cpupower = true +do_tools_perf = true +do_tools_perf_jvmti = true +do_tools_perf_python = true +do_tools_bpftool = true +do_tools_rtla = true + +do_dtbs = true diff --git a/debian.nvidia/tracking-bug b/debian.nvidia/tracking-bug new file mode 100644 index 0000000000000..53ded77eea8a2 --- /dev/null +++ b/debian.nvidia/tracking-bug @@ -0,0 +1 @@ +2158924 d2026.06.30-1 diff --git a/debian.nvidia/upstream-stable b/debian.nvidia/upstream-stable new file mode 100644 index 0000000000000..30e10948e88f9 --- /dev/null +++ b/debian.nvidia/upstream-stable @@ -0,0 +1,3 @@ +# The following upstream stable releases have been ported: +[upstream-stable] + linux-6.17.y = v6.17.1 diff --git a/debian.nvidia/variants b/debian.nvidia/variants new file mode 100644 index 0000000000000..cab2f62a9df90 --- /dev/null +++ b/debian.nvidia/variants @@ -0,0 +1,2 @@ +-7.0 +-- diff --git a/debian/control.d/flavour-module.stub b/debian/control.d/flavour-module.stub index 2810f83bb361f..4aa9ddbe76b95 100644 --- a/debian/control.d/flavour-module.stub +++ b/debian/control.d/flavour-module.stub @@ -4,6 +4,7 @@ Build-Profiles: Architecture: ARCH Section: kernel Priority: optional +Provides: ${MODULE:rprovides} Depends: ${misc:Depends}, linux-image-PKGVER-ABINUM-FLAVOUR | linux-image-unsigned-PKGVER-ABINUM-FLAVOUR, diff --git a/debian/debian.env b/debian/debian.env index be31a0c270197..2a9c07b235b89 100644 --- a/debian/debian.env +++ b/debian/debian.env @@ -1 +1 @@ -DEBIAN=debian.master +DEBIAN=debian.nvidia diff --git a/debian/rules b/debian/rules index 4f87ba2dcf796..ec843ccde5ef5 100755 --- a/debian/rules +++ b/debian/rules @@ -147,6 +147,10 @@ clean: debian/control debian/canonical-certs.pem debian/canonical-revoked-certs. rm -f debian/scripts/fix-filenames + # SUBSTVARS: rprovides for all DKMS packages + echo "linux:rprovides=$(foreach dkms,$(all_built-in_dkms_modules),$(foreach provides,$(dkms_$(dkms)_rprovides),$(provides)$(comma)))" >"debian/substvars" + echo "$(foreach dkms,$(all_standalone_dkms_modules),$(dkms):rprovides=$(strip $(foreach provides,$(dkms_$(dkms)_rprovides),$(provides)$(comma)))=NL=)" | sed -e 's/~(/ (/g' -e 's/, (/ (/g' -e 's/=NL= */\n/g' >>"debian/substvars" + .PHONY: distclean distclean: clean rm -rf debian/control debian/changelog diff --git a/debian/rules.d/0-common-vars.mk b/debian/rules.d/0-common-vars.mk index 1ea787d668dcc..17af44bc5239f 100644 --- a/debian/rules.d/0-common-vars.mk +++ b/debian/rules.d/0-common-vars.mk @@ -219,3 +219,51 @@ custom_override = $(or $($(1)_$(2)),$($(1))) # selftests that Ubuntu cares about ubuntu_selftests = breakpoints cpu-hotplug efivarfs memfd memory-hotplug mount net ptrace seccomp timers powerpc user ftrace + +# DKMS +all_dkms_modules = + +subst_paired = $(subst $(firstword $(subst =, ,$(1))),$(lastword $(subst =, ,$(1))),$(2)) +recursive_call = $(if $(2),$(call recursive_call,$(1),$(wordlist 2,$(words $(2)),$(2)),$(call $(1),$(firstword $(2)),$(3))),$(3)) + +$(foreach _line,$(shell gawk '{ OFS = "!"; $$1 = $$1; print }' $(DEBIAN)/dkms-versions), \ + $(eval _params = $(subst !, ,$(_line))) \ + $(eval _deb_pkgname = $(firstword $(_params))) \ + $(eval _deb_version = $(word 2,$(_params))) \ + $(if $(filter modulename=%,$(_params)), \ + $(eval _m = $(word 2,$(subst =, ,$(filter modulename=%,$(_params))))) \ + , \ + $(info modulename for $(_deb_pkgname) not specified in dkms-versions. Assume $(_deb_pkgname).) \ + $(eval _m = $(_deb_pkgname)) \ + ) \ + $(eval all_dkms_modules += $(_m)) \ + $(eval dkms_$(_m)_version = $(_deb_version)) \ + $(foreach _p,$(patsubst debpath=%,%,$(filter debpath=%,$(_params))), \ + $(eval dkms_$(_m)_debpath += $(strip \ + $(call recursive_call,subst_paired, \ + %module%=$(_m) \ + %package%=$(_deb_pkgname) \ + %version%=$(lastword $(subst :, ,$(_deb_version))) \ + , \ + $(_p) \ + ) \ + )) \ + ) \ + $(if $(dkms_$(_m)_debpath),,$(error debpath for $(_deb_pkgname) not specified.)) \ + $(if $(filter arch=%,$(_params)), \ + $(eval dkms_$(_m)_archs = $(patsubst arch=%,%,$(filter arch=%,$(_params)))) \ + , \ + $(eval dkms_$(_m)_archs = any) \ + ) \ + $(eval _rprovides_raw = $(filter rprovides=%,$(_params))) \ + $(eval dkms_$(_m)_rprovides = $(patsubst rprovides=%,%,$(_rprovides_raw))) \ + $(eval dkms_$(_m)_type = $(word 1,$(patsubst type=%,%,$(filter type=%,$(_params))) built-in)) \ + $(eval all_$(dkms_$(_m)_type)_dkms_modules += $(_m)) \ + $(if $(filter standalone,$(dkms_$(_m)_type)), \ + $(eval dkms_$(_m)_pkg_name = linux-modules-$(_m)-$(abi_release)) \ + $(eval dkms_$(_m)_subdir = ubuntu) \ + , \ + $(eval dkms_$(_m)_pkg_name = $(mods_pkg_name)) \ + $(eval dkms_$(_m)_subdir = kernel) \ + ) \ +) diff --git a/debian/rules.d/2-binary-arch.mk b/debian/rules.d/2-binary-arch.mk index a249f782bca8e..564923f55af31 100644 --- a/debian/rules.d/2-binary-arch.mk +++ b/debian/rules.d/2-binary-arch.mk @@ -499,7 +499,7 @@ define dh_all dh_shlibdeps -p$(1) $(shlibdeps_opts) dh_installdeb -p$(1) dh_installdebconf -p$(1) - $(lockme) dh_gencontrol -p$(1) -- -Vlinux:rprovides='$(rprovides)' $(2) + $(lockme) dh_gencontrol -p$(1) -- -Tdebian/substvars $(2) dh_md5sums -p$(1) dh_builddeb -p$(1) endef @@ -531,7 +531,6 @@ binary-%: pkgcloud = $(cloud_flavour_pkg_name)-$* $(foreach _m,$(all_dkms_modules), \ $(eval binary-%: enable_$(_m) = $$(filter true,$$(call custom_override,do_$(_m),$$*))) \ ) -binary-%: rprovides = $(foreach _m,$(all_built-in_dkms_modules),$(if $(enable_$(_m)),$(foreach _r,$(dkms_$(_m)_rprovides),$(_r)$(comma) ))) binary-%: $(stampdir)/stamp-install-% @echo Debug: $@ dh_testdir @@ -626,7 +625,7 @@ ifeq ($(do_tools_cpupower),true) endif ifeq ($(do_tools_perf),true) cd $(builddirpa)/tools/perf && \ - LLVM_CONFIG=llvm-config-$(LLVM_VERSION) $(kmake) prefix=/usr HAVE_CPLUS_DEMANGLE_SUPPORT=1 CROSS_COMPILE=$(CROSS_COMPILE) NO_LIBPERL=1 WERROR=0 + LLVM_CONFIG=llvm-config-$(LLVM_VERSION) $(kmake) prefix=/usr HAVE_CPLUS_DEMANGLE_SUPPORT=1 CROSS_COMPILE=$(CROSS_COMPILE) NO_LIBPERL=1 WERROR=0 $(if $(filter arm64,$(build_arch)),CORESIGHT=1) endif ifeq ($(do_tools_bpftool),true) $(kmake) CROSS_COMPILE=$(CROSS_COMPILE) -C $(builddirpa)/tools/bpf/bpftool diff --git a/debian/scripts/control-create b/debian/scripts/control-create index f96da1ec1168c..acdaa36ec5144 100755 --- a/debian/scripts/control-create +++ b/debian/scripts/control-create @@ -100,6 +100,33 @@ gen_per_flavour () { sed "${sed_common_patterns[@]}" \ -e "s/ARCH/${arch}/g" \ "debian/control.d/flavour-buildinfo.stub" + + while read -r package version extras + do + module="$package" + module_type= + + # Module arch parameters are skipped here, so a package section will + # be generated for each flavour, and its Architecture will be set to + # all architectures with that flavour. Even that is being generated, + # it doesn't follow all of them will be built. That's to work-around + # dkms_exclude/dkms_include that manipulates supported architectures + # in $(DEBIAN)/rules.d/$(arch).mk. + for param in $extras; do + case "$param" in + modulename=*) module="${param#modulename=}" ;; + type=*) module_type="${param#type=}" ;; + *) continue ;; + esac + done + + [ "$module_type" = "standalone" ] || continue + + sed "${sed_common_patterns[@]}" \ + -e "s/ARCH/${arch}/g" \ + -e "s/MODULE/${module}/g" \ + debian/control.d/flavour-module.stub + done < "${DEBIAN}/dkms-versions" } gen_common diff --git a/debian/scripts/dkms-build b/debian/scripts/dkms-build index 8e9adc56f03de..d64946a808e9b 100755 --- a/debian/scripts/dkms-build +++ b/debian/scripts/dkms-build @@ -171,7 +171,7 @@ echo "II: dkms-build building $package" fakeroot="" [ $(id -u) -ne 0 ] && fakeroot="/usr/bin/fakeroot" rc=0 -$fakeroot /usr/sbin/dkms build --no-prepare-kernel --no-clean-kernel \ +$fakeroot /usr/sbin/dkms build --force --no-prepare-kernel --no-clean-kernel \ -k "$abi_flavour" ${ARCH:+-a $ARCH} \ --sourcetree "$dkms_dir/source" \ --dkmstree "$dkms_dir/build" \ diff --git a/debian/scripts/misc/kconfig/annotations.py b/debian/scripts/misc/kconfig/annotations.py index 5f34410d4ef87..ba9db44964708 100644 --- a/debian/scripts/misc/kconfig/annotations.py +++ b/debian/scripts/misc/kconfig/annotations.py @@ -80,7 +80,7 @@ def _parse_body(self, data: str, parent=True): if not line: continue - # Catpure flavors of included files + # Capture flavors of included files if line.startswith("# FLAVOUR: "): self.include_flavour += line.split(" ")[2:] continue @@ -211,7 +211,7 @@ def _json_parse(self, data, is_included=False): self.include = data["attributes"]["include"] self.include_flavour = [] else: - # We are procesing an imported annotations, so merge all the + # We are processing an imported annotations, so merge all the # configs and attributes. try: self.config = data["config"] | self.config diff --git a/drivers/acpi/arm64/ffh.c b/drivers/acpi/arm64/ffh.c index 04380bab193df..8bce1070d3716 100644 --- a/drivers/acpi/arm64/ffh.c +++ b/drivers/acpi/arm64/ffh.c @@ -19,6 +19,9 @@ struct acpi_ffh_data { struct arm_smccc_1_2_regs *res); }; +static int (*ffh_custom_handler)(struct acpi_ffh_info *info, + acpi_integer *value, void *region_context); + int acpi_ffh_address_space_arch_setup(void *handler_ctxt, void **region_ctxt) { enum arm_smccc_conduit conduit; @@ -99,9 +102,38 @@ int acpi_ffh_address_space_arch_handler(acpi_integer *value, void *region_contex ffh_ctxt->invoke_ffh64_fn(r, r); memcpy(value, r, ffh_ctxt->info.length); } + } else if (ffh_custom_handler) { + int err = ffh_custom_handler(&ffh_ctxt->info, value, + region_context); + if (err) { + pr_err("ARM FFH custom offset handler returned error=%d\n", + err); + ret = AE_ERROR; + } } else { ret = AE_ERROR; } return ret; } + +int acpi_arm64_ffh_update_custom_offset_handler( + int (*handler)(struct acpi_ffh_info *info, acpi_integer *value, + void *region_context)) +{ + if (!handler) { + pr_debug("ARM FFH custom offset handler unregistered\n"); + ffh_custom_handler = NULL; + return 0; + } + + if (ffh_custom_handler) + pr_debug("ARM FFH custom offset handler updated\n"); + else + pr_debug("ARM FFH custom offset handler registered\n"); + + ffh_custom_handler = handler; + + return 0; +} +EXPORT_SYMBOL_GPL(acpi_arm64_ffh_update_custom_offset_handler); diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index f0e513e9ed5d3..053fc6765a592 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -177,12 +177,12 @@ __ATTR(_name, 0444, show_##_name, NULL) show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, highest_perf); show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, lowest_perf); show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, nominal_perf); +show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, reference_perf); show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, lowest_nonlinear_perf); show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, guaranteed_perf); show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, lowest_freq); show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, nominal_freq); -show_cppc_data(cppc_get_perf_ctrs, cppc_perf_fb_ctrs, reference_perf); show_cppc_data(cppc_get_perf_ctrs, cppc_perf_fb_ctrs, wraparound_time); /* Check for valid access_width, otherwise, fallback to using bit_width */ @@ -853,6 +853,16 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) } per_cpu(cpu_pcc_subspace_idx, pr->id) = pcc_subspace_id; + /* + * In CPPC v1, DESIRED_PERF is mandatory. In CPPC v2, it is optional + * only when AUTO_SEL_ENABLE is supported. + */ + if (!CPC_SUPPORTED(&cpc_ptr->cpc_regs[DESIRED_PERF]) && + (!osc_sb_cppc2_support_acked || + !CPC_SUPPORTED(&cpc_ptr->cpc_regs[AUTO_SEL_ENABLE]))) + pr_warn("Desired perf. register is mandatory if CPPC v2 is not supported " + "or autonomous selection is disabled\n"); + /* * Initialize the remaining cpc_regs as unsupported. * Example: In case FW exposes CPPC v2, the below loop will initialize @@ -1342,9 +1352,10 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps) { struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum); struct cpc_register_resource *highest_reg, *lowest_reg, - *lowest_non_linear_reg, *nominal_reg, *guaranteed_reg, - *low_freq_reg = NULL, *nom_freq_reg = NULL; - u64 high, low, guaranteed, nom, min_nonlinear, low_f = 0, nom_f = 0; + *lowest_non_linear_reg, *nominal_reg, *reference_reg, + *guaranteed_reg, *low_freq_reg = NULL, *nom_freq_reg = NULL; + u64 high, low, guaranteed, nom, ref, min_nonlinear, + low_f = 0, nom_f = 0; int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpunum); struct cppc_pcc_data *pcc_ss_data = NULL; int ret = 0, regs_in_pcc = 0; @@ -1358,6 +1369,7 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps) lowest_reg = &cpc_desc->cpc_regs[LOWEST_PERF]; lowest_non_linear_reg = &cpc_desc->cpc_regs[LOW_NON_LINEAR_PERF]; nominal_reg = &cpc_desc->cpc_regs[NOMINAL_PERF]; + reference_reg = &cpc_desc->cpc_regs[REFERENCE_PERF]; low_freq_reg = &cpc_desc->cpc_regs[LOWEST_FREQ]; nom_freq_reg = &cpc_desc->cpc_regs[NOMINAL_FREQ]; guaranteed_reg = &cpc_desc->cpc_regs[GUARANTEED_PERF]; @@ -1365,6 +1377,7 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps) /* Are any of the regs PCC ?*/ if (CPC_IN_PCC(highest_reg) || CPC_IN_PCC(lowest_reg) || CPC_IN_PCC(lowest_non_linear_reg) || CPC_IN_PCC(nominal_reg) || + (CPC_SUPPORTED(reference_reg) && CPC_IN_PCC(reference_reg)) || CPC_IN_PCC(low_freq_reg) || CPC_IN_PCC(nom_freq_reg) || CPC_IN_PCC(guaranteed_reg)) { if (pcc_ss_id < 0) { @@ -1381,35 +1394,66 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps) } } - cpc_read(cpunum, highest_reg, &high); + ret = cpc_read(cpunum, highest_reg, &high); + if (ret) + goto out_err; perf_caps->highest_perf = high; - cpc_read(cpunum, lowest_reg, &low); + ret = cpc_read(cpunum, lowest_reg, &low); + if (ret) + goto out_err; perf_caps->lowest_perf = low; - cpc_read(cpunum, nominal_reg, &nom); + ret = cpc_read(cpunum, nominal_reg, &nom); + if (ret) + goto out_err; perf_caps->nominal_perf = nom; + /* + * If reference perf register is not supported then we should + * use the nominal perf value + */ + if (CPC_SUPPORTED(reference_reg)) { + ret = cpc_read(cpunum, reference_reg, &ref); + if (ret) + goto out_err; + } else { + ref = nom; + } + perf_caps->reference_perf = ref; + if (guaranteed_reg->type != ACPI_TYPE_BUFFER || IS_NULL_REG(&guaranteed_reg->cpc_entry.reg)) { perf_caps->guaranteed_perf = 0; } else { - cpc_read(cpunum, guaranteed_reg, &guaranteed); + ret = cpc_read(cpunum, guaranteed_reg, &guaranteed); + if (ret) + goto out_err; perf_caps->guaranteed_perf = guaranteed; } - cpc_read(cpunum, lowest_non_linear_reg, &min_nonlinear); + ret = cpc_read(cpunum, lowest_non_linear_reg, &min_nonlinear); + if (ret) + goto out_err; perf_caps->lowest_nonlinear_perf = min_nonlinear; - if (!high || !low || !nom || !min_nonlinear) + if (!high || !low || !nom || !ref || !min_nonlinear) { ret = -EFAULT; + goto out_err; + } /* Read optional lowest and nominal frequencies if present */ - if (CPC_SUPPORTED(low_freq_reg)) - cpc_read(cpunum, low_freq_reg, &low_f); + if (CPC_SUPPORTED(low_freq_reg)) { + ret = cpc_read(cpunum, low_freq_reg, &low_f); + if (ret) + goto out_err; + } - if (CPC_SUPPORTED(nom_freq_reg)) - cpc_read(cpunum, nom_freq_reg, &nom_f); + if (CPC_SUPPORTED(nom_freq_reg)) { + ret = cpc_read(cpunum, nom_freq_reg, &nom_f); + if (ret) + goto out_err; + } perf_caps->lowest_freq = low_f; perf_caps->nominal_freq = nom_f; @@ -1431,20 +1475,10 @@ EXPORT_SYMBOL_GPL(cppc_get_perf_caps); bool cppc_perf_ctrs_in_pcc_cpu(unsigned int cpu) { struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu); - struct cpc_register_resource *ref_perf_reg; - - /* - * If reference perf register is not supported then we should use the - * nominal perf value - */ - ref_perf_reg = &cpc_desc->cpc_regs[REFERENCE_PERF]; - if (!CPC_SUPPORTED(ref_perf_reg)) - ref_perf_reg = &cpc_desc->cpc_regs[NOMINAL_PERF]; return CPC_IN_PCC(&cpc_desc->cpc_regs[DELIVERED_CTR]) || CPC_IN_PCC(&cpc_desc->cpc_regs[REFERENCE_CTR]) || - CPC_IN_PCC(&cpc_desc->cpc_regs[CTR_WRAP_TIME]) || - CPC_IN_PCC(ref_perf_reg); + CPC_IN_PCC(&cpc_desc->cpc_regs[CTR_WRAP_TIME]); } EXPORT_SYMBOL_GPL(cppc_perf_ctrs_in_pcc_cpu); @@ -1481,10 +1515,10 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs) { struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum); struct cpc_register_resource *delivered_reg, *reference_reg, - *ref_perf_reg, *ctr_wrap_reg; + *ctr_wrap_reg; int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpunum); struct cppc_pcc_data *pcc_ss_data = NULL; - u64 delivered, reference, ref_perf, ctr_wrap_time; + u64 delivered, reference, ctr_wrap_time; int ret = 0, regs_in_pcc = 0; if (!cpc_desc) { @@ -1494,19 +1528,11 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs) delivered_reg = &cpc_desc->cpc_regs[DELIVERED_CTR]; reference_reg = &cpc_desc->cpc_regs[REFERENCE_CTR]; - ref_perf_reg = &cpc_desc->cpc_regs[REFERENCE_PERF]; ctr_wrap_reg = &cpc_desc->cpc_regs[CTR_WRAP_TIME]; - /* - * If reference perf register is not supported then we should - * use the nominal perf value - */ - if (!CPC_SUPPORTED(ref_perf_reg)) - ref_perf_reg = &cpc_desc->cpc_regs[NOMINAL_PERF]; - /* Are any of the regs PCC ?*/ if (CPC_IN_PCC(delivered_reg) || CPC_IN_PCC(reference_reg) || - CPC_IN_PCC(ctr_wrap_reg) || CPC_IN_PCC(ref_perf_reg)) { + CPC_IN_PCC(ctr_wrap_reg)) { if (pcc_ss_id < 0) { pr_debug("Invalid pcc_ss_id\n"); return -ENODEV; @@ -1521,9 +1547,13 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs) } } - cpc_read(cpunum, delivered_reg, &delivered); - cpc_read(cpunum, reference_reg, &reference); - cpc_read(cpunum, ref_perf_reg, &ref_perf); + ret = cpc_read(cpunum, delivered_reg, &delivered); + if (ret) + goto out_err; + + ret = cpc_read(cpunum, reference_reg, &reference); + if (ret) + goto out_err; /* * Per spec, if ctr_wrap_time optional register is unsupported, then the @@ -1531,17 +1561,19 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs) * platform */ ctr_wrap_time = (u64)(~((u64)0)); - if (CPC_SUPPORTED(ctr_wrap_reg)) - cpc_read(cpunum, ctr_wrap_reg, &ctr_wrap_time); + if (CPC_SUPPORTED(ctr_wrap_reg)) { + ret = cpc_read(cpunum, ctr_wrap_reg, &ctr_wrap_time); + if (ret) + goto out_err; + } - if (!delivered || !reference || !ref_perf) { + if (!delivered || !reference) { ret = -EFAULT; goto out_err; } perf_fb_ctrs->delivered = delivered; perf_fb_ctrs->reference = reference; - perf_fb_ctrs->reference_perf = ref_perf; perf_fb_ctrs->wraparound_time = ctr_wrap_time; out_err: if (regs_in_pcc) @@ -1561,6 +1593,8 @@ int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable) struct cpc_register_resource *auto_sel_reg; struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu); struct cppc_pcc_data *pcc_ss_data = NULL; + bool autosel_ffh_sysmem; + bool epp_ffh_sysmem; int ret; if (!cpc_desc) { @@ -1571,6 +1605,11 @@ int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable) auto_sel_reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE]; epp_set_reg = &cpc_desc->cpc_regs[ENERGY_PERF]; + epp_ffh_sysmem = CPC_SUPPORTED(epp_set_reg) && + (CPC_IN_FFH(epp_set_reg) || CPC_IN_SYSTEM_MEMORY(epp_set_reg)); + autosel_ffh_sysmem = CPC_SUPPORTED(auto_sel_reg) && + (CPC_IN_FFH(auto_sel_reg) || CPC_IN_SYSTEM_MEMORY(auto_sel_reg)); + if (CPC_IN_PCC(epp_set_reg) || CPC_IN_PCC(auto_sel_reg)) { if (pcc_ss_id < 0) { pr_debug("Invalid pcc_ss_id for CPU:%d\n", cpu); @@ -1596,11 +1635,22 @@ int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable) ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE); up_write(&pcc_ss_data->pcc_lock); } else if (osc_cpc_flexible_adr_space_confirmed && - CPC_SUPPORTED(epp_set_reg) && CPC_IN_FFH(epp_set_reg)) { - ret = cpc_write(cpu, epp_set_reg, perf_ctrls->energy_perf); + (epp_ffh_sysmem || autosel_ffh_sysmem)) { + if (autosel_ffh_sysmem) { + ret = cpc_write(cpu, auto_sel_reg, enable); + if (ret) + return ret; + } + + if (epp_ffh_sysmem) { + ret = cpc_write(cpu, epp_set_reg, + perf_ctrls->energy_perf); + if (ret) + return ret; + } } else { ret = -ENOTSUPP; - pr_debug("_CPC in PCC and _CPC in FFH are not supported\n"); + pr_debug("_CPC in PCC/FFH/SystemMemory are not supported\n"); } return ret; @@ -1738,6 +1788,101 @@ int cppc_set_enable(int cpu, bool enable) } EXPORT_SYMBOL_GPL(cppc_set_enable); +/** + * cppc_get_perf - Get a CPU's performance controls. + * @cpu: CPU for which to get performance controls. + * @perf_ctrls: ptr to cppc_perf_ctrls. See cppc_acpi.h + * + * Return: 0 for success with perf_ctrls, -ERRNO otherwise. + */ +int cppc_get_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) +{ + struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu); + struct cpc_register_resource *desired_perf_reg, + *min_perf_reg, *max_perf_reg, + *energy_perf_reg, *auto_sel_reg; + u64 desired_perf = 0, min = 0, max = 0, energy_perf = 0, auto_sel = 0; + int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu); + struct cppc_pcc_data *pcc_ss_data = NULL; + int ret = 0, regs_in_pcc = 0; + + if (!cpc_desc) { + pr_debug("No CPC descriptor for CPU:%d\n", cpu); + return -ENODEV; + } + + if (!perf_ctrls) { + pr_debug("Invalid perf_ctrls pointer\n"); + return -EINVAL; + } + + desired_perf_reg = &cpc_desc->cpc_regs[DESIRED_PERF]; + min_perf_reg = &cpc_desc->cpc_regs[MIN_PERF]; + max_perf_reg = &cpc_desc->cpc_regs[MAX_PERF]; + energy_perf_reg = &cpc_desc->cpc_regs[ENERGY_PERF]; + auto_sel_reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE]; + + /* Are any of the regs PCC ?*/ + if (CPC_IN_PCC(desired_perf_reg) || CPC_IN_PCC(min_perf_reg) || + CPC_IN_PCC(max_perf_reg) || CPC_IN_PCC(energy_perf_reg) || + CPC_IN_PCC(auto_sel_reg)) { + if (pcc_ss_id < 0) { + pr_debug("Invalid pcc_ss_id for CPU:%d\n", cpu); + return -ENODEV; + } + pcc_ss_data = pcc_data[pcc_ss_id]; + regs_in_pcc = 1; + down_write(&pcc_ss_data->pcc_lock); + /* Ring doorbell once to update PCC subspace */ + if (send_pcc_cmd(pcc_ss_id, CMD_READ) < 0) { + ret = -EIO; + goto out_err; + } + } + + /* Read optional elements if present */ + if (CPC_SUPPORTED(max_perf_reg)) { + ret = cpc_read(cpu, max_perf_reg, &max); + if (ret) + goto out_err; + } + perf_ctrls->max_perf = max; + + if (CPC_SUPPORTED(min_perf_reg)) { + ret = cpc_read(cpu, min_perf_reg, &min); + if (ret) + goto out_err; + } + perf_ctrls->min_perf = min; + + if (CPC_SUPPORTED(desired_perf_reg)) { + ret = cpc_read(cpu, desired_perf_reg, &desired_perf); + if (ret) + goto out_err; + } + perf_ctrls->desired_perf = desired_perf; + + if (CPC_SUPPORTED(energy_perf_reg)) { + ret = cpc_read(cpu, energy_perf_reg, &energy_perf); + if (ret) + goto out_err; + } + perf_ctrls->energy_perf = energy_perf; + + if (CPC_SUPPORTED(auto_sel_reg)) { + ret = cpc_read(cpu, auto_sel_reg, &auto_sel); + if (ret) + goto out_err; + } + perf_ctrls->auto_sel = (bool)auto_sel; + +out_err: + if (regs_in_pcc) + up_write(&pcc_ss_data->pcc_lock); + return ret; +} +EXPORT_SYMBOL_GPL(cppc_get_perf); + /** * cppc_set_perf - Set a CPU's performance controls. * @cpu: CPU for which to set performance controls. @@ -1870,6 +2015,62 @@ int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) } EXPORT_SYMBOL_GPL(cppc_set_perf); +/** + * cppc_get_perf_limited - Get the Performance Limited register value. + * @cpu: CPU from which to get Performance Limited register. + * @perf_limited: Pointer to store the Performance Limited value. + * + * The returned value contains sticky status bits indicating platform-imposed + * performance limitations. + * + * Return: 0 for success, -EIO on failure, -EOPNOTSUPP if not supported. + */ +int cppc_get_perf_limited(int cpu, u64 *perf_limited) +{ + return cppc_get_reg_val(cpu, PERF_LIMITED, perf_limited); +} +EXPORT_SYMBOL_GPL(cppc_get_perf_limited); + +/** + * cppc_set_perf_limited() - Clear bits in the Performance Limited register. + * @cpu: CPU on which to write register. + * @bits_to_clear: Bitmask of bits to clear in the perf_limited register. + * + * The Performance Limited register contains two sticky bits set by platform: + * - Bit 0 (Desired_Excursion): Set when delivered performance is constrained + * below desired performance. Not used when Autonomous Selection is enabled. + * - Bit 1 (Minimum_Excursion): Set when delivered performance is constrained + * below minimum performance. + * + * These bits are sticky and remain set until OSPM explicitly clears them. + * This function only allows clearing bits (the platform sets them). + * + * Return: 0 for success, -EINVAL for invalid bits, -EIO on register + * access failure, -EOPNOTSUPP if not supported. + */ +int cppc_set_perf_limited(int cpu, u64 bits_to_clear) +{ + u64 current_val, new_val; + int ret; + + /* Only bits 0 and 1 are valid */ + if (bits_to_clear & ~CPPC_PERF_LIMITED_MASK) + return -EINVAL; + + if (!bits_to_clear) + return 0; + + ret = cppc_get_perf_limited(cpu, ¤t_val); + if (ret) + return ret; + + /* Clear the specified bits */ + new_val = current_val & ~bits_to_clear; + + return cppc_set_reg_val(cpu, PERF_LIMITED, new_val); +} +EXPORT_SYMBOL_GPL(cppc_set_perf_limited); + /** * cppc_get_transition_latency - returns frequency transition latency in ns * @cpu_num: CPU number for per_cpu(). diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index bb0c65a81da59..be32dafd129f5 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -754,6 +754,8 @@ static const struct usb_device_id quirks_table[] = { BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3608), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x13d3, 0x3609), .driver_info = BTUSB_MEDIATEK | + BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3613), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3627), .driver_info = BTUSB_MEDIATEK | diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 5aa9fcd80cf51..c2d9d409867db 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -757,20 +757,16 @@ static void amd_pstate_adjust_perf(unsigned int cpu, static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) { struct amd_cpudata *cpudata = policy->driver_data; - union perf_cached perf = READ_ONCE(cpudata->perf); - u32 nominal_freq, max_freq; + u32 nominal_freq; int ret = 0; nominal_freq = READ_ONCE(cpudata->nominal_freq); - max_freq = perf_to_freq(perf, cpudata->nominal_freq, perf.highest_perf); if (on) - policy->cpuinfo.max_freq = max_freq; + policy->cpuinfo.max_freq = cpudata->max_freq; else if (policy->cpuinfo.max_freq > nominal_freq) policy->cpuinfo.max_freq = nominal_freq; - policy->max = policy->cpuinfo.max_freq; - if (cppc_state == AMD_PSTATE_PASSIVE) { ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq); if (ret < 0) @@ -952,13 +948,15 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) WRITE_ONCE(cpudata->nominal_freq, nominal_freq); + /* max_freq is calculated according to (nominal_freq * highest_perf)/nominal_perf */ max_freq = perf_to_freq(perf, nominal_freq, perf.highest_perf); + WRITE_ONCE(cpudata->max_freq, max_freq); + lowest_nonlinear_freq = perf_to_freq(perf, nominal_freq, perf.lowest_nonlinear_perf); WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); /** * Below values need to be initialized correctly, otherwise driver will fail to load - * max_freq is calculated according to (nominal_freq * highest_perf)/nominal_perf * lowest_nonlinear_freq is a value between [min_freq, nominal_freq] * Check _CPC in ACPI table objects if any values are incorrect */ @@ -1018,12 +1016,9 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) perf = READ_ONCE(cpudata->perf); - policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, - cpudata->nominal_freq, - perf.lowest_perf); - policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf, - cpudata->nominal_freq, - perf.highest_perf); + policy->cpuinfo.min_freq = perf_to_freq(perf, cpudata->nominal_freq, + perf.lowest_perf); + policy->cpuinfo.max_freq = cpudata->max_freq; ret = amd_pstate_cppc_enable(policy); if (ret) @@ -1090,14 +1085,9 @@ static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, char *buf) { - struct amd_cpudata *cpudata; - union perf_cached perf; - - cpudata = policy->driver_data; - perf = READ_ONCE(cpudata->perf); + struct amd_cpudata *cpudata = policy->driver_data; - return sysfs_emit(buf, "%u\n", - perf_to_freq(perf, cpudata->nominal_freq, perf.highest_perf)); + return sysfs_emit(buf, "%u\n", cpudata->max_freq); } static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, @@ -1500,12 +1490,9 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) perf = READ_ONCE(cpudata->perf); - policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, - cpudata->nominal_freq, - perf.lowest_perf); - policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf, - cpudata->nominal_freq, - perf.highest_perf); + policy->cpuinfo.min_freq = perf_to_freq(perf, cpudata->nominal_freq, + perf.lowest_perf); + policy->cpuinfo.max_freq = cpudata->max_freq; policy->driver_data = cpudata; ret = amd_pstate_cppc_enable(policy); diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index cb45fdca27a6c..0d26e56b7938b 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -68,6 +68,7 @@ struct amd_aperf_mperf { * @min_limit_freq: Cached value of policy->min (in khz) * @max_limit_freq: Cached value of policy->max (in khz) * @nominal_freq: the frequency (in khz) that mapped to nominal_perf + * @max_freq: in ideal conditions the maximum frequency (in khz) possible frequency * @lowest_nonlinear_freq: the frequency (in khz) that mapped to lowest_nonlinear_perf * @cur: Difference of Aperf/Mperf/tsc count between last and current sample * @prev: Last Aperf/Mperf/tsc count value read from register @@ -94,6 +95,7 @@ struct amd_cpudata { u32 min_limit_freq; u32 max_limit_freq; u32 nominal_freq; + u32 max_freq; u32 lowest_nonlinear_freq; struct amd_aperf_mperf cur; diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index 011f35cb47b94..be28d7e6cf63e 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -28,6 +28,9 @@ static struct cpufreq_driver cppc_cpufreq_driver; +/* Autonomous Selection boot parameter */ +static bool auto_sel_mode; + #ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE static enum { FIE_UNSET = -1, @@ -50,7 +53,8 @@ struct cppc_freq_invariance { static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv); static struct kthread_worker *kworker_fie; -static int cppc_perf_from_fbctrs(struct cppc_perf_fb_ctrs *fb_ctrs_t0, +static int cppc_perf_from_fbctrs(u64 reference_perf, + struct cppc_perf_fb_ctrs *fb_ctrs_t0, struct cppc_perf_fb_ctrs *fb_ctrs_t1); /** @@ -70,7 +74,7 @@ static void __cppc_scale_freq_tick(struct cppc_freq_invariance *cppc_fi) struct cppc_perf_fb_ctrs fb_ctrs = {0}; struct cppc_cpudata *cpu_data; unsigned long local_freq_scale; - u64 perf; + u64 perf, ref_perf; cpu_data = cppc_fi->cpu_data; @@ -79,7 +83,9 @@ static void __cppc_scale_freq_tick(struct cppc_freq_invariance *cppc_fi) return; } - perf = cppc_perf_from_fbctrs(&cppc_fi->prev_perf_fb_ctrs, &fb_ctrs); + ref_perf = cpu_data->perf_caps.reference_perf; + perf = cppc_perf_from_fbctrs(ref_perf, + &cppc_fi->prev_perf_fb_ctrs, &fb_ctrs); if (!perf) return; @@ -287,6 +293,21 @@ static inline void cppc_freq_invariance_exit(void) } #endif /* CONFIG_ACPI_CPPC_CPUFREQ_FIE */ +static void cppc_cpufreq_update_perf_limits(struct cppc_cpudata *cpu_data, + struct cpufreq_policy *policy) +{ + struct cppc_perf_caps *caps = &cpu_data->perf_caps; + u32 min_perf, max_perf; + + min_perf = cppc_khz_to_perf(caps, policy->min); + max_perf = cppc_khz_to_perf(caps, policy->max); + + cpu_data->perf_ctrls.min_perf = + clamp_t(u32, min_perf, caps->lowest_perf, caps->highest_perf); + cpu_data->perf_ctrls.max_perf = + clamp_t(u32, max_perf, caps->lowest_perf, caps->highest_perf); +} + static int cppc_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int target_freq, unsigned int relation) @@ -298,6 +319,8 @@ static int cppc_cpufreq_set_target(struct cpufreq_policy *policy, cpu_data->perf_ctrls.desired_perf = cppc_khz_to_perf(&cpu_data->perf_caps, target_freq); + cppc_cpufreq_update_perf_limits(cpu_data, policy); + freqs.old = policy->cur; freqs.new = target_freq; @@ -322,8 +345,9 @@ static unsigned int cppc_cpufreq_fast_switch(struct cpufreq_policy *policy, desired_perf = cppc_khz_to_perf(&cpu_data->perf_caps, target_freq); cpu_data->perf_ctrls.desired_perf = desired_perf; - ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); + cppc_cpufreq_update_perf_limits(cpu_data, policy); + ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); if (ret) { pr_debug("Failed to set target on CPU:%d. ret:%d\n", cpu, ret); @@ -594,6 +618,12 @@ static struct cppc_cpudata *cppc_cpufreq_get_cpu_data(unsigned int cpu) goto free_mask; } + ret = cppc_get_perf(cpu, &cpu_data->perf_ctrls); + if (ret) { + pr_debug("Err reading CPU%d perf ctrls: ret:%d\n", cpu, ret); + goto free_mask; + } + return cpu_data; free_mask: @@ -628,13 +658,19 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy) caps = &cpu_data->perf_caps; policy->driver_data = cpu_data; + /* + * Enable CPPC for both OS-driven and autonomous modes. + * The Enable register is optional - some platforms may not support it + */ + ret = cppc_set_enable(cpu, true); + if (ret && ret != -EOPNOTSUPP) + pr_warn("Failed to enable CPPC for CPU%d (%d)\n", cpu, ret); + /* * Set min to lowest nonlinear perf to avoid any efficiency penalty (see * Section 8.4.7.1.1.5 of ACPI 6.1 spec) */ policy->min = cppc_perf_to_khz(caps, caps->lowest_nonlinear_perf); - policy->max = cppc_perf_to_khz(caps, policy->boost_enabled ? - caps->highest_perf : caps->nominal_perf); /* * Set cpuinfo.min_freq to Lowest to make the full range of performance @@ -642,7 +678,8 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy) * nonlinear perf */ policy->cpuinfo.min_freq = cppc_perf_to_khz(caps, caps->lowest_perf); - policy->cpuinfo.max_freq = policy->max; + policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, policy->boost_enabled ? + caps->highest_perf : caps->nominal_perf); policy->transition_delay_us = cppc_cpufreq_get_transition_delay_us(cpu); policy->shared_type = cpu_data->shared_type; @@ -681,11 +718,71 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy) policy->cur = cppc_perf_to_khz(caps, caps->highest_perf); cpu_data->perf_ctrls.desired_perf = caps->highest_perf; - ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); - if (ret) { - pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n", - caps->highest_perf, cpu, ret); - goto out; + /* + * Enable autonomous mode on first init if boot param is set. + * Check last_governor to detect first init and skip if auto_sel + * is already enabled. + */ + if (auto_sel_mode && policy->last_governor[0] == '\0' && + !cpu_data->perf_ctrls.auto_sel) { + /* Init min/max_perf from caps if not already set by HW. */ + if (!cpu_data->perf_ctrls.min_perf) + cpu_data->perf_ctrls.min_perf = caps->lowest_nonlinear_perf; + if (!cpu_data->perf_ctrls.max_perf) + cpu_data->perf_ctrls.max_perf = policy->boost_enabled ? + caps->highest_perf : caps->nominal_perf; + + cpu_data->perf_ctrls.desired_perf = + clamp_t(u32, cpu_data->perf_ctrls.desired_perf, + cpu_data->perf_ctrls.min_perf, + cpu_data->perf_ctrls.max_perf); + + policy->cur = cppc_perf_to_khz(caps, + cpu_data->perf_ctrls.desired_perf); + + /* EPP is optional - some platforms may not support it */ + ret = cppc_set_epp(cpu, CPPC_EPP_PERFORMANCE_PREF); + if (ret && ret != -EOPNOTSUPP) + pr_warn("Failed to set EPP for CPU%d (%d)\n", cpu, ret); + else if (!ret) + cpu_data->perf_ctrls.energy_perf = CPPC_EPP_PERFORMANCE_PREF; + + /* Program min/max/desired into CPPC regs before enabling auto_sel. */ + ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); + if (ret) { + pr_debug("Err setting perf for autonomous mode CPU:%d ret:%d\n", + cpu, ret); + goto out; + } + + ret = cppc_set_auto_sel(cpu, true); + if (ret && ret != -EOPNOTSUPP) { + pr_warn("Failed autonomous config for CPU%d (%d)\n", + cpu, ret); + goto out; + } + if (!ret) + cpu_data->perf_ctrls.auto_sel = true; + } + + if (cpu_data->perf_ctrls.auto_sel) { + /* Sync policy limits from HW when autonomous mode is active */ + policy->min = cppc_perf_to_khz(caps, + cpu_data->perf_ctrls.min_perf ?: + caps->lowest_nonlinear_perf); + policy->max = cppc_perf_to_khz(caps, + cpu_data->perf_ctrls.max_perf ?: + (policy->boost_enabled ? + caps->highest_perf : + caps->nominal_perf)); + } else { + /* Normal mode: governors control frequency */ + ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); + if (ret) { + pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n", + caps->highest_perf, cpu, ret); + goto out; + } } cppc_cpufreq_cpu_fie_init(policy); @@ -723,13 +820,11 @@ static inline u64 get_delta(u64 t1, u64 t0) return (u32)t1 - (u32)t0; } -static int cppc_perf_from_fbctrs(struct cppc_perf_fb_ctrs *fb_ctrs_t0, +static int cppc_perf_from_fbctrs(u64 reference_perf, + struct cppc_perf_fb_ctrs *fb_ctrs_t0, struct cppc_perf_fb_ctrs *fb_ctrs_t1) { u64 delta_reference, delta_delivered; - u64 reference_perf; - - reference_perf = fb_ctrs_t0->reference_perf; delta_reference = get_delta(fb_ctrs_t1->reference, fb_ctrs_t0->reference); @@ -766,7 +861,7 @@ static unsigned int cppc_cpufreq_get_rate(unsigned int cpu) struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); struct cppc_perf_fb_ctrs fb_ctrs_t0 = {0}, fb_ctrs_t1 = {0}; struct cppc_cpudata *cpu_data; - u64 delivered_perf; + u64 delivered_perf, reference_perf; int ret; if (!policy) @@ -783,7 +878,9 @@ static unsigned int cppc_cpufreq_get_rate(unsigned int cpu) return 0; } - delivered_perf = cppc_perf_from_fbctrs(&fb_ctrs_t0, &fb_ctrs_t1); + reference_perf = cpu_data->perf_caps.reference_perf; + delivered_perf = cppc_perf_from_fbctrs(reference_perf, + &fb_ctrs_t0, &fb_ctrs_t1); if (!delivered_perf) goto out_invalid_counters; @@ -807,17 +904,11 @@ static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state) { struct cppc_cpudata *cpu_data = policy->driver_data; struct cppc_perf_caps *caps = &cpu_data->perf_caps; - int ret; if (state) - policy->max = cppc_perf_to_khz(caps, caps->highest_perf); + policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, caps->highest_perf); else - policy->max = cppc_perf_to_khz(caps, caps->nominal_perf); - policy->cpuinfo.max_freq = policy->max; - - ret = freq_qos_update_request(policy->max_freq_req, policy->max); - if (ret < 0) - return ret; + policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, caps->nominal_perf); return 0; } @@ -849,6 +940,7 @@ static ssize_t show_auto_select(struct cpufreq_policy *policy, char *buf) static ssize_t store_auto_select(struct cpufreq_policy *policy, const char *buf, size_t count) { + struct cppc_cpudata *cpu_data = policy->driver_data; bool val; int ret; @@ -860,6 +952,29 @@ static ssize_t store_auto_select(struct cpufreq_policy *policy, if (ret) return ret; + cpu_data->perf_ctrls.auto_sel = val; + + if (val) { + u32 old_min_perf = cpu_data->perf_ctrls.min_perf; + u32 old_max_perf = cpu_data->perf_ctrls.max_perf; + + /* + * When enabling autonomous selection, program MIN_PERF and + * MAX_PERF from current policy limits so that the platform + * uses the correct performance bounds immediately. + */ + cppc_cpufreq_update_perf_limits(cpu_data, policy); + + ret = cppc_set_perf(policy->cpu, &cpu_data->perf_ctrls); + if (ret) { + cpu_data->perf_ctrls.min_perf = old_min_perf; + cpu_data->perf_ctrls.max_perf = old_max_perf; + cppc_set_auto_sel(policy->cpu, false); + cpu_data->perf_ctrls.auto_sel = false; + return ret; + } + } + return count; } @@ -910,19 +1025,48 @@ static ssize_t store_##_name(struct cpufreq_policy *policy, \ CPPC_CPUFREQ_ATTR_RW_U64(auto_act_window, cppc_get_auto_act_window, cppc_set_auto_act_window) -CPPC_CPUFREQ_ATTR_RW_U64(energy_performance_preference_val, - cppc_get_epp_perf, cppc_set_epp) +static ssize_t +show_energy_performance_preference_val(struct cpufreq_policy *policy, char *buf) +{ + return cppc_cpufreq_sysfs_show_u64(policy->cpu, cppc_get_epp_perf, buf); +} + +static ssize_t +store_energy_performance_preference_val(struct cpufreq_policy *policy, + const char *buf, size_t count) +{ + struct cppc_cpudata *cpu_data = policy->driver_data; + u64 val; + int ret; + + ret = kstrtou64(buf, 0, &val); + if (ret) + return ret; + + ret = cppc_set_epp(policy->cpu, val); + if (ret) + return ret; + + cpu_data->perf_ctrls.energy_perf = val; + + return count; +} + +CPPC_CPUFREQ_ATTR_RW_U64(perf_limited, cppc_get_perf_limited, + cppc_set_perf_limited) cpufreq_freq_attr_ro(freqdomain_cpus); cpufreq_freq_attr_rw(auto_select); cpufreq_freq_attr_rw(auto_act_window); cpufreq_freq_attr_rw(energy_performance_preference_val); +cpufreq_freq_attr_rw(perf_limited); static struct freq_attr *cppc_cpufreq_attr[] = { &freqdomain_cpus, &auto_select, &auto_act_window, &energy_performance_preference_val, + &perf_limited, NULL, }; @@ -958,10 +1102,18 @@ static int __init cppc_cpufreq_init(void) static void __exit cppc_cpufreq_exit(void) { + unsigned int cpu; + + for_each_present_cpu(cpu) + cppc_set_auto_sel(cpu, false); + cpufreq_unregister_driver(&cppc_cpufreq_driver); cppc_freq_invariance_exit(); } +module_param(auto_sel_mode, bool, 0444); +MODULE_PARM_DESC(auto_sel_mode, "Enable CPPC autonomous performance selection at boot"); + module_exit(cppc_cpufreq_exit); MODULE_AUTHOR("Ashwin Chaugule"); MODULE_DESCRIPTION("CPUFreq driver based on the ACPI CPPC v5.0+ spec"); diff --git a/drivers/cpufreq/cpufreq-nforce2.c b/drivers/cpufreq/cpufreq-nforce2.c index fbbbe501cf2dc..831102522ad64 100644 --- a/drivers/cpufreq/cpufreq-nforce2.c +++ b/drivers/cpufreq/cpufreq-nforce2.c @@ -355,8 +355,8 @@ static int nforce2_cpu_init(struct cpufreq_policy *policy) min_fsb = NFORCE2_MIN_FSB; /* cpuinfo and default policy values */ - policy->min = policy->cpuinfo.min_freq = min_fsb * fid * 100; - policy->max = policy->cpuinfo.max_freq = max_fsb * fid * 100; + policy->cpuinfo.min_freq = min_fsb * fid * 100; + policy->cpuinfo.max_freq = max_fsb * fid * 100; return 0; } diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 1f794524a1d92..a2fba4d3e5070 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -609,10 +609,19 @@ static int policy_set_boost(struct cpufreq_policy *policy, bool enable) policy->boost_enabled = enable; ret = cpufreq_driver->set_boost(policy, enable); - if (ret) + if (ret) { policy->boost_enabled = !policy->boost_enabled; + return ret; + } - return ret; + ret = freq_qos_update_request(&policy->boost_freq_req, policy->cpuinfo.max_freq); + if (ret < 0) { + policy->boost_enabled = !policy->boost_enabled; + cpufreq_driver->set_boost(policy, policy->boost_enabled); + return ret; + } + + return 0; } static ssize_t store_local_boost(struct cpufreq_policy *policy, @@ -760,7 +769,7 @@ static ssize_t store_##file_name \ if (ret) \ return ret; \ \ - ret = freq_qos_update_request(policy->object##_freq_req, val);\ + ret = freq_qos_update_request(&policy->object##_freq_req, val); \ return ret >= 0 ? count : ret; \ } @@ -1365,7 +1374,7 @@ static void cpufreq_policy_free(struct cpufreq_policy *policy) /* Cancel any pending policy->update work before freeing the policy. */ cancel_work_sync(&policy->update); - if (policy->max_freq_req) { + if (freq_qos_request_active(&policy->max_freq_req)) { /* * Remove max_freq_req after sending CPUFREQ_REMOVE_POLICY * notification, since CPUFREQ_CREATE_POLICY notification was @@ -1373,11 +1382,13 @@ static void cpufreq_policy_free(struct cpufreq_policy *policy) */ blocking_notifier_call_chain(&cpufreq_policy_notifier_list, CPUFREQ_REMOVE_POLICY, policy); - freq_qos_remove_request(policy->max_freq_req); + freq_qos_remove_request(&policy->max_freq_req); } - freq_qos_remove_request(policy->min_freq_req); - kfree(policy->min_freq_req); + if (freq_qos_request_active(&policy->min_freq_req)) + freq_qos_remove_request(&policy->min_freq_req); + if (freq_qos_request_active(&policy->boost_freq_req)) + freq_qos_remove_request(&policy->boost_freq_req); cpufreq_policy_put_kobj(policy); free_cpumask_var(policy->real_cpus); @@ -1386,6 +1397,47 @@ static void cpufreq_policy_free(struct cpufreq_policy *policy) kfree(policy); } +static int cpufreq_policy_init_qos(struct cpufreq_policy *policy) +{ + unsigned int min_freq, max_freq; + int ret; + + /* Use policy->min/max set by the driver as QoS requests. */ + min_freq = max(FREQ_QOS_MIN_DEFAULT_VALUE, policy->min); + if (policy->max) + max_freq = min(FREQ_QOS_MAX_DEFAULT_VALUE, policy->max); + else + max_freq = FREQ_QOS_MAX_DEFAULT_VALUE; + + /* + * If the driver didn't set policy->min/max, set them as + * they are used to clamp frequency requests. + */ + policy->min = policy->min ? policy->min : policy->cpuinfo.min_freq; + policy->max = policy->max ? policy->max : policy->cpuinfo.max_freq; + + if (policy->boost_supported) { + ret = freq_qos_add_request(&policy->constraints, + &policy->boost_freq_req, + FREQ_QOS_MAX, + policy->cpuinfo.max_freq); + if (ret < 0) + return ret; + } + + ret = freq_qos_add_request(&policy->constraints, &policy->min_freq_req, + FREQ_QOS_MIN, min_freq); + if (ret < 0) + return ret; + + ret = freq_qos_add_request(&policy->constraints, &policy->max_freq_req, + FREQ_QOS_MAX, max_freq); + if (ret < 0) + return ret; + + return ret; +} + static int cpufreq_policy_online(struct cpufreq_policy *policy, unsigned int cpu, bool new_policy) { @@ -1431,6 +1483,12 @@ static int cpufreq_policy_online(struct cpufreq_policy *policy, if (ret) goto out_offline_policy; + if (new_policy) { + ret = cpufreq_policy_init_qos(policy); + if (ret < 0) + goto out_offline_policy; + } + /* related_cpus should at least include policy->cpus. */ cpumask_copy(policy->related_cpus, policy->cpus); } @@ -1447,47 +1505,8 @@ static int cpufreq_policy_online(struct cpufreq_policy *policy, add_cpu_dev_symlink(policy, j, get_cpu_device(j)); } - policy->min_freq_req = kzalloc(2 * sizeof(*policy->min_freq_req), - GFP_KERNEL); - if (!policy->min_freq_req) { - ret = -ENOMEM; - goto out_destroy_policy; - } - - ret = freq_qos_add_request(&policy->constraints, - policy->min_freq_req, FREQ_QOS_MIN, - FREQ_QOS_MIN_DEFAULT_VALUE); - if (ret < 0) { - /* - * So we don't call freq_qos_remove_request() for an - * uninitialized request. - */ - kfree(policy->min_freq_req); - policy->min_freq_req = NULL; - goto out_destroy_policy; - } - - /* - * This must be initialized right here to avoid calling - * freq_qos_remove_request() on uninitialized request in case - * of errors. - */ - policy->max_freq_req = policy->min_freq_req + 1; - - ret = freq_qos_add_request(&policy->constraints, - policy->max_freq_req, FREQ_QOS_MAX, - FREQ_QOS_MAX_DEFAULT_VALUE); - if (ret < 0) { - policy->max_freq_req = NULL; - goto out_destroy_policy; - } - blocking_notifier_call_chain(&cpufreq_policy_notifier_list, CPUFREQ_CREATE_POLICY, policy); - } else { - ret = freq_qos_update_request(policy->max_freq_req, policy->max); - if (ret < 0) - goto out_destroy_policy; } if (cpufreq_driver->get && has_target()) { @@ -2789,16 +2808,10 @@ int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state) return -ENXIO; ret = cpufreq_frequency_table_cpuinfo(policy); - if (ret) { + if (ret) pr_err("%s: Policy frequency update failed\n", __func__); - return ret; - } - ret = freq_qos_update_request(policy->max_freq_req, policy->max); - if (ret < 0) - return ret; - - return 0; + return ret; } EXPORT_SYMBOL_GPL(cpufreq_boost_set_sw); diff --git a/drivers/cpufreq/freq_table.c b/drivers/cpufreq/freq_table.c index 5b364d8da4f92..ea994647abc88 100644 --- a/drivers/cpufreq/freq_table.c +++ b/drivers/cpufreq/freq_table.c @@ -49,16 +49,15 @@ int cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy) max_freq = freq; } - policy->min = policy->cpuinfo.min_freq = min_freq; - policy->max = max_freq; + policy->cpuinfo.min_freq = min_freq; /* * If the driver has set its own cpuinfo.max_freq above max_freq, leave * it as is. */ if (policy->cpuinfo.max_freq < max_freq) - policy->max = policy->cpuinfo.max_freq = max_freq; + policy->cpuinfo.max_freq = max_freq; - if (policy->min == ~0) + if (min_freq == ~0) return -EINVAL; else return 0; diff --git a/drivers/cpufreq/gx-suspmod.c b/drivers/cpufreq/gx-suspmod.c index d269a4f26f98e..d40c9e0bbb740 100644 --- a/drivers/cpufreq/gx-suspmod.c +++ b/drivers/cpufreq/gx-suspmod.c @@ -421,7 +421,7 @@ static int cpufreq_gx_cpu_init(struct cpufreq_policy *policy) policy->min = maxfreq / max_duration; else policy->min = maxfreq / POLICY_MIN_DIV; - policy->max = maxfreq; + policy->cpuinfo.min_freq = maxfreq / max_duration; policy->cpuinfo.max_freq = maxfreq; diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 11c58af419006..ed3fd134fb8b7 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -3049,9 +3049,6 @@ static int __intel_pstate_cpu_init(struct cpufreq_policy *policy) policy->cpuinfo.max_freq = READ_ONCE(global.no_turbo) ? cpu->pstate.max_freq : cpu->pstate.turbo_freq; - policy->min = policy->cpuinfo.min_freq; - policy->max = policy->cpuinfo.max_freq; - intel_pstate_init_acpi_perf_limits(policy); policy->fast_switch_possible = true; diff --git a/drivers/cpufreq/pcc-cpufreq.c b/drivers/cpufreq/pcc-cpufreq.c index ac2e90a65f0c4..0f185a13577f8 100644 --- a/drivers/cpufreq/pcc-cpufreq.c +++ b/drivers/cpufreq/pcc-cpufreq.c @@ -551,13 +551,11 @@ static int pcc_cpufreq_cpu_init(struct cpufreq_policy *policy) goto out; } - policy->max = policy->cpuinfo.max_freq = - ioread32(&pcch_hdr->nominal) * 1000; - policy->min = policy->cpuinfo.min_freq = - ioread32(&pcch_hdr->minimum_frequency) * 1000; + policy->cpuinfo.max_freq = ioread32(&pcch_hdr->nominal) * 1000; + policy->cpuinfo.min_freq = ioread32(&pcch_hdr->minimum_frequency) * 1000; - pr_debug("init: policy->max is %d, policy->min is %d\n", - policy->max, policy->min); + pr_debug("init: max_freq is %d, min_freq is %d\n", + policy->cpuinfo.max_freq, policy->cpuinfo.min_freq); out: return result; } diff --git a/drivers/cpufreq/pxa3xx-cpufreq.c b/drivers/cpufreq/pxa3xx-cpufreq.c index 50ff3b6a69000..06b27cbc59d6a 100644 --- a/drivers/cpufreq/pxa3xx-cpufreq.c +++ b/drivers/cpufreq/pxa3xx-cpufreq.c @@ -185,9 +185,8 @@ static int pxa3xx_cpufreq_init(struct cpufreq_policy *policy) int ret = -EINVAL; /* set default policy and cpuinfo */ - policy->min = policy->cpuinfo.min_freq = 104000; - policy->max = policy->cpuinfo.max_freq = - (cpu_is_pxa320()) ? 806000 : 624000; + policy->cpuinfo.min_freq = 104000; + policy->cpuinfo.max_freq = (cpu_is_pxa320()) ? 806000 : 624000; policy->cpuinfo.transition_latency = 1000; /* FIXME: 1 ms, assumed */ if (cpu_is_pxa300() || cpu_is_pxa310()) diff --git a/drivers/cpufreq/sh-cpufreq.c b/drivers/cpufreq/sh-cpufreq.c index 642ddb9ea217e..3c99d7009cbe2 100644 --- a/drivers/cpufreq/sh-cpufreq.c +++ b/drivers/cpufreq/sh-cpufreq.c @@ -124,10 +124,8 @@ static int sh_cpufreq_cpu_init(struct cpufreq_policy *policy) dev_notice(dev, "no frequency table found, falling back " "to rate rounding.\n"); - policy->min = policy->cpuinfo.min_freq = - (clk_round_rate(cpuclk, 1) + 500) / 1000; - policy->max = policy->cpuinfo.max_freq = - (clk_round_rate(cpuclk, ~0UL) + 500) / 1000; + policy->cpuinfo.min_freq = (clk_round_rate(cpuclk, 1) + 500) / 1000; + policy->cpuinfo.max_freq = (clk_round_rate(cpuclk, ~0UL) + 500) / 1000; } return 0; diff --git a/drivers/cpufreq/virtual-cpufreq.c b/drivers/cpufreq/virtual-cpufreq.c index 4159f31349b16..dc78b74409af4 100644 --- a/drivers/cpufreq/virtual-cpufreq.c +++ b/drivers/cpufreq/virtual-cpufreq.c @@ -164,10 +164,7 @@ static int virt_cpufreq_get_freq_info(struct cpufreq_policy *policy) policy->cpuinfo.min_freq = 1; policy->cpuinfo.max_freq = virt_cpufreq_get_perftbl_entry(policy->cpu, 0); - policy->min = policy->cpuinfo.min_freq; - policy->max = policy->cpuinfo.max_freq; - - policy->cur = policy->max; + policy->cur = policy->cpuinfo.max_freq; return 0; } diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c index f2f94d4d533e8..874b686743044 100644 --- a/drivers/firmware/arm_ffa/driver.c +++ b/drivers/firmware/arm_ffa/driver.c @@ -322,6 +322,10 @@ __ffa_partition_info_get(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, #define PART_INFO_ID_MASK GENMASK(15, 0) #define PART_INFO_EXEC_CXT_MASK GENMASK(31, 16) #define PART_INFO_PROPS_MASK GENMASK(63, 32) +#define FFA_PART_INFO_GET_REGS_FIRST_REG 3 +#define FFA_PART_INFO_GET_REGS_MIN_REGS_PER_DESC 3 +#define FFA_PART_INFO_GET_REGS_NUM_REGS \ + (sizeof(ffa_value_t) / sizeof_field(ffa_value_t, a0)) #define PART_INFO_ID(x) ((u16)(FIELD_GET(PART_INFO_ID_MASK, (x)))) #define PART_INFO_EXEC_CXT(x) ((u16)(FIELD_GET(PART_INFO_EXEC_CXT_MASK, (x)))) #define PART_INFO_PROPERTIES(x) ((u32)(FIELD_GET(PART_INFO_PROPS_MASK, (x)))) @@ -329,15 +333,13 @@ static int __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, struct ffa_partition_info *buffer, int num_parts) { - u16 buf_sz, start_idx, cur_idx, count = 0, prev_idx = 0, tag = 0; + u16 buf_sz, start_idx = 0, cur_idx, count = 0, tag = 0; struct ffa_partition_info *buf = buffer; ffa_value_t partition_info; do { __le64 *regs; - int idx; - - start_idx = prev_idx ? prev_idx + 1 : 0; + int idx, nr_desc, buf_idx, regs_per_desc, max_desc; invoke_ffa_fn((ffa_value_t){ .a0 = FFA_PARTITION_INFO_GET_REGS, @@ -353,15 +355,35 @@ __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, count = PARTITION_COUNT(partition_info.a2); if (!buffer || !num_parts) /* count only */ return count; + if (count > num_parts) + return -EINVAL; cur_idx = CURRENT_INDEX(partition_info.a2); - tag = UUID_INFO_TAG(partition_info.a2); + if (cur_idx < start_idx || cur_idx >= count) + return -EINVAL; + buf_sz = PARTITION_INFO_SZ(partition_info.a2); - if (buf_sz > sizeof(*buffer)) - buf_sz = sizeof(*buffer); + if (buf_sz % sizeof(*regs)) + return -EINVAL; + + regs_per_desc = buf_sz / sizeof(*regs); + if (regs_per_desc < FFA_PART_INFO_GET_REGS_MIN_REGS_PER_DESC) + return -EINVAL; + + nr_desc = cur_idx - start_idx + 1; + max_desc = (FFA_PART_INFO_GET_REGS_NUM_REGS - + FFA_PART_INFO_GET_REGS_FIRST_REG) / regs_per_desc; + if (nr_desc > max_desc) + return -EINVAL; + + buf_idx = buf - buffer; + if (buf_idx + nr_desc > num_parts) + return -EINVAL; + + tag = UUID_INFO_TAG(partition_info.a2); regs = (void *)&partition_info.a3; - for (idx = 0; idx < cur_idx - start_idx + 1; idx++, buf++) { + for (idx = 0; idx < nr_desc; idx++, buf++) { union { uuid_t uuid; u64 regs[2]; @@ -377,9 +399,9 @@ __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, buf->exec_ctxt = PART_INFO_EXEC_CXT(val); buf->properties = PART_INFO_PROPERTIES(val); uuid_copy(&buf->uuid, &uuid_regs.uuid); - regs += 3; + regs += regs_per_desc; } - prev_idx = cur_idx; + start_idx = cur_idx + 1; } while (cur_idx < (count - 1)); diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index 533e132e1bd8a..7ce80180802ce 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -96,6 +96,11 @@ bool efi_runtime_disabled(void) return disable_runtime; } +void __init efi_disable_runtime(void) +{ + disable_runtime = true; +} + bool __pure __efi_soft_reserve_enabled(void) { return !efi_enabled(EFI_MEM_NO_SOFT_RESERVE); diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index e386ffd009b7e..77b4dd7155c4f 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -84,6 +84,7 @@ lib-$(CONFIG_EFI_GENERIC_STUB) += efi-stub.o string.o intrinsics.o systable.o \ lib-$(CONFIG_ARM) += arm32-stub.o lib-$(CONFIG_ARM64) += kaslr.o arm64.o arm64-stub.o smbios.o +lib-$(CONFIG_ARM64_SECURE_LAUNCH) += arm64-slaunch.o lib-$(CONFIG_X86) += x86-stub.o smbios.o lib-$(CONFIG_X86_64) += x86-5lvl.o lib-$(CONFIG_RISCV) += kaslr.o riscv.o riscv-stub.o @@ -136,7 +137,7 @@ STUBCOPY_RELOC-$(CONFIG_ARM) := R_ARM_ABS # a verification pass to see if any absolute relocations exist in any of the # object files. # -STUBCOPY_FLAGS-$(CONFIG_ARM64) += --prefix-alloc-sections=.init \ +STUBCOPY_FLAGS-$(CONFIG_ARM64) += --prefix-alloc-sections=.efistub \ --prefix-symbols=__efistub_ STUBCOPY_RELOC-$(CONFIG_ARM64) := R_AARCH64_ABS diff --git a/drivers/firmware/efi/libstub/arm64-slaunch.c b/drivers/firmware/efi/libstub/arm64-slaunch.c new file mode 100644 index 0000000000000..5595fa9100d23 --- /dev/null +++ b/drivers/firmware/efi/libstub/arm64-slaunch.c @@ -0,0 +1,336 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * ARM64 DRTM Secure Launch — EFI stub component. Builds DRTM_PARAMETERS + * and issues DRTM_DYNAMIC_LAUNCH when the cmdline has "drtm=on". The SMC + * does not return on success: D-CRTM measures the image and ERETs to + * sl_entry. Copyright (c) 2025-2026, NVIDIA Corporation. + */ + +#include +#include +#include +#include + +#include "efistub.h" + +/* DRTM SMC IDs — duplicated here for EFI stub isolation */ +#define SL_DRTM_SMC_FEATURES 0xC4000111UL +#define SL_DRTM_SMC_DYNAMIC_LAUNCH 0xC4000114UL +#define SL_DRTM_PAGE_SIZE 0x1000 +#define SL_ROUND_UP_PAGE(x) (((x) + SL_DRTM_PAGE_SIZE - 1) & \ + ~(SL_DRTM_PAGE_SIZE - 1ULL)) + +/* + * Preamble<->DLME DTB-PA convention slot (mirrors SL_DLME_DTB_SLOT_OFFSET + * in arch/arm64/include/asm/drtm.h — keep them in sync). + */ +#define SL_DLME_DTB_SLOT_OFFSET (-8) + +/* From sl_stub.S — accessible via __efistub_ alias in image-vars.h */ +extern char sl_entry[]; + +/* + * DRTM Parameters (DEN0113 v1.2 §3.13 / Table 9) + * Struct must be packed — passed directly to TF-A via SMC. + */ +struct sl_drtm_params { + __le16 revision; + __le16 reserved; + __le32 launch_features; + __le64 dlme_region_address; + __le64 dlme_region_size; + __le64 dlme_image_start; + __le64 dlme_entry_point_offset; + __le64 dlme_image_size; + __le64 dlme_data_offset; + __le64 nw_dce_region_address; + __le64 nw_dce_region_size; + __le64 mem_prot_table_address; + __le64 mem_prot_table_size; +} __packed; + +static u64 sl_smc_ret(u64 fn, u64 arg1) +{ + register u64 x0 __asm__("x0") = fn; + register u64 x1 __asm__("x1") = arg1; + register u64 x2 __asm__("x2") = 0; + register u64 x3 __asm__("x3") = 0; + + asm volatile("smc #0" + : "+r"(x0), "+r"(x1), "+r"(x2), "+r"(x3) + : + : "x4", "x5", "x6", "x7", "x8", "x9", "x10", + "x11", "x12", "x13", "x14", "x15", "x16", "x17", + "memory"); + return x0; +} + +static void sl_smc(u64 fn, u64 arg1) +{ + sl_smc_ret(fn, arg1); +} + +/* Read CTR_EL0.DminLine and return cache line size in bytes. */ +static inline unsigned int sl_dcache_line_size(void) +{ + u64 ctr; + + asm volatile("mrs %0, ctr_el0" : "=r"(ctr)); + return 4U << ((ctr >> 16) & 0xfU); +} + +/* + * Clean (cvac) a range to PoC at cache-line granularity so TF-A sees + * the stub's freshly-written value from EL3. + */ +static inline void sl_dc_cvac_range(unsigned long start, unsigned long len) +{ + unsigned int line = sl_dcache_line_size(); + unsigned long mask = (unsigned long)line - 1UL; + unsigned long end = start + len; + unsigned long addr; + + start &= ~mask; + for (addr = start; addr < end; addr += line) + asm volatile("dc cvac, %0" : : "r"(addr) : "memory"); +} + +/* + * DLME data reserve (from DRTM_FEATURES) and whether D-CRTM advertised + * DRTM support. Consumed by arm64-stub.c and the launch gate in fdt.c. + */ +unsigned long sl_dlme_data_reserve; +bool sl_drtm_available; + +/* + * Query DRTM_FEATURES (DEN0113 v1.2 Table 6, feature 0x2) for the minimum + * DLME data size. Best-effort: failure leaves DRTM unavailable (normal + * boot). Called before ExitBootServices. + */ +void efi_slaunch_get_dlme_data_size(void) +{ + register u64 x0 __asm__("x0") = SL_DRTM_SMC_FEATURES; + register u64 x1 __asm__("x1") = (1ULL << 63) | 0x2; + register u64 x2 __asm__("x2") = 0; + register u64 x3 __asm__("x3") = 0; + u32 min_pages; + + asm volatile("smc #0" + : "+r"(x0), "+r"(x1), "+r"(x2), "+r"(x3) + : + : "x4", "x5", "x6", "x7", "x8", "x9", "x10", + "x11", "x12", "x13", "x14", "x15", "x16", "x17", + "memory"); + + /* DRTM_FEATURES success is x0 > 0; x1[31:0] = min DLME data pages. */ + if ((s64)x0 <= 0) + return; + min_pages = (u32)(x1 & 0xFFFFFFFF); + if (min_pages == 0) + return; + + sl_dlme_data_reserve = (unsigned long)min_pages * SL_DRTM_PAGE_SIZE; + sl_drtm_available = true; + efi_info("DRTM: min DLME data size %lu KB (%u pages)\n", + sl_dlme_data_reserve / 1024, min_pages); +} + +/* + * Zero the PE/COFF Optional Header ImageBase on the relocated kernel + * buffer (pre-EBS): LoadImage patched it with the load PA, diverging the + * D-CRTM-measured bytes from the on-disk Image. Pre-EBS because + * efi_remap_image() marked the header RO; flip RW, zero, restore RO. + */ +void efi_slaunch_scrub_imagebase(unsigned long kernel_addr) +{ + efi_guid_t guid = EFI_MEMORY_ATTRIBUTE_PROTOCOL_GUID; + efi_memory_attribute_protocol_t *memattr; + efi_status_t status; + u32 e_lfanew; + volatile u64 *image_base_ptr; + unsigned long page_base; + + if (!kernel_addr) + return; + + e_lfanew = *(volatile u32 *)((char *)kernel_addr + 0x3c); + image_base_ptr = (volatile u64 *)((char *)kernel_addr + + e_lfanew + 4 + 20 + 0x18); + page_base = (unsigned long)image_base_ptr & ~(SL_DRTM_PAGE_SIZE - 1UL); + + status = efi_bs_call(locate_protocol, &guid, NULL, (void **)&memattr); + if (status != EFI_SUCCESS) { + efi_warn("DRTM: no EFI_MEMORY_ATTRIBUTE_PROTOCOL; " + "skipping PE ImageBase scrub\n"); + return; + } + + status = memattr->clear_memory_attributes(memattr, page_base, + SL_DRTM_PAGE_SIZE, + EFI_MEMORY_RO); + if (status != EFI_SUCCESS) { + efi_warn("DRTM: clear EFI_MEMORY_RO failed for PE header page: 0x%lx\n", + status); + return; + } + + *image_base_ptr = 0; + sl_dc_cvac_range((unsigned long)image_base_ptr, 8); + asm volatile("dsb sy" : : : "memory"); + + status = memattr->set_memory_attributes(memattr, page_base, + SL_DRTM_PAGE_SIZE, + EFI_MEMORY_RO); + if (status != EFI_SUCCESS) + efi_warn("DRTM: restore EFI_MEMORY_RO on PE header page failed: 0x%lx\n", + status); + + efi_info("DRTM: PE ImageBase zeroed at 0x%lx\n", + (unsigned long)image_base_ptr); +} + +/* + * Token-aware cmdline match: true iff `tok` is a standalone + * whitespace-delimited word in `cmdline`, not a substring of another + * option (so "drtm=on" does not match "nodrtm=on" or "root=...drtm=on"). + */ +static bool sl_cmdline_token(const char *cmdline, const char *tok) +{ + size_t toklen = strlen(tok); + const char *p = cmdline; + + while ((p = strstr(p, tok)) != NULL) { + bool start_ok = (p == cmdline) || p[-1] == ' ' || p[-1] == '\t'; + bool end_ok = p[toklen] == '\0' || p[toklen] == ' ' || + p[toklen] == '\t'; + + if (start_ok && end_ok) + return true; + p += toklen; + } + return false; +} + +bool efi_slaunch_enabled(const char *cmdline) +{ + if (!cmdline) + return false; + /* + * A detected DRTM launch forces EFI runtime services off in + * slaunch_setup() (via disable_runtime), so no efi=noruntime token is + * needed here; drtm=on alone requests the launch. + */ + return sl_cmdline_token(cmdline, "drtm=on"); +} + +/* Canonical converted command line, recorded once by the stub entry. */ +static const char *sl_cmdline; + +/* + * Record the stub's canonical converted command line for the DRTM gates. + * Reusing it avoids a second efi_convert_cmdline(), which would measure + * the EFI LoadOptions a second time (duplicate PCR 9 event). + */ +void efi_slaunch_set_cmdline(const char *cmdline) +{ + sl_cmdline = cmdline; +} + +/* + * True iff the recorded command line requests a DRTM launch. Gates the + * DRTM_FEATURES probe: an SMC faults on a platform with no EL3 monitor, + * so it must not be issued on boots that never asked for a launch. + */ +bool efi_slaunch_requested(void) +{ + return efi_slaunch_enabled(sl_cmdline); +} + +/* + * TF-A requires DRTM_PARAMETERS to be 4KB-aligned; we are past + * ExitBootServices so cannot allocate — use a static buffer. + */ +static struct sl_drtm_params sl_params __aligned(SL_DRTM_PAGE_SIZE); + +void __noreturn efi_slaunch_drtm(unsigned long kernel_addr, + unsigned long fdt_addr) +{ + struct sl_drtm_params *params = &sl_params; + unsigned long image_size, kernel_memsize; + unsigned long dlme_data_offset; + unsigned long sl_entry_offset; + + /* + * Store DTB PA just below the DLME data area (dlme_data_offset - 8); + * sl_entry finds it via X0 + X1 - 8. Direct physical write avoids + * EFI-stub symbol resolution, which can fail under PIC/GOT. + */ + + /* Compute sl_entry offset from kernel image base */ + sl_entry_offset = (unsigned long)sl_entry - (unsigned long)_text; + + /* + * DLME region layout: [_text.._edata] measured image, [_edata.._end] + * BSS, then D-CRTM-populated DLME data after _end. FDT is separate + * (wherever efi_allocate_pages() put it). + */ + image_size = (unsigned long)(_edata - _text); + kernel_memsize = (unsigned long)(_end - _text); + dlme_data_offset = SL_ROUND_UP_PAGE(kernel_memsize) + SL_DLME_DTB_SLOT_GAP; + + /* + * Write DTB PA into the Preamble->DLME slot at (kernel_addr + + * dlme_data_offset + SL_DLME_DTB_SLOT_OFFSET); sl_entry reads it via + * X0 + X1 + SL_DLME_DTB_SLOT_OFFSET after the D-CRTM ERET. + */ + *(volatile u64 *)(kernel_addr + dlme_data_offset + + SL_DLME_DTB_SLOT_OFFSET) = fdt_addr; + + /* Build DRTM_PARAMETERS */ + params->revision = cpu_to_le16(DRTM_PARAMS_REVISION); + params->reserved = cpu_to_le16(0); + /* bits[5:3]=0: complete DMA protection. bit 7: request Secure-interrupt + * disable for the launch window (DEN0113 Table 9); the DLME re-enables + * post-launch via DRTM_ENABLE_SECURE_INTERRUPTS. */ + params->launch_features = cpu_to_le32(DRTM_LAUNCH_FEAT_MEM_PROT_ALL | DRTM_LAUNCH_FEAT_SEC_INT_DISABLE); + params->dlme_region_address = cpu_to_le64(kernel_addr); + params->dlme_region_size = cpu_to_le64(dlme_data_offset + sl_dlme_data_reserve); + params->dlme_image_start = cpu_to_le64(0); + params->dlme_entry_point_offset = cpu_to_le64(sl_entry_offset); + params->dlme_image_size = cpu_to_le64(image_size); + params->dlme_data_offset = cpu_to_le64(dlme_data_offset); + params->nw_dce_region_address = cpu_to_le64(0); + params->nw_dce_region_size = cpu_to_le64(0); + /* Complete DMA protection: table must be zero (DEN0113 v1.2 Table 9). */ + params->mem_prot_table_address = cpu_to_le64(0); + params->mem_prot_table_size = cpu_to_le64(0); + + /* + * Clean to DRAM what the D-CRTM reads after the SMC: the params + * struct and the DTB PA slot (outside TF-A's own DLME flush). + */ + sl_dc_cvac_range((unsigned long)params, sizeof(*params)); + sl_dc_cvac_range(kernel_addr + dlme_data_offset + + SL_DLME_DTB_SLOT_OFFSET, sizeof(u64)); + /* + * NOTE: the PE ImageBase scrub happens pre-EBS in + * efi_slaunch_scrub_imagebase(); the memory-attribute protocol used + * to unprotect the header page is unreachable after boot services exit. + */ + asm volatile("dsb sy" : : : "memory"); + + /* + * DRTM_DYNAMIC_LAUNCH — does not return on success. + * D-CRTM: measures kernel, populates DLME data, ERETs to sl_entry + */ + sl_smc(SL_DRTM_SMC_DYNAMIC_LAUNCH, (u64)params); + + /* + * The launch SMC returns only on failure (success ERETs to sl_entry). + * Boot services are gone here, so we cannot print; the pre-EBS notice + * in efi_boot_kernel() flagged that reaching this halt means the + * dynamic launch failed. + */ + for (;;) + asm volatile("wfi"); +} diff --git a/drivers/firmware/efi/libstub/arm64-stub.c b/drivers/firmware/efi/libstub/arm64-stub.c index 2c38693561475..72906e86cd6aa 100644 --- a/drivers/firmware/efi/libstub/arm64-stub.c +++ b/drivers/firmware/efi/libstub/arm64-stub.c @@ -36,11 +36,36 @@ efi_status_t handle_kernel_image(unsigned long *image_addr, kernel_codesize = __inittext_end - _text; kernel_memsize = kernel_size + (_end - _edata); *reserve_size = kernel_memsize; + +#ifdef CONFIG_ARM64_SECURE_LAUNCH + /* + * Reserve DLME data space (size from DRTM_FEATURES) after kernel + * BSS for D-CRTM to populate (address map, event log, etc.). Probe + * and reserve only when the command line requests a launch: the + * DRTM_FEATURES SMC would fault on a platform with no EL3 monitor. + */ + if (efi_slaunch_requested()) { + efi_slaunch_get_dlme_data_size(); + if (sl_drtm_available) + *reserve_size += SL_DLME_DTB_SLOT_GAP + + sl_dlme_data_reserve; + } +#endif *image_addr = (unsigned long)_text; - return efi_kaslr_relocate_kernel(image_addr, reserve_addr, reserve_size, - kernel_size, kernel_codesize, kernel_memsize, - efi_kaslr_get_phys_seed(image_handle)); + { + efi_status_t st; + + st = efi_kaslr_relocate_kernel(image_addr, reserve_addr, + reserve_size, kernel_size, + kernel_codesize, kernel_memsize, + efi_kaslr_get_phys_seed(image_handle)); +#ifdef CONFIG_ARM64_SECURE_LAUNCH + if (st == EFI_SUCCESS) + efi_slaunch_scrub_imagebase(*image_addr); +#endif + return st; + } } asmlinkage void primary_entry(void); diff --git a/drivers/firmware/efi/libstub/efi-stub-entry.c b/drivers/firmware/efi/libstub/efi-stub-entry.c index aa85e910fe595..fa3ec3fcc1f52 100644 --- a/drivers/firmware/efi/libstub/efi-stub-entry.c +++ b/drivers/firmware/efi/libstub/efi-stub-entry.c @@ -67,6 +67,11 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, if (status != EFI_SUCCESS) return status; +#ifdef CONFIG_ARM64_SECURE_LAUNCH + /* Record the canonical command line for the DRTM launch gates. */ + efi_slaunch_set_cmdline(cmdline_ptr); +#endif + efi_info("Booting Linux Kernel...\n"); status = handle_kernel_image(&image_addr, &image_size, diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h index fb532eb61ace7..7e9424645cd4f 100644 --- a/drivers/firmware/efi/libstub/efistub.h +++ b/drivers/firmware/efi/libstub/efistub.h @@ -1267,4 +1267,20 @@ void arch_accept_memory(phys_addr_t start, phys_addr_t end); efi_status_t efi_zboot_decompress_init(unsigned long *alloc_size); efi_status_t efi_zboot_decompress(u8 *out, unsigned long outlen); +#ifdef CONFIG_ARM64_SECURE_LAUNCH +bool efi_slaunch_enabled(const char *cmdline); +void efi_slaunch_set_cmdline(const char *cmdline); +bool efi_slaunch_requested(void); +void efi_slaunch_get_dlme_data_size(void); +void efi_slaunch_scrub_imagebase(unsigned long kernel_addr); +extern unsigned long sl_dlme_data_reserve; +/* Page gap below the DLME data region so the Preamble->DLME DTB-PA + * handoff slot (SL_DLME_DTB_SLOT_OFFSET) is dedicated scratch, not the + * kernel image tail. */ +#define SL_DLME_DTB_SLOT_GAP 0x1000 +extern bool sl_drtm_available; +void __noreturn efi_slaunch_drtm(unsigned long kernel_addr, + unsigned long fdt_addr); +#endif + #endif diff --git a/drivers/firmware/efi/libstub/fdt.c b/drivers/firmware/efi/libstub/fdt.c index 6c679da644dd6..3e79aa64bb830 100644 --- a/drivers/firmware/efi/libstub/fdt.c +++ b/drivers/firmware/efi/libstub/fdt.c @@ -353,6 +353,19 @@ efi_status_t efi_boot_kernel(void *handle, efi_loaded_image_t *image, unsigned long fdt_addr; efi_status_t status; +#ifdef CONFIG_ARM64_SECURE_LAUNCH + /* + * Announce the DRTM decision while boot services can still print: the + * launch runs after ExitBootServices with no console, so a failed + * launch only halts. The info breadcrumb is suppressed by "quiet" (and + * by the default log level on 7.0); the error survives "quiet". + */ + if (efi_slaunch_enabled(cmdline_ptr) && sl_drtm_available) + efi_info("DRTM: launching; a halt after this means it failed\n"); + else if (efi_slaunch_enabled(cmdline_ptr)) + efi_err("DRTM: firmware lacks DRTM support; booting normally\n"); +#endif + status = allocate_new_fdt_and_exit_boot(handle, image, &fdt_addr, cmdline_ptr); if (status != EFI_SUCCESS) { @@ -363,6 +376,12 @@ efi_status_t efi_boot_kernel(void *handle, efi_loaded_image_t *image, if (IS_ENABLED(CONFIG_ARM)) efi_handle_post_ebs_state(); +#ifdef CONFIG_ARM64_SECURE_LAUNCH + /* Requested + firmware-advertised launch; announced pre-EBS above. */ + if (efi_slaunch_enabled(cmdline_ptr) && sl_drtm_available) + efi_slaunch_drtm(kernel_addr, fdt_addr); +#endif + efi_enter_kernel(kernel_addr, fdt_addr, fdt_totalsize((void *)fdt_addr)); /* not reached */ } diff --git a/drivers/firmware/efi/libstub/kaslr.c b/drivers/firmware/efi/libstub/kaslr.c index 4bc963e999eb9..579e237d47cd5 100644 --- a/drivers/firmware/efi/libstub/kaslr.c +++ b/drivers/firmware/efi/libstub/kaslr.c @@ -128,10 +128,15 @@ efi_status_t efi_kaslr_relocate_kernel(unsigned long *image_addr, if (!check_image_region(*image_addr, kernel_memsize)) { efi_err("FIRMWARE BUG: Image BSS overlaps adjacent EFI memory region\n"); } else if (IS_ALIGNED(*image_addr, min_kimg_align) && - (unsigned long)_end < EFI_ALLOC_LIMIT) { + (unsigned long)_end < EFI_ALLOC_LIMIT && + *reserve_size == kernel_memsize) { /* * Just execute from wherever we were loaded by the - * UEFI PE/COFF loader if the placement is suitable. + * UEFI PE/COFF loader if the placement is suitable + * and nothing beyond the image is reserved: a tail + * (e.g. DRTM DLME data) lies beyond PE SizeOfImage, + * and LoadedImage does not expose the backing + * allocation extent, so relocate to allocate it. */ *reserve_size = 0; return EFI_SUCCESS; diff --git a/drivers/firmware/efi/libstub/zboot.lds b/drivers/firmware/efi/libstub/zboot.lds index 367907eb7d869..76f0c8031badc 100644 --- a/drivers/firmware/efi/libstub/zboot.lds +++ b/drivers/firmware/efi/libstub/zboot.lds @@ -61,3 +61,13 @@ PROVIDE(__efistub__gzdata_size = PROVIDE(__data_rawsize = ABSOLUTE(_edata - _data)); PROVIDE(__data_size = ABSOLUTE(_end - _data)); PROVIDE(__sbat_size = ABSOLUTE(_esbat - _sbat)); + +/* + * ARM64_SECURE_LAUNCH references these __efistub_ aliases, but sl_entry + * is not reachable in the compressed zboot path, so a drtm=on zboot build + * silently no-ops the DRTM launch. These stubs only let such a build link. + */ +PROVIDE_HIDDEN(__efistub__text = ADDR(.head)); +PROVIDE_HIDDEN(__efistub__end = _end); +PROVIDE_HIDDEN(__efistub__edata = _edata); +PROVIDE_HIDDEN(__efistub_sl_entry = ADDR(.head)); diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c index 39723928a0199..ae6b054f13e55 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.c +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -963,9 +963,10 @@ int xe_eu_stall_stream_open(struct drm_device *dev, u64 data, struct drm_file *f return -ENODEV; } - if (xe_observation_paranoid && !perfmon_capable()) { + ret = xe_observation_paranoid_check(); + if (ret) { drm_dbg(&xe->drm, "Insufficient privileges for EU stall monitoring\n"); - return -EACCES; + return ret; } /* Initialize and set default values */ diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index fa90441d30529..f5b2550d5048d 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -1676,9 +1676,10 @@ static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma) unsigned long start = vma->vm_start; int i, ret; - if (xe_observation_paranoid && !perfmon_capable()) { + ret = xe_observation_paranoid_check(); + if (ret) { drm_dbg(&stream->oa->xe->drm, "Insufficient privilege to map OA buffer\n"); - return -EACCES; + return ret; } /* Can mmap the entire OA buffer or nothing (no partial OA buffer mmaps) */ @@ -2068,10 +2069,12 @@ int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *f privileged_op = true; } - if (privileged_op && xe_observation_paranoid && !perfmon_capable()) { - drm_dbg(&oa->xe->drm, "Insufficient privileges to open xe OA stream\n"); - ret = -EACCES; - goto err_exec_q; + if (privileged_op) { + ret = xe_observation_paranoid_check(); + if (ret) { + drm_dbg(&oa->xe->drm, "Insufficient privileges to open xe OA stream\n"); + goto err_exec_q; + } } if (!param.exec_q && !param.sample) { @@ -2350,9 +2353,10 @@ int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *fi return -ENODEV; } - if (xe_observation_paranoid && !perfmon_capable()) { + err = xe_observation_paranoid_check(); + if (err) { drm_dbg(&oa->xe->drm, "Insufficient privileges to add xe OA config\n"); - return -EACCES; + return err; } err = copy_from_user(¶m, u64_to_user_ptr(data), sizeof(param)); @@ -2452,9 +2456,10 @@ int xe_oa_remove_config_ioctl(struct drm_device *dev, u64 data, struct drm_file return -ENODEV; } - if (xe_observation_paranoid && !perfmon_capable()) { + ret = xe_observation_paranoid_check(); + if (ret) { drm_dbg(&oa->xe->drm, "Insufficient privileges to remove xe OA config\n"); - return -EACCES; + return ret; } ret = get_user(arg, ptr); diff --git a/drivers/gpu/drm/xe/xe_observation.c b/drivers/gpu/drm/xe/xe_observation.c index e3f9b546207e4..39e05b9131a74 100644 --- a/drivers/gpu/drm/xe/xe_observation.c +++ b/drivers/gpu/drm/xe/xe_observation.c @@ -4,6 +4,7 @@ */ #include +#include #include #include @@ -12,9 +13,28 @@ #include "xe_oa.h" #include "xe_observation.h" -u32 xe_observation_paranoid = true; +static u32 xe_observation_paranoid = true; static struct ctl_table_header *sysctl_header; +/** + * xe_observation_paranoid_check - Gate access to xe observation streams. + * + * When the xe-specific observation_paranoid sysctl is enabled (the + * default), defer to perf_allow_cpu() so that access is governed by the + * same policy as system-wide perf CPU events: kernel.perf_event_paranoid + * plus the security_perf_event_open() LSM hook. When the sysctl has been + * cleared by a privileged user, observation is open to all callers. + * + * Return: 0 if access is permitted, a negative errno otherwise. + */ +int xe_observation_paranoid_check(void) +{ + if (!xe_observation_paranoid) + return 0; + + return perf_allow_cpu(); +} + static int xe_oa_ioctl(struct drm_device *dev, struct drm_xe_observation_param *arg, struct drm_file *file) { @@ -83,11 +103,13 @@ static const struct ctl_table observation_ctl_table[] = { }; /** - * xe_observation_sysctl_register - Register xe_observation_paranoid sysctl + * xe_observation_sysctl_register - Register the observation_paranoid sysctl * - * Normally only superuser/root can access observation stream - * data. However, superuser can set xe_observation_paranoid sysctl to 0 to - * allow non-privileged users to also access observation data. + * When dev.xe.observation_paranoid is set (the default), access to + * observation streams follows the system-wide perf_allow_cpu() policy: + * kernel.perf_event_paranoid plus the security_perf_event_open() LSM + * hook. A privileged user can clear the sysctl to bypass that gate and + * allow unprivileged access to observation data. * * Return: always returns 0 */ diff --git a/drivers/gpu/drm/xe/xe_observation.h b/drivers/gpu/drm/xe/xe_observation.h index 17816998e9666..73a03e03c96a7 100644 --- a/drivers/gpu/drm/xe/xe_observation.h +++ b/drivers/gpu/drm/xe/xe_observation.h @@ -11,8 +11,7 @@ struct drm_device; struct drm_file; -extern u32 xe_observation_paranoid; - +int xe_observation_paranoid_check(void); int xe_observation_ioctl(struct drm_device *dev, void *data, struct drm_file *file); int xe_observation_sysctl_register(void); void xe_observation_sysctl_unregister(void); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 4d00d796f0783..15861f96dd888 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1780,16 +1780,70 @@ static void arm_smmu_init_initial_stes(struct arm_smmu_ste *strtab, } } +static int arm_smmu_kdump_adopt_l2_strtab(struct arm_smmu_device *smmu, u32 sid, + phys_addr_t base, u32 span, + struct arm_smmu_strtab_l2 **l2table) +{ + struct arm_smmu_strtab_l2 *table; + size_t size; + + /* + * Only a coherent SMMU is supported at this moment. For a non-coherent + * SMMU that wants to support ARM_SMMU_OPT_KDUMP_ADOPT, try MEMREMAP_WC. + */ + if (WARN_ON(!(smmu->features & ARM_SMMU_FEAT_COHERENCY))) + return -EOPNOTSUPP; + + /* + * Retest the span in case the L1 descriptor has been overwritten since + * the adopt. Reject this master's insert; panic or SMMU-disable would + * either lose the vmcore or cascade aborts. Do not try to fix it, as it + * would break all other SIDs in the same bus (PCI case). The corruption + * blast radius is already bounded to that bus range. + */ + if (span != STRTAB_SPLIT + 1) { + dev_err(smmu->dev, + "kdump: L1[%u] span %u changed since adopt (was %u)\n", + arm_smmu_strtab_l1_idx(sid), span, STRTAB_SPLIT + 1); + return -EINVAL; + } + + size = (1UL << (span - 1)) * sizeof(struct arm_smmu_ste); + + table = devm_memremap(smmu->dev, base, size, MEMREMAP_WB); + if (IS_ERR(table)) { + dev_err(smmu->dev, + "kdump: failed to adopt l2 stream table for SID %u\n", + sid); + return PTR_ERR(table); + } + + *l2table = table; + return 0; +} + static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) { dma_addr_t l2ptr_dma; struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; struct arm_smmu_strtab_l2 **l2table; + u32 l1_idx = arm_smmu_strtab_l1_idx(sid); - l2table = &cfg->l2.l2ptrs[arm_smmu_strtab_l1_idx(sid)]; + l2table = &cfg->l2.l2ptrs[l1_idx]; if (*l2table) return 0; + /* Deferred adoption of the crashed kernel's L2 table */ + if (smmu->options & ARM_SMMU_OPT_KDUMP_ADOPT) { + u64 l2ptr = le64_to_cpu(cfg->l2.l1tab[l1_idx].l2ptr); + phys_addr_t base = l2ptr & STRTAB_L1_DESC_L2PTR_MASK; + u32 span = FIELD_GET(STRTAB_L1_DESC_SPAN, l2ptr); + + if (span && base) + return arm_smmu_kdump_adopt_l2_strtab(smmu, sid, base, + span, l2table); + } + *l2table = dmam_alloc_coherent(smmu->dev, sizeof(**l2table), &l2ptr_dma, GFP_KERNEL); if (!*l2table) { @@ -1801,8 +1855,7 @@ static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) arm_smmu_init_initial_stes((*l2table)->stes, ARRAY_SIZE((*l2table)->stes)); - arm_smmu_write_strtab_l1_desc(&cfg->l2.l1tab[arm_smmu_strtab_l1_idx(sid)], - l2ptr_dma); + arm_smmu_write_strtab_l1_desc(&cfg->l2.l1tab[l1_idx], l2ptr_dma); return 0; } @@ -2151,7 +2204,11 @@ static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev) static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev) { - arm_smmu_gerror_handler(irq, dev); + irqreturn_t ret = arm_smmu_gerror_handler(irq, dev); + + /* In kdump, EVTQ/PRIQ are disabled and there is no thread to wake */ + if (is_kdump_kernel()) + return ret; return IRQ_WAKE_THREAD; } @@ -3581,6 +3638,29 @@ static void arm_smmu_remove_master(struct arm_smmu_master *master) kfree(master->streams); } +static bool arm_smmu_is_attach_deferred(struct device *dev) +{ + struct arm_smmu_master *master = dev_iommu_priv_get(dev); + struct arm_smmu_device *smmu = master->smmu; + int i; + + if (!(smmu->options & ARM_SMMU_OPT_KDUMP_ADOPT)) + return false; + + for (i = 0; i < master->num_streams; i++) { + struct arm_smmu_ste *ste = + arm_smmu_get_step_for_sid(smmu, master->streams[i].id); + u64 ent0 = le64_to_cpu(ste->data[0]); + + /* Defer only when there might be in-flight DMAs */ + if ((ent0 & STRTAB_STE_0_V) && + FIELD_GET(STRTAB_STE_0_CFG, ent0) != STRTAB_STE_0_CFG_ABORT) + return true; + } + + return false; +} + static struct iommu_device *arm_smmu_probe_device(struct device *dev) { int ret; @@ -3730,6 +3810,23 @@ static int arm_smmu_def_domain_type(struct device *dev) if (IS_HISI_PTT_DEVICE(pdev)) return IOMMU_DOMAIN_IDENTITY; + + /* + * ASPEED BMC devices behind an AST1150 PCIe-to-PCI bridge + * receive DMA from BMC firmware using host physical addresses + * that bypass the kernel DMA API. Use identity mapping so + * the SMMU passes these transactions through untranslated. + */ + if (pdev->bus->self && + (pdev->bus->self->dev_flags & + PCI_DEV_FLAGS_PCI_BRIDGE_NO_ALIAS) && + pdev->bus->self->vendor == PCI_VENDOR_ID_ASPEED && + pdev->bus->self->device == 0x1150) + return IOMMU_DOMAIN_IDENTITY; + + if (pdev->vendor == PCI_VENDOR_ID_NVIDIA && + pdev->device >= 0x2E00 && pdev->device <= 0x2E3F) + return IOMMU_DOMAIN_DMA; } return 0; @@ -3743,6 +3840,7 @@ static const struct iommu_ops arm_smmu_ops = { .hw_info = arm_smmu_hw_info, .domain_alloc_sva = arm_smmu_sva_domain_alloc, .domain_alloc_paging_flags = arm_smmu_domain_alloc_paging_flags, + .is_attach_deferred = arm_smmu_is_attach_deferred, .probe_device = arm_smmu_probe_device, .release_device = arm_smmu_release_device, .device_group = arm_smmu_device_group, @@ -3922,10 +4020,204 @@ static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) return 0; } +static int arm_smmu_kdump_adopt_strtab_2lvl(struct arm_smmu_device *smmu, + u32 cfg_reg, phys_addr_t base) +{ + u32 log2size = FIELD_GET(STRTAB_BASE_CFG_LOG2SIZE, cfg_reg); + u32 split = FIELD_GET(STRTAB_BASE_CFG_SPLIT, cfg_reg); + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + u32 num_l1_ents; + size_t size; + int i; + + /* + * Only a coherent SMMU is supported at this moment. For a non-coherent + * SMMU that wants to support ARM_SMMU_OPT_KDUMP_ADOPT, try MEMREMAP_WC. + */ + if (WARN_ON(!(smmu->features & ARM_SMMU_FEAT_COHERENCY))) + return -EOPNOTSUPP; + + if (log2size < split || log2size > smmu->sid_bits) { + dev_err(smmu->dev, "kdump: log2size %u out of range [%u, %u]\n", + log2size, split, smmu->sid_bits); + return -EINVAL; + } + if (split != STRTAB_SPLIT) { + dev_err(smmu->dev, + "kdump: unsupported STRTAB_SPLIT %u (expected %u)\n", + split, STRTAB_SPLIT); + return -EINVAL; + } + + num_l1_ents = 1U << (log2size - split); + if (num_l1_ents > STRTAB_MAX_L1_ENTRIES) { + dev_err(smmu->dev, "kdump: l1 entries %u exceeds max %u\n", + num_l1_ents, STRTAB_MAX_L1_ENTRIES); + return -EINVAL; + } + + cfg->l2.num_l1_ents = num_l1_ents; + + size = num_l1_ents * sizeof(struct arm_smmu_strtab_l1); + cfg->l2.l1tab = memremap(base, size, MEMREMAP_WB); + if (!cfg->l2.l1tab) + return -ENOMEM; + + cfg->l2.l2ptrs = + kcalloc(num_l1_ents, sizeof(*cfg->l2.l2ptrs), GFP_KERNEL); + if (!cfg->l2.l2ptrs) + return -ENOMEM; + + for (i = 0; i < num_l1_ents; i++) { + u64 l2ptr = le64_to_cpu(cfg->l2.l1tab[i].l2ptr); + phys_addr_t l2_base = l2ptr & STRTAB_L1_DESC_L2PTR_MASK; + u32 span = FIELD_GET(STRTAB_L1_DESC_SPAN, l2ptr); + + if (!span || !l2_base) + continue; + + if (span != STRTAB_SPLIT + 1) { + dev_err(smmu->dev, + "kdump: L1[%u] unsupported span %u (vs %u)\n", + i, span, STRTAB_SPLIT + 1); + return -EINVAL; + } + + /* + * If the crashed kernel's l1 descriptors are deeply corrupted, + * blindly memremapping every l2 table here could lead to OOM. + * + * Defer the l2 memremap to arm_smmu_init_l2_strtab(), so peak + * memory is bounded by the kdump kernel's actual demand. + */ + } + + return 0; +} + +static int arm_smmu_kdump_adopt_strtab_linear(struct arm_smmu_device *smmu, + u32 cfg_reg, phys_addr_t base) +{ + u32 log2size = FIELD_GET(STRTAB_BASE_CFG_LOG2SIZE, cfg_reg); + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + unsigned int max_log2size; + size_t size; + + /* + * Only a coherent SMMU is supported at this moment. For a non-coherent + * SMMU that wants to support ARM_SMMU_OPT_KDUMP_ADOPT, try MEMREMAP_WC. + */ + if (WARN_ON(!(smmu->features & ARM_SMMU_FEAT_COHERENCY))) + return -EOPNOTSUPP; + + /* Cap the size at what the kdump kernel itself would have allocated */ + if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) + max_log2size = + ilog2(STRTAB_MAX_L1_ENTRIES * STRTAB_NUM_L2_STES); + else + max_log2size = smmu->sid_bits; + + /* cfg->linear.num_ents is unsigned int, so cap log2size at 31 */ + max_log2size = min(max_log2size, 31U); + if (log2size > max_log2size) { + dev_err(smmu->dev, "kdump: unsupported log2size %u (> %u)\n", + log2size, max_log2size); + return -EINVAL; + } + + /* + * We might end up with a num_ents != sid_bits, which is fine. In the + * ARM_SMMU_OPT_KDUMP_ADOPT case, arm_smmu_write_strtab() is bypassed. + */ + cfg->linear.num_ents = 1U << log2size; + + size = cfg->linear.num_ents * sizeof(struct arm_smmu_ste); + cfg->linear.table = memremap(base, size, MEMREMAP_WB); + if (!cfg->linear.table) + return -ENOMEM; + return 0; +} + +static void arm_smmu_kdump_adopt_cleanup(void *data) +{ + struct arm_smmu_device *smmu = data; + u32 cfg_reg = readl_relaxed(smmu->base + ARM_SMMU_STRTAB_BASE_CFG); + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + u32 fmt = FIELD_GET(STRTAB_BASE_CFG_FMT, cfg_reg); + + if (fmt == STRTAB_BASE_CFG_FMT_2LVL) { + kfree(cfg->l2.l2ptrs); + if (cfg->l2.l1tab) + memunmap(cfg->l2.l1tab); + } else if (fmt == STRTAB_BASE_CFG_FMT_LINEAR) { + if (cfg->linear.table) + memunmap(cfg->linear.table); + } +} + +static int arm_smmu_kdump_adopt_strtab(struct arm_smmu_device *smmu) +{ + u32 cfg_reg = readl_relaxed(smmu->base + ARM_SMMU_STRTAB_BASE_CFG); + u64 base_reg = readq_relaxed(smmu->base + ARM_SMMU_STRTAB_BASE); + u32 fmt = FIELD_GET(STRTAB_BASE_CFG_FMT, cfg_reg); + phys_addr_t base = base_reg & STRTAB_BASE_ADDR_MASK; + int ret; + + dev_info(smmu->dev, "kdump: adopting crashed kernel's stream table\n"); + + if (fmt == STRTAB_BASE_CFG_FMT_2LVL) { + /* + * Both kernels run on the same hardware, so it's impossible for + * kdump kernel to see the support for linear stream table only. + */ + if (WARN_ON(!(smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB))) + ret = -EINVAL; + else + ret = arm_smmu_kdump_adopt_strtab_2lvl(smmu, cfg_reg, + base); + } else if (fmt == STRTAB_BASE_CFG_FMT_LINEAR) { + /* + * In case that the old kernel for some reason used the linear + * format, enforce the same format to match the adopted table. + */ + ret = arm_smmu_kdump_adopt_strtab_linear(smmu, cfg_reg, base); + if (!ret) + smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB; + } else { + dev_err(smmu->dev, "kdump: invalid STRTAB format %u\n", fmt); + ret = -EINVAL; + } + + if (ret) { + arm_smmu_kdump_adopt_cleanup(smmu); + goto err; + } + + ret = devm_add_action_or_reset(smmu->dev, arm_smmu_kdump_adopt_cleanup, + smmu); + /* devm_add_action_or_reset ran the cleanup upon failure */ + if (ret) { + dev_warn(smmu->dev, "kdump: failed to set up cleanup action\n"); + goto err; + } + + return 0; + +err: + dev_warn(smmu->dev, "kdump: falling back to full reset\n"); + memset(&smmu->strtab_cfg, 0, sizeof(smmu->strtab_cfg)); + smmu->options &= ~ARM_SMMU_OPT_KDUMP_ADOPT; + return ret; +} + static int arm_smmu_init_strtab(struct arm_smmu_device *smmu) { int ret; + if ((smmu->options & ARM_SMMU_OPT_KDUMP_ADOPT) && + !arm_smmu_kdump_adopt_strtab(smmu)) + goto out; + if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) ret = arm_smmu_init_strtab_2lvl(smmu); else @@ -3933,6 +4225,7 @@ static int arm_smmu_init_strtab(struct arm_smmu_device *smmu) if (ret) return ret; +out: ida_init(&smmu->vmid_map); return 0; @@ -4057,6 +4350,21 @@ static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) arm_smmu_setup_msis(smmu); /* Request interrupt lines */ + irq = smmu->gerr_irq; + if (irq) { + ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler, + 0, "arm-smmu-v3-gerror", smmu); + if (ret < 0) + dev_warn(smmu->dev, "failed to enable gerror irq\n"); + } else { + dev_warn(smmu->dev, + "no gerr irq - errors will not be reported!\n"); + } + + /* No EVTQ/PRIQ interrupts in kdump -- queues are disabled */ + if (is_kdump_kernel()) + return; + irq = smmu->evtq.q.irq; if (irq) { ret = devm_request_threaded_irq(smmu->dev, irq, NULL, @@ -4069,16 +4377,6 @@ static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n"); } - irq = smmu->gerr_irq; - if (irq) { - ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler, - 0, "arm-smmu-v3-gerror", smmu); - if (ret < 0) - dev_warn(smmu->dev, "failed to enable gerror irq\n"); - } else { - dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n"); - } - if (smmu->features & ARM_SMMU_FEAT_PRI) { irq = smmu->priq.q.irq; if (irq) { @@ -4099,7 +4397,7 @@ static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) { int ret, irq; - u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN; + u32 irqen_flags = IRQ_CTRL_GERROR_IRQEN; /* Disable IRQs first */ ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL, @@ -4114,19 +4412,30 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) /* * Cavium ThunderX2 implementation doesn't support unique irq * lines. Use a single irq line for all the SMMUv3 interrupts. + * + * In kdump, EVTQ/PRIQ are disabled, so no threaded handling. */ - ret = devm_request_threaded_irq(smmu->dev, irq, - arm_smmu_combined_irq_handler, - arm_smmu_combined_irq_thread, - IRQF_ONESHOT, - "arm-smmu-v3-combined-irq", smmu); + if (is_kdump_kernel()) + ret = devm_request_irq(smmu->dev, irq, + arm_smmu_combined_irq_handler, 0, + "arm-smmu-v3-combined-irq", + smmu); + else + ret = devm_request_threaded_irq( + smmu->dev, irq, arm_smmu_combined_irq_handler, + arm_smmu_combined_irq_thread, IRQF_ONESHOT, + "arm-smmu-v3-combined-irq", smmu); if (ret < 0) dev_warn(smmu->dev, "failed to enable combined irq\n"); } else arm_smmu_setup_unique_irqs(smmu); - if (smmu->features & ARM_SMMU_FEAT_PRI) - irqen_flags |= IRQ_CTRL_PRIQ_IRQEN; + /* No EVTQ/PRIQ IRQ generation in kdump -- queues are disabled */ + if (!is_kdump_kernel()) { + irqen_flags |= IRQ_CTRL_EVTQ_IRQEN; + if (smmu->features & ARM_SMMU_FEAT_PRI) + irqen_flags |= IRQ_CTRL_PRIQ_IRQEN; + } /* Enable interrupt generation on the SMMU */ ret = arm_smmu_write_reg_sync(smmu, irqen_flags, @@ -4175,11 +4484,28 @@ static void arm_smmu_write_strtab(struct arm_smmu_device *smmu) static int arm_smmu_device_reset(struct arm_smmu_device *smmu) { int ret; - u32 reg, enables; + u32 reg, enables = 0; struct arm_smmu_cmdq_ent cmd; - /* Clear CR0 and sync (disables SMMU and queue processing) */ reg = readl_relaxed(smmu->base + ARM_SMMU_CR0); + + /* + * In a kdump case (set when CR0_SMMUEN=1 and !GERROR_SFM_ERR), retain + * CR0_SMMUEN to avoid aborting in-flight DMA, and CR0_ATSCHK to carry + * on the ATS-check policy. + * + * According to spec, updating STRTAB_BASE/CR1/CR2 when CR0_SMMUEN=1 is + * CONSTRAINED UNPREDICTABLE. So, skip those register updates and rely + * on the adopted stream table from the crashed kernel. + */ + if (smmu->options & ARM_SMMU_OPT_KDUMP_ADOPT) { + dev_info(smmu->dev, + "kdump: retaining SMMUEN for in-flight DMA\n"); + enables = reg & (CR0_SMMUEN | CR0_ATSCHK); + goto reset_queues; + } + + /* Clear CR0 and sync (disables SMMU and queue processing) */ if (reg & CR0_SMMUEN) { dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n"); arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0); @@ -4209,12 +4535,36 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu) /* Stream table */ arm_smmu_write_strtab(smmu); +reset_queues: + if (smmu->options & ARM_SMMU_OPT_KDUMP_ADOPT) { + /* Disable queues since arm_smmu_device_disable() was skipped */ + ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, + ARM_SMMU_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to disable queues\n"); + return ret; + } + } + + /* + * GERROR bits are latched. Read after queue disabling so that unhandled + * errors would be visible. Ack everything prior to re-enabling the CMDQ + * as a stale CMDQ_ERR would halt the CMDQ and new command will timeout. + */ + if (is_kdump_kernel()) { + u32 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR); + u32 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN); + + if ((gerror ^ gerrorn) & GERROR_ERR_MASK) + writel(gerror, smmu->base + ARM_SMMU_GERRORN); + } + /* Command queue */ writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE); writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD); writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS); - enables = CR0_CMDQEN; + enables |= CR0_CMDQEN; ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, ARM_SMMU_CR0ACK); if (ret) { @@ -4235,21 +4585,35 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu) cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL; arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); - /* Event queue */ - writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE); - writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD); - writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS); - - enables |= CR0_EVTQEN; - ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, - ARM_SMMU_CR0ACK); - if (ret) { - dev_err(smmu->dev, "failed to enable event queue\n"); - return ret; + /* + * Event queue + * + * Do not enable in a kdump case, as the crashed kernel's CDs and page + * tables might be corrupted, triggering event spamming. + */ + if (!is_kdump_kernel()) { + writeq_relaxed(smmu->evtq.q.q_base, + smmu->base + ARM_SMMU_EVTQ_BASE); + writel_relaxed(smmu->evtq.q.llq.prod, + smmu->page1 + ARM_SMMU_EVTQ_PROD); + writel_relaxed(smmu->evtq.q.llq.cons, + smmu->page1 + ARM_SMMU_EVTQ_CONS); + + enables |= CR0_EVTQEN; + ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, + ARM_SMMU_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to enable event queue\n"); + return ret; + } } - /* PRI queue */ - if (smmu->features & ARM_SMMU_FEAT_PRI) { + /* + * PRI queue + * + * Do not enable in a kdump case, as we cannot serve page requests. + */ + if (!is_kdump_kernel() && (smmu->features & ARM_SMMU_FEAT_PRI)) { writeq_relaxed(smmu->priq.q.q_base, smmu->base + ARM_SMMU_PRIQ_BASE); writel_relaxed(smmu->priq.q.llq.prod, @@ -4282,9 +4646,6 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu) return ret; } - if (is_kdump_kernel()) - enables &= ~(CR0_EVTQEN | CR0_PRIQEN); - /* Enable the SMMU interface */ enables |= CR0_SMMUEN; ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, @@ -4365,6 +4726,33 @@ static void arm_smmu_get_httu(struct arm_smmu_device *smmu, u32 reg) hw_features, fw_features); } +static void arm_smmu_device_hw_probe_kdump(struct arm_smmu_device *smmu) +{ + u32 gerror, gerrorn, active; + + /* No adoption if SMMU is disabled (i.e., there is no in-flight DMA) */ + if (!(readl_relaxed(smmu->base + ARM_SMMU_CR0) & CR0_SMMUEN)) + return; + + /* For now, only support a coherent SMMU that works with MEMREMAP_WB */ + if (!(smmu->features & ARM_SMMU_FEAT_COHERENCY)) { + dev_warn(smmu->dev, + "kdump: non-coherent SMMU unsupported; reset to block all DMAs\n"); + return; + } + + gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR); + gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN); + active = gerror ^ gerrorn; + if (active & GERROR_SFM_ERR) { + dev_warn(smmu->dev, + "kdump: SMMU in Service Failure Mode, must reset\n"); + return; + } + + smmu->options |= ARM_SMMU_OPT_KDUMP_ADOPT; +} + static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) { u32 reg; @@ -4579,6 +4967,10 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) dev_info(smmu->dev, "oas %lu-bit (features 0x%08x)\n", smmu->oas, smmu->features); + + if (is_kdump_kernel()) + arm_smmu_device_hw_probe_kdump(smmu); + return 0; } @@ -4743,6 +5135,14 @@ static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu) struct list_head rmr_list; struct iommu_resv_region *e; + /* + * Kdump adoption keeps the crashed kernel's table live. Rewriting the + * adopted STE here could expose an in-flight fetch to a transient V=0 + * entry, or change Cfg=translate to Cfg=bypass. Must skip here. + */ + if (smmu->options & ARM_SMMU_OPT_KDUMP_ADOPT) + return; + INIT_LIST_HEAD(&rmr_list); iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); @@ -4759,10 +5159,7 @@ static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu) continue; } - /* - * STE table is not programmed to HW, see - * arm_smmu_initial_bypass_stes() - */ + /* The fresh stream table is not yet live. */ arm_smmu_make_bypass_ste(smmu, arm_smmu_get_step_for_sid(smmu, rmr->sids[i])); } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 3c6d65d36164f..cc25d0b3d3e8d 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -774,6 +774,7 @@ struct arm_smmu_device { #define ARM_SMMU_OPT_MSIPOLL (1 << 2) #define ARM_SMMU_OPT_CMDQ_FORCE_SYNC (1 << 3) #define ARM_SMMU_OPT_TEGRA241_CMDQV (1 << 4) +#define ARM_SMMU_OPT_KDUMP_ADOPT (1 << 5) u32 options; struct arm_smmu_cmdq cmdq; diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 0208e5897c299..1ebf9631c1825 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -75,6 +75,7 @@ #define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63) #define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53) +#define ARM_LPAE_PTE_CONT (((arm_lpae_iopte)1) << 52) #define ARM_LPAE_PTE_DBM (((arm_lpae_iopte)1) << 51) #define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10) #define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8) @@ -320,6 +321,27 @@ static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries, sizeof(*ptep) * num_entries, DMA_TO_DEVICE); } +static int arm_lpae_cont_ptes(int lvl, struct arm_lpae_io_pgtable *data) +{ + switch (ARM_LPAE_GRANULE(data)) { + case SZ_4K: + if (lvl >= 1) + return 16; + break; + case SZ_16K: + if (lvl == 2) + return 32; + else if (lvl == 3) + return 128; + break; + case SZ_64K: + if (lvl >= 2) + return 32; + break; + } + return 1; +} + static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg, int num_entries) { for (int i = 0; i < num_entries; i++) @@ -329,13 +351,35 @@ static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cf __arm_lpae_sync_pte(ptep, num_entries, cfg); } +static bool arm_lpae_use_contpte(struct arm_lpae_io_pgtable *data, + unsigned long iova, phys_addr_t paddr, + int lvl, int num_entries, int i) +{ + size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data); + int cont_ptes = arm_lpae_cont_ptes(lvl, data); + int contmask = cont_ptes - 1; + int contpte_addr_mask = sz * cont_ptes - 1; + int map_idx_start, tbl_idx; + + if ((paddr & contpte_addr_mask) != (iova & contpte_addr_mask)) + return false; + + map_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data); + tbl_idx = map_idx_start + i; + if (((tbl_idx & contmask) <= i) && + (tbl_idx < ((map_idx_start + num_entries) & ~contmask))) + return true; + + return false; +} + static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, struct iommu_iotlb_gather *gather, unsigned long iova, size_t size, size_t pgcount, int lvl, arm_lpae_iopte *ptep); static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, - phys_addr_t paddr, arm_lpae_iopte prot, + unsigned long iova, phys_addr_t paddr, arm_lpae_iopte prot, int lvl, int num_entries, arm_lpae_iopte *ptep) { arm_lpae_iopte pte = prot; @@ -349,7 +393,9 @@ static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, pte |= ARM_LPAE_PTE_TYPE_BLOCK; for (i = 0; i < num_entries; i++) - ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data); + ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data) | + (arm_lpae_use_contpte(data, iova, paddr, lvl, num_entries, i) ? + ARM_LPAE_PTE_CONT : 0); if (!cfg->coherent_walk) __arm_lpae_sync_pte(ptep, num_entries, cfg); @@ -383,7 +429,7 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, } } - __arm_lpae_init_pte(data, paddr, prot, lvl, num_entries, ptep); + __arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep); return 0; } diff --git a/drivers/iommu/iommufd/pages.c b/drivers/iommu/iommufd/pages.c index 9b49f0c5b4599..27a1628e92a08 100644 --- a/drivers/iommu/iommufd/pages.c +++ b/drivers/iommu/iommufd/pages.c @@ -711,9 +711,10 @@ static void batch_unpin(struct pfn_batch *batch, struct iopt_pages *pages, size_t to_unpin = min_t(size_t, npages, batch->npfns[cur] - first_page_off); - unpin_user_page_range_dirty_lock( - pfn_to_page(batch->pfns[cur] + first_page_off), - to_unpin, pages->writable); + if (pfn_valid(batch->pfns[cur] + first_page_off)) + unpin_user_page_range_dirty_lock( + pfn_to_page(batch->pfns[cur] + first_page_off), + to_unpin, pages->writable); iopt_pages_sub_npinned(pages, to_unpin); cur++; first_page_off = 0; @@ -873,6 +874,41 @@ static long pin_memfd_pages(struct pfn_reader_user *user, unsigned long start, return npages_out; } +static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, + unsigned long vaddr, unsigned long *pfn, + bool write_fault) +{ + struct follow_pfnmap_args args = { .vma = vma, .address = vaddr }; + int ret; + + ret = follow_pfnmap_start(&args); + if (ret) { + bool unlocked = false; + + ret = fixup_user_fault(mm, vaddr, + FAULT_FLAG_REMOTE | + (write_fault ? FAULT_FLAG_WRITE : 0), + &unlocked); + if (unlocked) + return -EAGAIN; + + if (ret) + return ret; + + ret = follow_pfnmap_start(&args); + if (ret) + return ret; + } + + if (write_fault && !args.writable) + ret = -EFAULT; + else + *pfn = args.pfn; + + follow_pfnmap_end(&args); + return ret; +} + static int pfn_reader_user_pin(struct pfn_reader_user *user, struct iopt_pages *pages, unsigned long start_index, @@ -941,6 +977,42 @@ static int pfn_reader_user_pin(struct pfn_reader_user *user, user->gup_flags, user->upages, &user->locked); } + + if (rc < 0) { + struct vm_area_struct *vma; + unsigned long vaddr; + unsigned long pfn; + int pinned = 0; + + /* fast path above doesn't hold the lock */ + if (!user->locked) + mmap_read_lock(pages->source_mm); + vaddr = untagged_addr_remote(pages->source_mm, uptr); +retry: + vma = vma_lookup(pages->source_mm, vaddr); + if (vma && vma->vm_flags & VM_PFNMAP) { + do { + rc = follow_fault_pfn(vma, pages->source_mm, vaddr, + &pfn, pages->writable); + if (rc == -EAGAIN) + goto retry; + if (!rc) { + if (!pfn_valid(pfn)) { + user->upages[pinned] = pfn_to_page(pfn); + pinned += 1; + vaddr += PAGE_SIZE; + } else { + rc = -EFAULT; + } + } + } while (pinned < npages && vaddr < vma->vm_end && !rc); + } + if (pinned) + rc = pinned; + if (!user->locked) + mmap_read_unlock(pages->source_mm); + } + if (rc <= 0) { if (WARN_ON(!rc)) return -EFAULT; @@ -1313,7 +1385,8 @@ static void pfn_reader_release_pins(struct pfn_reader *pfns) user->upages_start; if (!user->file) { - unpin_user_pages(user->upages + start_index, npages); + if (pfn_valid(page_to_pfn(user->upages[0]))) + unpin_user_pages(user->upages + start_index, npages); } else { long n = user->ufolios_len / sizeof(*user->ufolios); diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 20f13b686ab22..3dd47cb01bb16 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -1634,7 +1634,13 @@ static int gic_irq_domain_translate(struct irq_domain *d, if(fwspec->param_count != 2) return -EINVAL; - if (fwspec->param[0] < 16) { + /* + * Below check was added on assumption that MAX_IPI + * value will not be greater than 8. + */ + BUILD_BUG_ON(MAX_IPI > 8); + + if (fwspec->param[0] < MAX_IPI) { pr_err(FW_BUG "Illegal GSI%d translation request\n", fwspec->param[0]); return -EINVAL; diff --git a/drivers/net/ethernet/microchip/lan743x_ethtool.c b/drivers/net/ethernet/microchip/lan743x_ethtool.c index 8a3c1ecc7866c..9195419ecee02 100644 --- a/drivers/net/ethernet/microchip/lan743x_ethtool.c +++ b/drivers/net/ethernet/microchip/lan743x_ethtool.c @@ -1079,6 +1079,13 @@ static int lan743x_ethtool_set_eee(struct net_device *netdev, return phylink_ethtool_set_eee(adapter->phylink, eee); } +static int lan743x_ethtool_nway_reset(struct net_device *netdev) +{ + struct lan743x_adapter *adapter = netdev_priv(netdev); + + return phylink_ethtool_nway_reset(adapter->phylink); +} + static int lan743x_ethtool_set_link_ksettings(struct net_device *netdev, const struct ethtool_link_ksettings *cmd) @@ -1369,6 +1376,7 @@ const struct ethtool_ops lan743x_ethtool_ops = { .set_rxfh = lan743x_ethtool_set_rxfh, .get_rxfh_fields = lan743x_ethtool_get_rxfh_fields, .get_ts_info = lan743x_ethtool_get_ts_info, + .nway_reset = lan743x_ethtool_nway_reset, .get_eee = lan743x_ethtool_get_eee, .set_eee = lan743x_ethtool_set_eee, .get_link_ksettings = lan743x_ethtool_get_link_ksettings, diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c index f0b5dd752f084..1cdce35e14239 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.c +++ b/drivers/net/ethernet/microchip/lan743x_main.c @@ -28,9 +28,15 @@ #define RFE_RD_FIFO_TH_3_DWORDS 0x3 +static bool pci11x1x_is_a0(struct lan743x_adapter *adapter) +{ + u32 dev_rev = adapter->csr.id_rev & ID_REV_CHIP_REV_MASK_; + return dev_rev == ID_REV_CHIP_REV_PCI11X1X_A0_; +} + static void pci11x1x_strap_get_status(struct lan743x_adapter *adapter) { - u32 chip_rev; + u32 fpga_rev; u32 cfg_load; u32 hw_cfg; u32 strap; @@ -47,18 +53,19 @@ static void pci11x1x_strap_get_status(struct lan743x_adapter *adapter) cfg_load = lan743x_csr_read(adapter, ETH_SYS_CONFIG_LOAD_STARTED_REG); lan743x_hs_syslock_release(adapter); hw_cfg = lan743x_csr_read(adapter, HW_CFG); - - if (cfg_load & GEN_SYS_LOAD_STARTED_REG_ETH_ || - hw_cfg & HW_CFG_RST_PROTECT_) { - strap = lan743x_csr_read(adapter, STRAP_READ); + strap = lan743x_csr_read(adapter, STRAP_READ); + if ((pci11x1x_is_a0(adapter) && + (cfg_load & GEN_SYS_LOAD_STARTED_REG_ETH_ || + hw_cfg & HW_CFG_RST_PROTECT_)) || + (strap & STRAP_READ_USE_SGMII_EN_)) { if (strap & STRAP_READ_SGMII_EN_) adapter->is_sgmii_en = true; else adapter->is_sgmii_en = false; } else { - chip_rev = lan743x_csr_read(adapter, FPGA_REV); - if (chip_rev) { - if (chip_rev & FPGA_SGMII_OP) + fpga_rev = lan743x_csr_read(adapter, FPGA_REV); + if (fpga_rev) { + if (fpga_rev & FPGA_SGMII_OP) adapter->is_sgmii_en = true; else adapter->is_sgmii_en = false; @@ -101,9 +108,9 @@ static int lan743x_pci_init(struct lan743x_adapter *adapter, if (ret) goto return_error; - netif_info(adapter, probe, adapter->netdev, - "PCI: Vendor ID = 0x%04X, Device ID = 0x%04X\n", - pdev->vendor, pdev->device); + dev_dbg(&adapter->pdev->dev, + "PCI: Vendor ID = 0x%04X, Device ID = 0x%04X\n", + pdev->vendor, pdev->device); bars = pci_select_bars(pdev, IORESOURCE_MEM); if (!test_bit(0, &bars)) goto disable_device; @@ -185,10 +192,10 @@ static int lan743x_csr_init(struct lan743x_adapter *adapter) csr->id_rev = lan743x_csr_read(adapter, ID_REV); csr->fpga_rev = lan743x_csr_read(adapter, FPGA_REV); - netif_info(adapter, probe, adapter->netdev, - "ID_REV = 0x%08X, FPGA_REV = %d.%d\n", - csr->id_rev, FPGA_REV_GET_MAJOR_(csr->fpga_rev), - FPGA_REV_GET_MINOR_(csr->fpga_rev)); + dev_dbg(&adapter->pdev->dev, + "ID_REV = 0x%08X, FPGA_REV = %d.%d\n", + csr->id_rev, FPGA_REV_GET_MAJOR_(csr->fpga_rev), + FPGA_REV_GET_MINOR_(csr->fpga_rev)); if (!ID_REV_IS_VALID_CHIP_ID_(csr->id_rev)) return -ENODEV; @@ -946,8 +953,8 @@ int lan743x_sgmii_read(struct lan743x_adapter *adapter, u8 mmd, u16 addr) u32 val; if (mmd > 31) { - netif_err(adapter, probe, adapter->netdev, - "%s mmd should <= 31\n", __func__); + dev_err(&adapter->pdev->dev, + "%s mmd should <= 31\n", __func__); return -EINVAL; } @@ -976,8 +983,8 @@ static int lan743x_sgmii_write(struct lan743x_adapter *adapter, int ret; if (mmd > 31) { - netif_err(adapter, probe, adapter->netdev, - "%s mmd should <= 31\n", __func__); + dev_err(&adapter->pdev->dev, + "%s mmd should <= 31\n", __func__); return -EINVAL; } mutex_lock(&adapter->sgmii_rw_lock); @@ -1208,8 +1215,37 @@ static void lan743x_mac_set_address(struct lan743x_adapter *adapter, lan743x_csr_write(adapter, MAC_RX_ADDRH, addr_hi); ether_addr_copy(adapter->mac_address, addr); - netif_info(adapter, drv, adapter->netdev, - "MAC address set to %pM\n", addr); + dev_dbg(&adapter->pdev->dev, "MAC address set to %pM\n", addr); +} + +static void lan743x_mac_rx_enable_fse(struct lan743x_adapter *adapter) +{ + u32 mac_rx; + bool rxen; + + mac_rx = lan743x_csr_read(adapter, MAC_RX); + if (mac_rx & MAC_RX_FSE_) + return; + + rxen = mac_rx & MAC_RX_RXEN_; + if (rxen) { + mac_rx &= ~MAC_RX_RXEN_; + lan743x_csr_write(adapter, MAC_RX, mac_rx); + lan743x_csr_wait_for_bit(adapter, MAC_RX, MAC_RX_RXD_, + 1, 1000, 20000, 100); + } + + /* Per AN2948, hardware prevents modification of the FSE bit while the + * MAC receiver is enabled (RXEN bit set). Use separate register write + * to assert the FSE bit before enabling the RXEN bit in MAC_RX + */ + mac_rx |= MAC_RX_FSE_; + lan743x_csr_write(adapter, MAC_RX, mac_rx); + + if (rxen) { + mac_rx |= MAC_RX_RXEN_; + lan743x_csr_write(adapter, MAC_RX, mac_rx); + } } static int lan743x_mac_init(struct lan743x_adapter *adapter) @@ -1251,6 +1287,8 @@ static int lan743x_mac_init(struct lan743x_adapter *adapter) lan743x_mac_set_address(adapter, adapter->mac_address); eth_hw_addr_set(netdev, adapter->mac_address); + lan743x_mac_rx_enable_fse(adapter); + return 0; } @@ -1363,8 +1401,8 @@ static void lan743x_phy_interface_select(struct lan743x_adapter *adapter) else adapter->phy_interface = PHY_INTERFACE_MODE_RGMII; - netif_dbg(adapter, drv, adapter->netdev, - "selected phy interface: 0x%X\n", adapter->phy_interface); + dev_dbg(&adapter->pdev->dev, + "selected phy interface: 0x%X\n", adapter->phy_interface); } static void lan743x_rfe_open(struct lan743x_adapter *adapter) @@ -3161,7 +3199,7 @@ static int lan743x_phylink_create(struct lan743x_adapter *adapter) } adapter->phylink = pl; - netdev_dbg(netdev, "lan743x phylink created"); + dev_dbg(&adapter->pdev->dev, "lan743x phylink created"); return 0; } @@ -3574,30 +3612,26 @@ static int lan743x_mdiobus_init(struct lan743x_adapter *adapter) adapter->mdiobus->priv = (void *)adapter; if (adapter->is_pci11x1x) { if (adapter->is_sgmii_en) { - netif_dbg(adapter, drv, adapter->netdev, - "SGMII operation\n"); + dev_dbg(&adapter->pdev->dev, "SGMII operation\n"); adapter->mdiobus->read = lan743x_mdiobus_read_c22; adapter->mdiobus->write = lan743x_mdiobus_write_c22; adapter->mdiobus->read_c45 = lan743x_mdiobus_read_c45; adapter->mdiobus->write_c45 = lan743x_mdiobus_write_c45; adapter->mdiobus->name = "lan743x-mdiobus-c45"; - netif_dbg(adapter, drv, adapter->netdev, - "lan743x-mdiobus-c45\n"); + dev_dbg(&adapter->pdev->dev, "lan743x-mdiobus-c45\n"); } else { - netif_dbg(adapter, drv, adapter->netdev, - "RGMII operation\n"); + dev_dbg(&adapter->pdev->dev, "RGMII operation\n"); // Only C22 support when RGMII I/F adapter->mdiobus->read = lan743x_mdiobus_read_c22; adapter->mdiobus->write = lan743x_mdiobus_write_c22; adapter->mdiobus->name = "lan743x-mdiobus"; - netif_dbg(adapter, drv, adapter->netdev, - "lan743x-mdiobus\n"); + dev_dbg(&adapter->pdev->dev, "lan743x-mdiobus\n"); } } else { adapter->mdiobus->read = lan743x_mdiobus_read_c22; adapter->mdiobus->write = lan743x_mdiobus_write_c22; adapter->mdiobus->name = "lan743x-mdiobus"; - netif_dbg(adapter, drv, adapter->netdev, "lan743x-mdiobus\n"); + dev_dbg(&adapter->pdev->dev, "lan743x-mdiobus\n"); } snprintf(adapter->mdiobus->id, MII_BUS_ID_SIZE, @@ -3689,8 +3723,7 @@ static int lan743x_pcidev_probe(struct pci_dev *pdev, ret = lan743x_phylink_create(adapter); if (ret < 0) { - netif_err(adapter, probe, netdev, - "failed to setup phylink (%d)\n", ret); + dev_err(&pdev->dev, "failed to setup phylink (%d)\n", ret); goto cleanup_mdiobus; } diff --git a/drivers/net/ethernet/microchip/lan743x_main.h b/drivers/net/ethernet/microchip/lan743x_main.h index 02a28b7091630..1573c8f9c9937 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.h +++ b/drivers/net/ethernet/microchip/lan743x_main.h @@ -27,6 +27,7 @@ #define ID_REV_CHIP_REV_MASK_ (0x0000FFFF) #define ID_REV_CHIP_REV_A0_ (0x00000000) #define ID_REV_CHIP_REV_B0_ (0x00000010) +#define ID_REV_CHIP_REV_PCI11X1X_A0_ (0x000000A0) #define ID_REV_CHIP_REV_PCI11X1X_B0_ (0x000000B0) #define FPGA_REV (0x04) @@ -181,6 +182,7 @@ #define MAC_RX (0x104) #define MAC_RX_MAX_SIZE_SHIFT_ (16) #define MAC_RX_MAX_SIZE_MASK_ (0x3FFF0000) +#define MAC_RX_FSE_ BIT(2) #define MAC_RX_RXD_ BIT(1) #define MAC_RX_RXEN_ BIT(0) diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index 786186c9a115f..1c5fafb5b108c 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -78,7 +78,7 @@ static void mana_gd_init_registers(struct pci_dev *pdev) { struct gdma_context *gc = pci_get_drvdata(pdev); - if (gc->is_pf) + if (gc->is_pf && !gc->is_pf2) mana_gd_init_pf_regs(pdev); else mana_gd_init_vf_regs(pdev); @@ -1956,7 +1956,7 @@ static void mana_gd_cleanup(struct pci_dev *pdev) static bool mana_is_pf(unsigned short dev_id) { - return dev_id == MANA_PF_DEVICE_ID; + return dev_id == MANA_PF_DEVICE_ID || dev_id == MANA_PF2_DEVICE_ID; } static int mana_gd_probe(struct pci_dev *pdev, const struct pci_device_id *ent) @@ -2003,6 +2003,8 @@ static int mana_gd_probe(struct pci_dev *pdev, const struct pci_device_id *ent) gc->numa_node = dev_to_node(&pdev->dev); gc->is_pf = mana_is_pf(pdev->device); + gc->is_pf2 = (pdev->device == MANA_PF2_DEVICE_ID); + gc->bar0_va = bar0_va; gc->dev = &pdev->dev; xa_init(&gc->irq_contexts); @@ -2176,6 +2178,7 @@ static void mana_gd_shutdown(struct pci_dev *pdev) static const struct pci_device_id mana_id_table[] = { { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF_DEVICE_ID) }, + { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF2_DEVICE_ID) }, { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_VF_DEVICE_ID) }, { } }; diff --git a/drivers/net/ethernet/realtek/Kconfig b/drivers/net/ethernet/realtek/Kconfig index 9b0f4f9631dba..ca58b70f6f317 100644 --- a/drivers/net/ethernet/realtek/Kconfig +++ b/drivers/net/ethernet/realtek/Kconfig @@ -126,4 +126,6 @@ config RTASE To compile this driver as a module, choose M here: the module will be called rtase. This is recommended. +source "drivers/net/ethernet/realtek/r8127/Kconfig" + endif # NET_VENDOR_REALTEK diff --git a/drivers/net/ethernet/realtek/Makefile b/drivers/net/ethernet/realtek/Makefile index 12a9c399f40c6..4e1d21508e260 100644 --- a/drivers/net/ethernet/realtek/Makefile +++ b/drivers/net/ethernet/realtek/Makefile @@ -9,3 +9,4 @@ r8169-y += r8169_main.o r8169_firmware.o r8169_phy_config.o r8169-$(CONFIG_R8169_LEDS) += r8169_leds.o obj-$(CONFIG_R8169) += r8169.o obj-$(CONFIG_RTASE) += rtase/ +obj-$(CONFIG_R8127) += r8127/ diff --git a/drivers/net/ethernet/realtek/r8127/Kconfig b/drivers/net/ethernet/realtek/r8127/Kconfig new file mode 100644 index 0000000000000..e5a8e399390cd --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/Kconfig @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: GPL-2.0-only +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ + +################################################################################ +# This product is covered by one or more of the following patents: +# US6,570,884, US6,115,776, and US6,327,625. +################################################################################ + +config R8127 + tristate "RealTek RTL-8127 PCI 10 Gigabit Ethernet Adapter support" + depends on PCI + help + This is a driver for the 10 Gigabit Ethernet PCI network cards based on + the RTL-8127 chips. If you have one of those, say Y here. + + To compile this driver as a module, choose M here: the module + will be called r8127. This is recommended. + diff --git a/drivers/net/ethernet/realtek/r8127/Makefile b/drivers/net/ethernet/realtek/r8127/Makefile new file mode 100755 index 0000000000000..d7cf5f65a9d8e --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/Makefile @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: GPL-2.0-only +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ + +################################################################################ +# This product is covered by one or more of the following patents: +# US6,570,884, US6,115,776, and US6,327,625. +################################################################################ + +ccflags-y += -DCONFIG_SOC_LAN -DCONFIG_R8127_NAPI -DCONFIG_R8127_VLAN -DCONFIG_ASPM -DENABLE_S5WOL -DENABLE_EEE -DENABLE_TX_NO_CLOSE -DENABLE_GIGA_LITE +obj-$(CONFIG_R8127) += r8127.o +r8127-y := r8127_n.o rtl_eeprom.o rtltool.o diff --git a/drivers/net/ethernet/realtek/r8127/r8127.h b/drivers/net/ethernet/realtek/r8127/r8127.h new file mode 100755 index 0000000000000..fccb974bc08c4 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/r8127.h @@ -0,0 +1,3068 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#ifndef __R8127_H +#define __R8127_H + +//#include +#include +#include +#include +#include "r8127_dash.h" +#include "r8127_realwow.h" +#ifdef ENABLE_PTP_SUPPORT +#include "r8127_ptp.h" +#endif +#include "r8127_rss.h" +#ifdef ENABLE_LIB_SUPPORT +#include "r8127_lib.h" +#endif + +#ifndef fallthrough +#define fallthrough +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) +#define netif_xmit_stopped netif_tx_queue_stopped +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,7,0) +#ifndef MDIO_AN_EEE_ADV_100TX +#define MDIO_AN_EEE_ADV_100TX 0x0002 /* Advertise 100TX EEE cap */ +#endif +#ifndef MDIO_AN_EEE_ADV_1000T +#define MDIO_AN_EEE_ADV_1000T 0x0004 /* Advertise 1000T EEE cap */ +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0) +#define MDIO_EEE_100TX MDIO_AN_EEE_ADV_100TX /* 100TX EEE cap */ +#define MDIO_EEE_1000T MDIO_AN_EEE_ADV_1000T /* 1000T EEE cap */ +#define MDIO_EEE_10GT 0x0008 /* 10GT EEE cap */ +#define MDIO_EEE_1000KX 0x0010 /* 1000KX EEE cap */ +#define MDIO_EEE_10GKX4 0x0020 /* 10G KX4 EEE cap */ +#define MDIO_EEE_10GKR 0x0040 /* 10G KR EEE cap */ +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0) */ + +static inline u32 mmd_eee_adv_to_ethtool_adv_t(u16 eee_adv) +{ + u32 adv = 0; + + if (eee_adv & MDIO_EEE_100TX) + adv |= ADVERTISED_100baseT_Full; + if (eee_adv & MDIO_EEE_1000T) + adv |= ADVERTISED_1000baseT_Full; + if (eee_adv & MDIO_EEE_10GT) + adv |= ADVERTISED_10000baseT_Full; + if (eee_adv & MDIO_EEE_1000KX) + adv |= ADVERTISED_1000baseKX_Full; + if (eee_adv & MDIO_EEE_10GKX4) + adv |= ADVERTISED_10000baseKX4_Full; + if (eee_adv & MDIO_EEE_10GKR) + adv |= ADVERTISED_10000baseKR_Full; + + return adv; +} + +static inline u16 ethtool_adv_to_mmd_eee_adv_t(u32 adv) +{ + u16 reg = 0; + + if (adv & ADVERTISED_100baseT_Full) + reg |= MDIO_EEE_100TX; + if (adv & ADVERTISED_1000baseT_Full) + reg |= MDIO_EEE_1000T; + if (adv & ADVERTISED_10000baseT_Full) + reg |= MDIO_EEE_10GT; + if (adv & ADVERTISED_1000baseKX_Full) + reg |= MDIO_EEE_1000KX; + if (adv & ADVERTISED_10000baseKX4_Full) + reg |= MDIO_EEE_10GKX4; + if (adv & ADVERTISED_10000baseKR_Full) + reg |= MDIO_EEE_10GKR; + + return reg; +} +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,7,0) */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,9,0) +static inline bool skb_transport_header_was_set(const struct sk_buff *skb) +{ + return skb->transport_header != ~0U; +} +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,9,0) */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,20,0) +static inline void linkmode_set_bit(int nr, volatile unsigned long *addr) +{ + __set_bit(nr, addr); +} + +static inline void linkmode_clear_bit(int nr, volatile unsigned long *addr) +{ + __clear_bit(nr, addr); +} + +static inline int linkmode_test_bit(int nr, volatile unsigned long *addr) +{ + return test_bit(nr, addr); +} +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4,20,0) */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,0,0) +static inline void linkmode_mod_bit(int nr, volatile unsigned long *addr, + int set) +{ + if (set) + linkmode_set_bit(nr, addr); + else + linkmode_clear_bit(nr, addr); +} +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5,0,0) */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,3,0) +static inline +ssize_t strscpy(char *dest, const char *src, size_t count) +{ + long res = 0; + + if (count == 0) + return -E2BIG; + + while (count) { + char c; + + c = src[res]; + dest[res] = c; + if (!c) + return res; + res++; + count--; + } + + /* Hit buffer length without finding a NUL; force NUL-termination. */ + if (res) + dest[res-1] = '\0'; + + return -E2BIG; +} +#endif + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0)) +static inline unsigned char *skb_checksum_start(const struct sk_buff *skb) +{ +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,22)) + return skb->head + skb->csum_start; +#else /* < 2.6.22 */ + return skb_transport_header(skb); +#endif +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) +static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue, + unsigned int bytes) +{} +static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue, + unsigned int pkts, + unsigned int bytes) +{} +static inline void netdev_tx_reset_queue(struct netdev_queue *q) {} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,8,0) +static inline void fsleep(unsigned long usecs) +{ + if (usecs <= 10) + udelay(usecs); + else if (usecs <= 20000) + usleep_range(usecs, 2 * usecs); + else + msleep(DIV_ROUND_UP(usecs, 1000)); +} +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5,8,0) */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,2,0) +#define netdev_xmit_more() (0) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,8,0) +#define netif_testing_on(dev) +#define netif_testing_off(dev) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,2,0) +#define netdev_sw_irq_coalesce_default_on(dev) +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(6,2,0) */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,32) +typedef int netdev_tx_t; +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,12,0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,1,9) +static inline bool page_is_pfmemalloc(struct page *page) +{ + /* + * Page index cannot be this large so this must be + * a pfmemalloc page. + */ + return page->index == -1UL; +} +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4,1,9) */ +static inline bool dev_page_is_reusable(struct page *page) +{ + return likely(page_to_nid(page) == numa_mem_id() && + !page_is_pfmemalloc(page)); +} +#endif + +/* +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0)&& !defined(ENABLE_LIB_SUPPORT) +#define RTL_USE_NEW_INTR_API +#endif +*/ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,10,0) +#define dma_map_page_attrs(dev, page, offset, size, dir, attrs) \ + dma_map_page(dev, page, offset, size, dir) +#define dma_unmap_page_attrs(dev, page, size, dir, attrs) \ + dma_unmap_page(dev, page, size, dir) +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,10,0) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) +#define page_ref_inc(page) atomic_inc(&page->_count) +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,4,216) +#define page_ref_count(page) atomic_read(&page->_count) +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,4,216) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) +#define skb_transport_offset(skb) (skb->h.raw - skb->data) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,26) +#define device_set_wakeup_enable(dev, val) do {} while (0) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,14,0) +static inline void ether_addr_copy(u8 *dst, const u8 *src) +{ + u16 *a = (u16 *)dst; + const u16 *b = (const u16 *)src; + + a[0] = b[0]; + a[1] = b[1]; + a[2] = b[2]; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,15,0) +#define IS_ERR_OR_NULL(ptr) (!ptr) +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,13,0) +#define reinit_completion(x) ((x)->done = 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,39) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,32) +#define pm_runtime_mark_last_busy(x) +#define pm_runtime_put_autosuspend(x) pm_runtime_put(x) +#define pm_runtime_put_sync_autosuspend(x) pm_runtime_put_sync(x) + +static inline bool pm_runtime_suspended(struct device *dev) +{ + return dev->power.runtime_status == RPM_SUSPENDED + && !dev->power.disable_depth; +} + +static inline bool pm_runtime_active(struct device *dev) +{ + return dev->power.runtime_status == RPM_ACTIVE + || dev->power.disable_depth; +} +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,36) +#define queue_delayed_work(long_wq, work, delay) schedule_delayed_work(work, delay) +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,34) +#define netif_printk(priv, type, level, netdev, fmt, args...) \ + do { \ + if (netif_msg_##type(priv)) \ + printk(level "%s: " fmt,(netdev)->name , ##args); \ + } while (0) + +#define netif_emerg(priv, type, netdev, fmt, args...) \ + netif_printk(priv, type, KERN_EMERG, netdev, fmt, ##args) +#define netif_alert(priv, type, netdev, fmt, args...) \ + netif_printk(priv, type, KERN_ALERT, netdev, fmt, ##args) +#define netif_crit(priv, type, netdev, fmt, args...) \ + netif_printk(priv, type, KERN_CRIT, netdev, fmt, ##args) +#define netif_err(priv, type, netdev, fmt, args...) \ + netif_printk(priv, type, KERN_ERR, netdev, fmt, ##args) +#define netif_warn(priv, type, netdev, fmt, args...) \ + netif_printk(priv, type, KERN_WARNING, netdev, fmt, ##args) +#define netif_notice(priv, type, netdev, fmt, args...) \ + netif_printk(priv, type, KERN_NOTICE, netdev, fmt, ##args) +#define netif_info(priv, type, netdev, fmt, args...) \ + netif_printk(priv, type, KERN_INFO, (netdev), fmt, ##args) +#endif +#endif +#endif +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15) +#define setup_timer(_timer, _function, _data) \ +do { \ + (_timer)->function = _function; \ + (_timer)->data = _data; \ + init_timer(_timer); \ +} while (0) +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,0,0) +#if defined(skb_vlan_tag_present) && !defined(vlan_tx_tag_present) +#define vlan_tx_tag_present skb_vlan_tag_present +#endif +#if defined(skb_vlan_tag_get) && !defined(vlan_tx_tag_get) +#define vlan_tx_tag_get skb_vlan_tag_get +#endif +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,0,0) + +#define RTL_ALLOC_SKB_INTR(napi, length) dev_alloc_skb(length) +#define R8127_USE_NAPI_ALLOC_SKB 0 +#ifdef CONFIG_R8127_NAPI +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0) +#undef RTL_ALLOC_SKB_INTR +#define RTL_ALLOC_SKB_INTR(napi, length) napi_alloc_skb(napi, length) +#undef R8127_USE_NAPI_ALLOC_SKB +#define R8127_USE_NAPI_ALLOC_SKB 1 +#endif +#endif + +#define RTL_BUILD_SKB_INTR(data, frag_size) build_skb(data, frag_size) +#ifdef CONFIG_R8127_NAPI +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,12,0) +#undef RTL_BUILD_SKB_INTR +#define RTL_BUILD_SKB_INTR(data, frag_size) napi_build_skb(data, frag_size) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0) +#define eth_random_addr(addr) random_ether_addr(addr) +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0) +#define netdev_features_t u32 +#endif +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,5,0) +#define NETIF_F_ALL_CSUM NETIF_F_CSUM_MASK +#else +#ifndef NETIF_F_ALL_CSUM +#define NETIF_F_ALL_CSUM NETIF_F_CSUM_MASK +#endif +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,37) +#define ENABLE_R8127_PROCFS +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,11,0) +#define ENABLE_R8127_SYSFS +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) +#define NETIF_F_HW_VLAN_RX NETIF_F_HW_VLAN_CTAG_RX +#define NETIF_F_HW_VLAN_TX NETIF_F_HW_VLAN_CTAG_TX +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,8,0) +#define __devinit +#define __devexit +#define __devexit_p(func) func +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) +#define CHECKSUM_PARTIAL CHECKSUM_HW +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) +#define irqreturn_t void +#define IRQ_HANDLED 1 +#define IRQ_NONE 0 +#define IRQ_RETVAL(x) +#endif + +#ifndef NETIF_F_RXALL +#define NETIF_F_RXALL 0 +#endif + +#ifndef NETIF_F_RXFCS +#define NETIF_F_RXFCS 0 +#endif + +#if !defined(HAVE_FREE_NETDEV) && (LINUX_VERSION_CODE < KERNEL_VERSION(3,1,0)) +#define free_netdev(x) kfree(x) +#endif + +#ifndef SET_NETDEV_DEV +#define SET_NETDEV_DEV(net, pdev) +#endif + +#ifndef SET_MODULE_OWNER +#define SET_MODULE_OWNER(dev) +#endif + +#ifndef SA_SHIRQ +#define SA_SHIRQ IRQF_SHARED +#endif + +#ifndef NETIF_F_GSO +#define gso_size tso_size +#define gso_segs tso_segs +#endif + +#ifndef PCI_VENDOR_ID_DLINK +#define PCI_VENDOR_ID_DLINK 0x1186 +#endif + +#ifndef dma_mapping_error +#define dma_mapping_error(a,b) 0 +#endif + +#ifndef netif_err +#define netif_err(a,b,c,d) +#endif + +#ifndef AUTONEG_DISABLE +#define AUTONEG_DISABLE 0x00 +#endif + +#ifndef AUTONEG_ENABLE +#define AUTONEG_ENABLE 0x01 +#endif + +#ifndef BMCR_SPEED1000 +#define BMCR_SPEED1000 0x0040 +#endif + +#ifndef BMCR_SPEED100 +#define BMCR_SPEED100 0x2000 +#endif + +#ifndef BMCR_SPEED10 +#define BMCR_SPEED10 0x0000 +#endif + +#ifndef SPEED_UNKNOWN +#define SPEED_UNKNOWN -1 +#endif + +#ifndef DUPLEX_UNKNOWN +#define DUPLEX_UNKNOWN 0xff +#endif + +#ifndef SUPPORTED_Pause +#define SUPPORTED_Pause (1 << 13) +#endif + +#ifndef SUPPORTED_Asym_Pause +#define SUPPORTED_Asym_Pause (1 << 14) +#endif + +#ifndef MDIO_EEE_100TX +#define MDIO_EEE_100TX 0x0002 +#endif + +#ifndef MDIO_EEE_1000T +#define MDIO_EEE_1000T 0x0004 +#endif + +#ifndef MDIO_EEE_2_5GT +#define MDIO_EEE_2_5GT 0x0001 +#endif + +#ifndef MDIO_EEE_5GT +#define MDIO_EEE_5GT 0x0002 +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0) +#define ethtool_keee ethtool_eee +#define rtl8127_ethtool_adv_to_mmd_eee_adv_cap1_t ethtool_adv_to_mmd_eee_adv_t +static inline u32 rtl8127_ethtool_adv_to_mmd_eee_adv_cap2_t(u32 adv) +{ + u32 result = 0; + + if (adv & SUPPORTED_2500baseX_Full) + result |= MDIO_EEE_2_5GT; + + return result; +} +#else +#define rtl8127_ethtool_adv_to_mmd_eee_adv_cap1_t linkmode_to_mii_eee_cap1_t +#define rtl8127_ethtool_adv_to_mmd_eee_adv_cap2_t linkmode_to_mii_eee_cap2_t +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0) */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29) +#ifdef CONFIG_NET_POLL_CONTROLLER +#define RTL_NET_POLL_CONTROLLER dev->poll_controller=rtl8127_netpoll +#else +#define RTL_NET_POLL_CONTROLLER +#endif + +#ifdef CONFIG_R8127_VLAN +#define RTL_SET_VLAN dev->vlan_rx_register=rtl8127_vlan_rx_register +#else +#define RTL_SET_VLAN +#endif + +#define RTL_NET_DEVICE_OPS(ops) dev->open=rtl8127_open; \ + dev->hard_start_xmit=rtl8127_start_xmit; \ + dev->get_stats=rtl8127_get_stats; \ + dev->stop=rtl8127_close; \ + dev->tx_timeout=rtl8127_tx_timeout; \ + dev->set_multicast_list=rtl8127_set_rx_mode; \ + dev->change_mtu=rtl8127_change_mtu; \ + dev->set_mac_address=rtl8127_set_mac_address; \ + dev->do_ioctl=rtl8127_do_ioctl; \ + RTL_NET_POLL_CONTROLLER; \ + RTL_SET_VLAN; +#else +#define RTL_NET_DEVICE_OPS(ops) dev->netdev_ops=&ops +#endif + +#ifndef FALSE +#define FALSE 0 +#endif + +#ifndef TRUE +#define TRUE 1 +#endif + +#ifndef false +#define false 0 +#endif + +#ifndef true +#define true 1 +#endif + +//Hardware will continue interrupt 10 times after interrupt finished. +#define RTK_KEEP_INTERRUPT_COUNT (10) + +//the low 32 bit address of receive buffer must be 8-byte alignment. +#ifndef NET_IP_ALIGN +#define NET_IP_ALIGN 2 +#endif +#define R8127_RX_ALIGN NET_IP_ALIGN + +#ifdef CONFIG_R8127_NAPI +#define NAPI_SUFFIX "-NAPI" +#else +#define NAPI_SUFFIX "" +#endif +#if defined(ENABLE_DASH_PRINTER_SUPPORT) +#define DASH_SUFFIX "-PRINTER" +#elif defined(ENABLE_DASH_SUPPORT) +#define DASH_SUFFIX "-DASH" +#else +#define DASH_SUFFIX "" +#endif + +#if defined(ENABLE_REALWOW_SUPPORT) +#define REALWOW_SUFFIX "-REALWOW" +#else +#define REALWOW_SUFFIX "" +#endif + +#if defined(ENABLE_PTP_SUPPORT) +#define PTP_SUFFIX "-PTP" +#else +#define PTP_SUFFIX "" +#endif + +#if defined(ENABLE_RSS_SUPPORT) +#define RSS_SUFFIX "-RSS" +#else +#define RSS_SUFFIX "" +#endif + +#define RTL8127_VERSION "11.014.00" NAPI_SUFFIX DASH_SUFFIX REALWOW_SUFFIX PTP_SUFFIX RSS_SUFFIX +#define MODULENAME "r8127" +#define PFX MODULENAME ": " + +#define GPL_CLAIM "\ +r8127 Copyright (C) 2025 Realtek NIC software team \n \ +This program comes with ABSOLUTELY NO WARRANTY; for details, please see . \n \ +This is free software, and you are welcome to redistribute it under certain conditions; see . \n" + +#ifdef RTL8127_DEBUG +#define assert(expr) \ + if(!(expr)) { \ + printk("Assertion failed! %s,%s,%s,line=%d\n", \ + #expr,__FILE__,__FUNCTION__,__LINE__); \ + } +#define dprintk(fmt, args...) do { printk(PFX fmt, ## args); } while (0) +#else +#define assert(expr) do {} while (0) +#define dprintk(fmt, args...) do {} while (0) +#endif /* RTL8127_DEBUG */ + +#define R8127_MSG_DEFAULT \ + (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_IFUP | NETIF_MSG_IFDOWN) + +#ifdef CONFIG_R8127_NAPI +#define rtl8127_rx_hwaccel_skb vlan_hwaccel_receive_skb +#define rtl8127_rx_quota(count, quota) min(count, quota) +#else +#define rtl8127_rx_hwaccel_skb vlan_hwaccel_rx +#define rtl8127_rx_quota(count, quota) count +#endif + +/* MAC address length */ +#ifndef MAC_ADDR_LEN +#define MAC_ADDR_LEN 6 +#endif + +#ifndef MAC_PROTOCOL_LEN +#define MAC_PROTOCOL_LEN 2 +#endif + +#ifndef ETH_FCS_LEN +#define ETH_FCS_LEN 4 +#endif + +#ifndef NETIF_F_TSO6 +#define NETIF_F_TSO6 0 +#endif + +#define Reserved2_data 7 +#define RX_DMA_BURST_unlimited 7 /* Maximum PCI burst, '7' is unlimited */ +#define RX_DMA_BURST_512 5 +#define RX_DMA_BURST_256 4 +#define TX_DMA_BURST_unlimited 7 +#define TX_DMA_BURST_1024 6 +#define TX_DMA_BURST_512 5 +#define TX_DMA_BURST_256 4 +#define TX_DMA_BURST_128 3 +#define TX_DMA_BURST_64 2 +#define TX_DMA_BURST_32 1 +#define TX_DMA_BURST_16 0 +#define Reserved1_data 0x3F +#define RxPacketMaxSize 0x3FE8 /* 16K - 1 - ETH_HLEN - VLAN - CRC... */ +#define Jumbo_Frame_1k ETH_DATA_LEN +#define Jumbo_Frame_2k (2*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN) +#define Jumbo_Frame_3k (3*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN) +#define Jumbo_Frame_4k (4*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN) +#define Jumbo_Frame_5k (5*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN) +#define Jumbo_Frame_6k (6*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN) +#define Jumbo_Frame_7k (7*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN) +#define Jumbo_Frame_8k (8*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN) +#define Jumbo_Frame_9k (9*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN) +#define InterFrameGap 0x03 /* 3 means InterFrameGap = the shortest one */ +#define RxEarly_off_V1 (0x07 << 11) +#define RxEarly_off_V2 (1 << 11) +#define Rx_Single_fetch_V2 (1 << 14) +#define Rx_Close_Multiple (1 << 21) +#define Rx_Fetch_Number_8 (1 << 30) + +#define R8127_REGS_SIZE (256) +#define R8127_MAC_REGS_SIZE (256) +#define R8127_PHY_REGS_SIZE (16*2) +#define R8127_EPHY_REGS_SIZE (31*2) +#define R8127_ERI_REGS_SIZE (0x100) +#define R8127_REGS_DUMP_SIZE (0x400) +#define R8127_PCI_REGS_SIZE (0x100) +#define R8127_NAPI_WEIGHT 64 + +#define R8127_MAX_MSIX_VEC_8125A 4 +#define R8127_MAX_MSIX_VEC_8125B 32 +#define R8127_MAX_MSIX_VEC_8125D 32 +#define R8127_MIN_MSIX_VEC_8125B 22 +#define R8127_MIN_MSIX_VEC_8125BP 31 +#define R8127_MIN_MSIX_VEC_8125D 20 +#define R8127_MIN_MSIX_VEC_8127 30 +#define R8127_MAX_MSIX_VEC 32 +#define R8127_MAX_RX_QUEUES_VEC_V3 (16) +#define R8127_MAX_RX_QUEUES_VEC_V4 (8) + +#define RTL8127_TX_TIMEOUT (6 * HZ) +#define RTL8127_LINK_TIMEOUT (1 * HZ) +#define RTL8127_ESD_TIMEOUT (2 * HZ) + +#define rtl8127_rx_page_size(order) (PAGE_SIZE << order) + +#define MAX_NUM_TX_DESC 1024 /* Maximum number of Tx descriptor registers */ +#define MAX_NUM_RX_DESC 1024 /* Maximum number of Rx descriptor registers */ + +#define MIN_NUM_TX_DESC 256 /* Minimum number of Tx descriptor registers */ +#define MIN_NUM_RX_DESC 256 /* Minimum number of Rx descriptor registers */ + +#define NUM_TX_DESC MAX_NUM_TX_DESC /* Number of Tx descriptor registers */ +#define NUM_RX_DESC MAX_NUM_RX_DESC /* Number of Rx descriptor registers */ + +#ifdef ENABLE_DOUBLE_VLAN +#define RX_BUF_SIZE 0x05F6 /* 0x05F6(1526) = 1514 + 8(double vlan) + 4(crc) bytes */ +#define RT_VALN_HLEN 8 /* 8(double vlan) bytes */ +#else +#define RX_BUF_SIZE 0x05F2 /* 0x05F2(1522) = 1514 + 4(single vlan) + 4(crc) bytes */ +#define RT_VALN_HLEN 4 /* 4(single vlan) bytes */ +#endif + +#define R8127_MAX_TX_QUEUES (2) +#define R8127_MAX_RX_QUEUES_V2 (4) +#define R8127_MAX_RX_QUEUES_V3 (16) +#define R8127_MAX_RX_QUEUES R8127_MAX_RX_QUEUES_V3 +#define R8127_MAX_QUEUES R8127_MAX_RX_QUEUES + +#define OCP_STD_PHY_BASE 0xa400 + +//Channel Wait Count +#define R8127_CHANNEL_WAIT_COUNT (20000) +#define R8127_CHANNEL_WAIT_TIME (1) // 1us +#define R8127_CHANNEL_EXIT_DELAY_TIME (20) //20us + +#ifdef ENABLE_LIB_SUPPORT +#define R8127_MULTI_RX_Q(tp) 0 +#else +#define R8127_MULTI_RX_Q(tp) (tp->num_rx_rings > 1) +#endif + +#define NODE_ADDRESS_SIZE 6 + +#define SHORT_PACKET_PADDING_BUF_SIZE 256 + +#define RTK_MAGIC_DEBUG_VALUE 0x0badbeef + +/* write/read MMIO register */ +#define RTL_W8(tp, reg, val8) writeb((val8), tp->mmio_addr + (reg)) +#define RTL_W16(tp, reg, val16) writew((val16), tp->mmio_addr + (reg)) +#define RTL_W32(tp, reg, val32) writel((val32), tp->mmio_addr + (reg)) +#define RTL_R8(tp, reg) readb(tp->mmio_addr + (reg)) +#define RTL_R16(tp, reg) readw(tp->mmio_addr + (reg)) +#define RTL_R32(tp, reg) ((unsigned long) readl(tp->mmio_addr + (reg))) + +#ifndef DMA_64BIT_MASK +#define DMA_64BIT_MASK 0xffffffffffffffffULL +#endif + +#ifndef DMA_32BIT_MASK +#define DMA_32BIT_MASK 0x00000000ffffffffULL +#endif + +#ifndef NETDEV_TX_OK +#define NETDEV_TX_OK 0 /* driver took care of packet */ +#endif + +#ifndef NETDEV_TX_BUSY +#define NETDEV_TX_BUSY 1 /* driver tx path was busy*/ +#endif + +#ifndef NETDEV_TX_LOCKED +#define NETDEV_TX_LOCKED -1t /* driver tx lock was already taken */ +#endif + +#ifndef ADVERTISED_Pause +#define ADVERTISED_Pause (1 << 13) +#endif + +#ifndef ADVERTISED_Asym_Pause +#define ADVERTISED_Asym_Pause (1 << 14) +#endif + +#ifndef ADVERTISE_PAUSE_CAP +#define ADVERTISE_PAUSE_CAP 0x400 +#endif + +#ifndef ADVERTISE_PAUSE_ASYM +#define ADVERTISE_PAUSE_ASYM 0x800 +#endif + +#ifndef MII_CTRL1000 +#define MII_CTRL1000 0x09 +#endif + +#ifndef ADVERTISE_1000FULL +#define ADVERTISE_1000FULL 0x200 +#endif + +#ifndef ADVERTISE_1000HALF +#define ADVERTISE_1000HALF 0x100 +#endif + +#ifndef BIT_ULL +#define BIT_ULL(nr) (1ULL << (nr)) +#endif + +#ifndef ADVERTISED_2500baseX_Full +#define ADVERTISED_2500baseX_Full 0x8000 +#endif +#define RTK_ADVERTISED_5000baseX_Full BIT_ULL(48) +#define RTK_SUPPORTED_5000baseX_Full BIT_ULL(48) + +#define RTK_ADVERTISE_2500FULL 0x80 +#define RTK_ADVERTISE_5000FULL 0x100 +#define RTK_ADVERTISE_10000FULL 0x1000 +#define RTK_LPA_ADVERTISE_2500FULL 0x20 +#define RTK_LPA_ADVERTISE_5000FULL 0x40 +#define RTK_LPA_ADVERTISE_10000FULL 0x800 + +#define RTK_EEE_ADVERTISE_2500FULL BIT(0) +#define RTK_EEE_ADVERTISE_5000FULL BIT(1) +#define RTK_LPA_EEE_ADVERTISE_2500FULL BIT(0) +#define RTK_LPA_EEE_ADVERTISE_5000FULL BIT(1) + +/* Tx NO CLOSE */ +#define MAX_TX_NO_CLOSE_DESC_PTR_V2 0x10000 +#define MAX_TX_NO_CLOSE_DESC_PTR_MASK_V2 0xFFFF +#define MAX_TX_NO_CLOSE_DESC_PTR_V3 0x100000000 +#define MAX_TX_NO_CLOSE_DESC_PTR_MASK_V3 0xFFFFFFFF +#define MAX_TX_NO_CLOSE_DESC_PTR_V4 0x80000000 +#define MAX_TX_NO_CLOSE_DESC_PTR_MASK_V4 0x7FFFFFFF +#define TX_NO_CLOSE_SW_PTR_MASK_V2 0x1FFFF + +#ifndef ETH_MIN_MTU +#define ETH_MIN_MTU 68 +#endif + +#define D0_SPEED_UP_SPEED_DISABLE 0 +#define D0_SPEED_UP_SPEED_1000 1 +#define D0_SPEED_UP_SPEED_2500 2 +#define D0_SPEED_UP_SPEED_5000 3 +#define D0_SPEED_UP_SPEED_10000 4 + +#define RTL8127_MAC_MCU_PAGE_SIZE 256 //256 words + +#ifndef WRITE_ONCE +#define WRITE_ONCE(var, val) (*((volatile typeof(val) *)(&(var))) = (val)) +#endif +#ifndef READ_ONCE +#define READ_ONCE(var) (*((volatile typeof(var) *)(&(var)))) +#endif + +#ifndef SPEED_5000 +#define SPEED_5000 5000 +#endif + +#ifndef SPEED_10000 +#define SPEED_10000 10000 +#endif + +#define R8127_LINK_STATE_OFF 0 +#define R8127_LINK_STATE_ON 1 +#define R8127_LINK_STATE_UNKNOWN 2 + +/*****************************************************************************/ + +//#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,3) +#if ((LINUX_VERSION_CODE < KERNEL_VERSION(2,4,27)) || \ + ((LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)) && \ + (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,3)))) +/* copied from linux kernel 2.6.20 include/linux/netdev.h */ +#define NETDEV_ALIGN 32 +#define NETDEV_ALIGN_CONST (NETDEV_ALIGN - 1) + +static inline void *netdev_priv(struct net_device *dev) +{ + return (char *)dev + ((sizeof(struct net_device) + + NETDEV_ALIGN_CONST) + & ~NETDEV_ALIGN_CONST); +} +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,3) + +/*****************************************************************************/ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) +#define RTLDEV tp +#else +#define RTLDEV dev +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) +/*****************************************************************************/ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) +typedef struct net_device *napi_ptr; +typedef int *napi_budget; + +#define napi dev +#define RTL_NAPI_CONFIG(ndev, priv, function, weig) ndev->poll=function; \ + ndev->weight=weig; +#define RTL_NAPI_QUOTA(budget, ndev) min(*budget, ndev->quota) +#define RTL_GET_PRIV(stuct_ptr, priv_struct) netdev_priv(stuct_ptr) +#define RTL_GET_NETDEV(priv_ptr) +#define RTL_RX_QUOTA(budget) *budget +#define RTL_NAPI_QUOTA_UPDATE(ndev, work_done, budget) *budget -= work_done; \ + ndev->quota -= work_done; +#define RTL_NETIF_RX_COMPLETE(dev, napi, work_done) netif_rx_complete(dev) +#define RTL_NETIF_RX_SCHEDULE_PREP(dev, napi) netif_rx_schedule_prep(dev) +#define __RTL_NETIF_RX_SCHEDULE(dev, napi) __netif_rx_schedule(dev) +#define RTL_NAPI_RETURN_VALUE work_done >= work_to_do +#define RTL_NAPI_ENABLE(dev, napi) netif_poll_enable(dev) +#define RTL_NAPI_DISABLE(dev, napi) netif_poll_disable(dev) +#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1)) +#else +typedef struct napi_struct *napi_ptr; +typedef int napi_budget; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,1,0) +#define RTL_NAPI_CONFIG(ndev, priv, function, weight) netif_napi_add_weight(ndev, &priv->napi, function, weight) +#else +#define RTL_NAPI_CONFIG(ndev, priv, function, weight) netif_napi_add(ndev, &priv->napi, function, weight) +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(6,1,0) +#define RTL_NAPI_QUOTA(budget, ndev) min(budget, budget) +#define RTL_GET_PRIV(stuct_ptr, priv_struct) container_of(stuct_ptr, priv_struct, stuct_ptr) +#define RTL_GET_NETDEV(priv_ptr) struct net_device *dev = priv_ptr->dev; +#define RTL_RX_QUOTA(budget) budget +#define RTL_NAPI_QUOTA_UPDATE(ndev, work_done, budget) +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29) +#define RTL_NETIF_RX_COMPLETE(dev, napi, work_done) netif_rx_complete(dev, napi) +#define RTL_NETIF_RX_SCHEDULE_PREP(dev, napi) netif_rx_schedule_prep(dev, napi) +#define __RTL_NETIF_RX_SCHEDULE(dev, napi) __netif_rx_schedule(dev, napi) +#endif +#if LINUX_VERSION_CODE == KERNEL_VERSION(2,6,29) +#define RTL_NETIF_RX_COMPLETE(dev, napi, work_done) netif_rx_complete(napi) +#define RTL_NETIF_RX_SCHEDULE_PREP(dev, napi) netif_rx_schedule_prep(napi) +#define __RTL_NETIF_RX_SCHEDULE(dev, napi) __netif_rx_schedule(napi) +#endif +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,29) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0) +#define RTL_NETIF_RX_COMPLETE(dev, napi, work_done) napi_complete_done(napi, work_done) +#else +#define RTL_NETIF_RX_COMPLETE(dev, napi, work_done) napi_complete(napi) +#endif +#define RTL_NETIF_RX_SCHEDULE_PREP(dev, napi) napi_schedule_prep(napi) +#define __RTL_NETIF_RX_SCHEDULE(dev, napi) __napi_schedule(napi) +#endif +#define RTL_NAPI_RETURN_VALUE work_done +#define RTL_NAPI_ENABLE(dev, napi) napi_enable(napi) +#define RTL_NAPI_DISABLE(dev, napi) napi_disable(napi) +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,27) +#define RTL_NAPI_DEL(priv) +#else +#define RTL_NAPI_DEL(priv) netif_napi_del(&priv->napi) +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,27) + +/*****************************************************************************/ +#ifdef CONFIG_R8127_NAPI +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0) +#define RTL_NAPI_CONSUME_SKB_ANY(skb, budget) napi_consume_skb(skb, budget) +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0) +#define RTL_NAPI_CONSUME_SKB_ANY(skb, budget) dev_consume_skb_any(skb); +#else +#define RTL_NAPI_CONSUME_SKB_ANY(skb, budget) dev_kfree_skb_any(skb); +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0) +#else //CONFIG_R8127_NAPI +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0) +#define RTL_NAPI_CONSUME_SKB_ANY(skb, budget) dev_consume_skb_any(skb); +#else +#define RTL_NAPI_CONSUME_SKB_ANY(skb, budget) dev_kfree_skb_any(skb); +#endif +#endif //CONFIG_R8127_NAPI + +/*****************************************************************************/ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,9) +#ifdef __CHECKER__ +#define __iomem __attribute__((noderef, address_space(2))) +extern void __chk_io_ptr(void __iomem *); +#define __bitwise __attribute__((bitwise)) +#else +#define __iomem +#define __chk_io_ptr(x) (void)0 +#define __bitwise +#endif +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,9) + +/*****************************************************************************/ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,8) +#ifdef __CHECKER__ +#define __force __attribute__((force)) +#else +#define __force +#endif +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,8) + +#ifndef module_param +#define module_param(v,t,p) MODULE_PARM(v, "i"); +#endif + +#ifndef PCI_DEVICE +#define PCI_DEVICE(vend,dev) \ + .vendor = (vend), .device = (dev), \ + .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID +#endif + +/*****************************************************************************/ +/* 2.5.28 => 2.4.23 */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,28)) + +static inline void _kc_synchronize_irq(void) +{ + synchronize_irq(); +} +#undef synchronize_irq +#define synchronize_irq(X) _kc_synchronize_irq() + +#include +#define work_struct tq_struct +#undef INIT_WORK +#define INIT_WORK(a,b,c) INIT_TQUEUE(a,(void (*)(void *))b,c) +#undef container_of +#define container_of list_entry +#define schedule_work schedule_task +#define flush_scheduled_work flush_scheduled_tasks +#endif /* 2.5.28 => 2.4.17 */ + +/*****************************************************************************/ +/* 2.6.4 => 2.6.0 */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,4)) +#define MODULE_VERSION(_version) MODULE_INFO(version, _version) +#endif /* 2.6.4 => 2.6.0 */ +/*****************************************************************************/ +/* 2.6.0 => 2.5.28 */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)) +#define MODULE_INFO(version, _version) +#ifndef CONFIG_E1000_DISABLE_PACKET_SPLIT +#define CONFIG_E1000_DISABLE_PACKET_SPLIT 1 +#endif + +#define pci_set_consistent_dma_mask(dev,mask) 1 + +#undef dev_put +#define dev_put(dev) __dev_put(dev) + +#ifndef skb_fill_page_desc +#define skb_fill_page_desc _kc_skb_fill_page_desc +extern void _kc_skb_fill_page_desc(struct sk_buff *skb, int i, struct page *page, int off, int size); +#endif + +#ifndef pci_dma_mapping_error +#define pci_dma_mapping_error _kc_pci_dma_mapping_error +static inline int _kc_pci_dma_mapping_error(dma_addr_t dma_addr) +{ + return dma_addr == 0; +} +#endif + +#undef ALIGN +#define ALIGN(x,a) (((x)+(a)-1)&~((a)-1)) + +#endif /* 2.6.0 => 2.5.28 */ + +/*****************************************************************************/ +/* 2.4.22 => 2.4.17 */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,4,22)) +#define pci_name(x) ((x)->slot_name) +#endif /* 2.4.22 => 2.4.17 */ + +/*****************************************************************************/ +/* 2.6.5 => 2.6.0 */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,5)) +#define pci_dma_sync_single_for_cpu pci_dma_sync_single +#define pci_dma_sync_single_for_device pci_dma_sync_single_for_cpu +#endif /* 2.6.5 => 2.6.0 */ + +/*****************************************************************************/ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) +/* + * initialize a work-struct's func and data pointers: + */ +#define PREPARE_WORK(_work, _func, _data) \ + do { \ + (_work)->func = _func; \ + (_work)->data = _data; \ + } while (0) + +#endif +/*****************************************************************************/ +/* 2.6.4 => 2.6.0 */ +#if ((LINUX_VERSION_CODE < KERNEL_VERSION(2,4,25) && \ + LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22)) || \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) && \ + LINUX_VERSION_CODE < KERNEL_VERSION(2,6,4))) +#define ETHTOOL_OPS_COMPAT +#endif /* 2.6.4 => 2.6.0 */ + +/*****************************************************************************/ +/* Installations with ethtool version without eeprom, adapter id, or statistics + * support */ + +#ifndef ETH_GSTRING_LEN +#define ETH_GSTRING_LEN 32 +#endif + +#ifndef ETHTOOL_GSTATS +#define ETHTOOL_GSTATS 0x1d +#undef ethtool_drvinfo +#define ethtool_drvinfo k_ethtool_drvinfo +struct k_ethtool_drvinfo { + u32 cmd; + char driver[32]; + char version[32]; + char fw_version[32]; + char bus_info[32]; + char reserved1[32]; + char reserved2[16]; + u32 n_stats; + u32 testinfo_len; + u32 eedump_len; + u32 regdump_len; +}; + +struct ethtool_stats { + u32 cmd; + u32 n_stats; + u64 data[0]; +}; +#endif /* ETHTOOL_GSTATS */ + +#ifndef ETHTOOL_PHYS_ID +#define ETHTOOL_PHYS_ID 0x1c +#endif /* ETHTOOL_PHYS_ID */ + +#ifndef ETHTOOL_GSTRINGS +#define ETHTOOL_GSTRINGS 0x1b +enum ethtool_stringset { + ETH_SS_TEST = 0, + ETH_SS_STATS, +}; +struct ethtool_gstrings { + u32 cmd; /* ETHTOOL_GSTRINGS */ + u32 string_set; /* string set id e.c. ETH_SS_TEST, etc*/ + u32 len; /* number of strings in the string set */ + u8 data[0]; +}; +#endif /* ETHTOOL_GSTRINGS */ + +#ifndef ETHTOOL_TEST +#define ETHTOOL_TEST 0x1a +enum ethtool_test_flags { + ETH_TEST_FL_OFFLINE = (1 << 0), + ETH_TEST_FL_FAILED = (1 << 1), +}; +struct ethtool_test { + u32 cmd; + u32 flags; + u32 reserved; + u32 len; + u64 data[0]; +}; +#endif /* ETHTOOL_TEST */ + +#ifndef ETHTOOL_GEEPROM +#define ETHTOOL_GEEPROM 0xb +#undef ETHTOOL_GREGS +struct ethtool_eeprom { + u32 cmd; + u32 magic; + u32 offset; + u32 len; + u8 data[0]; +}; + +struct ethtool_value { + u32 cmd; + u32 data; +}; +#endif /* ETHTOOL_GEEPROM */ + +#ifndef ETHTOOL_GLINK +#define ETHTOOL_GLINK 0xa +#endif /* ETHTOOL_GLINK */ + +#ifndef ETHTOOL_GREGS +#define ETHTOOL_GREGS 0x00000004 /* Get NIC registers */ +#define ethtool_regs _kc_ethtool_regs +/* for passing big chunks of data */ +struct _kc_ethtool_regs { + u32 cmd; + u32 version; /* driver-specific, indicates different chips/revs */ + u32 len; /* bytes */ + u8 data[0]; +}; +#endif /* ETHTOOL_GREGS */ + +#ifndef ETHTOOL_GMSGLVL +#define ETHTOOL_GMSGLVL 0x00000007 /* Get driver message level */ +#endif +#ifndef ETHTOOL_SMSGLVL +#define ETHTOOL_SMSGLVL 0x00000008 /* Set driver msg level, priv. */ +#endif +#ifndef ETHTOOL_NWAY_RST +#define ETHTOOL_NWAY_RST 0x00000009 /* Restart autonegotiation, priv */ +#endif +#ifndef ETHTOOL_GLINK +#define ETHTOOL_GLINK 0x0000000a /* Get link status */ +#endif +#ifndef ETHTOOL_GEEPROM +#define ETHTOOL_GEEPROM 0x0000000b /* Get EEPROM data */ +#endif +#ifndef ETHTOOL_SEEPROM +#define ETHTOOL_SEEPROM 0x0000000c /* Set EEPROM data */ +#endif +#ifndef ETHTOOL_GCOALESCE +#define ETHTOOL_GCOALESCE 0x0000000e /* Get coalesce config */ +/* for configuring coalescing parameters of chip */ +#define ethtool_coalesce _kc_ethtool_coalesce +struct _kc_ethtool_coalesce { + u32 cmd; /* ETHTOOL_{G,S}COALESCE */ + + /* How many usecs to delay an RX interrupt after + * a packet arrives. If 0, only rx_max_coalesced_frames + * is used. + */ + u32 rx_coalesce_usecs; + + /* How many packets to delay an RX interrupt after + * a packet arrives. If 0, only rx_coalesce_usecs is + * used. It is illegal to set both usecs and max frames + * to zero as this would cause RX interrupts to never be + * generated. + */ + u32 rx_max_coalesced_frames; + + /* Same as above two parameters, except that these values + * apply while an IRQ is being serviced by the host. Not + * all cards support this feature and the values are ignored + * in that case. + */ + u32 rx_coalesce_usecs_irq; + u32 rx_max_coalesced_frames_irq; + + /* How many usecs to delay a TX interrupt after + * a packet is sent. If 0, only tx_max_coalesced_frames + * is used. + */ + u32 tx_coalesce_usecs; + + /* How many packets to delay a TX interrupt after + * a packet is sent. If 0, only tx_coalesce_usecs is + * used. It is illegal to set both usecs and max frames + * to zero as this would cause TX interrupts to never be + * generated. + */ + u32 tx_max_coalesced_frames; + + /* Same as above two parameters, except that these values + * apply while an IRQ is being serviced by the host. Not + * all cards support this feature and the values are ignored + * in that case. + */ + u32 tx_coalesce_usecs_irq; + u32 tx_max_coalesced_frames_irq; + + /* How many usecs to delay in-memory statistics + * block updates. Some drivers do not have an in-memory + * statistic block, and in such cases this value is ignored. + * This value must not be zero. + */ + u32 stats_block_coalesce_usecs; + + /* Adaptive RX/TX coalescing is an algorithm implemented by + * some drivers to improve latency under low packet rates and + * improve throughput under high packet rates. Some drivers + * only implement one of RX or TX adaptive coalescing. Anything + * not implemented by the driver causes these values to be + * silently ignored. + */ + u32 use_adaptive_rx_coalesce; + u32 use_adaptive_tx_coalesce; + + /* When the packet rate (measured in packets per second) + * is below pkt_rate_low, the {rx,tx}_*_low parameters are + * used. + */ + u32 pkt_rate_low; + u32 rx_coalesce_usecs_low; + u32 rx_max_coalesced_frames_low; + u32 tx_coalesce_usecs_low; + u32 tx_max_coalesced_frames_low; + + /* When the packet rate is below pkt_rate_high but above + * pkt_rate_low (both measured in packets per second) the + * normal {rx,tx}_* coalescing parameters are used. + */ + + /* When the packet rate is (measured in packets per second) + * is above pkt_rate_high, the {rx,tx}_*_high parameters are + * used. + */ + u32 pkt_rate_high; + u32 rx_coalesce_usecs_high; + u32 rx_max_coalesced_frames_high; + u32 tx_coalesce_usecs_high; + u32 tx_max_coalesced_frames_high; + + /* How often to do adaptive coalescing packet rate sampling, + * measured in seconds. Must not be zero. + */ + u32 rate_sample_interval; +}; +#endif /* ETHTOOL_GCOALESCE */ + +#ifndef ETHTOOL_SCOALESCE +#define ETHTOOL_SCOALESCE 0x0000000f /* Set coalesce config. */ +#endif +#ifndef ETHTOOL_GRINGPARAM +#define ETHTOOL_GRINGPARAM 0x00000010 /* Get ring parameters */ +/* for configuring RX/TX ring parameters */ +#define ethtool_ringparam _kc_ethtool_ringparam +struct _kc_ethtool_ringparam { + u32 cmd; /* ETHTOOL_{G,S}RINGPARAM */ + + /* Read only attributes. These indicate the maximum number + * of pending RX/TX ring entries the driver will allow the + * user to set. + */ + u32 rx_max_pending; + u32 rx_mini_max_pending; + u32 rx_jumbo_max_pending; + u32 tx_max_pending; + + /* Values changeable by the user. The valid values are + * in the range 1 to the "*_max_pending" counterpart above. + */ + u32 rx_pending; + u32 rx_mini_pending; + u32 rx_jumbo_pending; + u32 tx_pending; +}; +#endif /* ETHTOOL_GRINGPARAM */ + +#ifndef ETHTOOL_SRINGPARAM +#define ETHTOOL_SRINGPARAM 0x00000011 /* Set ring parameters, priv. */ +#endif +#ifndef ETHTOOL_GPAUSEPARAM +#define ETHTOOL_GPAUSEPARAM 0x00000012 /* Get pause parameters */ +/* for configuring link flow control parameters */ +#define ethtool_pauseparam _kc_ethtool_pauseparam +struct _kc_ethtool_pauseparam { + u32 cmd; /* ETHTOOL_{G,S}PAUSEPARAM */ + + /* If the link is being auto-negotiated (via ethtool_cmd.autoneg + * being true) the user may set 'autonet' here non-zero to have the + * pause parameters be auto-negotiated too. In such a case, the + * {rx,tx}_pause values below determine what capabilities are + * advertised. + * + * If 'autoneg' is zero or the link is not being auto-negotiated, + * then {rx,tx}_pause force the driver to use/not-use pause + * flow control. + */ + u32 autoneg; + u32 rx_pause; + u32 tx_pause; +}; +#endif /* ETHTOOL_GPAUSEPARAM */ + +#ifndef ETHTOOL_SPAUSEPARAM +#define ETHTOOL_SPAUSEPARAM 0x00000013 /* Set pause parameters. */ +#endif +#ifndef ETHTOOL_GRXCSUM +#define ETHTOOL_GRXCSUM 0x00000014 /* Get RX hw csum enable (ethtool_value) */ +#endif +#ifndef ETHTOOL_SRXCSUM +#define ETHTOOL_SRXCSUM 0x00000015 /* Set RX hw csum enable (ethtool_value) */ +#endif +#ifndef ETHTOOL_GTXCSUM +#define ETHTOOL_GTXCSUM 0x00000016 /* Get TX hw csum enable (ethtool_value) */ +#endif +#ifndef ETHTOOL_STXCSUM +#define ETHTOOL_STXCSUM 0x00000017 /* Set TX hw csum enable (ethtool_value) */ +#endif +#ifndef ETHTOOL_GSG +#define ETHTOOL_GSG 0x00000018 /* Get scatter-gather enable +* (ethtool_value) */ +#endif +#ifndef ETHTOOL_SSG +#define ETHTOOL_SSG 0x00000019 /* Set scatter-gather enable +* (ethtool_value). */ +#endif +#ifndef ETHTOOL_TEST +#define ETHTOOL_TEST 0x0000001a /* execute NIC self-test, priv. */ +#endif +#ifndef ETHTOOL_GSTRINGS +#define ETHTOOL_GSTRINGS 0x0000001b /* get specified string set */ +#endif +#ifndef ETHTOOL_PHYS_ID +#define ETHTOOL_PHYS_ID 0x0000001c /* identify the NIC */ +#endif +#ifndef ETHTOOL_GSTATS +#define ETHTOOL_GSTATS 0x0000001d /* get NIC-specific statistics */ +#endif +#ifndef ETHTOOL_GTSO +#define ETHTOOL_GTSO 0x0000001e /* Get TSO enable (ethtool_value) */ +#endif +#ifndef ETHTOOL_STSO +#define ETHTOOL_STSO 0x0000001f /* Set TSO enable (ethtool_value) */ +#endif + +#ifndef ETHTOOL_BUSINFO_LEN +#define ETHTOOL_BUSINFO_LEN 32 +#endif + +/*****************************************************************************/ + +enum RTL8127_registers { + MAC0 = 0x00, /* Ethernet hardware address. */ + MAC4 = 0x04, + MAR0 = 0x08, /* Multicast filter. */ + CounterAddrLow = 0x10, + CounterAddrHigh = 0x14, + CustomLED = 0x18, + TxDescStartAddrLow = 0x20, + TxDescStartAddrHigh = 0x24, + TxHDescStartAddrLow = 0x28, + TxHDescStartAddrHigh = 0x2c, + FLASH = 0x30, + INT_CFG0_8125 = 0x34, + ERSR = 0x36, + ChipCmd = 0x37, + TxPoll = 0x38, + IntrMask = 0x3C, + IntrStatus = 0x3E, + TxConfig = 0x40, + RxConfig = 0x44, + TCTR = 0x48, + Cfg9346 = 0x50, + Config0 = 0x51, + Config1 = 0x52, + Config2 = 0x53, + Config3 = 0x54, + Config4 = 0x55, + Config5 = 0x56, + TDFNR = 0x57, + TimeInt0 = 0x58, + TimeInt1 = 0x5C, + PHYAR = 0x60, + CSIDR = 0x64, + CSIAR = 0x68, + PHYstatus = 0x6C, + MACDBG = 0x6D, + GPIO = 0x6E, + PMCH = 0x6F, + ERIDR = 0x70, + ERIAR = 0x74, + INT_CFG1_8125 = 0x7A, + EPHY_RXER_NUM = 0x7C, + EPHYAR = 0x80, + TimeInt2 = 0x8C, + OCPDR = 0xB0, + MACOCP = 0xB0, + OCPAR = 0xB4, + SecMAC0 = 0xB4, + SecMAC4 = 0xB8, + PHYOCP = 0xB8, + DBG_reg = 0xD1, + TwiCmdReg = 0xD2, + MCUCmd_reg = 0xD3, + RxMaxSize = 0xDA, + EFUSEAR = 0xDC, + CPlusCmd = 0xE0, + IntrMitigate = 0xE2, + RxDescAddrLow = 0xE4, + RxDescAddrHigh = 0xE8, + MTPS = 0xEC, + FuncEvent = 0xF0, + PPSW = 0xF2, + FuncEventMask = 0xF4, + TimeInt3 = 0xF4, + FuncPresetState = 0xF8, + CMAC_IBCR0 = 0xF8, + CMAC_IBCR2 = 0xF9, + CMAC_IBIMR0 = 0xFA, + CMAC_IBISR0 = 0xFB, + FuncForceEvent = 0xFC, + //8125 + IMR0_8125 = 0x38, + ISR0_8125 = 0x3C, + TPPOLL_8125 = 0x90, + IMR1_8125 = 0x800, + ISR1_8125 = 0x802, + IMR2_8125 = 0x804, + ISR2_8125 = 0x806, + IMR3_8125 = 0x808, + ISR3_8125 = 0x80A, + BACKUP_ADDR0_8125 = 0x19E0, + BACKUP_ADDR1_8125 = 0X19E4, + TCTR0_8125 = 0x0048, + TCTR1_8125 = 0x004C, + TCTR2_8125 = 0x0088, + TCTR3_8125 = 0x001C, + TIMER_INT0_8125 = 0x0058, + TIMER_INT1_8125 = 0x005C, + TIMER_INT2_8125 = 0x008C, + TIMER_INT3_8125 = 0x00F4, + INT_MITI_V2_0_RX = 0x0A00, + INT_MITI_V2_0_TX = 0x0A02, + INT_MITI_V2_1_RX = 0x0A08, + INT_MITI_V2_1_TX = 0x0A0A, + IMR_V2_CLEAR_REG_8125 = 0x0D00, + ISR_V2_8125 = 0x0D04, + IMR_V2_SET_REG_8125 = 0x0D0C, + TDU_STA_8125 = 0x0D08, + RDU_STA_8125 = 0x0D0A, + IMR_V4_L2_CLEAR_REG_8125 = 0x0D10, + IMR_V4_L2_SET_REG_8125 = 0x0D18, + ISR_V4_L2_8125 = 0x0D14, + SW_TAIL_PTR0_8125BP = 0x0D30, + SW_TAIL_PTR1_8125BP = 0x0D38, + HW_CLO_PTR0_8125BP = 0x0D34, + HW_CLO_PTR1_8125BP = 0x0D3C, + DOUBLE_VLAN_CONFIG = 0x1000, + TX_NEW_CTRL = 0x203E, + TNPDS_Q1_LOW_8125 = 0x2100, + PLA_TXQ0_IDLE_CREDIT = 0x2500, + PLA_TXQ1_IDLE_CREDIT = 0x2504, + SW_TAIL_PTR0_8125 = 0x2800, + HW_CLO_PTR0_8125 = 0x2802, + SW_TAIL_PTR0_8126 = 0x2800, + HW_CLO_PTR0_8126 = 0x2800, + RDSAR_Q1_LOW_8125 = 0x4000, + RSS_CTRL_8125 = 0x4500, + Q_NUM_CTRL_8125 = 0x4800, + RSS_KEY_8125 = 0x4600, + RSS_INDIRECTION_TBL_8125_V2 = 0x4700, + EEE_TXIDLE_TIMER_8125 = 0x6048, + PTP_CTRL_8125 = 0x6800, + PTP_STATUS_8125 = 0x6802, + PTP_ISR_8125 = 0x6804, + PTP_IMR_8125 = 0x6805, + PTP_TIME_CORRECT_CMD_8125 = 0x6806, + PTP_SOFT_CONFIG_Time_NS_8125 = 0x6808, + PTP_SOFT_CONFIG_Time_S_8125 = 0x680C, + PTP_SOFT_CONFIG_Time_Sign = 0x6812, + PTP_LOCAL_Time_SUB_NS_8125 = 0x6814, + PTP_LOCAL_Time_NS_8125 = 0x6818, + PTP_LOCAL_Time_S_8125 = 0x681C, + PTP_Time_SHIFTER_S_8125 = 0x6856, + PPS_RISE_TIME_NS_8125 = 0x68A0, + PPS_RISE_TIME_S_8125 = 0x68A4, + PTP_EGRESS_TIME_BASE_NS_8125 = 0XCF20, + PTP_EGRESS_TIME_BASE_S_8125 = 0XCF24, + PTP_CTL = 0xE400, + PTP_INER = 0xE402, + PTP_INSR = 0xE404, + PTP_SYNCE_CTL = 0xE406, + PTP_GEN_CFG = 0xE408, + PTP_CLK_CFG_8126 = 0xE410, + PTP_CFG_NS_LO_8126 = 0xE412, + PTP_CFG_NS_HI_8126 = 0xE414, + PTP_CFG_S_LO_8126 = 0xE416, + PTP_CFG_S_MI_8126 = 0xE418, + PTP_CFG_S_HI_8126 = 0xE41A, + PTP_TAI_CFG = 0xE420, + PTP_TAI_TS_S_LO = 0xE42A, + PTP_TAI_TS_S_HI = 0xE42C, + PTP_TRX_TS_STA = 0xE430, + PTP_TRX_TS_NS_LO = 0xE446, + PTP_TRX_TS_NS_HI = 0xE448, + PTP_TRX_TS_S_LO = 0xE44A, + PTP_TRX_TS_S_MI = 0xE44C, + PTP_TRX_TS_S_HI = 0xE44E, + + //TCAM + TCAM_NOTVALID_ADDR = 0xA000, + TCAM_VALID_ADDR = 0xA800, + TCAM_MAC_ADDR = 448, + TCAM_VLAN_TAG = 496, + //TCAM V2 + TCAM_NOTVALID_ADDR_V2 = 0xA000, + TCAM_VALID_ADDR_V2 = 0xB000, + TCAM_MAC_ADDR_V2 = 0x00, + TCAM_VLAN_TAG_V2 = 0x03, +}; + +enum RTL8127_register_content { + /* InterruptStatusBits */ + SYSErr = 0x8000, + PCSTimeout = 0x4000, + SWInt = 0x0100, + TxDescUnavail = 0x0080, + RxFIFOOver = 0x0040, + LinkChg = 0x0020, + RxDescUnavail = 0x0010, + TxErr = 0x0008, + TxOK = 0x0004, + RxErr = 0x0002, + RxOK = 0x0001, + RxDU1 = 0x0002, + RxOK1 = 0x0001, + + /* RxStatusDesc */ + RxRWT = (1 << 22), + RxRES = (1 << 21), + RxRUNT = (1 << 20), + RxCRC = (1 << 19), + + RxRWT_V3 = (1 << 18), + RxRES_V3 = (1 << 20), + RxRUNT_V3 = (1 << 19), + RxCRC_V3 = (1 << 17), + + RxRES_V4 = (1 << 22), + RxRUNT_V4 = (1 << 21), + RxCRC_V4 = (1 << 20), + + /* ChipCmdBits */ + StopReq = 0x80, + CmdReset = 0x10, + CmdRxEnb = 0x08, + CmdTxEnb = 0x04, + RxBufEmpty = 0x01, + + /* Cfg9346Bits */ + Cfg9346_EEM_MASK = 0xC0, + Cfg9346_Lock = 0x00, + Cfg9346_Unlock = 0xC0, + Cfg9346_EEDO = (1 << 0), + Cfg9346_EEDI = (1 << 1), + Cfg9346_EESK = (1 << 2), + Cfg9346_EECS = (1 << 3), + Cfg9346_EEM0 = (1 << 6), + Cfg9346_EEM1 = (1 << 7), + + /* rx_mode_bits */ + AcceptErr = 0x20, + AcceptRunt = 0x10, + AcceptBroadcast = 0x08, + AcceptMulticast = 0x04, + AcceptMyPhys = 0x02, + AcceptAllPhys = 0x01, + AcceppVlanPhys = 0x8000, + + /* Transmit Priority Polling*/ + HPQ = 0x80, + NPQ = 0x40, + FSWInt = 0x01, + + /* RxConfigBits */ + Reserved2_shift = 13, + RxCfgDMAShift = 8, + EnableRxDescV3 = (1 << 24), + EnableRxDescV4_1 = (1 << 24), + EnableOuterVlan = (1 << 23), + EnableInnerVlan = (1 << 22), + RxCfg_128_int_en = (1 << 15), + RxCfg_fet_multi_en = (1 << 14), + RxCfg_half_refetch = (1 << 13), + RxCfg_pause_slot_en = (1 << 11), + RxCfg_9356SEL = (1 << 6), + EnableRxDescV4_0 = (1 << 1), //not in rcr + + /* TxConfigBits */ + TxInterFrameGapShift = 24, + TxDMAShift = 8, /* DMA burst value (0-7) is shift this many bits */ + TxMACLoopBack = (1 << 17), /* MAC loopback */ + + /* Config1 register */ + LEDS1 = (1 << 7), + LEDS0 = (1 << 6), + Speed_down = (1 << 4), + MEMMAP = (1 << 3), + IOMAP = (1 << 2), + VPD = (1 << 1), + PMEnable = (1 << 0), /* Power Management Enable */ + + /* Config2 register */ + PMSTS_En = (1 << 5), + + /* Config3 register */ + Isolate_en = (1 << 12), /* Isolate enable */ + MagicPacket = (1 << 5), /* Wake up when receives a Magic Packet */ + LinkUp = (1 << 4), /* This bit is reserved in RTL8125B.*/ + /* Wake up when the cable connection is re-established */ + ECRCEN = (1 << 3), /* This bit is reserved in RTL8125B*/ + Jumbo_En0 = (1 << 2), /* This bit is reserved in RTL8125B*/ + RDY_TO_L23 = (1 << 1), /* This bit is reserved in RTL8125B*/ + Beacon_en = (1 << 0), /* This bit is reserved in RTL8125B*/ + + /* Config4 register */ + Jumbo_En1 = (1 << 1), /* This bit is reserved in RTL8125B*/ + + /* Config5 register */ + BWF = (1 << 6), /* Accept Broadcast wakeup frame */ + MWF = (1 << 5), /* Accept Multicast wakeup frame */ + UWF = (1 << 4), /* Accept Unicast wakeup frame */ + LanWake = (1 << 1), /* LanWake enable/disable */ + PMEStatus = (1 << 0), /* PME status can be reset by PCI RST# */ + + /* CPlusCmd */ + EnableBist = (1 << 15), + Macdbgo_oe = (1 << 14), + Normal_mode = (1 << 13), + Force_halfdup = (1 << 12), + Force_rxflow_en = (1 << 11), + Force_txflow_en = (1 << 10), + Cxpl_dbg_sel = (1 << 9),//This bit is reserved in RTL8125B + ASF = (1 << 8),//This bit is reserved in RTL8125C + PktCntrDisable = (1 << 7), + RxVlan = (1 << 6), + RxChkSum = (1 << 5), + Macdbgo_sel = 0x001C, + INTT_0 = 0x0000, + INTT_1 = 0x0001, + INTT_2 = 0x0002, + INTT_3 = 0x0003, + + /* rtl8127_PHYstatus */ + PowerSaveStatus = 0x80, + _1000bpsL = 0x80000, + _10000bpsF = 0x4000, + _10000bpsL = 0x2000, + _5000bpsF = 0x1000, + _5000bpsL = 0x800, + _2500bpsF = 0x400, + _2500bpsL = 0x200, + TxFlowCtrl = 0x40, + RxFlowCtrl = 0x20, + _1000bpsF = 0x10, + _100bps = 0x08, + _10bps = 0x04, + LinkStatus = 0x02, + FullDup = 0x01, + + /* DBG_reg */ + Fix_Nak_1 = (1 << 4), + Fix_Nak_2 = (1 << 3), + DBGPIN_E2 = (1 << 0), + + /* ResetCounterCommand */ + CounterReset = 0x1, + /* DumpCounterCommand */ + CounterDump = 0x8, + + /* PHY access */ + PHYAR_Flag = 0x80000000, + PHYAR_Write = 0x80000000, + PHYAR_Read = 0x00000000, + PHYAR_Reg_Mask = 0x1f, + PHYAR_Reg_shift = 16, + PHYAR_Data_Mask = 0xffff, + + /* EPHY access */ + EPHYAR_Flag = 0x80000000, + EPHYAR_Write = 0x80000000, + EPHYAR_Read = 0x00000000, + EPHYAR_Reg_Mask = 0x3f, + EPHYAR_Reg_Mask_v2 = 0x7f, + EPHYAR_Reg_shift = 16, + EPHYAR_Data_Mask = 0xffff, + EPHYAR_EXT_ADDR = 0x0ffe, + + /* CSI access */ + CSIAR_Flag = 0x80000000, + CSIAR_Write = 0x80000000, + CSIAR_Read = 0x00000000, + CSIAR_ByteEn = 0x0f, + CSIAR_ByteEn_shift = 12, + CSIAR_Addr_Mask = 0x0fff, + + /* ERI access */ + ERIAR_Flag = 0x80000000, + ERIAR_Write = 0x80000000, + ERIAR_Read = 0x00000000, + ERIAR_Addr_Align = 4, /* ERI access register address must be 4 byte alignment */ + ERIAR_ExGMAC = 0, + ERIAR_MSIX = 1, + ERIAR_ASF = 2, + ERIAR_OOB = 2, + ERIAR_Type_shift = 16, + ERIAR_ByteEn = 0x0f, + ERIAR_ByteEn_shift = 12, + + /* OCP GPHY access */ + OCPDR_Write = 0x80000000, + OCPDR_Read = 0x00000000, + OCPDR_Reg_Mask = 0xFF, + OCPDR_Data_Mask = 0xFFFF, + OCPDR_GPHY_Reg_shift = 16, + OCPAR_Flag = 0x80000000, + OCPAR_GPHY_Write = 0x8000F060, + OCPAR_GPHY_Read = 0x0000F060, + OCPR_Write = 0x80000000, + OCPR_Read = 0x00000000, + OCPR_Addr_Reg_shift = 16, + OCPR_Flag = 0x80000000, + OCP_STD_PHY_BASE_PAGE = 0x0A40, + + /* MCU Command */ + Now_is_oob = (1 << 7), + Txfifo_empty = (1 << 5), + Rxfifo_empty = (1 << 4), + + /* E-FUSE access */ + EFUSE_WRITE = 0x80000000, + EFUSE_WRITE_OK = 0x00000000, + EFUSE_READ = 0x00000000, + EFUSE_READ_OK = 0x80000000, + EFUSE_WRITE_V3 = 0x40000000, + EFUSE_WRITE_OK_V3 = 0x00000000, + EFUSE_READ_V3 = 0x80000000, + EFUSE_READ_OK_V3 = 0x00000000, + EFUSE_Reg_Mask = 0x03FF, + EFUSE_Reg_Shift = 8, + EFUSE_Check_Cnt = 300, + EFUSE_READ_FAIL = 0xFF, + EFUSE_Data_Mask = 0x000000FF, + + /* GPIO */ + GPIO_en = (1 << 0), + + /* PTP */ + PTP_ISR_TOK = (1 << 1), + PTP_ISR_TER = (1 << 2), + PTP_EXEC_CMD = (1 << 7), + PTP_ADJUST_TIME_NS_NEGATIVE = (1 << 30), + PTP_ADJUST_TIME_S_NEGATIVE = (1ULL << 48), + PTP_SOFT_CONFIG_TIME_NS_NEGATIVE = (1 << 30), + PTP_SOFT_CONFIG_TIME_S_NEGATIVE = (1ULL << 48), + + /* New Interrupt Bits */ + INT_CFG0_ENABLE_8125 = (1 << 0), + INT_CFG0_TIMEOUT0_BYPASS_8125 = (1 << 1), + INT_CFG0_MITIGATION_BYPASS_8125 = (1 << 2), + INT_CFG0_RDU_BYPASS_8126 = (1 << 4), + INT_CFG0_MSIX_ENTRY_NUM_MODE = (1 << 5), + INT_CFG0_AUTO_CLEAR_IMR = (1 << 5), + INT_CFG0_AVOID_MISS_INTR = (1 << 6), + ISRIMR_V2_ROK_Q0 = (1 << 0), + ISRIMR_TOK_Q0 = (1 << 16), + ISRIMR_TOK_Q1 = (1 << 18), + ISRIMR_V2_LINKCHG = (1 << 21), + + ISRIMR_V4_ROK_Q0 = (1 << 0), + ISRIMR_V4_LINKCHG = (1 << 29), + + ISRIMR_V5_ROK_Q0 = (1 << 0), + ISRIMR_V5_TOK_Q0 = (1 << 16), + ISRIMR_V5_TOK_Q1 = (1 << 17), + ISRIMR_V5_LINKCHG = (1 << 18), + + ISRIMR_V6_ROK_Q0 = (1 << 0), + ISRIMR_V6_TOK_Q0 = (1 << 8), + ISRIMR_V6_TOK_Q1 = (1 << 9), + ISRIMR_V6_LINKCHG = (1 << 29), + + /* Magic Number */ + RTL8127_MAGIC_NUMBER = 0x0badbadbadbadbadull, +}; + +enum _DescStatusBit { + DescOwn = (1 << 31), /* Descriptor is owned by NIC */ + RingEnd = (1 << 30), /* End of descriptor ring */ + FirstFrag = (1 << 29), /* First segment of a packet */ + LastFrag = (1 << 28), /* Final segment of a packet */ + + DescOwn_V3 = (DescOwn), /* Descriptor is owned by NIC */ + RingEnd_V3 = (RingEnd), /* End of descriptor ring */ + FirstFrag_V3 = (1 << 25), /* First segment of a packet */ + LastFrag_V3 = (1 << 24), /* Final segment of a packet */ + + DescOwn_V4 = (DescOwn), /* Descriptor is owned by NIC */ + RingEnd_V4 = (RingEnd), /* End of descriptor ring */ + FirstFrag_V4 = (FirstFrag), /* First segment of a packet */ + LastFrag_V4 = (LastFrag), /* Final segment of a packet */ + + /* Tx private */ + /*------ offset 0 of tx descriptor ------*/ + LargeSend = (1 << 27), /* TCP Large Send Offload (TSO) */ + GiantSendv4 = (1 << 26), /* TCP Giant Send Offload V4 (GSOv4) */ + GiantSendv6 = (1 << 25), /* TCP Giant Send Offload V6 (GSOv6) */ + LargeSend_DP = (1 << 16), /* TCP Large Send Offload (TSO) */ + MSSShift = 16, /* MSS value position */ + MSSMask = 0x7FFU, /* MSS value 11 bits */ + TxIPCS = (1 << 18), /* Calculate IP checksum */ + TxUDPCS = (1 << 17), /* Calculate UDP/IP checksum */ + TxTCPCS = (1 << 16), /* Calculate TCP/IP checksum */ + TxVlanTag = (1 << 17), /* Add VLAN tag */ + + /*@@@@@@ offset 4 of tx descriptor => bits for RTL8125 only begin @@@@@@*/ + TxUDPCS_C = (1 << 31), /* Calculate UDP/IP checksum */ + TxTCPCS_C = (1 << 30), /* Calculate TCP/IP checksum */ + TxIPCS_C = (1 << 29), /* Calculate IP checksum */ + TxIPV6F_C = (1 << 28), /* Indicate it is an IPv6 packet */ + /*@@@@@@ offset 4 of tx descriptor => bits for RTL8125 only end @@@@@@*/ + + + /* Rx private */ + /*------ offset 0 of rx descriptor ------*/ + PID1 = (1 << 18), /* Protocol ID bit 1/2 */ + PID0 = (1 << 17), /* Protocol ID bit 2/2 */ + +#define RxProtoUDP (PID1) +#define RxProtoTCP (PID0) +#define RxProtoIP (PID1 | PID0) +#define RxProtoMask RxProtoIP + + RxIPF = (1 << 16), /* IP checksum failed */ + RxUDPF = (1 << 15), /* UDP/IP checksum failed */ + RxTCPF = (1 << 14), /* TCP/IP checksum failed */ + RxVlanTag = (1 << 16), /* VLAN tag available */ + + /*@@@@@@ offset 0 of rx descriptor => bits for RTL8125 only begin @@@@@@*/ + RxUDPT = (1 << 18), + RxTCPT = (1 << 17), + /*@@@@@@ offset 0 of rx descriptor => bits for RTL8125 only end @@@@@@*/ + + /*@@@@@@ offset 4 of rx descriptor => bits for RTL8125 only begin @@@@@@*/ + RxV6F = (1 << 31), + RxV4F = (1 << 30), + /*@@@@@@ offset 4 of rx descriptor => bits for RTL8125 only end @@@@@@*/ + + + PID1_v3 = (1 << 29), /* Protocol ID bit 1/2 */ + PID0_v3 = (1 << 28), /* Protocol ID bit 2/2 */ + +#define RxProtoUDP_v3 (PID1_v3) +#define RxProtoTCP_v3 (PID0_v3) +#define RxProtoIP_v3 (PID1_v3 | PID0_v3) +#define RxProtoMask_v3 RxProtoIP_v3 + + RxIPF_v3 = (1 << 26), /* IP checksum failed */ + RxUDPF_v3 = (1 << 25), /* UDP/IP checksum failed */ + RxTCPF_v3 = (1 << 24), /* TCP/IP checksum failed */ + RxSCTPF_v3 = (1 << 23), /* SCTP checksum failed */ + RxVlanTag_v3 = (RxVlanTag), /* VLAN tag available */ + + /*@@@@@@ offset 0 of rx descriptor => bits for RTL8125 only begin @@@@@@*/ + RxUDPT_v3 = (1 << 29), + RxTCPT_v3 = (1 << 28), + RxSCTP_v3 = (1 << 27), + /*@@@@@@ offset 0 of rx descriptor => bits for RTL8125 only end @@@@@@*/ + + /*@@@@@@ offset 4 of rx descriptor => bits for RTL8125 only begin @@@@@@*/ + RxV6F_v3 = (RxV6F), + RxV4F_v3 = (RxV4F), + /*@@@@@@ offset 4 of rx descriptor => bits for RTL8125 only end @@@@@@*/ + + RxIPF_v4 = (1 << 17), /* IP checksum failed */ + RxUDPF_v4 = (1 << 16), /* UDP/IP checksum failed */ + RxTCPF_v4 = (1 << 15), /* TCP/IP checksum failed */ + RxSCTPF_v4 = (1 << 19), /* SCTP checksum failed */ + RxVlanTag_v4 = (RxVlanTag), /* VLAN tag available */ + + /*@@@@@@ offset 0 of rx descriptor => bits for RTL8125 only begin @@@@@@*/ + RxUDPT_v4 = (1 << 19), + RxTCPT_v4 = (1 << 18), + RxSCTP_v4 = (1 << 19), + /*@@@@@@ offset 0 of rx descriptor => bits for RTL8125 only end @@@@@@*/ + + /*@@@@@@ offset 4 of rx descriptor => bits for RTL8125 only begin @@@@@@*/ + RxV6F_v4 = (RxV6F), + RxV4F_v4 = (RxV4F), + /*@@@@@@ offset 4 of rx descriptor => bits for RTL8125 only end @@@@@@*/ +}; + +enum features { +// RTL_FEATURE_WOL = (1 << 0), + RTL_FEATURE_MSI = (1 << 1), + RTL_FEATURE_MSIX = (1 << 2), +}; + +enum wol_capability { + WOL_DISABLED = 0, + WOL_ENABLED = 1 +}; + +enum bits { + BIT_0 = (1 << 0), + BIT_1 = (1 << 1), + BIT_2 = (1 << 2), + BIT_3 = (1 << 3), + BIT_4 = (1 << 4), + BIT_5 = (1 << 5), + BIT_6 = (1 << 6), + BIT_7 = (1 << 7), + BIT_8 = (1 << 8), + BIT_9 = (1 << 9), + BIT_10 = (1 << 10), + BIT_11 = (1 << 11), + BIT_12 = (1 << 12), + BIT_13 = (1 << 13), + BIT_14 = (1 << 14), + BIT_15 = (1 << 15), + BIT_16 = (1 << 16), + BIT_17 = (1 << 17), + BIT_18 = (1 << 18), + BIT_19 = (1 << 19), + BIT_20 = (1 << 20), + BIT_21 = (1 << 21), + BIT_22 = (1 << 22), + BIT_23 = (1 << 23), + BIT_24 = (1 << 24), + BIT_25 = (1 << 25), + BIT_26 = (1 << 26), + BIT_27 = (1 << 27), + BIT_28 = (1 << 28), + BIT_29 = (1 << 29), + BIT_30 = (1 << 30), + BIT_31 = (1 << 31) +}; + +#define RTL8127_CP_NUM 4 +#define RTL8127_MAX_SUPPORT_CP_LEN 110 + +enum rtl8127_cp_status { + rtl8127_cp_normal = 0, + rtl8127_cp_short, + rtl8127_cp_open, + rtl8127_cp_mismatch, + rtl8127_cp_unknown +}; + +enum efuse { + EFUSE_NOT_SUPPORT = 0, + EFUSE_SUPPORT_V1, + EFUSE_SUPPORT_V2, + EFUSE_SUPPORT_V3, + EFUSE_SUPPORT_V4, +}; +#define RsvdMask 0x3fffc000 +#define RsvdMaskV3 0x3fff8000 +#define RsvdMaskV4 RsvdMaskV3 + +struct TxDesc { + u32 opts1; + u32 opts2; + u64 addr; + u32 reserved0; + u32 reserved1; + u32 reserved2; + u32 reserved3; +}; + +struct RxDesc { + u32 opts1; + u32 opts2; + u64 addr; +}; + +struct RxDescV3 { + union { + struct { + u32 rsv1; + u32 rsv2; + } RxDescDDWord1; + }; + + union { + struct { + u32 RSSResult; + u16 HeaderBufferLen; + u16 HeaderInfo; + } RxDescNormalDDWord2; + + struct { + u32 rsv5; + u32 rsv6; + } RxDescDDWord2; + }; + + union { + u64 addr; + + struct { + u32 TimeStampLow; + u32 TimeStampHigh; + } RxDescTimeStamp; + + struct { + u32 rsv8; + u32 rsv9; + } RxDescDDWord3; + }; + + union { + struct { + u32 opts2; + u32 opts1; + } RxDescNormalDDWord4; + + struct { + u16 TimeStampHHigh; + u16 rsv11; + u32 opts1; + } RxDescPTPDDWord4; + }; +}; + +struct RxDescV4 { + union { + u64 addr; + + struct { + u32 RSSInfo; + u32 RSSResult; + } RxDescNormalDDWord1; + }; + + struct { + u32 opts2; + u32 opts1; + } RxDescNormalDDWord2; +}; + +enum rxdesc_type { + RXDESC_TYPE_NORMAL=0, + RXDESC_TYPE_NEXT, + RXDESC_TYPE_PTP, + RXDESC_TYPE_MAX +}; + +//Rx Desc Type +enum rx_desc_ring_type { + RX_DESC_RING_TYPE_UNKNOWN=0, + RX_DESC_RING_TYPE_1, + RX_DESC_RING_TYPE_2, + RX_DESC_RING_TYPE_3, + RX_DESC_RING_TYPE_4, + RX_DESC_RING_TYPE_MAX +}; + +enum rx_desc_len { + RX_DESC_LEN_TYPE_1 = (sizeof(struct RxDesc)), + RX_DESC_LEN_TYPE_3 = (sizeof(struct RxDescV3)), + RX_DESC_LEN_TYPE_4 = (sizeof(struct RxDescV4)) +}; + +struct ring_info { + struct sk_buff *skb; + u32 len; + unsigned int bytecount; + unsigned short gso_segs; + u8 __pad[sizeof(void *) - sizeof(u32)]; +}; + +struct pci_resource { + u8 cmd; + u8 cls; + u16 io_base_h; + u16 io_base_l; + u16 mem_base_h; + u16 mem_base_l; + u8 ilr; + u16 resv_0x1c_h; + u16 resv_0x1c_l; + u16 resv_0x20_h; + u16 resv_0x20_l; + u16 resv_0x24_h; + u16 resv_0x24_l; + u16 resv_0x2c_h; + u16 resv_0x2c_l; + u32 pci_sn_l; + u32 pci_sn_h; +}; + +enum r8127_flag { + R8127_FLAG_DOWN = 0, + R8127_FLAG_TASK_RESET_PENDING, + R8127_FLAG_TASK_ESD_CHECK_PENDING, + R8127_FLAG_TASK_LINKCHG_CHECK_PENDING, + R8127_FLAG_MAX +}; + +enum r8127_sysfs_flag { + R8127_SYSFS_RTL_ADV = 0, + R8127_SYSFS_FLAG_MAX +}; + +struct rtl8127_tx_ring { + void* priv; + struct net_device *netdev; + u32 index; + u32 cur_tx; /* Index into the Tx descriptor buffer of next Rx pkt. */ + u32 dirty_tx; + u32 num_tx_desc; /* Number of Tx descriptor registers */ + struct TxDesc *TxDescArray; /* 256-aligned Tx descriptor ring */ + dma_addr_t TxPhyAddr; + u32 TxDescAllocSize; + struct ring_info tx_skb[MAX_NUM_TX_DESC]; /* Tx data buffers */ + + u32 NextHwDesCloPtr; + u32 BeginHwDesCloPtr; + + u16 hw_clo_ptr_reg; + u16 sw_tail_ptr_reg; + + u16 tdsar_reg; /* Transmit Descriptor Start Address */ +}; + +struct rtl8127_rx_buffer { + struct page *page; + u32 page_offset; + dma_addr_t dma; + void* data; + struct sk_buff *skb; +}; + +struct rtl8127_rx_ring { + void* priv; + struct net_device *netdev; + u32 index; + u32 cur_rx; /* Index into the Rx descriptor buffer of next Rx pkt. */ + u32 dirty_rx; + u32 num_rx_desc; /* Number of Rx descriptor registers */ + struct RxDesc *RxDescArray; /* 256-aligned Rx descriptor ring */ + u32 RxDescAllocSize; + u64 RxDescPhyAddr[MAX_NUM_RX_DESC]; /* Rx desc physical address*/ + dma_addr_t RxPhyAddr; +#ifdef ENABLE_PAGE_REUSE + struct rtl8127_rx_buffer rx_buffer[MAX_NUM_RX_DESC]; + u16 rx_offset; +#else + struct sk_buff *Rx_skbuff[MAX_NUM_RX_DESC]; /* Rx data buffers */ +#endif //ENABLE_PAGE_REUSE + + u16 rdsar_reg; /* Receive Descriptor Start Address */ +}; + +struct r8127_napi { +#ifdef CONFIG_R8127_NAPI +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,24) + struct napi_struct napi; +#endif +#endif + void* priv; + int index; +}; + +struct r8127_irq { + irq_handler_t handler; + unsigned int vector; + u8 requested; + char name[IFNAMSIZ + 10]; +}; + +#pragma pack(1) +struct rtl8127_regs { + //00 + u8 mac_id[6]; + u16 reg_06; + u8 mar[8]; + //10 + u64 dtccr; + u16 ledsel0; + u16 legreg; + u32 tctr3; + //20 + u32 txq0_dsc_st_addr_0; + u32 txq0_dsc_st_addr_2; + u64 reg_28; + //30 + u16 rit; + u16 ritc; + u16 reg_34; + u8 reg_36; + u8 command; + u32 imr0; + u32 isr0; + //40 + u32 tcr; + u32 rcr; + u32 tctr0; + u32 tctr1; + //50 + u8 cr93c46; + u8 config0; + u8 config1; + u8 config2; + u8 config3; + u8 config4; + u8 config5; + u8 tdfnr; + u32 timer_int0; + u32 timer_int1; + //60 + u32 gphy_mdcmdio; + u32 csidr; + u32 csiar; + u16 phy_status; + u8 config6; + u8 pmch; + //70 + u32 eridr; + u32 eriar; + u16 config7; + u16 reg_7a; + u32 ephy_rxerr_cnt; + //80 + u32 ephy_mdcmdio; + u16 ledsel2; + u16 ledsel1; + u32 tctr2; + u32 timer_int2; + //90 + u8 tppoll0; + u8 reg_91; + u16 reg_92; + u16 led_feature; + u16 ledsel3; + u16 eee_led_config; + u16 reg_9a; + u32 reg_9c; + //a0 + u32 reg_a0; + u32 reg_a4; + u32 reg_a8; + u32 reg_ac; + //b0 + u32 patch_dbg; + u32 reg_b4; + u32 gphy_ocp; + u32 reg_bc; + //c0 + u32 reg_c0; + u32 reg_c4; + u32 reg_c8; + u16 otp_cmd; + u16 otp_pg_config; + //d0 + u16 phy_pwr; + u8 twsi_ctrl; + u8 oob_ctrl; + u16 mac_dbgo; + u16 mac_dbg; + u16 reg_d8; + u16 rms; + u32 efuse_data; + //e0 + u16 cplus_cmd; + u16 reg_e2; + u32 rxq0_dsc_st_addr_0; + u32 rxq0_dsc_st_addr_2; + u16 reg_ec; + u16 tx10midle_cnt; + //f0 + u16 misc0; + u16 misc1; + u32 timer_int3; + u32 cmac_ib; + u16 reg_fc; + u16 sw_rst; +}; +#pragma pack() + +struct rtl8127_regs_save { + union { + u8 mac_io[R8127_MAC_REGS_SIZE]; + + struct rtl8127_regs mac_reg; + }; + u16 pcie_phy[R8127_EPHY_REGS_SIZE/2]; + u16 eth_phy[R8127_PHY_REGS_SIZE/2]; + u32 eri_reg[R8127_ERI_REGS_SIZE/4]; + u32 pci_reg[R8127_PCI_REGS_SIZE/4]; + u16 sw_tail_ptr_reg[R8127_MAX_TX_QUEUES]; + u16 hw_clo_ptr_reg[R8127_MAX_TX_QUEUES]; + + //ktime_t begin_ktime; + //ktime_t end_ktime; + //u64 duration_ns; + + u16 sw0_tail_ptr; + u16 next_hwq0_clo_ptr; + u16 sw1_tail_ptr; + u16 next_hwq1_clo_ptr; + + u16 int_miti_rxq0; + u16 int_miti_txq0; + u16 int_miti_rxq1; + u16 int_miti_txq1; + u8 int_config; + u32 imr_new; + u32 isr_new; + + u8 tdu_status; + u16 rdu_status; + + u16 tc_mode; + + u32 txq1_dsc_st_addr_0; + u32 txq1_dsc_st_addr_2; + + u32 pla_tx_q0_idle_credit; + u32 pla_tx_q1_idle_credit; + + u32 rxq1_dsc_st_addr_0; + u32 rxq1_dsc_st_addr_2; + + u32 rss_ctrl; + u8 rss_key[RTL8127_RSS_KEY_SIZE]; + u8 rss_i_table[RTL8127_MAX_INDIRECTION_TABLE_ENTRIES]; + u16 rss_queue_num_sel_r; +}; + +struct rtl8127_counters { + /* legacy */ + u64 tx_packets; + u64 rx_packets; + u64 tx_errors; + u32 rx_errors; + u16 rx_missed; + u16 align_errors; + u32 tx_one_collision; + u32 tx_multi_collision; + u64 rx_unicast; + u64 rx_broadcast; + u32 rx_multicast; + u16 tx_aborted; + u16 tx_underrun; + + /* extended */ + u64 tx_octets; + u64 rx_octets; + u64 rx_multicast64; + u64 tx_unicast64; + u64 tx_broadcast64; + u64 tx_multicast64; + u32 tx_pause_on; + u32 tx_pause_off; + u32 tx_pause_all; + u32 tx_deferred; + u32 tx_late_collision; + u32 tx_all_collision; + u32 tx_aborted32; + u32 align_errors32; + u32 rx_frame_too_long; + u32 rx_runt; + u32 rx_pause_on; + u32 rx_pause_off; + u32 rx_pause_all; + u32 rx_unknown_opcode; + u32 rx_mac_error; + u32 tx_underrun32; + u32 rx_mac_missed; + u32 rx_tcam_dropped; + u32 tdu; + u32 rdu; +}; + +/* Flow Control Settings */ +enum rtl8127_fc_mode { + rtl8127_fc_none = 0, + rtl8127_fc_rx_pause, + rtl8127_fc_tx_pause, + rtl8127_fc_full, + rtl8127_fc_default +}; + +enum rtl8127_state_t { + __RTL8127_TESTING = 0, + __RTL8127_RESETTING, + __RTL8127_DOWN, + __RTL8127_PTP_TX_IN_PROGRESS, +}; + +#define RTL_FLAG_RX_HWTSTAMP_ENABLED BIT_0 + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0) +struct ethtool_eee { + __u32 cmd; + __u32 supported; + __u32 advertised; + __u32 lp_advertised; + __u32 eee_active; + __u32 eee_enabled; + __u32 tx_lpi_enabled; + __u32 tx_lpi_timer; + __u32 reserved[2]; +}; +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0) */ + +struct rtl8127_private { + void __iomem *mmio_addr; /* memory map physical address */ + struct pci_dev *pci_dev; /* Index of PCI device */ + struct net_device *dev; + struct r8127_napi r8127napi[R8127_MAX_MSIX_VEC]; + struct r8127_irq irq_tbl[R8127_MAX_MSIX_VEC]; + unsigned int irq_nvecs; + unsigned int max_irq_nvecs; + unsigned int min_irq_nvecs; + unsigned int hw_supp_irq_nvecs; + //struct msix_entry msix_entries[R8127_MAX_MSIX_VEC]; + struct net_device_stats stats; /* statistics of net device */ + unsigned long state; + u32 flags; + + u32 msg_enable; + u32 tx_tcp_csum_cmd; + u32 tx_udp_csum_cmd; + u32 tx_ip_csum_cmd; + u32 tx_ipv6_csum_cmd; + int max_jumbo_frame_size; + int chipset; + u32 mcfg; + //u32 cur_rx; /* Index into the Rx descriptor buffer of next Rx pkt. */ + //u32 cur_tx; /* Index into the Tx descriptor buffer of next Rx pkt. */ + //u32 dirty_rx; + //u32 dirty_tx; + //struct TxDesc *TxDescArray; /* 256-aligned Tx descriptor ring */ + //struct RxDesc *RxDescArray; /* 256-aligned Rx descriptor ring */ + //dma_addr_t TxPhyAddr; + //dma_addr_t RxPhyAddr; + //struct sk_buff *Rx_skbuff[MAX_NUM_RX_DESC]; /* Rx data buffers */ + //struct ring_info tx_skb[MAX_NUM_TX_DESC]; /* Tx data buffers */ + unsigned rx_buf_sz; +#ifdef ENABLE_PAGE_REUSE + unsigned rx_buf_page_order; + unsigned rx_buf_page_size; + u32 page_reuse_fail_cnt; +#endif //ENABLE_PAGE_REUSE + u16 HwSuppNumTxQueues; + u16 HwSuppNumRxQueues; + unsigned int num_tx_rings; + unsigned int num_rx_rings; + struct rtl8127_tx_ring tx_ring[R8127_MAX_TX_QUEUES]; + struct rtl8127_rx_ring rx_ring[R8127_MAX_RX_QUEUES]; +#ifdef ENABLE_LIB_SUPPORT + struct blocking_notifier_head lib_nh; + struct rtl8127_ring lib_tx_ring[R8127_MAX_TX_QUEUES]; + struct rtl8127_ring lib_rx_ring[R8127_MAX_RX_QUEUES]; +#endif + //struct timer_list esd_timer; + //struct timer_list link_timer; + struct pci_resource pci_cfg_space; + unsigned int esd_flag; + unsigned int pci_cfg_is_read; + unsigned int rtl8127_rx_config; + u16 rms; + u16 cp_cmd; + u32 intr_mask; + u32 timer_intr_mask; + u16 isr_reg[R8127_MAX_MSIX_VEC]; + u16 imr_reg[R8127_MAX_MSIX_VEC]; + int phy_auto_nego_reg; + int phy_1000_ctrl_reg; + int phy_2500_ctrl_reg; + u8 org_mac_addr[NODE_ADDRESS_SIZE]; + struct rtl8127_counters *tally_vaddr; + dma_addr_t tally_paddr; + +#ifdef CONFIG_R8127_VLAN + struct vlan_group *vlgrp; +#endif + u8 wol_enabled; + u32 wol_opts; + u8 efuse_ver; + u8 eeprom_type; + u8 autoneg; + u8 duplex; + u32 speed; + u64 advertising; + enum rtl8127_fc_mode fcpause; + u32 HwSuppMaxPhyLinkSpeed; + u16 eeprom_len; + u16 cur_page; + u32 bios_setting; + + int (*set_speed)(struct net_device *, u8 autoneg, u32 speed, u8 duplex, u64 adv); +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) + void (*get_settings)(struct net_device *, struct ethtool_cmd *); +#else + void (*get_settings)(struct net_device *, struct ethtool_link_ksettings *); +#endif + void (*phy_reset_enable)(struct net_device *); + unsigned int (*phy_reset_pending)(struct net_device *); + unsigned int (*link_ok)(struct net_device *); +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) + struct work_struct reset_task; + struct work_struct esd_task; + struct work_struct linkchg_task; +#else + struct delayed_work reset_task; + struct delayed_work esd_task; + struct delayed_work linkchg_task; +#endif + DECLARE_BITMAP(task_flags, R8127_FLAG_MAX); + unsigned features; + + u8 org_pci_offset_99; + u8 org_pci_offset_180; + u8 issue_offset_99_event; + + u8 org_pci_offset_80; + u8 org_pci_offset_81; + u8 use_timer_interrupt; + + u32 keep_intr_cnt; + + u8 HwIcVerUnknown; + u8 NotWrRamCodeToMicroP; + u8 NotWrMcuPatchCode; + u8 HwHasWrRamCodeToMicroP; + + u16 sw_ram_code_ver; + u16 hw_ram_code_ver; + + u8 rtk_enable_diag; + + u8 ShortPacketSwChecksum; + + u8 UseSwPaddingShortPkt; + + u8 RequireAdcBiasPatch; + u16 AdcBiasPatchIoffset; + + u8 RequireAdjustUpsTxLinkPulseTiming; + u16 SwrCnt1msIni; + + u8 HwSuppNowIsOobVer; + + u8 RequiredSecLanDonglePatch; + + u8 RequirePhyMdiSwapPatch; + + u32 HwFiberModeVer; + u32 HwFiberStat; + u8 HwSwitchMdiToFiber; + + u16 NicCustLedValue; + + u8 HwSuppMagicPktVer; + + u8 HwSuppLinkChgWakeUpVer; + + u8 HwSuppCheckPhyDisableModeVer; + + u8 random_mac; + + u16 phy_reg_aner; + u16 phy_reg_anlpar; + u16 phy_reg_gbsr; + u16 phy_reg_status_2500; + + u32 HwPcieSNOffset; + + u32 MaxTxDescPtrMask; + u8 HwSuppTxNoCloseVer; + u8 EnableTxNoClose; + + u8 HwSuppIsrVer; + u8 HwCurrIsrVer; + + u8 HwSuppIntMitiVer; + + u8 HwSuppExtendTallyCounterVer; + + u8 check_keep_link_speed; + u8 resume_not_chg_speed; + + u8 HwSuppD0SpeedUpVer; + u8 D0SpeedUpSpeed; + + u8 ring_lib_enabled; + + const char *fw_name; + struct rtl8127_fw *rtl_fw; + u32 ocp_base; + + //Dash+++++++++++++++++ + u8 HwSuppDashVer; + u8 DASH; + u8 dash_printer_enabled; + u8 HwPkgDet; + u8 AllowAccessDashOcp; + void __iomem *mapped_cmac_ioaddr; /* mapped cmac memory map physical address */ + void __iomem *cmac_ioaddr; /* cmac memory map physical address */ + +#ifdef ENABLE_DASH_SUPPORT + u16 AfterRecvFromFwBufLen; + u8 AfterRecvFromFwBuf[RECV_FROM_FW_BUF_SIZE]; + u16 AfterSendToFwBufLen; + u8 AfterSendToFwBuf[SEND_TO_FW_BUF_SIZE]; + u16 SendToFwBufferLen; + u32 SizeOfSendToFwBuffer; + u32 SizeOfSendToFwBufferMemAlloc; + u32 NumOfSendToFwBuffer; + + u8 OobReq; + u8 OobAck; + u32 OobReqComplete; + u32 OobAckComplete; + + u8 RcvFwReqSysOkEvt; + u8 RcvFwDashOkEvt; + u8 SendFwHostOkEvt; + + u8 DashFwDisableRx; + + void *UnalignedSendToFwBufferVa; + void *SendToFwBuffer; + u64 SendToFwBufferPhy; + u8 SendingToFw; + dma_addr_t UnalignedSendToFwBufferPa; + PTX_DASH_SEND_FW_DESC TxDashSendFwDesc; + u64 TxDashSendFwDescPhy; + u8 *UnalignedTxDashSendFwDescVa; + u32 SizeOfTxDashSendFwDescMemAlloc; + u32 SizeOfTxDashSendFwDesc; + u32 NumTxDashSendFwDesc; + u32 CurrNumTxDashSendFwDesc; + u32 LastSendNumTxDashSendFwDesc; + dma_addr_t UnalignedTxDashSendFwDescPa; + + u32 NumRecvFromFwBuffer; + u32 SizeOfRecvFromFwBuffer; + u32 SizeOfRecvFromFwBufferMemAlloc; + void *RecvFromFwBuffer; + u64 RecvFromFwBufferPhy; + + void *UnalignedRecvFromFwBufferVa; + dma_addr_t UnalignedRecvFromFwBufferPa; + PRX_DASH_FROM_FW_DESC RxDashRecvFwDesc; + u64 RxDashRecvFwDescPhy; + u8 *UnalignedRxDashRecvFwDescVa; + u32 SizeOfRxDashRecvFwDescMemAlloc; + u32 SizeOfRxDashRecvFwDesc; + u32 NumRxDashRecvFwDesc; + u32 CurrNumRxDashRecvFwDesc; + dma_addr_t UnalignedRxDashRecvFwDescPa; + u8 DashReqRegValue; + u16 HostReqValue; + + u32 CmacResetIsrCounter; + u8 CmacResetIntr; + u8 CmacResetting; + u8 CmacOobIssueCmacReset; + u32 CmacResetbyFwCnt; + +#if defined(ENABLE_DASH_PRINTER_SUPPORT) + struct completion fw_ack; + struct completion fw_req; + struct completion fw_host_ok; +#endif + //Dash----------------- +#endif //ENABLE_DASH_SUPPORT + + //Realwow++++++++++++++ + u8 HwSuppKCPOffloadVer; + + u8 EnableDhcpTimeoutWake; + u8 EnableTeredoOffload; + u8 EnableKCPOffload; +#ifdef ENABLE_REALWOW_SUPPORT + u32 DhcpTimeout; + MP_KCP_INFO MpKCPInfo; + //Realwow-------------- +#endif //ENABLE_REALWOW_SUPPORT + + struct ethtool_keee eee; + +#ifdef ENABLE_R8127_PROCFS + //Procfs support + struct proc_dir_entry *proc_dir; + struct proc_dir_entry *proc_dir_debug; + struct proc_dir_entry *proc_dir_test; +#endif +#ifdef ENABLE_R8127_SYSFS + //sysfs support + DECLARE_BITMAP(sysfs_flag, R8127_SYSFS_FLAG_MAX); + u32 testmode; +#endif + u8 HwSuppRxDescType; + u8 InitRxDescType; + u16 RxDescLength; //V1 16 Byte V2 32 Bytes + + spinlock_t phy_lock; + + u8 HwSuppPtpVer; + u8 EnablePtp; +#ifdef ENABLE_PTP_SUPPORT + u32 tx_hwtstamp_timeouts; + u32 tx_hwtstamp_skipped; + struct work_struct ptp_tx_work; + struct sk_buff *ptp_tx_skb; + struct hwtstamp_config hwtstamp_config; + unsigned long ptp_tx_start; + struct ptp_clock_info ptp_clock_info; + struct ptp_clock *ptp_clock; + u8 syncE_en; + u8 pps_enable; + struct hrtimer pps_timer; +#endif + + u8 HwSuppRssVer; + u8 EnableRss; + u16 HwSuppIndirTblEntries; +#ifdef ENABLE_RSS_SUPPORT + u32 rss_flags; + /* Receive Side Scaling settings */ + u8 rss_key[RTL8127_RSS_KEY_SIZE]; + u8 rss_indir_tbl[RTL8127_MAX_INDIRECTION_TABLE_ENTRIES]; + u32 rss_options; +#endif + + u8 HwSuppMacMcuVer; + u16 MacMcuPageSize; + u64 hw_mcu_patch_code_ver; + u64 bin_mcu_patch_code_ver; + + u8 HwSuppTcamVer; + + u16 TcamNotValidReg; + u16 TcamValidReg; + u16 TcamMaAddrcOffset; + u16 TcamVlanTagOffset; +}; + +#ifdef ENABLE_LIB_SUPPORT +static inline unsigned int +rtl8127_num_lib_tx_rings(struct rtl8127_private *tp) +{ + int count, i; + + for (count = 0, i = tp->num_tx_rings; i < tp->HwSuppNumTxQueues; i++) + if(tp->lib_tx_ring[i].enabled) + count++; + + return count; +} + +static inline unsigned int +rtl8127_num_lib_rx_rings(struct rtl8127_private *tp) +{ + int count, i; + + for (count = 0, i = tp->num_rx_rings; i < tp->HwSuppNumRxQueues; i++) + if(tp->lib_rx_ring[i].enabled) + count++; + + return count; +} + +#else +static inline unsigned int +rtl8127_num_lib_tx_rings(struct rtl8127_private *tp) +{ + return 0; +} + +static inline unsigned int +rtl8127_num_lib_rx_rings(struct rtl8127_private *tp) +{ + return 0; +} +#endif + +static inline unsigned int +rtl8127_tot_tx_rings(struct rtl8127_private *tp) +{ + return tp->num_tx_rings + rtl8127_num_lib_tx_rings(tp); +} + +static inline unsigned int +rtl8127_tot_rx_rings(struct rtl8127_private *tp) +{ + return tp->num_rx_rings + rtl8127_num_lib_rx_rings(tp); +} + +static inline struct netdev_queue *txring_txq(const struct rtl8127_tx_ring *ring) +{ + return netdev_get_tx_queue(ring->netdev, ring->index); +} + +enum eetype { + EEPROM_TYPE_NONE=0, + EEPROM_TYPE_93C46, + EEPROM_TYPE_93C56, + EEPROM_TWSI +}; + +enum mcfg { + CFG_METHOD_1=1, + CFG_METHOD_2, + CFG_METHOD_DEFAULT, + CFG_METHOD_MAX +}; + +#define LSO_32K 32000 +#define LSO_64K 64000 + +#define NIC_MIN_PHYS_BUF_COUNT (2) +#define NIC_MAX_PHYS_BUF_COUNT_LSO_64K (24) +#define NIC_MAX_PHYS_BUF_COUNT_LSO2 (16*4) + +#define GTTCPHO_SHIFT 18 +#define GTTCPHO_MAX 0x70U +#define GTPKTSIZE_MAX 0x3ffffU +#define TCPHO_SHIFT 18 +#define TCPHO_MAX 0x3ffU +#define LSOPKTSIZE_MAX 0xffffU +#define MSS_MAX 0x07ffu /* MSS value */ + +#define OOB_CMD_RESET 0x00 +#define OOB_CMD_DRIVER_START 0x05 +#define OOB_CMD_DRIVER_STOP 0x06 +#define OOB_CMD_SET_IPMAC 0x41 + +#define WAKEUP_MAGIC_PACKET_NOT_SUPPORT (0) +#define WAKEUP_MAGIC_PACKET_V1 (1) +#define WAKEUP_MAGIC_PACKET_V2 (2) +#define WAKEUP_MAGIC_PACKET_V3 (3) + +//Ram Code Version +#define NIC_RAMCODE_VERSION_CFG_METHOD_1 (0x0015) +#define NIC_RAMCODE_VERSION_CFG_METHOD_2 (0x0015) + +//hwoptimize +#define HW_PATCH_SOC_LAN (BIT_0) +#define HW_PATCH_SAMSUNG_LAN_DONGLE (BIT_2) + +static const u16 other_q_intr_mask = (RxOK1 | RxDU1); + +void rtl8127_mdio_write(struct rtl8127_private *tp, u16 RegAddr, u16 value); +void rtl8127_mdio_prot_write(struct rtl8127_private *tp, u32 RegAddr, u32 value); +void rtl8127_mdio_prot_direct_write_phy_ocp(struct rtl8127_private *tp, u32 RegAddr, u32 value); +u32 rtl8127_mdio_read(struct rtl8127_private *tp, u16 RegAddr); +u32 rtl8127_mdio_prot_read(struct rtl8127_private *tp, u32 RegAddr); +u32 rtl8127_mdio_prot_direct_read_phy_ocp(struct rtl8127_private *tp, u32 RegAddr); +void rtl8127_ephy_write(struct rtl8127_private *tp, int RegAddr, int value); +void rtl8127_mac_ocp_write(struct rtl8127_private *tp, u16 reg_addr, u16 value); +u16 rtl8127_mac_ocp_read(struct rtl8127_private *tp, u16 reg_addr); +void rtl8127_clear_eth_phy_bit(struct rtl8127_private *tp, u8 addr, u16 mask); +void rtl8127_set_eth_phy_bit(struct rtl8127_private *tp, u8 addr, u16 mask); +void rtl8127_ocp_write(struct rtl8127_private *tp, u16 addr, u8 len, u32 data); +void rtl8127_oob_notify(struct rtl8127_private *tp, u8 cmd); +void rtl8127_init_ring_indexes(struct rtl8127_private *tp); +void rtl8127_oob_mutex_lock(struct rtl8127_private *tp); +u32 rtl8127_ocp_read(struct rtl8127_private *tp, u16 addr, u8 len); +u32 rtl8127_ocp_read_with_oob_base_address(struct rtl8127_private *tp, u16 addr, u8 len, u32 base_address); +u32 rtl8127_ocp_write_with_oob_base_address(struct rtl8127_private *tp, u16 addr, u8 len, u32 value, u32 base_address); +u32 rtl8127_eri_read(struct rtl8127_private *tp, int addr, int len, int type); +u32 rtl8127_eri_read_with_oob_base_address(struct rtl8127_private *tp, int addr, int len, int type, u32 base_address); +int rtl8127_eri_write(struct rtl8127_private *tp, int addr, int len, u32 value, int type); +int rtl8127_eri_write_with_oob_base_address(struct rtl8127_private *tp, int addr, int len, u32 value, int type, u32 base_address); +u16 rtl8127_ephy_read(struct rtl8127_private *tp, int RegAddr); +void rtl8127_wait_txrx_fifo_empty(struct net_device *dev); +void rtl8127_enable_now_is_oob(struct rtl8127_private *tp); +void rtl8127_disable_now_is_oob(struct rtl8127_private *tp); +void rtl8127_oob_mutex_unlock(struct rtl8127_private *tp); +void rtl8127_dash2_disable_tx(struct rtl8127_private *tp); +void rtl8127_dash2_enable_tx(struct rtl8127_private *tp); +void rtl8127_dash2_disable_rx(struct rtl8127_private *tp); +void rtl8127_dash2_enable_rx(struct rtl8127_private *tp); +void rtl8127_hw_disable_mac_mcu_bps(struct net_device *dev); +void rtl8127_mark_to_asic(struct rtl8127_private *tp, struct RxDesc *desc, u32 rx_buf_sz); +void rtl8127_mark_as_last_descriptor(struct rtl8127_private *tp, struct RxDesc *desc); + +static inline void +rtl8127_make_unusable_by_asic(struct rtl8127_private *tp, + struct RxDesc *desc) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + ((struct RxDescV3 *)desc)->addr = RTL8127_MAGIC_NUMBER; + ((struct RxDescV3 *)desc)->RxDescNormalDDWord4.opts1 &= ~cpu_to_le32(DescOwn | RsvdMaskV3); + break; + case RX_DESC_RING_TYPE_4: + ((struct RxDescV4 *)desc)->addr = RTL8127_MAGIC_NUMBER; + ((struct RxDescV4 *)desc)->RxDescNormalDDWord2.opts1 &= ~cpu_to_le32(DescOwn | RsvdMaskV4); + break; + default: + desc->addr = RTL8127_MAGIC_NUMBER; + desc->opts1 &= ~cpu_to_le32(DescOwn | RsvdMask); + break; + } +} + +static inline struct RxDesc* +rtl8127_get_rxdesc(struct rtl8127_private *tp, struct RxDesc *RxDescBase, u32 const cur_rx) +{ + return (struct RxDesc*)((u8*)RxDescBase + (cur_rx * tp->RxDescLength)); +} + +static inline void +rtl8127_disable_hw_interrupt_v2(struct rtl8127_private *tp, + u32 message_id) +{ + RTL_W32(tp, IMR_V2_CLEAR_REG_8125, BIT(message_id)); +} + +static inline void +rtl8127_enable_hw_interrupt_v2(struct rtl8127_private *tp, u32 message_id) +{ + RTL_W32(tp, IMR_V2_SET_REG_8125, BIT(message_id)); +} + +int rtl8127_open(struct net_device *dev); +int rtl8127_close(struct net_device *dev); +void rtl8127_hw_config(struct net_device *dev); +void rtl8127_hw_set_timer_int(struct rtl8127_private *tp, u32 message_id, u8 timer_intmiti_val); +void rtl8127_set_rx_q_num(struct rtl8127_private *tp, unsigned int num_rx_queues); +void rtl8127_set_tx_q_num(struct rtl8127_private *tp, unsigned int num_tx_queues); +void rtl8127_enable_mcu(struct rtl8127_private *tp, bool enable); +void rtl8127_hw_start(struct net_device *dev); +void rtl8127_hw_reset(struct net_device *dev); +void rtl8127_tx_clear(struct rtl8127_private *tp); +void rtl8127_rx_clear(struct rtl8127_private *tp); +int rtl8127_init_ring(struct net_device *dev); +void rtl8127_hw_set_rx_packet_filter(struct net_device *dev); +void rtl8127_enable_hw_linkchg_interrupt(struct rtl8127_private *tp); +int rtl8127_dump_tally_counter(struct rtl8127_private *tp, dma_addr_t paddr); +void rtl8127_enable_napi(struct rtl8127_private *tp); +void _rtl8127_wait_for_quiescence(struct net_device *dev); + +void rtl8127_mdio_direct_write_phy_ocp(struct rtl8127_private *tp, u16 RegAddr,u16 value); +u32 rtl8127_mdio_direct_read_phy_ocp(struct rtl8127_private *tp, u16 RegAddr); +void rtl8127_clear_and_set_eth_phy_ocp_bit(struct rtl8127_private *tp, u16 addr, u16 clearmask, u16 setmask); +void rtl8127_clear_eth_phy_ocp_bit(struct rtl8127_private *tp, u16 addr, u16 mask); +void rtl8127_set_eth_phy_ocp_bit(struct rtl8127_private *tp, u16 addr, u16 mask); + +void rtl8127_clear_mac_ocp_bit(struct rtl8127_private *tp, u16 addr, u16 mask); + +#ifndef ENABLE_LIB_SUPPORT +static inline void rtl8127_lib_reset_prepare(struct rtl8127_private *tp) { } +static inline void rtl8127_lib_reset_complete(struct rtl8127_private *tp) { } +#endif + +#define HW_SUPPORT_CHECK_PHY_DISABLE_MODE(_M) ((_M)->HwSuppCheckPhyDisableModeVer > 0) +#define HW_HAS_WRITE_PHY_MCU_RAM_CODE(_M) (((_M)->HwHasWrRamCodeToMicroP == TRUE) ? 1 : 0) +#define HW_SUPPORT_D0_SPEED_UP(_M) ((_M)->HwSuppD0SpeedUpVer > 0) +#define HW_SUPPORT_MAC_MCU(_M) ((_M)->HwSuppMacMcuVer > 0) +#define HW_SUPPORT_TCAM(_M) ((_M)->HwSuppTcamVer > 0) + +#define HW_SUPP_PHY_LINK_SPEED_GIGA(_M) ((_M)->HwSuppMaxPhyLinkSpeed >= 1000) +#define HW_SUPP_PHY_LINK_SPEED_2500M(_M) ((_M)->HwSuppMaxPhyLinkSpeed >= 2500) +#define HW_SUPP_PHY_LINK_SPEED_5000M(_M) ((_M)->HwSuppMaxPhyLinkSpeed >= 5000) +#define HW_SUPP_PHY_LINK_SPEED_10000M(_M) ((_M)->HwSuppMaxPhyLinkSpeed >= 10000) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,34) +#define netdev_mc_count(dev) ((dev)->mc_count) +#define netdev_mc_empty(dev) (netdev_mc_count(dev) == 0) +#define netdev_for_each_mc_addr(mclist, dev) \ + for (mclist = dev->mc_list; mclist; mclist = mclist->next) +#endif + +#endif /* __R8127_H */ diff --git a/drivers/net/ethernet/realtek/r8127/r8127_dash.h b/drivers/net/ethernet/realtek/r8127/r8127_dash.h new file mode 100755 index 0000000000000..0f6a3d150a005 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/r8127_dash.h @@ -0,0 +1,261 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#ifndef _LINUX_R8127_DASH_H +#define _LINUX_R8127_DASH_H + +#include + +#define SIOCDEVPRIVATE_RTLDASH SIOCDEVPRIVATE+2 + +enum rtl_dash_cmd { + RTL_DASH_ARP_NS_OFFLOAD = 0, + RTL_DASH_SET_OOB_IPMAC, + RTL_DASH_NOTIFY_OOB, + + RTL_DASH_SEND_BUFFER_DATA_TO_DASH_FW, + RTL_DASH_CHECK_SEND_BUFFER_TO_DASH_FW_COMPLETE, + RTL_DASH_GET_RCV_FROM_FW_BUFFER_DATA, + RTL_DASH_OOB_REQ, + RTL_DASH_OOB_ACK, + RTL_DASH_DETACH_OOB_REQ, + RTL_DASH_DETACH_OOB_ACK, + + RTL_FW_SET_IPV4 = 0x10, + RTL_FW_GET_IPV4, + RTL_FW_SET_IPV6, + RTL_FW_GET_IPV6, + RTL_FW_SET_EXT_SNMP, + RTL_FW_GET_EXT_SNMP, + RTL_FW_SET_WAKEUP_PATTERN, + RTL_FW_GET_WAKEUP_PATTERN, + RTL_FW_DEL_WAKEUP_PATTERN, + + RTLT_DASH_COMMAND_INVALID, +}; + +struct rtl_dash_ip_mac { + struct sockaddr ifru_addr; + struct sockaddr ifru_netmask; + struct sockaddr ifru_hwaddr; +}; + +struct rtl_dash_ioctl_struct { + __u32 cmd; + __u32 offset; + __u32 len; + union { + __u32 data; + void *data_buffer; + }; +}; + +struct settings_ipv4 { + __u32 IPv4addr; + __u32 IPv4mask; + __u32 IPv4Gateway; +}; + +struct settings_ipv6 { + __u32 reserved; + __u32 prefixLen; + __u16 IPv6addr[8]; + __u16 IPv6Gateway[8]; +}; + +struct settings_ext_snmp { + __u16 index; + __u16 oid_get_len; + __u8 oid_for_get[24]; + __u8 reserved0[26]; + __u16 value_len; + __u8 value[256]; + __u8 supported; + __u8 reserved1[27]; +}; + +struct wakeup_pattern { + __u8 index; + __u8 valid; + __u8 start; + __u8 length; + __u8 name[36]; + __u8 mask[16]; + __u8 pattern[128]; + __u32 reserved[2]; +}; + +typedef struct _RX_DASH_FROM_FW_DESC { + u16 length; + u8 statusLowByte; + u8 statusHighByte; + u32 resv; + u64 BufferAddress; +} +RX_DASH_FROM_FW_DESC, *PRX_DASH_FROM_FW_DESC; + +typedef struct _TX_DASH_SEND_FW_DESC { + u16 length; + u8 statusLowByte; + u8 statusHighByte; + u32 resv; + u64 BufferAddress; +} +TX_DASH_SEND_FW_DESC, *PTX_DASH_SEND_FW_DESC; + +typedef struct _OSOOBHdr { + u32 len; + u8 type; + u8 flag; + u8 hostReqV; + u8 res; +} +OSOOBHdr, *POSOOBHdr; + +typedef struct _RX_DASH_BUFFER_TYPE_2 { + OSOOBHdr oobhdr; + u8 RxDataBuffer[0]; +} +RX_DASH_BUFFER_TYPE_2, *PRX_DASH_BUFFER_TYPE_2; + +#define ALIGN_8 (0x7) +#define ALIGN_16 (0xf) +#define ALIGN_32 (0x1f) +#define ALIGN_64 (0x3f) +#define ALIGN_256 (0xff) +#define ALIGN_4096 (0xfff) + +#define OCP_REG_CONFIG0 (0x10) +#define OCP_REG_CONFIG0_REV_F (0xB8) +#define OCP_REG_DASH_POLL (0x30) +#define OCP_REG_HOST_REQ (0x34) +#define OCP_REG_DASH_REQ (0x35) +#define OCP_REG_CR (0x36) +#define OCP_REG_DMEMSTA (0x38) +#define OCP_REG_GPHYAR (0x60) + + +#define OCP_REG_CONFIG0_DASHEN BIT_15 +#define OCP_REG_CONFIG0_OOBRESET BIT_14 +#define OCP_REG_CONFIG0_APRDY BIT_13 +#define OCP_REG_CONFIG0_FIRMWARERDY BIT_12 +#define OCP_REG_CONFIG0_DRIVERRDY BIT_11 +#define OCP_REG_CONFIG0_OOB_WDT BIT_9 +#define OCP_REG_CONFIG0_DRV_WAIT_OOB BIT_8 +#define OCP_REG_CONFIG0_TLSEN BIT_7 + +#define HW_DASH_SUPPORT_DASH(_M) ((_M)->HwSuppDashVer > 0) +#define HW_DASH_SUPPORT_TYPE_1(_M) ((_M)->HwSuppDashVer == 1) +#define HW_DASH_SUPPORT_TYPE_2(_M) ((_M)->HwSuppDashVer == 2) +#define HW_DASH_SUPPORT_TYPE_3(_M) ((_M)->HwSuppDashVer == 3) + +#define RECV_FROM_FW_BUF_SIZE (1520) +#define SEND_TO_FW_BUF_SIZE (1520) + +#define RX_DASH_FROM_FW_OWN BIT_15 +#define TX_DASH_SEND_FW_OWN BIT_15 +#define TX_DASH_SEND_FW_OWN_HIGHBYTE BIT_7 + +#define TXS_CC3_0 (BIT_0|BIT_1|BIT_2|BIT_3) +#define TXS_EXC BIT_4 +#define TXS_LNKF BIT_5 +#define TXS_OWC BIT_6 +#define TXS_TES BIT_7 +#define TXS_UNF BIT_9 +#define TXS_LGSEN BIT_11 +#define TXS_LS BIT_12 +#define TXS_FS BIT_13 +#define TXS_EOR BIT_14 +#define TXS_OWN BIT_15 + +#define TPPool_HRDY 0x20 + +#define HostReqReg (0xC0) +#define SystemMasterDescStartAddrLow (0xF0) +#define SystemMasterDescStartAddrHigh (0xF4) +#define SystemSlaveDescStartAddrLow (0xF8) +#define SystemSlaveDescStartAddrHigh (0xFC) + +//DASH Request Type +#define WSMANREG 0x01 +#define OSPUSHDATA 0x02 + +#define RXS_OWN BIT_15 +#define RXS_EOR BIT_14 +#define RXS_FS BIT_13 +#define RXS_LS BIT_12 + +#define ISRIMR_DP_DASH_OK BIT_15 +#define ISRIMR_DP_HOST_OK BIT_13 +#define ISRIMR_DP_REQSYS_OK BIT_11 + +#define ISRIMR_DASH_INTR_EN BIT_12 +#define ISRIMR_DASH_INTR_CMAC_RESET BIT_15 + +#define ISRIMR_DASH_TYPE2_ROK BIT_0 +#define ISRIMR_DASH_TYPE2_RDU BIT_1 +#define ISRIMR_DASH_TYPE2_TOK BIT_2 +#define ISRIMR_DASH_TYPE2_TDU BIT_3 +#define ISRIMR_DASH_TYPE2_TX_FIFO_FULL BIT_4 +#define ISRIMR_DASH_TYPE2_TX_DISABLE_IDLE BIT_5 +#define ISRIMR_DASH_TYPE2_RX_DISABLE_IDLE BIT_6 + +#define CMAC_OOB_STOP 0x25 +#define CMAC_OOB_INIT 0x26 +#define CMAC_OOB_RESET 0x2a + +#define NO_BASE_ADDRESS 0x00000000 +#define RTL8168FP_OOBMAC_BASE 0xBAF70000 +#define RTL8168FP_CMAC_IOBASE 0xBAF20000 +#define RTL8168FP_KVM_BASE 0xBAF80400 +#define CMAC_SYNC_REG 0x20 +#define CMAC_RXDESC_OFFSET 0x90 //RX: 0x90 - 0x98 +#define CMAC_TXDESC_OFFSET 0x98 //TX: 0x98 - 0x9F + +/* cmac write/read MMIO register */ +#define RTL_CMAC_W8(tp, reg, val8) writeb ((val8), tp->cmac_ioaddr + (reg)) +#define RTL_CMAC_W16(tp, reg, val16) writew ((val16), tp->cmac_ioaddr + (reg)) +#define RTL_CMAC_W32(tp, reg, val32) writel ((val32), tp->cmac_ioaddr + (reg)) +#define RTL_CMAC_R8(tp, reg) readb (tp->cmac_ioaddr + (reg)) +#define RTL_CMAC_R16(tp, reg) readw (tp->cmac_ioaddr + (reg)) +#define RTL_CMAC_R32(tp, reg) ((unsigned long) readl (tp->cmac_ioaddr + (reg))) + +int rtl8127_dash_ioctl(struct net_device *dev, struct ifreq *ifr); +void HandleDashInterrupt(struct net_device *dev); +int AllocateDashShareMemory(struct net_device *dev); +void FreeAllocatedDashShareMemory(struct net_device *dev); +void DashHwInit(struct net_device *dev); + + +#endif /* _LINUX_R8127_DASH_H */ diff --git a/drivers/net/ethernet/realtek/r8127/r8127_firmware.h b/drivers/net/ethernet/realtek/r8127/r8127_firmware.h new file mode 100755 index 0000000000000..6b1acea98a3f8 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/r8127_firmware.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#ifndef _LINUX_R8127_FIRMWARE_H +#define _LINUX_R8127_FIRMWARE_H + +#include +#include + +struct rtl8127_private; +typedef void (*rtl8127_fw_write_t)(struct rtl8127_private *tp, u16 reg, u16 val); +typedef u32 (*rtl8127_fw_read_t)(struct rtl8127_private *tp, u16 reg); + +#define RTL8127_VER_SIZE 32 + +struct rtl8127_fw { + rtl8127_fw_write_t phy_write; + rtl8127_fw_read_t phy_read; + rtl8127_fw_write_t mac_mcu_write; + rtl8127_fw_read_t mac_mcu_read; + const struct firmware *fw; + const char *fw_name; + struct device *dev; + + char version[RTL8127_VER_SIZE]; + + struct rtl8127_fw_phy_action { + __le32 *code; + size_t size; + } phy_action; +}; + +int rtl8127_fw_request_firmware(struct rtl8127_fw *rtl_fw); +void rtl8127_fw_release_firmware(struct rtl8127_fw *rtl_fw); +void rtl8127_fw_write_firmware(struct rtl8127_private *tp, struct rtl8127_fw *rtl_fw); + +#endif /* _LINUX_R8127_FIRMWARE_H */ diff --git a/drivers/net/ethernet/realtek/r8127/r8127_n.c b/drivers/net/ethernet/realtek/r8127/r8127_n.c new file mode 100755 index 0000000000000..03d99ae8d3853 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/r8127_n.c @@ -0,0 +1,17752 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +/* + * This driver is modified from r8169.c in Linux kernel 2.6.18 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,22) +#include +#include +#endif +#include +#include +#include +#include + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0) +#include +#endif +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,37) +#include +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) +#define dev_printk(A,B,fmt,args...) printk(A fmt,##args) +#else +#include +#include +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,31) +#include +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,4,10) +#include +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6,4,10) */ + +#include +#include + +#include "r8127.h" +#include "rtl_eeprom.h" +#include "rtltool.h" +#include "r8127_firmware.h" + +#ifdef ENABLE_R8127_PROCFS +#include +#include +#endif + +#define FIRMWARE_8127_1 "rtl_nic/rtl8127-1.fw" +#define FIRMWARE_8127_2 "rtl_nic/rtl8127-2.fw" + +static const struct { + const char *name; + const char *fw_name; +} rtl_chip_fw_infos[] = { + /* PCI-E devices. */ + [CFG_METHOD_1] = {"RTL8127", FIRMWARE_8127_1}, + [CFG_METHOD_2] = {"RTL8127", FIRMWARE_8127_2}, + [CFG_METHOD_DEFAULT] = {"Unknown", }, +}; + +#define _R(NAME,MAC,RCR,MASK,JumFrameSz) \ + { .name = NAME, .mcfg = MAC, .RCR_Cfg = RCR, .RxConfigMask = MASK, .jumbo_frame_sz = JumFrameSz } + +static const struct { + const char *name; + u8 mcfg; + u32 RCR_Cfg; + u32 RxConfigMask; /* Clears the bits supported by this chip */ + u32 jumbo_frame_sz; +} rtl_chip_info[] = { + _R("RTL8127", + CFG_METHOD_1, + Rx_Fetch_Number_8 | Rx_Close_Multiple | RxCfg_pause_slot_en | EnableInnerVlan | EnableOuterVlan | (RX_DMA_BURST_512 << RxCfgDMAShift), + 0xff7e5880, + Jumbo_Frame_9k), + + _R("RTL8127", + CFG_METHOD_2, + Rx_Fetch_Number_8 | Rx_Close_Multiple | RxCfg_pause_slot_en | EnableInnerVlan | EnableOuterVlan | (RX_DMA_BURST_512 << RxCfgDMAShift), + 0xff7e5880, + Jumbo_Frame_9k), + + _R("Unknown", + CFG_METHOD_DEFAULT, + (RX_DMA_BURST_512 << RxCfgDMAShift), + 0xff7e5880, + Jumbo_Frame_1k) +}; +#undef _R + + +static struct pci_device_id rtl8127_pci_tbl[] = { + { PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0x8127), }, + { PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0x0E10), }, + {0,}, +}; + +MODULE_DEVICE_TABLE(pci, rtl8127_pci_tbl); + +static int use_dac = 1; +static int timer_count = 0x2600; +static int timer_count_v2 = (0x2600 / 0x200); + +static struct { + u32 msg_enable; +} debug = { -1 }; + +static unsigned int speed_mode = SPEED_10000; +static unsigned int duplex_mode = DUPLEX_FULL; +static unsigned int autoneg_mode = AUTONEG_ENABLE; +#ifdef CONFIG_ASPM +static int aspm = 1; +#else +static int aspm = 0; +#endif +#ifdef ENABLE_S5WOL +static int s5wol = 1; +#else +static int s5wol = 0; +#endif +#ifdef ENABLE_S5_KEEP_CURR_MAC +static int s5_keep_curr_mac = 1; +#else +static int s5_keep_curr_mac = 0; +#endif +#ifdef ENABLE_EEE +static int eee_enable = 1; +#else +static int eee_enable = 0; +#endif +#ifdef CONFIG_SOC_LAN +static ulong hwoptimize = HW_PATCH_SOC_LAN; +#else +static ulong hwoptimize = 0; +#endif +#ifdef ENABLE_S0_MAGIC_PACKET +static int s0_magic_packet = 1; +#else +static int s0_magic_packet = 0; +#endif +#ifdef ENABLE_TX_NO_CLOSE +static int tx_no_close_enable = 1; +#else +static int tx_no_close_enable = 0; +#endif +#ifdef DISABLE_WOL_SUPPORT +static int disable_wol_support = 1; +#else +static int disable_wol_support = 0; +#endif +#ifdef ENABLE_DOUBLE_VLAN +static int enable_double_vlan = 1; +#else +static int enable_double_vlan = 0; +#endif +#ifdef ENABLE_GIGA_LITE +static int eee_giga_lite = 1; +#else +static int eee_giga_lite = 0; +#endif + +MODULE_AUTHOR("Realtek and the Linux r8127 crew "); +MODULE_DESCRIPTION("Realtek r8127 Ethernet controller driver"); + +module_param(speed_mode, uint, 0); +MODULE_PARM_DESC(speed_mode, "force phy operation. Deprecated by ethtool (8)."); + +module_param(duplex_mode, uint, 0); +MODULE_PARM_DESC(duplex_mode, "force phy operation. Deprecated by ethtool (8)."); + +module_param(autoneg_mode, uint, 0); +MODULE_PARM_DESC(autoneg_mode, "force phy operation. Deprecated by ethtool (8)."); + +module_param(aspm, int, 0); +MODULE_PARM_DESC(aspm, "Enable ASPM."); + +module_param(s5wol, int, 0); +MODULE_PARM_DESC(s5wol, "Enable Shutdown Wake On Lan."); + +module_param(s5_keep_curr_mac, int, 0); +MODULE_PARM_DESC(s5_keep_curr_mac, "Enable Shutdown Keep Current MAC Address."); + +module_param(use_dac, int, 0); +MODULE_PARM_DESC(use_dac, "Enable PCI DAC. Unsafe on 32 bit PCI slot."); + +module_param(timer_count, int, 0); +MODULE_PARM_DESC(timer_count, "Timer Interrupt Interval."); + +module_param(eee_enable, int, 0); +MODULE_PARM_DESC(eee_enable, "Enable Energy Efficient Ethernet."); + +module_param(hwoptimize, ulong, 0); +MODULE_PARM_DESC(hwoptimize, "Enable HW optimization function."); + +module_param(s0_magic_packet, int, 0); +MODULE_PARM_DESC(s0_magic_packet, "Enable S0 Magic Packet."); + +module_param(tx_no_close_enable, int, 0); +MODULE_PARM_DESC(tx_no_close_enable, "Enable TX No Close."); + +module_param(disable_wol_support, int, 0); +MODULE_PARM_DESC(disable_wol_support, "Disable PM support."); + +module_param(enable_double_vlan, int, 0); +MODULE_PARM_DESC(enable_double_vlan, "Enable Double VLAN."); + +module_param(eee_giga_lite, int, 0); +MODULE_PARM_DESC(eee_giga_lite, "Enable Giga Lite."); + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) +module_param_named(debug, debug.msg_enable, int, 0); +MODULE_PARM_DESC(debug, "Debug verbosity level (0=none, ..., 16=all)"); +#endif//LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + +MODULE_LICENSE("GPL"); +#ifdef ENABLE_USE_FIRMWARE_FILE +MODULE_FIRMWARE(FIRMWARE_8127_1); +MODULE_FIRMWARE(FIRMWARE_8127_2); +#endif + +MODULE_VERSION(RTL8127_VERSION); + +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) +static void rtl8127_esd_timer(unsigned long __opaque); +#else +static void rtl8127_esd_timer(struct timer_list *t); +#endif +*/ +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) +static void rtl8127_link_timer(unsigned long __opaque); +#else +static void rtl8127_link_timer(struct timer_list *t); +#endif +*/ + +static netdev_tx_t rtl8127_start_xmit(struct sk_buff *skb, struct net_device *dev); +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) +static irqreturn_t rtl8127_interrupt(int irq, void *dev_instance, struct pt_regs *regs); +#else +static irqreturn_t rtl8127_interrupt(int irq, void *dev_instance); +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) +static irqreturn_t rtl8127_interrupt_msix(int irq, void *dev_instance, struct pt_regs *regs); +#else +static irqreturn_t rtl8127_interrupt_msix(int irq, void *dev_instance); +#endif +static void rtl8127_set_rx_mode(struct net_device *dev); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0) +static void rtl8127_tx_timeout(struct net_device *dev, unsigned int txqueue); +#else +static void rtl8127_tx_timeout(struct net_device *dev); +#endif +static int rtl8127_rx_interrupt(struct net_device *, struct rtl8127_private *, struct rtl8127_rx_ring *, napi_budget); +static int rtl8127_tx_interrupt(struct rtl8127_tx_ring *ring, int budget); +static int rtl8127_tx_interrupt_with_vector(struct rtl8127_private *tp, const int message_id, int budget); +static void rtl8127_wait_for_quiescence(struct net_device *dev); +static int rtl8127_change_mtu(struct net_device *dev, int new_mtu); +static void rtl8127_down(struct net_device *dev); + +static int rtl8127_set_mac_address(struct net_device *dev, void *p); +static void rtl8127_rar_set(struct rtl8127_private *tp, const u8 *addr); +static void rtl8127_desc_addr_fill(struct rtl8127_private *); +static void rtl8127_tx_desc_init(struct rtl8127_private *tp); +static void rtl8127_rx_desc_init(struct rtl8127_private *tp); + +static u16 rtl8127_get_hw_phy_mcu_code_ver(struct rtl8127_private *tp); +static void rtl8127_phy_power_up(struct net_device *dev); +static void rtl8127_phy_power_down(struct net_device *dev); +static int rtl8127_set_speed(struct net_device *dev, u8 autoneg, u32 speed, u8 duplex, u64 adv); +static bool rtl8127_set_phy_mcu_patch_request(struct rtl8127_private *tp); +static bool rtl8127_clear_phy_mcu_patch_request(struct rtl8127_private *tp); + +#ifdef CONFIG_R8127_NAPI +static int rtl8127_poll(napi_ptr napi, napi_budget budget); +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) +static void rtl8127_reset_task(void *_data); +static void rtl8127_esd_task(void *_data); +static void rtl8127_linkchg_task(void *_data); +#else +static void rtl8127_reset_task(struct work_struct *work); +static void rtl8127_esd_task(struct work_struct *work); +static void rtl8127_linkchg_task(struct work_struct *work); +#endif +static void rtl8127_schedule_reset_work(struct rtl8127_private *tp); +static void rtl8127_schedule_esd_work(struct rtl8127_private *tp); +static void rtl8127_schedule_linkchg_work(struct rtl8127_private *tp); +static void rtl8127_init_all_schedule_work(struct rtl8127_private *tp); +static void rtl8127_cancel_all_schedule_work(struct rtl8127_private *tp); + +static inline struct device *tp_to_dev(struct rtl8127_private *tp) +{ + return &tp->pci_dev->dev; +} + +#if ((LINUX_VERSION_CODE < KERNEL_VERSION(4,7,0) && \ + LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,00))) +void ethtool_convert_legacy_u32_to_link_mode(unsigned long *dst, + u32 legacy_u32) +{ + bitmap_zero(dst, __ETHTOOL_LINK_MODE_MASK_NBITS); + dst[0] = legacy_u32; +} + +bool ethtool_convert_link_mode_to_legacy_u32(u32 *legacy_u32, + const unsigned long *src) +{ + bool retval = true; + + /* TODO: following test will soon always be true */ + if (__ETHTOOL_LINK_MODE_MASK_NBITS > 32) { + __ETHTOOL_DECLARE_LINK_MODE_MASK(ext); + + bitmap_zero(ext, __ETHTOOL_LINK_MODE_MASK_NBITS); + bitmap_fill(ext, 32); + bitmap_complement(ext, ext, __ETHTOOL_LINK_MODE_MASK_NBITS); + if (bitmap_intersects(ext, src, + __ETHTOOL_LINK_MODE_MASK_NBITS)) { + /* src mask goes beyond bit 31 */ + retval = false; + } + } + *legacy_u32 = src[0]; + return retval; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) + +#ifndef LPA_1000FULL +#define LPA_1000FULL 0x0800 +#endif + +#ifndef LPA_1000HALF +#define LPA_1000HALF 0x0400 +#endif + +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,4,0) +static inline void eth_hw_addr_random(struct net_device *dev) +{ + random_ether_addr(dev->dev_addr); +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) +#undef ethtool_ops +#define ethtool_ops _kc_ethtool_ops + +struct _kc_ethtool_ops { + int (*get_settings)(struct net_device *, struct ethtool_cmd *); + int (*set_settings)(struct net_device *, struct ethtool_cmd *); + void (*get_drvinfo)(struct net_device *, struct ethtool_drvinfo *); + int (*get_regs_len)(struct net_device *); + void (*get_regs)(struct net_device *, struct ethtool_regs *, void *); + void (*get_wol)(struct net_device *, struct ethtool_wolinfo *); + int (*set_wol)(struct net_device *, struct ethtool_wolinfo *); + u32 (*get_msglevel)(struct net_device *); + void (*set_msglevel)(struct net_device *, u32); + int (*nway_reset)(struct net_device *); + u32 (*get_link)(struct net_device *); + int (*get_eeprom_len)(struct net_device *); + int (*get_eeprom)(struct net_device *, struct ethtool_eeprom *, u8 *); + int (*set_eeprom)(struct net_device *, struct ethtool_eeprom *, u8 *); + int (*get_coalesce)(struct net_device *, struct ethtool_coalesce *); + int (*set_coalesce)(struct net_device *, struct ethtool_coalesce *); + void (*get_ringparam)(struct net_device *, struct ethtool_ringparam *); + int (*set_ringparam)(struct net_device *, struct ethtool_ringparam *); + void (*get_pauseparam)(struct net_device *, + struct ethtool_pauseparam*); + int (*set_pauseparam)(struct net_device *, + struct ethtool_pauseparam*); + u32 (*get_rx_csum)(struct net_device *); + int (*set_rx_csum)(struct net_device *, u32); + u32 (*get_tx_csum)(struct net_device *); + int (*set_tx_csum)(struct net_device *, u32); + u32 (*get_sg)(struct net_device *); + int (*set_sg)(struct net_device *, u32); + u32 (*get_tso)(struct net_device *); + int (*set_tso)(struct net_device *, u32); + int (*self_test_count)(struct net_device *); + void (*self_test)(struct net_device *, struct ethtool_test *, u64 *); + void (*get_strings)(struct net_device *, u32 stringset, u8 *); + int (*phys_id)(struct net_device *, u32); + int (*get_stats_count)(struct net_device *); + void (*get_ethtool_stats)(struct net_device *, struct ethtool_stats *, + u64 *); +} *ethtool_ops = NULL; + +#undef SET_ETHTOOL_OPS +#define SET_ETHTOOL_OPS(netdev, ops) (ethtool_ops = (ops)) + +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) +#ifndef SET_ETHTOOL_OPS +#define SET_ETHTOOL_OPS(netdev,ops) \ + ((netdev)->ethtool_ops = (ops)) +#endif //SET_ETHTOOL_OPS +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) + +//#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,5) +#ifndef netif_msg_init +#define netif_msg_init _kc_netif_msg_init +/* copied from linux kernel 2.6.20 include/linux/netdevice.h */ +static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits) +{ + /* use default */ + if (debug_value < 0 || debug_value >= (sizeof(u32) * 8)) + return default_msg_enable_bits; + if (debug_value == 0) /* no output */ + return 0; + /* set low N bits */ + return (1 << debug_value) - 1; +} + +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,5) + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,22) +static inline void eth_copy_and_sum (struct sk_buff *dest, + const unsigned char *src, + int len, int base) +{ + skb_copy_to_linear_data(dest, src, len); +} +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,22) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,7) +/* copied from linux kernel 2.6.20 /include/linux/time.h */ +/* Parameters used to convert the timespec values: */ +#define MSEC_PER_SEC 1000L + +/* copied from linux kernel 2.6.20 /include/linux/jiffies.h */ +/* + * Change timeval to jiffies, trying to avoid the + * most obvious overflows.. + * + * And some not so obvious. + * + * Note that we don't want to return MAX_LONG, because + * for various timeout reasons we often end up having + * to wait "jiffies+1" in order to guarantee that we wait + * at _least_ "jiffies" - so "jiffies+1" had better still + * be positive. + */ +#define MAX_JIFFY_OFFSET ((~0UL >> 1)-1) + +/* + * Convert jiffies to milliseconds and back. + * + * Avoid unnecessary multiplications/divisions in the + * two most common HZ cases: + */ +static inline unsigned int _kc_jiffies_to_msecs(const unsigned long j) +{ +#if HZ <= MSEC_PER_SEC && !(MSEC_PER_SEC % HZ) + return (MSEC_PER_SEC / HZ) * j; +#elif HZ > MSEC_PER_SEC && !(HZ % MSEC_PER_SEC) + return (j + (HZ / MSEC_PER_SEC) - 1)/(HZ / MSEC_PER_SEC); +#else + return (j * MSEC_PER_SEC) / HZ; +#endif +} + +static inline unsigned long _kc_msecs_to_jiffies(const unsigned int m) +{ + if (m > _kc_jiffies_to_msecs(MAX_JIFFY_OFFSET)) + return MAX_JIFFY_OFFSET; +#if HZ <= MSEC_PER_SEC && !(MSEC_PER_SEC % HZ) + return (m + (MSEC_PER_SEC / HZ) - 1) / (MSEC_PER_SEC / HZ); +#elif HZ > MSEC_PER_SEC && !(HZ % MSEC_PER_SEC) + return m * (HZ / MSEC_PER_SEC); +#else + return (m * HZ + MSEC_PER_SEC - 1) / MSEC_PER_SEC; +#endif +} +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,7) + + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,11) + +/* copied from linux kernel 2.6.12.6 /include/linux/pm.h */ +typedef int __bitwise pci_power_t; + +/* copied from linux kernel 2.6.12.6 /include/linux/pci.h */ +typedef u32 __bitwise pm_message_t; + +#define PCI_D0 ((pci_power_t __force) 0) +#define PCI_D1 ((pci_power_t __force) 1) +#define PCI_D2 ((pci_power_t __force) 2) +#define PCI_D3hot ((pci_power_t __force) 3) +#define PCI_D3cold ((pci_power_t __force) 4) +#define PCI_POWER_ERROR ((pci_power_t __force) -1) + +/* copied from linux kernel 2.6.12.6 /drivers/pci/pci.c */ +/** + * pci_choose_state - Choose the power state of a PCI device + * @dev: PCI device to be suspended + * @state: target sleep state for the whole system. This is the value + * that is passed to suspend() function. + * + * Returns PCI power state suitable for given device and given system + * message. + */ + +pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state) +{ + if (!pci_find_capability(dev, PCI_CAP_ID_PM)) + return PCI_D0; + + switch (state) { + case 0: + return PCI_D0; + case 3: + return PCI_D3hot; + default: + printk("They asked me for state %d\n", state); +// BUG(); + } + return PCI_D0; +} +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,11) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,9) +/** + * msleep_interruptible - sleep waiting for waitqueue interruptions + * @msecs: Time in milliseconds to sleep for + */ +#define msleep_interruptible _kc_msleep_interruptible +unsigned long _kc_msleep_interruptible(unsigned int msecs) +{ + unsigned long timeout = _kc_msecs_to_jiffies(msecs); + + while (timeout && !signal_pending(current)) { + set_current_state(TASK_INTERRUPTIBLE); + timeout = schedule_timeout(timeout); + } + return _kc_jiffies_to_msecs(timeout); +} +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,9) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,7) +/* copied from linux kernel 2.6.20 include/linux/sched.h */ +#ifndef __sched +#define __sched __attribute__((__section__(".sched.text"))) +#endif + +/* copied from linux kernel 2.6.20 kernel/timer.c */ +signed long __sched schedule_timeout_uninterruptible(signed long timeout) +{ + __set_current_state(TASK_UNINTERRUPTIBLE); + return schedule_timeout(timeout); +} + +/* copied from linux kernel 2.6.20 include/linux/mii.h */ +#undef if_mii +#define if_mii _kc_if_mii +static inline struct mii_ioctl_data *if_mii(struct ifreq *rq) +{ + return (struct mii_ioctl_data *) &rq->ifr_ifru; +} +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,7) + +static u32 rtl8127_read_thermal_sensor(struct rtl8127_private *tp) +{ + u16 ts_digout; + + ts_digout = rtl8127_mdio_direct_read_phy_ocp(tp, 0xBD84); + ts_digout &= 0x3ff; + + return ts_digout; +} + +int rtl8127_dump_tally_counter(struct rtl8127_private *tp, dma_addr_t paddr) +{ + u32 cmd; + u32 WaitCnt; + int retval = -1; + + RTL_W32(tp, CounterAddrHigh, (u64)paddr >> 32); + cmd = (u64)paddr & DMA_BIT_MASK(32); + RTL_W32(tp, CounterAddrLow, cmd); + RTL_W32(tp, CounterAddrLow, cmd | CounterDump); + + WaitCnt = 0; + while (RTL_R32(tp, CounterAddrLow) & CounterDump) { + udelay(10); + + WaitCnt++; + if (WaitCnt > 20) + break; + } + + if (WaitCnt <= 20) + retval = 0; + + return retval; +} + +static u32 +rtl8127_get_hw_clo_ptr(struct rtl8127_tx_ring *ring) +{ + struct rtl8127_private *tp = ring->priv; + + switch (tp->HwSuppTxNoCloseVer) { + case 3: + return RTL_R16(tp, ring->hw_clo_ptr_reg); + case 4: + case 5: + case 6: + return RTL_R32(tp, ring->hw_clo_ptr_reg); + default: +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + WARN_ON(1); +#endif + return 0; + } +} + +static u32 +rtl8127_get_sw_tail_ptr(struct rtl8127_tx_ring *ring) +{ + struct rtl8127_private *tp = ring->priv; + + switch (tp->HwSuppTxNoCloseVer) { + case 3: + return RTL_R16(tp, ring->sw_tail_ptr_reg); + case 4: + case 5: + case 6: + return RTL_R32(tp, ring->sw_tail_ptr_reg); + default: +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + WARN_ON(1); +#endif + return 0; + } +} + +static u32 +rtl8127_get_phy_status(struct rtl8127_private *tp) +{ + return RTL_R32(tp, PHYstatus); +} + +static bool +rtl8127_sysfs_testmode_on(struct rtl8127_private *tp) +{ +#ifdef ENABLE_R8127_SYSFS + return !!tp->testmode; +#else + return 1; +#endif +} + +static u32 rtl8127_convert_link_speed(u32 status) +{ + u32 speed = SPEED_UNKNOWN; + + if (status & LinkStatus) { + if (status & _10000bpsF) + speed = SPEED_10000; + else if (status & (_5000bpsF | _10000bpsL)) + speed = SPEED_5000; + else if (status & (_2500bpsF | _5000bpsL)) + speed = SPEED_2500; + else if (status & (_1000bpsF | _2500bpsL | _1000bpsL)) + speed = SPEED_1000; + else if (status & _100bps) + speed = SPEED_100; + else if (status & _10bps) + speed = SPEED_10; + } + + return speed; +} + +static void rtl8127_mdi_swap(struct rtl8127_private *tp) +{ + int i; + u16 reg, val, mdi_reverse; + u16 tps_p0, tps_p1, tps_p2, tps_p3, tps_p3_p0; + + switch (tp->mcfg) { + default: + return; + }; + + tps_p3_p0 = rtl8127_mac_ocp_read(tp, 0xD440) & 0xF000; + tps_p3 = !!(tps_p3_p0 & BIT_15); + tps_p2 = !!(tps_p3_p0 & BIT_14); + tps_p1 = !!(tps_p3_p0 & BIT_13); + tps_p0 = !!(tps_p3_p0 & BIT_12); + mdi_reverse = rtl8127_mac_ocp_read(tp, 0xD442); + + if ((mdi_reverse & BIT_5) && tps_p3_p0 == 0xA000) + return; + + if (!(mdi_reverse & BIT_5)) + val = tps_p0 << 8 | + tps_p1 << 9 | + tps_p2 << 10 | + tps_p3 << 11; + else + val = tps_p3 << 8 | + tps_p2 << 9 | + tps_p1 << 10 | + tps_p0 << 11; + + for (i=8; i<12; i++) { + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, reg); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + BIT(i), + val & BIT(i)); + } +} + +static int rtl8127_vcd_test(struct rtl8127_private *tp) +{ + u16 val; + u32 wait_cnt; + int ret = -1; + + rtl8127_mdi_swap(tp); + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA422, BIT(0)); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA422, 0x00F0); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA422, BIT(0)); + + wait_cnt = 0; + do { + mdelay(1); + val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA422); + wait_cnt++; + } while (!(val & BIT_15) && (wait_cnt < 5000)); + + if (wait_cnt == 5000) + goto exit; + + ret = 0; + +exit: + return ret; +} + +static void rtl8127_get_cp_len(struct rtl8127_private *tp, + int cp_len[RTL8127_CP_NUM]) +{ + int i; + u32 status; + int tmp_cp_len; + + status = rtl8127_get_phy_status(tp); + if (status & LinkStatus) { + if (status & _10bps) { + tmp_cp_len = -1; + } else if (status & (_100bps | _1000bpsF)) { + tmp_cp_len = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA880);; + } else if (status & (_10000bpsF | _10000bpsL | _5000bpsF | + _5000bpsL | _2500bpsF | _2500bpsL)) { + tmp_cp_len = rtl8127_mdio_direct_read_phy_ocp(tp, 0xAC2E);; + tmp_cp_len >>= 5; + } else + tmp_cp_len = 0; + } else + tmp_cp_len = 0; + + if (tmp_cp_len > 0) + tmp_cp_len &= 0xff; + for (i=0; i RTL8127_MAX_SUPPORT_CP_LEN) + cp_len[i] = RTL8127_MAX_SUPPORT_CP_LEN; + + return; +} + +static int __rtl8127_get_cp_status(u16 val) +{ + switch (val) { + case 0x0060: + return rtl8127_cp_normal; + case 0x0048: + case 0x0042: + return rtl8127_cp_open; + case 0x0050: + case 0x0044: + return rtl8127_cp_short; + default: + return rtl8127_cp_normal; + } +} + +static int _rtl8127_get_cp_status(struct rtl8127_private *tp, u8 pair_num) +{ + u16 val; + int cp_status = rtl8127_cp_unknown; + + if (pair_num > 3) + goto exit; + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8026 + 4 * pair_num); + val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA438); + + cp_status = __rtl8127_get_cp_status(val); + +exit: + return cp_status; +} + +static const char * rtl8127_get_cp_status_string(int cp_status) +{ + switch(cp_status) { + case rtl8127_cp_normal: + return "normal "; + case rtl8127_cp_short: + return "short "; + case rtl8127_cp_open: + return "open "; + case rtl8127_cp_mismatch: + return "mismatch"; + default: + return "unknown "; + } +} + +static u16 rtl8127_get_cp_pp(struct rtl8127_private *tp, u8 pair_num) +{ + u16 pp = 0; + + if (pair_num > 3) + goto exit; + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8028 + 4 * pair_num); + pp = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA438); + + pp &= 0x3fff; + pp /= 80; + +exit: + return pp; +} + +static void rtl8127_get_cp_status(struct rtl8127_private *tp, + int cp_status[RTL8127_CP_NUM], + bool poe_mode) +{ + u32 status; + int i; + + status = rtl8127_get_phy_status(tp); + if (status & LinkStatus && !(status & (_10bps | _100bps))) { + for (i=0; i= KERNEL_VERSION(3,10,0) +static int proc_get_driver_variable(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + struct rtl8127_private *tp = netdev_priv(dev); + + seq_puts(m, "\nDump Driver Variable\n"); + + rtnl_lock(); + + seq_puts(m, "Variable\tValue\n----------\t-----\n"); + seq_printf(m, "MODULENAME\t%s\n", MODULENAME); + seq_printf(m, "driver version\t%s\n", RTL8127_VERSION); + seq_printf(m, "mcfg\t%d\n", tp->mcfg); + seq_printf(m, "chipset\t%d\n", tp->chipset); + seq_printf(m, "chipset_name\t%s\n", rtl_chip_info[tp->chipset].name); + seq_printf(m, "mtu\t%d\n", dev->mtu); + seq_printf(m, "NUM_RX_DESC\t0x%x\n", tp->rx_ring[0].num_rx_desc); + seq_printf(m, "cur_rx0\t0x%x\n", tp->rx_ring[0].cur_rx); + seq_printf(m, "dirty_rx0\t0x%x\n", tp->rx_ring[0].dirty_rx); + seq_printf(m, "cur_rx1\t0x%x\n", tp->rx_ring[1].cur_rx); + seq_printf(m, "dirty_rx1\t0x%x\n", tp->rx_ring[1].dirty_rx); + seq_printf(m, "cur_rx2\t0x%x\n", tp->rx_ring[2].cur_rx); + seq_printf(m, "dirty_rx2\t0x%x\n", tp->rx_ring[2].dirty_rx); + seq_printf(m, "cur_rx3\t0x%x\n", tp->rx_ring[3].cur_rx); + seq_printf(m, "dirty_rx3\t0x%x\n", tp->rx_ring[3].dirty_rx); + seq_printf(m, "NUM_TX_DESC\t0x%x\n", tp->tx_ring[0].num_tx_desc); + seq_printf(m, "cur_tx0\t0x%x\n", tp->tx_ring[0].cur_tx); + seq_printf(m, "dirty_tx0\t0x%x\n", tp->tx_ring[0].dirty_tx); + seq_printf(m, "cur_tx1\t0x%x\n", tp->tx_ring[1].cur_tx); + seq_printf(m, "dirty_tx1\t0x%x\n", tp->tx_ring[1].dirty_tx); + seq_printf(m, "rx_buf_sz\t0x%x\n", tp->rx_buf_sz); +#ifdef ENABLE_PAGE_REUSE + seq_printf(m, "rx_buf_page_order\t0x%x\n", tp->rx_buf_page_order); + seq_printf(m, "rx_buf_page_size\t0x%x\n", tp->rx_buf_page_size); + seq_printf(m, "page_reuse_fail_cnt\t0x%x\n", tp->page_reuse_fail_cnt); +#endif //ENABLE_PAGE_REUSE + seq_printf(m, "esd_flag\t0x%x\n", tp->esd_flag); + seq_printf(m, "pci_cfg_is_read\t0x%x\n", tp->pci_cfg_is_read); + seq_printf(m, "rtl8127_rx_config\t0x%x\n", tp->rtl8127_rx_config); + seq_printf(m, "cp_cmd\t0x%x\n", tp->cp_cmd); + seq_printf(m, "intr_mask\t0x%x\n", tp->intr_mask); + seq_printf(m, "timer_intr_mask\t0x%x\n", tp->timer_intr_mask); + seq_printf(m, "wol_enabled\t0x%x\n", tp->wol_enabled); + seq_printf(m, "wol_opts\t0x%x\n", tp->wol_opts); + seq_printf(m, "efuse_ver\t0x%x\n", tp->efuse_ver); + seq_printf(m, "eeprom_type\t0x%x\n", tp->eeprom_type); + seq_printf(m, "autoneg\t0x%x\n", tp->autoneg); + seq_printf(m, "duplex\t0x%x\n", tp->duplex); + seq_printf(m, "speed\t%d\n", tp->speed); + seq_printf(m, "advertising\t0x%llx\n", tp->advertising); + seq_printf(m, "eeprom_len\t0x%x\n", tp->eeprom_len); + seq_printf(m, "cur_page\t0x%x\n", tp->cur_page); + seq_printf(m, "features\t0x%x\n", tp->features); + seq_printf(m, "org_pci_offset_99\t0x%x\n", tp->org_pci_offset_99); + seq_printf(m, "org_pci_offset_180\t0x%x\n", tp->org_pci_offset_180); + seq_printf(m, "issue_offset_99_event\t0x%x\n", tp->issue_offset_99_event); + seq_printf(m, "org_pci_offset_80\t0x%x\n", tp->org_pci_offset_80); + seq_printf(m, "org_pci_offset_81\t0x%x\n", tp->org_pci_offset_81); + seq_printf(m, "use_timer_interrupt\t0x%x\n", tp->use_timer_interrupt); + seq_printf(m, "HwIcVerUnknown\t0x%x\n", tp->HwIcVerUnknown); + seq_printf(m, "NotWrRamCodeToMicroP\t0x%x\n", tp->NotWrRamCodeToMicroP); + seq_printf(m, "NotWrMcuPatchCode\t0x%x\n", tp->NotWrMcuPatchCode); + seq_printf(m, "HwHasWrRamCodeToMicroP\t0x%x\n", tp->HwHasWrRamCodeToMicroP); + seq_printf(m, "sw_ram_code_ver\t0x%x\n", tp->sw_ram_code_ver); + seq_printf(m, "hw_ram_code_ver\t0x%x\n", tp->hw_ram_code_ver); + seq_printf(m, "rtk_enable_diag\t0x%x\n", tp->rtk_enable_diag); + seq_printf(m, "ShortPacketSwChecksum\t0x%x\n", tp->ShortPacketSwChecksum); + seq_printf(m, "UseSwPaddingShortPkt\t0x%x\n", tp->UseSwPaddingShortPkt); + seq_printf(m, "RequireAdcBiasPatch\t0x%x\n", tp->RequireAdcBiasPatch); + seq_printf(m, "AdcBiasPatchIoffset\t0x%x\n", tp->AdcBiasPatchIoffset); + seq_printf(m, "RequireAdjustUpsTxLinkPulseTiming\t0x%x\n", tp->RequireAdjustUpsTxLinkPulseTiming); + seq_printf(m, "SwrCnt1msIni\t0x%x\n", tp->SwrCnt1msIni); + seq_printf(m, "HwSuppNowIsOobVer\t0x%x\n", tp->HwSuppNowIsOobVer); + seq_printf(m, "HwFiberModeVer\t0x%x\n", tp->HwFiberModeVer); + seq_printf(m, "HwFiberStat\t0x%x\n", tp->HwFiberStat); + seq_printf(m, "HwSwitchMdiToFiber\t0x%x\n", tp->HwSwitchMdiToFiber); + seq_printf(m, "NicCustLedValue\t0x%x\n", tp->NicCustLedValue); + seq_printf(m, "RequiredSecLanDonglePatch\t0x%x\n", tp->RequiredSecLanDonglePatch); + seq_printf(m, "HwSuppDashVer\t0x%x\n", tp->HwSuppDashVer); + seq_printf(m, "DASH\t0x%x\n", tp->DASH); + seq_printf(m, "dash_printer_enabled\t0x%x\n", tp->dash_printer_enabled); + seq_printf(m, "HwSuppKCPOffloadVer\t0x%x\n", tp->HwSuppKCPOffloadVer); + seq_printf(m, "speed_mode\t0x%x\n", speed_mode); + seq_printf(m, "duplex_mode\t0x%x\n", duplex_mode); + seq_printf(m, "autoneg_mode\t0x%x\n", autoneg_mode); + seq_printf(m, "aspm\t0x%x\n", aspm); + seq_printf(m, "s5wol\t0x%x\n", s5wol); + seq_printf(m, "s5_keep_curr_mac\t0x%x\n", s5_keep_curr_mac); + seq_printf(m, "eee_enable\t0x%x\n", tp->eee.eee_enabled); + seq_printf(m, "hwoptimize\t0x%lx\n", hwoptimize); + seq_printf(m, "proc_init_num\t0x%x\n", proc_init_num); + seq_printf(m, "s0_magic_packet\t0x%x\n", s0_magic_packet); + seq_printf(m, "disable_wol_support\t0x%x\n", disable_wol_support); + seq_printf(m, "enable_double_vlan\t0x%x\n", enable_double_vlan); + seq_printf(m, "eee_giga_lite\t0x%x\n", eee_giga_lite); + seq_printf(m, "HwSuppMagicPktVer\t0x%x\n", tp->HwSuppMagicPktVer); + seq_printf(m, "HwSuppLinkChgWakeUpVer\t0x%x\n", tp->HwSuppLinkChgWakeUpVer); + seq_printf(m, "HwSuppD0SpeedUpVer\t0x%x\n", tp->HwSuppD0SpeedUpVer); + seq_printf(m, "D0SpeedUpSpeed\t0x%x\n", tp->D0SpeedUpSpeed); + seq_printf(m, "HwSuppCheckPhyDisableModeVer\t0x%x\n", tp->HwSuppCheckPhyDisableModeVer); + seq_printf(m, "HwPkgDet\t0x%x\n", tp->HwPkgDet); + seq_printf(m, "HwSuppTxNoCloseVer\t0x%x\n", tp->HwSuppTxNoCloseVer); + seq_printf(m, "EnableTxNoClose\t0x%x\n", tp->EnableTxNoClose); + seq_printf(m, "NextHwDesCloPtr0\t0x%x\n", tp->tx_ring[0].NextHwDesCloPtr); + seq_printf(m, "BeginHwDesCloPtr0\t0x%x\n", tp->tx_ring[0].BeginHwDesCloPtr); + seq_printf(m, "hw_clo_ptr_reg0\t0x%x\n", rtl8127_get_hw_clo_ptr(&tp->tx_ring[0])); + seq_printf(m, "sw_tail_ptr_reg0\t0x%x\n", rtl8127_get_sw_tail_ptr(&tp->tx_ring[0])); + seq_printf(m, "NextHwDesCloPtr1\t0x%x\n", tp->tx_ring[1].NextHwDesCloPtr); + seq_printf(m, "BeginHwDesCloPtr1\t0x%x\n", tp->tx_ring[1].BeginHwDesCloPtr); + seq_printf(m, "hw_clo_ptr_reg1\t0x%x\n", rtl8127_get_hw_clo_ptr(&tp->tx_ring[1])); + seq_printf(m, "sw_tail_ptr_reg1\t0x%x\n", rtl8127_get_sw_tail_ptr(&tp->tx_ring[1])); + seq_printf(m, "InitRxDescType\t0x%x\n", tp->InitRxDescType); + seq_printf(m, "RxDescLength\t0x%x\n", tp->RxDescLength); + seq_printf(m, "num_rx_rings\t0x%x\n", tp->num_rx_rings); + seq_printf(m, "num_tx_rings\t0x%x\n", tp->num_tx_rings); + seq_printf(m, "tot_rx_rings\t0x%x\n", rtl8127_tot_rx_rings(tp)); + seq_printf(m, "tot_tx_rings\t0x%x\n", rtl8127_tot_tx_rings(tp)); + seq_printf(m, "HwSuppNumRxQueues\t0x%x\n", tp->HwSuppNumRxQueues); + seq_printf(m, "HwSuppNumTxQueues\t0x%x\n", tp->HwSuppNumTxQueues); + seq_printf(m, "EnableRss\t0x%x\n", tp->EnableRss); + seq_printf(m, "EnablePtp\t0x%x\n", tp->EnablePtp); + seq_printf(m, "min_irq_nvecs\t0x%x\n", tp->min_irq_nvecs); + seq_printf(m, "irq_nvecs\t0x%x\n", tp->irq_nvecs); + seq_printf(m, "hw_supp_irq_nvecs\t0x%x\n", tp->hw_supp_irq_nvecs); + seq_printf(m, "ring_lib_enabled\t0x%x\n", tp->ring_lib_enabled); + seq_printf(m, "HwSuppIsrVer\t0x%x\n", tp->HwSuppIsrVer); + seq_printf(m, "HwCurrIsrVer\t0x%x\n", tp->HwCurrIsrVer); + seq_printf(m, "HwSuppMacMcuVer\t0x%x\n", tp->HwSuppMacMcuVer); + seq_printf(m, "MacMcuPageSize\t0x%x\n", tp->MacMcuPageSize); + seq_printf(m, "hw_mcu_patch_code_ver\t0x%llx\n", tp->hw_mcu_patch_code_ver); + seq_printf(m, "bin_mcu_patch_code_ver\t0x%llx\n", tp->bin_mcu_patch_code_ver); +#ifdef ENABLE_PTP_SUPPORT + seq_printf(m, "tx_hwtstamp_timeouts\t0x%x\n", tp->tx_hwtstamp_timeouts); + seq_printf(m, "tx_hwtstamp_skipped\t0x%x\n", tp->tx_hwtstamp_skipped); +#endif + seq_printf(m, "random_mac\t0x%x\n", tp->random_mac); + seq_printf(m, "org_mac_addr\t%pM\n", tp->org_mac_addr); +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,13) + seq_printf(m, "perm_addr\t%pM\n", dev->perm_addr); +#endif + seq_printf(m, "dev_addr\t%pM\n", dev->dev_addr); + + rtnl_unlock(); + + seq_putc(m, '\n'); + return 0; +} + +static int proc_get_tally_counter(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + struct rtl8127_private *tp = netdev_priv(dev); + struct rtl8127_counters *counters; + dma_addr_t paddr; + + seq_puts(m, "\nDump Tally Counter\n"); + + rtnl_lock(); + + counters = tp->tally_vaddr; + paddr = tp->tally_paddr; + if (!counters) { + seq_puts(m, "\nDump Tally Counter Fail\n"); + goto out_unlock; + } + + rtl8127_dump_tally_counter(tp, paddr); + + seq_puts(m, "Statistics\tValue\n----------\t-----\n"); + seq_printf(m, "tx_packets\t%lld\n", le64_to_cpu(counters->tx_packets)); + seq_printf(m, "rx_packets\t%lld\n", le64_to_cpu(counters->rx_packets)); + seq_printf(m, "tx_errors\t%lld\n", le64_to_cpu(counters->tx_errors)); + seq_printf(m, "rx_errors\t%d\n", le32_to_cpu(counters->rx_errors)); + seq_printf(m, "rx_missed\t%d\n", le16_to_cpu(counters->rx_missed)); + seq_printf(m, "align_errors\t%d\n", le16_to_cpu(counters->align_errors)); + seq_printf(m, "tx_one_collision\t%d\n", le32_to_cpu(counters->tx_one_collision)); + seq_printf(m, "tx_multi_collision\t%d\n", le32_to_cpu(counters->tx_multi_collision)); + seq_printf(m, "rx_unicast\t%lld\n", le64_to_cpu(counters->rx_unicast)); + seq_printf(m, "rx_broadcast\t%lld\n", le64_to_cpu(counters->rx_broadcast)); + seq_printf(m, "rx_multicast\t%d\n", le32_to_cpu(counters->rx_multicast)); + seq_printf(m, "tx_aborted\t%d\n", le16_to_cpu(counters->tx_aborted)); + seq_printf(m, "tx_underrun\t%d\n", le16_to_cpu(counters->tx_underrun)); + + seq_printf(m, "tx_octets\t%lld\n", le64_to_cpu(counters->tx_octets)); + seq_printf(m, "rx_octets\t%lld\n", le64_to_cpu(counters->rx_octets)); + seq_printf(m, "rx_multicast64\t%lld\n", le64_to_cpu(counters->rx_multicast64)); + seq_printf(m, "tx_unicast64\t%lld\n", le64_to_cpu(counters->tx_unicast64)); + seq_printf(m, "tx_broadcast64\t%lld\n", le64_to_cpu(counters->tx_broadcast64)); + seq_printf(m, "tx_multicast64\t%lld\n", le64_to_cpu(counters->tx_multicast64)); + seq_printf(m, "tx_pause_on\t%d\n", le32_to_cpu(counters->tx_pause_on)); + seq_printf(m, "tx_pause_off\t%d\n", le32_to_cpu(counters->tx_pause_off)); + seq_printf(m, "tx_pause_all\t%d\n", le32_to_cpu(counters->tx_pause_all)); + seq_printf(m, "tx_deferred\t%d\n", le32_to_cpu(counters->tx_deferred)); + seq_printf(m, "tx_late_collision\t%d\n", le32_to_cpu(counters->tx_late_collision)); + seq_printf(m, "tx_all_collision\t%d\n", le32_to_cpu(counters->tx_all_collision)); + seq_printf(m, "tx_aborted32\t%d\n", le32_to_cpu(counters->tx_aborted32)); + seq_printf(m, "align_errors32\t%d\n", le32_to_cpu(counters->align_errors32)); + seq_printf(m, "rx_frame_too_long\t%d\n", le32_to_cpu(counters->rx_frame_too_long)); + seq_printf(m, "rx_runt\t%d\n", le32_to_cpu(counters->rx_runt)); + seq_printf(m, "rx_pause_on\t%d\n", le32_to_cpu(counters->rx_pause_on)); + seq_printf(m, "rx_pause_off\t%d\n", le32_to_cpu(counters->rx_pause_off)); + seq_printf(m, "rx_pause_all\t%d\n", le32_to_cpu(counters->rx_pause_all)); + seq_printf(m, "rx_unknown_opcode\t%d\n", le32_to_cpu(counters->rx_unknown_opcode)); + seq_printf(m, "rx_mac_error\t%d\n", le32_to_cpu(counters->rx_mac_error)); + seq_printf(m, "tx_underrun32\t%d\n", le32_to_cpu(counters->tx_underrun32)); + seq_printf(m, "rx_mac_missed\t%d\n", le32_to_cpu(counters->rx_mac_missed)); + seq_printf(m, "rx_tcam_dropped\t%d\n", le32_to_cpu(counters->rx_tcam_dropped)); + seq_printf(m, "tdu\t%d\n", le32_to_cpu(counters->tdu)); + seq_printf(m, "rdu\t%d\n", le32_to_cpu(counters->rdu)); + + seq_putc(m, '\n'); + +out_unlock: + rtnl_unlock(); + + return 0; +} + +static int proc_get_registers(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + int i, n, max = R8127_MAC_REGS_SIZE; + u8 byte_rd; + struct rtl8127_private *tp = netdev_priv(dev); + void __iomem *ioaddr = tp->mmio_addr; + + seq_puts(m, "\nDump MAC Registers\n"); + seq_puts(m, "Offset\tValue\n------\t-----\n"); + + for (n = 0; n < max;) { + seq_printf(m, "\n0x%04x:\t", n); + + rtnl_lock(); + + for (i = 0; i < 16 && n < max; i++, n++) { + byte_rd = readb(ioaddr + n); + seq_printf(m, "%02x ", byte_rd); + } + + rtnl_unlock(); + } + + rtnl_lock(); + + max = 0xB00; + for (n = 0xA00; n < max;) { + seq_printf(m, "\n0x%04x:\t", n); + + for (i = 0; i < 16 && n < max; i++, n++) { + byte_rd = readb(ioaddr + n); + seq_printf(m, "%02x ", byte_rd); + } + } + + max = 0xD40; + for (n = 0xD00; n < max;) { + seq_printf(m, "\n0x%04x:\t", n); + + for (i = 0; i < 16 && n < max; i++, n++) { + byte_rd = readb(ioaddr + n); + seq_printf(m, "%02x ", byte_rd); + } + } + + max = 0x2840; + for (n = 0x2800; n < max;) { + seq_printf(m, "\n0x%04x:\t", n); + + for (i = 0; i < 16 && n < max; i++, n++) { + byte_rd = readb(ioaddr + n); + seq_printf(m, "%02x ", byte_rd); + } + } + + rtnl_unlock(); + + seq_putc(m, '\n'); + return 0; +} + +static int proc_get_pcie_phy(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + int i, n, max = R8127_EPHY_REGS_SIZE/2; + u16 word_rd; + struct rtl8127_private *tp = netdev_priv(dev); + + seq_puts(m, "\nDump PCIE PHY\n"); + seq_puts(m, "\nOffset\tValue\n------\t-----\n "); + + rtnl_lock(); + + for (n = 0; n < max;) { + seq_printf(m, "\n0x%02x:\t", n); + + for (i = 0; i < 8 && n < max; i++, n++) { + word_rd = rtl8127_ephy_read(tp, n); + seq_printf(m, "%04x ", word_rd); + } + } + + rtnl_unlock(); + + seq_putc(m, '\n'); + return 0; +} + +static int proc_get_eth_phy(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + int i, n, max = R8127_PHY_REGS_SIZE/2; + unsigned long flags; + u16 word_rd; + struct rtl8127_private *tp = netdev_priv(dev); + + seq_puts(m, "\nDump Ethernet PHY\n"); + seq_puts(m, "\nOffset\tValue\n------\t-----\n "); + + spin_lock_irqsave(&tp->phy_lock, flags); + + seq_puts(m, "\n####################page 0##################\n "); + rtl8127_mdio_write(tp, 0x1f, 0x0000); + for (n = 0; n < max;) { + seq_printf(m, "\n0x%02x:\t", n); + + for (i = 0; i < 8 && n < max; i++, n++) { + word_rd = rtl8127_mdio_read(tp, n); + seq_printf(m, "%04x ", word_rd); + } + } + + seq_puts(m, "\n####################extra reg##################\n "); + n = 0xA400; + seq_printf(m, "\n0x%02x:\t", n); + for (i = 0; i < 8; i++, n+=2) { + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + seq_printf(m, "%04x ", word_rd); + } + + n = 0xA410; + seq_printf(m, "\n0x%02x:\t", n); + for (i = 0; i < 3; i++, n+=2) { + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + seq_printf(m, "%04x ", word_rd); + } + + n = 0xA434; + seq_printf(m, "\n0x%02x:\t", n); + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + seq_printf(m, "%04x ", word_rd); + + n = 0xA5D0; + seq_printf(m, "\n0x%02x:\t", n); + for (i = 0; i < 4; i++, n+=2) { + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + seq_printf(m, "%04x ", word_rd); + } + + n = 0xA61A; + seq_printf(m, "\n0x%02x:\t", n); + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + seq_printf(m, "%04x ", word_rd); + + n = 0xA6D0; + seq_printf(m, "\n0x%02x:\t", n); + for (i = 0; i < 3; i++, n+=2) { + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + seq_printf(m, "%04x ", word_rd); + } + + spin_unlock_irqrestore(&tp->phy_lock, flags); + + seq_putc(m, '\n'); + return 0; +} + +static int proc_get_extended_registers(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + int i, n, max = R8127_ERI_REGS_SIZE; + u32 dword_rd; + struct rtl8127_private *tp = netdev_priv(dev); + + seq_puts(m, "\nDump Extended Registers\n"); + seq_puts(m, "\nOffset\tValue\n------\t-----\n "); + + rtnl_lock(); + + for (n = 0; n < max;) { + seq_printf(m, "\n0x%02x:\t", n); + + for (i = 0; i < 4 && n < max; i++, n+=4) { + dword_rd = rtl8127_eri_read(tp, n, 4, ERIAR_ExGMAC); + seq_printf(m, "%08x ", dword_rd); + } + } + + rtnl_unlock(); + + seq_putc(m, '\n'); + return 0; +} + +static int proc_get_pci_registers(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + int i, n, max = R8127_PCI_REGS_SIZE; + u32 dword_rd; + struct rtl8127_private *tp = netdev_priv(dev); + + seq_puts(m, "\nDump PCI Registers\n"); + seq_puts(m, "\nOffset\tValue\n------\t-----\n "); + + rtnl_lock(); + + for (n = 0; n < max;) { + seq_printf(m, "\n0x%03x:\t", n); + + for (i = 0; i < 4 && n < max; i++, n+=4) { + pci_read_config_dword(tp->pci_dev, n, &dword_rd); + seq_printf(m, "%08x ", dword_rd); + } + } + + n = 0x110; + pci_read_config_dword(tp->pci_dev, n, &dword_rd); + seq_printf(m, "\n0x%03x:\t%08x ", n, dword_rd); + n = 0x70c; + pci_read_config_dword(tp->pci_dev, n, &dword_rd); + seq_printf(m, "\n0x%03x:\t%08x ", n, dword_rd); + + rtnl_unlock(); + + seq_putc(m, '\n'); + return 0; +} + +static int proc_get_temperature(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + struct rtl8127_private *tp = netdev_priv(dev); + u16 ts_digout, tj, fah; + + seq_puts(m, "\nChip Temperature\n"); + + rtnl_lock(); + + if (!rtl8127_sysfs_testmode_on(tp)) { + seq_puts(m, "\nPlease turn on ""/sys/class/net//rtk_adv/testmode"".\n\n"); + rtnl_unlock(); + return 0; + } + + netif_testing_on(dev); + ts_digout = rtl8127_read_thermal_sensor(tp); + netif_testing_off(dev); + + rtnl_unlock(); + + tj = ts_digout / 2; + if (ts_digout <= 512) { + tj = ts_digout / 2; + seq_printf(m, "Cel:%d\n", tj); + fah = tj * (9/5) + 32; + seq_printf(m, "Fah:%d\n", fah); + } else { + tj = (512 - ((ts_digout / 2) - 512)) / 2; + seq_printf(m, "Cel:-%d\n", tj); + fah = tj * (9/5) + 32; + seq_printf(m, "Fah:-%d\n", fah); + } + + seq_putc(m, '\n'); + return 0; +} + +static int _proc_get_cable_info(struct seq_file *m, void *v, bool poe_mode) +{ + int i; + u32 status; + int cp_status[RTL8127_CP_NUM]; + int cp_len[RTL8127_CP_NUM] = {0}; + struct net_device *dev = m->private; + struct rtl8127_private *tp = netdev_priv(dev); + const char *pair_str[RTL8127_CP_NUM] = {"1-2", "3-6", "4-5", "7-8"}; + int ret; + + rtnl_lock(); + + if (!rtl8127_sysfs_testmode_on(tp)) { + seq_puts(m, "\nPlease turn on ""/sys/class/net//rtk_adv/testmode"".\n\n"); + ret = 0; + goto error_unlock; + } + + rtl8127_mdio_write(tp, 0x1F, 0x0000); + if (rtl8127_mdio_read(tp, MII_BMCR) & BMCR_PDOWN) { + ret = -EIO; + goto error_unlock; + } + + netif_testing_on(dev); + + status = rtl8127_get_phy_status(tp); + if (status & LinkStatus) + seq_printf(m, "\nlink speed:%d", + rtl8127_convert_link_speed(status)); + else + seq_puts(m, "\nlink status:off"); + + rtl8127_get_cp_len(tp, cp_len); + + rtl8127_get_cp_status(tp, cp_status, poe_mode); + + seq_puts(m, "\npair\tlength\tstatus \tpp\n"); + + for (i=0; iprivate; + struct rtl8127_private *tp = netdev_priv(dev); + int i; + + rtnl_lock(); + + for (i = 0; i < tp->num_rx_rings; i++) { + struct rtl8127_rx_ring *ring = &tp->rx_ring[i]; + + if (!ring) + continue; + + seq_printf(m, "\ndump rx %d desc:%d\n", i, ring->num_rx_desc); + + _proc_dump_desc(m, (void*)ring->RxDescArray, ring->RxDescAllocSize); + } + +#ifdef ENABLE_LIB_SUPPORT + if (rtl8127_num_lib_rx_rings(tp) > 0) { + for (i = 0; i < tp->HwSuppNumRxQueues; i++) { + struct rtl8127_ring *lib_ring = &tp->lib_rx_ring[i]; + if (lib_ring->enabled) { + seq_printf(m, "\ndump lib rx %d desc:%d\n", i, + lib_ring->ring_size); + _proc_dump_desc(m, (void*)lib_ring->desc_addr, + lib_ring->desc_size); + } + } + } +#endif //ENABLE_LIB_SUPPORT + + rtnl_unlock(); + + seq_putc(m, '\n'); + return 0; +} + +static int proc_dump_tx_desc(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + struct rtl8127_private *tp = netdev_priv(dev); + int i; + + rtnl_lock(); + + for (i = 0; i < tp->num_tx_rings; i++) { + struct rtl8127_tx_ring *ring = &tp->tx_ring[i]; + + if (!ring) + continue; + + seq_printf(m, "\ndump tx %d desc:%d\n", i, ring->num_tx_desc); + + _proc_dump_desc(m, (void*)ring->TxDescArray, ring->TxDescAllocSize); + } + +#ifdef ENABLE_LIB_SUPPORT + if (rtl8127_num_lib_tx_rings(tp) > 0) { + for (i = 0; i < tp->HwSuppNumTxQueues; i++) { + struct rtl8127_ring *lib_ring = &tp->lib_tx_ring[i]; + if (lib_ring->enabled) { + seq_printf(m, "\ndump lib tx %d desc:%d\n", i, + lib_ring->ring_size); + _proc_dump_desc(m, (void*)lib_ring->desc_addr, + lib_ring->desc_size); + } + } + } +#endif //ENABLE_LIB_SUPPORT + + rtnl_unlock(); + + seq_putc(m, '\n'); + return 0; +} + +static int proc_dump_msix_tbl(struct seq_file *m, void *v) +{ + int i, j; + void __iomem *ioaddr; + struct net_device *dev = m->private; + struct rtl8127_private *tp = netdev_priv(dev); + + /* ioremap MMIO region */ + ioaddr = ioremap(pci_resource_start(tp->pci_dev, 4), pci_resource_len(tp->pci_dev, 4)); + if (!ioaddr) + return -EFAULT; + + rtnl_lock(); + + seq_printf(m, "\ndump MSI-X Table. Total Entry %d. \n", tp->hw_supp_irq_nvecs); + + for (i=0; ihw_supp_irq_nvecs; i++) { + seq_printf(m, "\n%04x ", i); + for (j=0; j<4; j++) + seq_printf(m, "%08x ", + readl(ioaddr + i*0x10 + 4*j)); + } + + rtnl_unlock(); + + iounmap(ioaddr); + + seq_putc(m, '\n'); + return 0; +} + +#else //LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) + +static int proc_get_driver_variable(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + struct net_device *dev = data; + struct rtl8127_private *tp = netdev_priv(dev); + int len = 0; + + len += snprintf(page + len, count - len, + "\nDump Driver Driver\n"); + + rtnl_lock(); + + len += snprintf(page + len, count - len, + "Variable\tValue\n----------\t-----\n"); + + len += snprintf(page + len, count - len, + "MODULENAME\t%s\n" + "driver version\t%s\n" + "mcfg\t%d\n" + "chipset\t%d\n" + "chipset_name\t%s\n" + "mtu\t%d\n" + "NUM_RX_DESC\t0x%x\n" + "cur_rx0\t0x%x\n" + "dirty_rx0\t0x%x\n" + "cur_rx1\t0x%x\n" + "dirty_rx1\t0x%x\n" + "cur_rx2\t0x%x\n" + "dirty_rx2\t0x%x\n" + "cur_rx3\t0x%x\n" + "dirty_rx3\t0x%x\n" + "NUM_TX_DESC\t0x%x\n" + "cur_tx0\t0x%x\n" + "dirty_tx0\t0x%x\n" + "cur_tx1\t0x%x\n" + "dirty_tx1\t0x%x\n" + "rx_buf_sz\t0x%x\n" +#ifdef ENABLE_PAGE_REUSE + "rx_buf_page_order\t0x%x\n" + "rx_buf_page_size\t0x%x\n" + "page_reuse_fail_cnt\t0x%x\n" +#endif //ENABLE_PAGE_REUSE + "esd_flag\t0x%x\n" + "pci_cfg_is_read\t0x%x\n" + "rtl8127_rx_config\t0x%x\n" + "cp_cmd\t0x%x\n" + "intr_mask\t0x%x\n" + "timer_intr_mask\t0x%x\n" + "wol_enabled\t0x%x\n" + "wol_opts\t0x%x\n" + "efuse_ver\t0x%x\n" + "eeprom_type\t0x%x\n" + "autoneg\t0x%x\n" + "duplex\t0x%x\n" + "speed\t%d\n" + "advertising\t0x%llx\n" + "eeprom_len\t0x%x\n" + "cur_page\t0x%x\n" + "features\t0x%x\n" + "org_pci_offset_99\t0x%x\n" + "org_pci_offset_180\t0x%x\n" + "issue_offset_99_event\t0x%x\n" + "org_pci_offset_80\t0x%x\n" + "org_pci_offset_81\t0x%x\n" + "use_timer_interrupt\t0x%x\n" + "HwIcVerUnknown\t0x%x\n" + "NotWrRamCodeToMicroP\t0x%x\n" + "NotWrMcuPatchCode\t0x%x\n" + "HwHasWrRamCodeToMicroP\t0x%x\n" + "sw_ram_code_ver\t0x%x\n" + "hw_ram_code_ver\t0x%x\n" + "rtk_enable_diag\t0x%x\n" + "ShortPacketSwChecksum\t0x%x\n" + "UseSwPaddingShortPkt\t0x%x\n" + "RequireAdcBiasPatch\t0x%x\n" + "AdcBiasPatchIoffset\t0x%x\n" + "RequireAdjustUpsTxLinkPulseTiming\t0x%x\n" + "SwrCnt1msIni\t0x%x\n" + "HwSuppNowIsOobVer\t0x%x\n" + "HwFiberModeVer\t0x%x\n" + "HwFiberStat\t0x%x\n" + "HwSwitchMdiToFiber\t0x%x\n" + "NicCustLedValue\t0x%x\n" + "RequiredSecLanDonglePatch\t0x%x\n" + "HwSuppDashVer\t0x%x\n" + "DASH\t0x%x\n" + "dash_printer_enabled\t0x%x\n" + "HwSuppKCPOffloadVer\t0x%x\n" + "speed_mode\t0x%x\n" + "duplex_mode\t0x%x\n" + "autoneg_mode\t0x%x\n" + "aspm\t0x%x\n" + "s5wol\t0x%x\n" + "s5_keep_curr_mac\t0x%x\n" + "eee_enable\t0x%x\n" + "hwoptimize\t0x%lx\n" + "proc_init_num\t0x%x\n" + "s0_magic_packet\t0x%x\n" + "disable_wol_support\t0x%x\n" + "enable_double_vlan\t0x%x\n" + "eee_giga_lite\t0x%x\n" + "HwSuppMagicPktVer\t0x%x\n" + "HwSuppLinkChgWakeUpVer\t0x%x\n" + "HwSuppD0SpeedUpVer\t0x%x\n" + "D0SpeedUpSpeed\t0x%x\n" + "HwSuppCheckPhyDisableModeVer\t0x%x\n" + "HwPkgDet\t0x%x\n" + "HwSuppTxNoCloseVer\t0x%x\n" + "EnableTxNoClose\t0x%x\n" + "NextHwDesCloPtr0\t0x%x\n" + "BeginHwDesCloPtr0\t0x%x\n" + "hw_clo_ptr_reg0\t0x%x\n" + "sw_tail_ptr_reg0\t0x%x\n" + "NextHwDesCloPtr1\t0x%x\n" + "BeginHwDesCloPtr1\t0x%x\n" + "hw_clo_ptr_reg1\t0x%x\n" + "sw_tail_ptr_reg1\t0x%x\n" + "InitRxDescType\t0x%x\n" + "RxDescLength\t0x%x\n" + "num_rx_rings\t0x%x\n" + "num_tx_rings\t0x%x\n" + "tot_rx_rings\t0x%x\n" + "tot_tx_rings\t0x%x\n" + "HwSuppNumRxQueues\t0x%x\n" + "HwSuppNumTxQueues\t0x%x\n" + "EnableRss\t0x%x\n" + "EnablePtp\t0x%x\n" + "min_irq_nvecs\t0x%x\n" + "irq_nvecs\t0x%x\n" + "hw_supp_irq_nvecs\t0x%x\n" + "ring_lib_enabled\t0x%x\n" + "HwSuppIsrVer\t0x%x\n" + "HwCurrIsrVer\t0x%x\n" + "HwSuppMacMcuVer\t0x%x\n" + "MacMcuPageSize\t0x%x\n" + "hw_mcu_patch_code_ver\t0x%llx\n" + "bin_mcu_patch_code_ver\t0x%llx\n" +#ifdef ENABLE_PTP_SUPPORT + "tx_hwtstamp_timeouts\t0x%x\n" + "tx_hwtstamp_skipped\t0x%x\n" +#endif + "random_mac\t0x%x\n" + "org_mac_addr\t%pM\n" +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,13) + "perm_addr\t%pM\n" +#endif + "dev_addr\t%pM\n", + MODULENAME, + RTL8127_VERSION, + tp->mcfg, + tp->chipset, + rtl_chip_info[tp->chipset].name, + dev->mtu, + tp->rx_ring[0].num_rx_desc, + tp->rx_ring[0].cur_rx, + tp->rx_ring[0].dirty_rx, + tp->rx_ring[1].cur_rx, + tp->rx_ring[1].dirty_rx, + tp->rx_ring[2].cur_rx, + tp->rx_ring[2].dirty_rx, + tp->rx_ring[3].cur_rx, + tp->rx_ring[3].dirty_rx, + tp->tx_ring[0].num_tx_desc, + tp->tx_ring[0].cur_tx, + tp->tx_ring[0].dirty_tx, + tp->tx_ring[1].cur_tx, + tp->tx_ring[1].dirty_tx, + tp->rx_buf_sz, +#ifdef ENABLE_PAGE_REUSE + tp->rx_buf_page_order, + tp->rx_buf_page_size, + tp->page_reuse_fail_cnt, +#endif //ENABLE_PAGE_REUSE + tp->esd_flag, + tp->pci_cfg_is_read, + tp->rtl8127_rx_config, + tp->cp_cmd, + tp->intr_mask, + tp->timer_intr_mask, + tp->wol_enabled, + tp->wol_opts, + tp->efuse_ver, + tp->eeprom_type, + tp->autoneg, + tp->duplex, + tp->speed, + tp->advertising, + tp->eeprom_len, + tp->cur_page, + tp->features, + tp->org_pci_offset_99, + tp->org_pci_offset_180, + tp->issue_offset_99_event, + tp->org_pci_offset_80, + tp->org_pci_offset_81, + tp->use_timer_interrupt, + tp->HwIcVerUnknown, + tp->NotWrRamCodeToMicroP, + tp->NotWrMcuPatchCode, + tp->HwHasWrRamCodeToMicroP, + tp->sw_ram_code_ver, + tp->hw_ram_code_ver, + tp->rtk_enable_diag, + tp->ShortPacketSwChecksum, + tp->UseSwPaddingShortPkt, + tp->RequireAdcBiasPatch, + tp->AdcBiasPatchIoffset, + tp->RequireAdjustUpsTxLinkPulseTiming, + tp->SwrCnt1msIni, + tp->HwSuppNowIsOobVer, + tp->HwFiberModeVer, + tp->HwFiberStat, + tp->HwSwitchMdiToFiber, + tp->NicCustLedValue, + tp->RequiredSecLanDonglePatch, + tp->HwSuppDashVer, + tp->DASH, + tp->dash_printer_enabled, + tp->HwSuppKCPOffloadVer, + speed_mode, + duplex_mode, + autoneg_mode, + aspm, + s5wol, + s5_keep_curr_mac, + tp->eee.eee_enabled, + hwoptimize, + proc_init_num, + s0_magic_packet, + disable_wol_support, + enable_double_vlan, + eee_giga_lite, + tp->HwSuppMagicPktVer, + tp->HwSuppLinkChgWakeUpVer, + tp->HwSuppD0SpeedUpVer, + tp->D0SpeedUpSpeed, + tp->HwSuppCheckPhyDisableModeVer, + tp->HwPkgDet, + tp->HwSuppTxNoCloseVer, + tp->EnableTxNoClose, + tp->tx_ring[0].NextHwDesCloPtr, + tp->tx_ring[0].BeginHwDesCloPtr, + rtl8127_get_hw_clo_ptr(&tp->tx_ring[0]), + rtl8127_get_sw_tail_ptr(&tp->tx_ring[0]), + tp->tx_ring[1].NextHwDesCloPtr, + tp->tx_ring[1].BeginHwDesCloPtr, + rtl8127_get_hw_clo_ptr(&tp->tx_ring[1]), + rtl8127_get_sw_tail_ptr(&tp->tx_ring[1]), + tp->InitRxDescType, + tp->RxDescLength, + tp->num_rx_rings, + tp->num_tx_rings, + rtl8127_tot_rx_rings(tp), + rtl8127_tot_tx_rings(tp), + tp->HwSuppNumRxQueues, + tp->HwSuppNumTxQueues, + tp->EnableRss, + tp->EnablePtp, + tp->min_irq_nvecs, + tp->irq_nvecs, + tp->hw_supp_irq_nvecs, + tp->ring_lib_enabled, + tp->HwSuppIsrVer, + tp->HwCurrIsrVer, + tp->HwSuppMacMcuVer, + tp->MacMcuPageSize, + tp->hw_mcu_patch_code_ver, + tp->bin_mcu_patch_code_ver, +#ifdef ENABLE_PTP_SUPPORT + tp->tx_hwtstamp_timeouts, + tp->tx_hwtstamp_skipped, +#endif + tp->random_mac, + tp->org_mac_addr, +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,13) + dev->perm_addr, +#endif + dev->dev_addr); + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + return len; +} + +static int proc_get_tally_counter(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + struct net_device *dev = data; + struct rtl8127_private *tp = netdev_priv(dev); + struct rtl8127_counters *counters; + dma_addr_t paddr; + int len = 0; + + len += snprintf(page + len, count - len, + "\nDump Tally Counter\n"); + + rtnl_lock(); + + counters = tp->tally_vaddr; + paddr = tp->tally_paddr; + if (!counters) { + len += snprintf(page + len, count - len, + "\nDump Tally Counter Fail\n"); + goto out_unlock; + } + + rtl8127_dump_tally_counter(tp, paddr); + + len += snprintf(page + len, count - len, + "Statistics\tValue\n----------\t-----\n"); + + len += snprintf(page + len, count - len, + "tx_packets\t%lld\n" + "rx_packets\t%lld\n" + "tx_errors\t%lld\n" + "rx_errors\t%d\n" + "rx_missed\t%d\n" + "align_errors\t%d\n" + "tx_one_collision\t%d\n" + "tx_multi_collision\t%d\n" + "rx_unicast\t%lld\n" + "rx_broadcast\t%lld\n" + "rx_multicast\t%d\n" + "tx_aborted\t%d\n" + "tx_underrun\t%d\n" + + "tx_octets\t%lld\n" + "rx_octets\t%lld\n" + "rx_multicast64\t%lld\n" + "tx_unicast64\t%lld\n" + "tx_broadcast64\t%lld\n" + "tx_multicast64\t%lld\n" + "tx_pause_on\t%d\n" + "tx_pause_off\t%d\n" + "tx_pause_all\t%d\n" + "tx_deferred\t%d\n" + "tx_late_collision\t%d\n" + "tx_all_collision\t%d\n" + "tx_aborted32\t%d\n" + "align_errors32\t%d\n" + "rx_frame_too_long\t%d\n" + "rx_runt\t%d\n" + "rx_pause_on\t%d\n" + "rx_pause_off\t%d\n" + "rx_pause_all\t%d\n" + "rx_unknown_opcode\t%d\n" + "rx_mac_error\t%d\n" + "tx_underrun32\t%d\n" + "rx_mac_missed\t%d\n" + "rx_tcam_dropped\t%d\n" + "tdu\t%d\n" + "rdu\t%d\n", + le64_to_cpu(counters->tx_packets), + le64_to_cpu(counters->rx_packets), + le64_to_cpu(counters->tx_errors), + le32_to_cpu(counters->rx_errors), + le16_to_cpu(counters->rx_missed), + le16_to_cpu(counters->align_errors), + le32_to_cpu(counters->tx_one_collision), + le32_to_cpu(counters->tx_multi_collision), + le64_to_cpu(counters->rx_unicast), + le64_to_cpu(counters->rx_broadcast), + le32_to_cpu(counters->rx_multicast), + le16_to_cpu(counters->tx_aborted), + le16_to_cpu(counters->tx_underrun), + + le64_to_cpu(counters->tx_octets), + le64_to_cpu(counters->rx_octets), + le64_to_cpu(counters->rx_multicast64), + le64_to_cpu(counters->tx_unicast64), + le64_to_cpu(counters->tx_broadcast64), + le64_to_cpu(counters->tx_multicast64), + le32_to_cpu(counters->tx_pause_on), + le32_to_cpu(counters->tx_pause_off), + le32_to_cpu(counters->tx_pause_all), + le32_to_cpu(counters->tx_deferred), + le32_to_cpu(counters->tx_late_collision), + le32_to_cpu(counters->tx_all_collision), + le32_to_cpu(counters->tx_aborted32), + le32_to_cpu(counters->align_errors32), + le32_to_cpu(counters->rx_frame_too_long), + le32_to_cpu(counters->rx_runt), + le32_to_cpu(counters->rx_pause_on), + le32_to_cpu(counters->rx_pause_off), + le32_to_cpu(counters->rx_pause_all), + le32_to_cpu(counters->rx_unknown_opcode), + le32_to_cpu(counters->rx_mac_error), + le32_to_cpu(counters->tx_underrun32), + le32_to_cpu(counters->rx_mac_missed), + le32_to_cpu(counters->rx_tcam_dropped), + le32_to_cpu(counters->tdu), + le32_to_cpu(counters->rdu)); + + len += snprintf(page + len, count - len, "\n"); +out_unlock: + rtnl_unlock(); + + *eof = 1; + return len; +} + +static int proc_get_registers(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + struct net_device *dev = data; + int i, n, max = R8127_MAC_REGS_SIZE; + u8 byte_rd; + struct rtl8127_private *tp = netdev_priv(dev); + void __iomem *ioaddr = tp->mmio_addr; + int len = 0; + + len += snprintf(page + len, count - len, + "\nDump MAC Registers\n" + "Offset\tValue\n------\t-----\n"); + + for (n = 0; n < max;) { + len += snprintf(page + len, count - len, + "\n0x%04x:\t", + n); + + rtnl_lock(); + + for (i = 0; i < 16 && n < max; i++, n++) { + byte_rd = readb(ioaddr + n); + len += snprintf(page + len, count - len, + "%02x ", + byte_rd); + } + + rtnl_unlock(); + } + + rtnl_lock(); + + max = 0xB00; + for (n = 0xA00; n < max;) { + len += snprintf(page + len, count - len, + "\n0x%04x:\t", + n); + + for (i = 0; i < 16 && n < max; i++, n++) { + byte_rd = readb(ioaddr + n); + len += snprintf(page + len, count - len, + "%02x ", + byte_rd); + } + } + + max = 0xD40; + for (n = 0xD00; n < max;) { + len += snprintf(page + len, count - len, + "\n0x%04x:\t", + n); + + for (i = 0; i < 16 && n < max; i++, n++) { + byte_rd = readb(ioaddr + n); + len += snprintf(page + len, count - len, + "%02x ", + byte_rd); + } + } + + max = 0x2840; + for (n = 0x2800; n < max;) { + len += snprintf(page + len, count - len, + "\n0x%04x:\t", + n); + + for (i = 0; i < 16 && n < max; i++, n++) { + byte_rd = readb(ioaddr + n); + len += snprintf(page + len, count - len, + "%02x ", + byte_rd); + } + } + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + return len; +} + +static int proc_get_pcie_phy(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + struct net_device *dev = data; + int i, n, max = R8127_EPHY_REGS_SIZE/2; + u16 word_rd; + struct rtl8127_private *tp = netdev_priv(dev); + int len = 0; + + len += snprintf(page + len, count - len, + "\nDump PCIE PHY\n" + "Offset\tValue\n------\t-----\n"); + + rtnl_lock(); + + for (n = 0; n < max;) { + len += snprintf(page + len, count - len, + "\n0x%02x:\t", + n); + + for (i = 0; i < 8 && n < max; i++, n++) { + word_rd = rtl8127_ephy_read(tp, n); + len += snprintf(page + len, count - len, + "%04x ", + word_rd); + } + } + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + return len; +} + +static int proc_get_eth_phy(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + struct net_device *dev = data; + int i, n, max = R8127_PHY_REGS_SIZE/2; + u16 word_rd; + struct rtl8127_private *tp = netdev_priv(dev); + int len = 0; + + len += snprintf(page + len, count - len, + "\nDump Ethernet PHY\n" + "Offset\tValue\n------\t-----\n"); + + rtnl_lock(); + + len += snprintf(page + len, count - len, + "\n####################page 0##################\n"); + rtl8127_mdio_write(tp, 0x1f, 0x0000); + for (n = 0; n < max;) { + len += snprintf(page + len, count - len, + "\n0x%02x:\t", + n); + + for (i = 0; i < 8 && n < max; i++, n++) { + word_rd = rtl8127_mdio_read(tp, n); + len += snprintf(page + len, count - len, + "%04x ", + word_rd); + } + } + + len += snprintf(page + len, count - len, + "\n####################extra reg##################\n"); + n = 0xA400; + len += snprintf(page + len, count - len, + "\n0x%02x:\t", + n); + for (i = 0; i < 8; i++, n+=2) { + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + len += snprintf(page + len, count - len, + "%04x ", + word_rd); + } + + n = 0xA410; + len += snprintf(page + len, count - len, + "\n0x%02x:\t", + n); + for (i = 0; i < 3; i++, n+=2) { + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + len += snprintf(page + len, count - len, + "%04x ", + word_rd); + } + + n = 0xA434; + len += snprintf(page + len, count - len, + "\n0x%02x:\t", + n); + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + len += snprintf(page + len, count - len, + "%04x ", + word_rd); + + n = 0xA5D0; + len += snprintf(page + len, count - len, + "\n0x%02x:\t", + n); + for (i = 0; i < 4; i++, n+=2) { + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + len += snprintf(page + len, count - len, + "%04x ", + word_rd); + } + + n = 0xA61A; + len += snprintf(page + len, count - len, + "\n0x%02x:\t", + n); + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + len += snprintf(page + len, count - len, + "%04x ", + word_rd); + + n = 0xA6D0; + len += snprintf(page + len, count - len, + "\n0x%02x:\t", + n); + for (i = 0; i < 3; i++, n+=2) { + word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n); + len += snprintf(page + len, count - len, + "%04x ", + word_rd); + } + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + return len; +} + +static int proc_get_extended_registers(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + struct net_device *dev = data; + int i, n, max = R8127_ERI_REGS_SIZE; + u32 dword_rd; + struct rtl8127_private *tp = netdev_priv(dev); + int len = 0; + + len += snprintf(page + len, count - len, + "\nDump Extended Registers\n" + "Offset\tValue\n------\t-----\n"); + + rtnl_lock(); + + for (n = 0; n < max;) { + len += snprintf(page + len, count - len, + "\n0x%02x:\t", + n); + + for (i = 0; i < 4 && n < max; i++, n+=4) { + dword_rd = rtl8127_eri_read(tp, n, 4, ERIAR_ExGMAC); + len += snprintf(page + len, count - len, + "%08x ", + dword_rd); + } + } + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + return len; +} + +static int proc_get_pci_registers(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + struct net_device *dev = data; + int i, n, max = R8127_PCI_REGS_SIZE; + u32 dword_rd; + struct rtl8127_private *tp = netdev_priv(dev); + int len = 0; + + len += snprintf(page + len, count - len, + "\nDump PCI Registers\n" + "Offset\tValue\n------\t-----\n"); + + rtnl_lock(); + + for (n = 0; n < max;) { + len += snprintf(page + len, count - len, + "\n0x%03x:\t", + n); + + for (i = 0; i < 4 && n < max; i++, n+=4) { + pci_read_config_dword(tp->pci_dev, n, &dword_rd); + len += snprintf(page + len, count - len, + "%08x ", + dword_rd); + } + } + + n = 0x110; + pci_read_config_dword(tp->pci_dev, n, &dword_rd); + len += snprintf(page + len, count - len, + "\n0x%03x:\t%08x ", + n, + dword_rd); + n = 0x70c; + pci_read_config_dword(tp->pci_dev, n, &dword_rd); + len += snprintf(page + len, count - len, + "\n0x%03x:\t%08x ", + n, + dword_rd); + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + return len; +} + +static int proc_get_temperature(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + struct net_device *dev = data; + struct rtl8127_private *tp = netdev_priv(dev); + u16 ts_digout, tj, fah; + int len = 0; + + len += snprintf(page + len, count - len, + "\nChip Temperature\n"); + + rtnl_lock(); + + if (!rtl8127_sysfs_testmode_on(tp)) { + len += snprintf(page + len, count - len, + "\nPlease turn on ""/sys/class/net//rtk_adv/testmode"".\n\n"); + goto out_unlock; + } + + ts_digout = rtl8127_read_thermal_sensor(tp); + + tj = ts_digout / 2; + if (ts_digout <= 512) { + tj = ts_digout / 2; + len += snprintf(page + len, count - len, + "Cel:%d\n", + tj); + fah = tj * (9/5) + 32; + len += snprintf(page + len, count - len, + "Fah:%d\n", + fah); + + } else { + tj = (512 - ((ts_digout / 2) - 512)) / 2; + len += snprintf(page + len, count - len, + "Cel:-%d\n", + tj); + fah = tj * (9/5) + 32; + len += snprintf(page + len, count - len, + "Fah:-%d\n", + fah); + } + + len += snprintf(page + len, count - len, "\n"); + +out_unlock: + rtnl_unlock(); + + *eof = 1; + return len; +} + +static int _proc_get_cable_info(char *page, char **start, + off_t offset, int count, + int *eof, void *data, + bool poe_mode) +{ + int i; + u16 status; + int len = 0; + struct net_device *dev = data; + int cp_status[RTL8127_CP_NUM] = {0}; + int cp_len[RTL8127_CP_NUM] = {0}; + struct rtl8127_private *tp = netdev_priv(dev); + const char *pair_str[RTL8127_CP_NUM] = {"1-2", "3-6", "4-5", "7-8"}; + + switch (tp->mcfg) { + default: + return -EOPNOTSUPP; + } + + spin_lock_irqsave(&tp->phy_lock, flags); + + if (!rtl8127_sysfs_testmode_on(tp)) { + len += snprintf(page + len, count - len, + "\nPlease turn on ""/sys/class/net//rtk_adv/testmode"".\n\n"); + goto out_unlock; + } + + status = RTL_R16(tp, PHYstatus); + if (status & LinkStatus) + len += snprintf(page + len, count - len, + "\nlink speed:%d", + rtl8127_convert_link_speed(status)); + else + len += snprintf(page + len, count - len, + "\nlink status:off"); + + rtl8127_get_cp_len(tp, cp_len); + + rtl8127_get_cp_status(tp, cp_status, poe_mode); + + len += snprintf(page + len, count - len, + "\npair\tlength\tstatus \tpp\n"); + + for (i=0; iphy_lock, flags); + + *eof = 1; + return len; +} + +static int proc_get_cable_info(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + return _proc_get_cable_info(page, start, offset, count, eof, data, 0); +} + +static int proc_get_poe_cable_info(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + return _proc_get_cable_info(page, start, offset, count, eof, data, 1); +} + +static void _proc_dump_desc(char *page, int *page_len, int *count, void *desc_base, + u32 alloc_size) +{ + u32 *pdword; + int i, len; + + if (desc_base == NULL || + alloc_size == 0) + return; + + len = *page_len; + pdword = (u32*)desc_base; + for (i=0; i<(alloc_size/4); i++) { + if (!(i % 4)) + len += snprintf(page + len, *count - len, + "\n%04x ", + i); + len += snprintf(page + len, *count - len, + "%08x ", + pdword[i]); + } + + len += snprintf(page + len, *count - len, "\n"); + + *page_len = len; + return; +} + +static int proc_dump_rx_desc(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + int i; + int len = 0; + struct net_device *dev = data; + struct rtl8127_private *tp = netdev_priv(dev); + + rtnl_lock(); + + for (i = 0; i < tp->num_rx_rings; i++) { + struct rtl8127_rx_ring *ring = &tp->rx_ring[i]; + + if (!ring) + continue; + + len += snprintf(page + len, count - len, + "\ndump rx %d desc:%d", + i, ring->num_rx_desc); + + _proc_dump_desc(page, &len, &count, + ring->RxDescArray, + ring->RxDescAllocSize); + } + +#ifdef ENABLE_LIB_SUPPORT + if (rtl8127_num_lib_rx_rings(tp) > 0) { + for (i = 0; i < tp->HwSuppNumRxQueues; i++) { + struct rtl8127_ring *lib_ring = &tp->lib_rx_ring[i]; + if (lib_ring->enabled) { + len += snprintf(page + len, count - len, + "\ndump lib rx %d desc:%d", + i, + ring->ring_size); + _proc_dump_desc(page, &len, &count, + (void*)lib_ring->desc_addr, + lib_ring->desc_size); + } + } + } +#endif //ENABLE_LIB_SUPPORT + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + + return len; +} + +static int proc_dump_tx_desc(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + int len = 0; + struct net_device *dev = data; + struct rtl8127_private *tp = netdev_priv(dev); + int i; + + rtnl_lock(); + + for (i = 0; i < tp->num_tx_rings; i++) { + struct rtl8127_tx_ring *ring = &tp->tx_ring[i]; + + if (!ring) + continue; + + len += snprintf(page + len, count - len, + "\ndump tx desc:%d", + ring->num_tx_desc); + + _proc_dump_desc(page, &len, &count, + ring->TxDescArray, + ring->TxDescAllocSize); + } + +#ifdef ENABLE_LIB_SUPPORT + if (rtl8127_num_lib_tx_rings(tp) > 0) { + for (i = 0; i < tp->HwSuppNumTxQueues; i++) { + struct rtl8127_ring *lib_ring = &tp->lib_tx_ring[i]; + if (lib_ring->enabled) { + len += snprintf(page + len, count - len, + "\ndump lib tx %d desc:%d", + i, + ring->ring_size); + _proc_dump_desc(page, &len, &count, + (void*)lib_ring->desc_addr, + lib_ring->desc_size); + } + } + } +#endif //ENABLE_LIB_SUPPORT + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + + return len; +} + +static int proc_dump_msix_tbl(char *page, char **start, + off_t offset, int count, + int *eof, void *data) +{ + int i, j; + int len = 0; + void __iomem *ioaddr; + struct net_device *dev = data; + struct rtl8127_private *tp = netdev_priv(dev); + + /* ioremap MMIO region */ + ioaddr = ioremap(pci_resource_start(tp->pci_dev, 4), pci_resource_len(tp->pci_dev, 4)); + if (!ioaddr) + return -EFAULT; + + rtnl_lock(); + + len += snprintf(page + len, count - len, + "\ndump MSI-X Table. Total Entry %d. \n", + tp->hw_supp_irq_nvecs); + + for (i=0; ihw_supp_irq_nvecs; i++) { + len += snprintf(page + len, count - len, + "\n%04x ", i); + for (j=0; j<4; j++) + len += snprintf(page + len, count - len, "%08x ", + readl(ioaddr + i*0x10 + 4*j)); + } + + rtnl_unlock(); + + len += snprintf(page + len, count - len, "\n"); + + *eof = 1; + return 0; +} + +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) + +static void rtl8127_proc_module_init(void) +{ + //create /proc/net/r8127 +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,32) + rtl8127_proc = proc_mkdir(MODULENAME, init_net.proc_net); +#else + rtl8127_proc = proc_mkdir(MODULENAME, proc_net); +#endif + if (!rtl8127_proc) + dprintk("cannot create %s proc entry \n", MODULENAME); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) +/* + * seq_file wrappers for procfile show routines. + */ +static int rtl8127_proc_open(struct inode *inode, struct file *file) +{ + struct net_device *dev = proc_get_parent_data(inode); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0) + int (*show)(struct seq_file *, void *) = pde_data(inode); +#else + int (*show)(struct seq_file *, void *) = PDE_DATA(inode); +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0) + + return single_open(file, show, dev); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0) +static const struct proc_ops rtl8127_proc_fops = { + .proc_open = rtl8127_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, +}; +#else +static const struct file_operations rtl8127_proc_fops = { + .open = rtl8127_proc_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; +#endif + +#endif + +/* + * Table of proc files we need to create. + */ +struct rtl8127_proc_file { + char name[16]; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) + int (*show)(struct seq_file *, void *); +#else + int (*show)(char *, char **, off_t, int, int *, void *); +#endif +}; + +static const struct rtl8127_proc_file rtl8127_debug_proc_files[] = { + { "driver_var", &proc_get_driver_variable }, + { "tally", &proc_get_tally_counter }, + { "registers", &proc_get_registers }, + { "pcie_phy", &proc_get_pcie_phy }, + { "eth_phy", &proc_get_eth_phy }, + { "ext_regs", &proc_get_extended_registers }, + { "pci_regs", &proc_get_pci_registers }, + { "tx_desc", &proc_dump_tx_desc }, + { "rx_desc", &proc_dump_rx_desc }, + { "msix_tbl", &proc_dump_msix_tbl }, + { "", NULL } +}; + +static const struct rtl8127_proc_file rtl8127_test_proc_files[] = { + { "temp", &proc_get_temperature }, + { "cdt", &proc_get_cable_info }, + { "cdt_poe", &proc_get_poe_cable_info }, + { "", NULL } +}; + +#define R8127_PROC_DEBUG_DIR "debug" +#define R8127_PROC_TEST_DIR "test" + +static void rtl8127_proc_init(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + const struct rtl8127_proc_file *f; + struct proc_dir_entry *dir; + + if (!rtl8127_proc) + return; + + if (tp->proc_dir_debug || tp->proc_dir_test) + return; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) + dir = proc_mkdir_data(dev->name, 0, rtl8127_proc, dev); + if (!dir) { + printk("Unable to initialize /proc/net/%s/%s\n", + MODULENAME, dev->name); + return; + } + tp->proc_dir = dir; + proc_init_num++; + + /* create debug entry */ + dir = proc_mkdir_data(R8127_PROC_DEBUG_DIR, 0, tp->proc_dir, dev); + if (!dir) { + printk("Unable to initialize /proc/net/%s/%s/%s\n", + MODULENAME, dev->name, R8127_PROC_DEBUG_DIR); + return; + } + + tp->proc_dir_debug = dir; + for (f = rtl8127_debug_proc_files; f->name[0]; f++) { + if (!proc_create_data(f->name, S_IFREG | S_IRUGO, dir, + &rtl8127_proc_fops, f->show)) { + printk("Unable to initialize " + "/proc/net/%s/%s/%s/%s\n", + MODULENAME, dev->name, R8127_PROC_DEBUG_DIR, + f->name); + return; + } + } + + /* create test entry */ + dir = proc_mkdir_data(R8127_PROC_TEST_DIR, 0, tp->proc_dir, dev); + if (!dir) { + printk("Unable to initialize /proc/net/%s/%s/%s\n", + MODULENAME, dev->name, R8127_PROC_TEST_DIR); + return; + } + + tp->proc_dir_test = dir; + for (f = rtl8127_test_proc_files; f->name[0]; f++) { + if (!proc_create_data(f->name, S_IFREG | S_IRUGO, dir, + &rtl8127_proc_fops, f->show)) { + printk("Unable to initialize " + "/proc/net/%s/%s/%s/%s\n", + MODULENAME, dev->name, R8127_PROC_TEST_DIR, + f->name); + return; + } + } +#else + dir = proc_mkdir(dev->name, rtl8127_proc); + if (!dir) { + printk("Unable to initialize /proc/net/%s/%s\n", + MODULENAME, dev->name); + return; + } + + tp->proc_dir = dir; + proc_init_num++; + + /* create debug entry */ + dir = proc_mkdir(R8127_PROC_DEBUG_DIR, tp->proc_dir); + if (!dir) { + printk("Unable to initialize /proc/net/%s/%s/%s\n", + MODULENAME, dev->name, R8127_PROC_DEBUG_DIR); + return; + } + + tp->proc_dir_debug = dir; + for (f = rtl8127_debug_proc_files; f->name[0]; f++) { + if (!create_proc_read_entry(f->name, S_IFREG | S_IRUGO, + dir, f->show, dev)) { + printk("Unable to initialize " + "/proc/net/%s/%s/%s/%s\n", + MODULENAME, dev->name, R8127_PROC_DEBUG_DIR, + f->name); + return; + } + } + + /* create test entry */ + dir = proc_mkdir(R8127_PROC_TEST_DIR, tp->proc_dir); + if (!dir) { + printk("Unable to initialize /proc/net/%s/%s/%s\n", + MODULENAME, dev->name, R8127_PROC_TEST_DIR); + return; + } + + tp->proc_dir_test = dir; + for (f = rtl8127_test_proc_files; f->name[0]; f++) { + if (!create_proc_read_entry(f->name, S_IFREG | S_IRUGO, + dir, f->show, dev)) { + printk("Unable to initialize " + "/proc/net/%s/%s/%s/%s\n", + MODULENAME, dev->name, R8127_PROC_TEST_DIR, + f->name); + return; + } + } +#endif +} + +static void rtl8127_proc_remove(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (tp->proc_dir) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) + remove_proc_subtree(dev->name, rtl8127_proc); +#else + const struct rtl8127_proc_file *f; + struct rtl8127_private *tp = netdev_priv(dev); + + if (tp->proc_dir_debug) { + for (f = rtl8127_debug_proc_files; f->name[0]; f++) + remove_proc_entry(f->name, tp->proc_dir_debug); + remove_proc_entry(R8127_PROC_DEBUG_DIR, tp->proc_dir); + } + + if (tp->proc_dir_test) { + for (f = rtl8127_test_proc_files; f->name[0]; f++) + remove_proc_entry(f->name, tp->proc_dir_test); + remove_proc_entry(R8127_PROC_TEST_DIR, tp->proc_dir); + } + + remove_proc_entry(dev->name, rtl8127_proc); +#endif + proc_init_num--; + + tp->proc_dir_debug = NULL; + tp->proc_dir_test = NULL; + tp->proc_dir = NULL; + } +} + +#endif //ENABLE_R8127_PROCFS + +#ifdef ENABLE_R8127_SYSFS +/**************************************************************************** +* -----------------------------SYSFS STUFF------------------------- +***************************************************************************** +*/ +static ssize_t testmode_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct net_device *netdev = to_net_dev(dev); + struct rtl8127_private *tp = netdev_priv(netdev); + + sprintf(buf, "%u\n", tp->testmode); + + return strlen(buf); +} + +static ssize_t testmode_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct net_device *netdev = to_net_dev(dev); + struct rtl8127_private *tp = netdev_priv(netdev); + u32 testmode; + + if (sscanf(buf, "%u\n", &testmode) != 1) + return -EINVAL; + + if (tp->testmode != testmode) { + rtnl_lock(); + tp->testmode = testmode; + rtnl_unlock(); + } + + return count; +} + +static DEVICE_ATTR_RW(testmode); + +static struct attribute *rtk_adv_attrs[] = { + &dev_attr_testmode.attr, + NULL +}; + +static struct attribute_group rtk_adv_grp = { + .name = "rtl_adv", + .attrs = rtk_adv_attrs, +}; + +static void rtl8127_sysfs_init(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int ret; + + /* init rtl_adv */ +#ifdef ENABLE_LIB_SUPPORT + tp->testmode = 0; +#else + tp->testmode = 1; +#endif //ENABLE_LIB_SUPPORT + + ret = sysfs_create_group(&dev->dev.kobj, &rtk_adv_grp); + if (ret < 0) + netif_warn(tp, probe, dev, "create rtk_adv_grp fail\n"); + else + set_bit(R8127_SYSFS_RTL_ADV, tp->sysfs_flag); +} + +static void rtl8127_sysfs_remove(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (test_and_clear_bit(R8127_SYSFS_RTL_ADV, tp->sysfs_flag)) + sysfs_remove_group(&dev->dev.kobj, &rtk_adv_grp); +} +#endif //ENABLE_R8127_SYSFS + +static inline u16 map_phy_ocp_addr(u16 PageNum, u8 RegNum) +{ + u16 OcpPageNum = 0; + u8 OcpRegNum = 0; + u16 OcpPhyAddress = 0; + + if (PageNum == 0) { + OcpPageNum = OCP_STD_PHY_BASE_PAGE + (RegNum / 8); + OcpRegNum = 0x10 + (RegNum % 8); + } else { + OcpPageNum = PageNum; + OcpRegNum = RegNum; + } + + OcpPageNum <<= 4; + + if (OcpRegNum < 16) { + OcpPhyAddress = 0; + } else { + OcpRegNum -= 16; + OcpRegNum <<= 1; + + OcpPhyAddress = OcpPageNum + OcpRegNum; + } + + + return OcpPhyAddress; +} + +static void mdio_real_direct_write_phy_ocp(struct rtl8127_private *tp, + u16 RegAddr, + u16 value) +{ + u32 data32; + int i; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18) + WARN_ON_ONCE(RegAddr % 2); +#endif + data32 = RegAddr/2; + data32 <<= OCPR_Addr_Reg_shift; + data32 |= OCPR_Write | value; + + RTL_W32(tp, PHYOCP, data32); + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { + udelay(R8127_CHANNEL_WAIT_TIME); + + if (!(RTL_R32(tp, PHYOCP) & OCPR_Flag)) + break; + } +} + +void rtl8127_mdio_direct_write_phy_ocp(struct rtl8127_private *tp, + u16 RegAddr, + u16 value) +{ + if (tp->rtk_enable_diag) + return; + + mdio_real_direct_write_phy_ocp(tp, RegAddr, value); +} + +/* +static void rtl8127_mdio_write_phy_ocp(struct rtl8127_private *tp, + u16 PageNum, + u32 RegAddr, + u32 value) +{ + u16 ocp_addr; + + ocp_addr = map_phy_ocp_addr(PageNum, RegAddr); + + rtl8127_mdio_direct_write_phy_ocp(tp, ocp_addr, value); +} +*/ + +static void rtl8127_mdio_real_write_phy_ocp(struct rtl8127_private *tp, + u16 PageNum, + u32 RegAddr, + u32 value) +{ + u16 ocp_addr; + + ocp_addr = map_phy_ocp_addr(PageNum, RegAddr); + + mdio_real_direct_write_phy_ocp(tp, ocp_addr, value); +} + +static void mdio_real_write(struct rtl8127_private *tp, + u16 RegAddr, + u16 value) +{ + if (RegAddr == 0x1F) { + tp->cur_page = value; + return; + } + rtl8127_mdio_real_write_phy_ocp(tp, tp->cur_page, RegAddr, value); +} + +void rtl8127_mdio_write(struct rtl8127_private *tp, + u16 RegAddr, + u16 value) +{ + if (tp->rtk_enable_diag) + return; + + mdio_real_write(tp, RegAddr, value); +} + +void rtl8127_mdio_prot_write(struct rtl8127_private *tp, + u32 RegAddr, + u32 value) +{ + mdio_real_write(tp, RegAddr, value); +} + +void rtl8127_mdio_prot_direct_write_phy_ocp(struct rtl8127_private *tp, + u32 RegAddr, + u32 value) +{ + mdio_real_direct_write_phy_ocp(tp, RegAddr, value); +} + +static u32 mdio_real_direct_read_phy_ocp(struct rtl8127_private *tp, + u16 RegAddr) +{ + u32 data32; + int i, value = 0; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18) + WARN_ON_ONCE(RegAddr % 2); +#endif + data32 = RegAddr/2; + data32 <<= OCPR_Addr_Reg_shift; + + RTL_W32(tp, PHYOCP, data32); + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { + udelay(R8127_CHANNEL_WAIT_TIME); + + if (RTL_R32(tp, PHYOCP) & OCPR_Flag) + break; + } + value = RTL_R32(tp, PHYOCP) & OCPDR_Data_Mask; + + return value; +} + +u32 rtl8127_mdio_direct_read_phy_ocp(struct rtl8127_private *tp, + u16 RegAddr) +{ + if (tp->rtk_enable_diag) + return 0xffffffff; + + return mdio_real_direct_read_phy_ocp(tp, RegAddr); +} + +/* +static u32 rtl8127_mdio_read_phy_ocp(struct rtl8127_private *tp, + u16 PageNum, + u32 RegAddr) +{ + u16 ocp_addr; + + ocp_addr = map_phy_ocp_addr(PageNum, RegAddr); + + return rtl8127_mdio_direct_read_phy_ocp(tp, ocp_addr); +} +*/ + +static u32 rtl8127_mdio_real_read_phy_ocp(struct rtl8127_private *tp, + u16 PageNum, + u32 RegAddr) +{ + u16 ocp_addr; + + ocp_addr = map_phy_ocp_addr(PageNum, RegAddr); + + return mdio_real_direct_read_phy_ocp(tp, ocp_addr); +} + +static u32 mdio_real_read(struct rtl8127_private *tp, + u16 RegAddr) +{ + return rtl8127_mdio_real_read_phy_ocp(tp, tp->cur_page, RegAddr); +} + +u32 rtl8127_mdio_read(struct rtl8127_private *tp, + u16 RegAddr) +{ + if (tp->rtk_enable_diag) + return 0xffffffff; + + return mdio_real_read(tp, RegAddr); +} + +u32 rtl8127_mdio_prot_read(struct rtl8127_private *tp, + u32 RegAddr) +{ + return mdio_real_read(tp, RegAddr); +} + +u32 rtl8127_mdio_prot_direct_read_phy_ocp(struct rtl8127_private *tp, + u32 RegAddr) +{ + return mdio_real_direct_read_phy_ocp(tp, RegAddr); +} + +static void rtl8127_clear_and_set_eth_phy_bit(struct rtl8127_private *tp, u8 addr, u16 clearmask, u16 setmask) +{ + u16 PhyRegValue; + + PhyRegValue = rtl8127_mdio_read(tp, addr); + PhyRegValue &= ~clearmask; + PhyRegValue |= setmask; + rtl8127_mdio_write(tp, addr, PhyRegValue); +} + +void rtl8127_clear_eth_phy_bit(struct rtl8127_private *tp, u8 addr, u16 mask) +{ + rtl8127_clear_and_set_eth_phy_bit(tp, + addr, + mask, + 0); +} + +void rtl8127_set_eth_phy_bit(struct rtl8127_private *tp, u8 addr, u16 mask) +{ + rtl8127_clear_and_set_eth_phy_bit(tp, + addr, + 0, + mask); +} + +void rtl8127_clear_and_set_eth_phy_ocp_bit(struct rtl8127_private *tp, u16 addr, u16 clearmask, u16 setmask) +{ + u16 PhyRegValue; + + PhyRegValue = rtl8127_mdio_direct_read_phy_ocp(tp, addr); + PhyRegValue &= ~clearmask; + PhyRegValue |= setmask; + rtl8127_mdio_direct_write_phy_ocp(tp, addr, PhyRegValue); +} + +void rtl8127_clear_eth_phy_ocp_bit(struct rtl8127_private *tp, u16 addr, u16 mask) +{ + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + addr, + mask, + 0); +} + +void rtl8127_set_eth_phy_ocp_bit(struct rtl8127_private *tp, u16 addr, u16 mask) +{ + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + addr, + 0, + mask); +} + +void rtl8127_mac_ocp_write(struct rtl8127_private *tp, u16 reg_addr, u16 value) +{ + u32 data32; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18) + WARN_ON_ONCE(reg_addr % 2); +#endif + + data32 = reg_addr/2; + data32 <<= OCPR_Addr_Reg_shift; + data32 += value; + data32 |= OCPR_Write; + + RTL_W32(tp, MACOCP, data32); +} + +u16 rtl8127_mac_ocp_read(struct rtl8127_private *tp, u16 reg_addr) +{ + u32 data32; + u16 data16 = 0; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18) + WARN_ON_ONCE(reg_addr % 2); +#endif + + data32 = reg_addr/2; + data32 <<= OCPR_Addr_Reg_shift; + + RTL_W32(tp, MACOCP, data32); + data16 = (u16)RTL_R32(tp, MACOCP); + + return data16; +} + +#ifdef ENABLE_USE_FIRMWARE_FILE +static void mac_mcu_write(struct rtl8127_private *tp, u16 reg, u16 value) +{ + if (reg == 0x1f) { + tp->ocp_base = value << 4; + return; + } + + rtl8127_mac_ocp_write(tp, tp->ocp_base + reg, value); +} + +static u32 mac_mcu_read(struct rtl8127_private *tp, u16 reg) +{ + return rtl8127_mac_ocp_read(tp, tp->ocp_base + reg); +} +#endif + +static void +rtl8127_clear_set_mac_ocp_bit( + struct rtl8127_private *tp, + u16 addr, + u16 clearmask, + u16 setmask +) +{ + u16 PhyRegValue; + + PhyRegValue = rtl8127_mac_ocp_read(tp, addr); + PhyRegValue &= ~clearmask; + PhyRegValue |= setmask; + rtl8127_mac_ocp_write(tp, addr, PhyRegValue); +} + +void +rtl8127_clear_mac_ocp_bit( + struct rtl8127_private *tp, + u16 addr, + u16 mask +) +{ + rtl8127_clear_set_mac_ocp_bit(tp, + addr, + mask, + 0); +} + +static void +rtl8127_set_mac_ocp_bit( + struct rtl8127_private *tp, + u16 addr, + u16 mask +) +{ + rtl8127_clear_set_mac_ocp_bit(tp, + addr, + 0, + mask); +} + +u32 rtl8127_ocp_read_with_oob_base_address(struct rtl8127_private *tp, u16 addr, u8 len, const u32 base_address) +{ + return rtl8127_eri_read_with_oob_base_address(tp, addr, len, ERIAR_OOB, base_address); +} + +u32 rtl8127_ocp_read(struct rtl8127_private *tp, u16 addr, u8 len) +{ + u32 value = 0; + + if (!tp->AllowAccessDashOcp) + return 0xffffffff; + + if (HW_DASH_SUPPORT_TYPE_2(tp)) + value = rtl8127_ocp_read_with_oob_base_address(tp, addr, len, NO_BASE_ADDRESS); + else if (HW_DASH_SUPPORT_TYPE_3(tp)) + value = rtl8127_ocp_read_with_oob_base_address(tp, addr, len, RTL8168FP_OOBMAC_BASE); + + return value; +} + +u32 rtl8127_ocp_write_with_oob_base_address(struct rtl8127_private *tp, u16 addr, u8 len, u32 value, const u32 base_address) +{ + return rtl8127_eri_write_with_oob_base_address(tp, addr, len, value, ERIAR_OOB, base_address); +} + +void rtl8127_ocp_write(struct rtl8127_private *tp, u16 addr, u8 len, u32 value) +{ + if (!tp->AllowAccessDashOcp) + return; + + if (HW_DASH_SUPPORT_TYPE_2(tp)) + rtl8127_ocp_write_with_oob_base_address(tp, addr, len, value, NO_BASE_ADDRESS); + else if (HW_DASH_SUPPORT_TYPE_3(tp)) + rtl8127_ocp_write_with_oob_base_address(tp, addr, len, value, RTL8168FP_OOBMAC_BASE); +} + +void rtl8127_oob_mutex_lock(struct rtl8127_private *tp) +{ + u8 reg_16, reg_a0; + u32 wait_cnt_0, wait_Cnt_1; + u16 ocp_reg_mutex_ib; + u16 ocp_reg_mutex_oob; + u16 ocp_reg_mutex_prio; + + if (!tp->DASH) + return; + + switch (tp->mcfg) { + default: + return; + } + + rtl8127_ocp_write(tp, ocp_reg_mutex_ib, 1, BIT_0); + reg_16 = rtl8127_ocp_read(tp, ocp_reg_mutex_oob, 1); + wait_cnt_0 = 0; + while(reg_16) { + reg_a0 = rtl8127_ocp_read(tp, ocp_reg_mutex_prio, 1); + if (reg_a0) { + rtl8127_ocp_write(tp, ocp_reg_mutex_ib, 1, 0x00); + reg_a0 = rtl8127_ocp_read(tp, ocp_reg_mutex_prio, 1); + wait_Cnt_1 = 0; + while(reg_a0) { + reg_a0 = rtl8127_ocp_read(tp, ocp_reg_mutex_prio, 1); + + wait_Cnt_1++; + + if (wait_Cnt_1 > 2000) + break; + }; + rtl8127_ocp_write(tp, ocp_reg_mutex_ib, 1, BIT_0); + + } + reg_16 = rtl8127_ocp_read(tp, ocp_reg_mutex_oob, 1); + + wait_cnt_0++; + + if (wait_cnt_0 > 2000) + break; + }; +} + +void rtl8127_oob_mutex_unlock(struct rtl8127_private *tp) +{ + //u16 ocp_reg_mutex_ib; + //u16 ocp_reg_mutex_oob; + //u16 ocp_reg_mutex_prio; + + if (!tp->DASH) + return; + + switch (tp->mcfg) { + default: + return; + } + + //rtl8127_ocp_write(tp, ocp_reg_mutex_prio, 1, BIT_0); + //rtl8127_ocp_write(tp, ocp_reg_mutex_ib, 1, 0x00); +} + +static bool +rtl8127_is_allow_access_dash_ocp(struct rtl8127_private *tp) +{ + bool allow_access = false; + + if (!HW_DASH_SUPPORT_DASH(tp)) + goto exit; + + allow_access = true; + switch (tp->mcfg) { + default: + goto exit; + } +exit: + return allow_access; +} + +static int rtl8127_check_dash(struct rtl8127_private *tp) +{ + if (!tp->AllowAccessDashOcp) + return 0; + + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) { + if (rtl8127_ocp_read(tp, 0x128, 1) & BIT_0) + return 1; + } + + return 0; +} + +void rtl8127_dash2_disable_tx(struct rtl8127_private *tp) +{ + if (!tp->DASH) + return; + + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) { + u16 WaitCnt; + u8 TmpUchar; + + //Disable oob Tx + RTL_CMAC_W8(tp, CMAC_IBCR2, RTL_CMAC_R8(tp, CMAC_IBCR2) & ~(BIT_0)); + WaitCnt = 0; + + //wait oob tx disable + do { + TmpUchar = RTL_CMAC_R8(tp, CMAC_IBISR0); + + if (TmpUchar & ISRIMR_DASH_TYPE2_TX_DISABLE_IDLE) { + break; + } + + fsleep(50); + WaitCnt++; + } while(WaitCnt < 2000); + + //Clear ISRIMR_DASH_TYPE2_TX_DISABLE_IDLE + RTL_CMAC_W8(tp, CMAC_IBISR0, RTL_CMAC_R8(tp, CMAC_IBISR0) | ISRIMR_DASH_TYPE2_TX_DISABLE_IDLE); + } +} + +void rtl8127_dash2_enable_tx(struct rtl8127_private *tp) +{ + if (!tp->DASH) + return; + + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) + RTL_CMAC_W8(tp, CMAC_IBCR2, RTL_CMAC_R8(tp, CMAC_IBCR2) | BIT_0); +} + +void rtl8127_dash2_disable_rx(struct rtl8127_private *tp) +{ + if (!tp->DASH) + return; + + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) + RTL_CMAC_W8(tp, CMAC_IBCR0, RTL_CMAC_R8(tp, CMAC_IBCR0) & ~(BIT_0)); +} + +void rtl8127_dash2_enable_rx(struct rtl8127_private *tp) +{ + if (!tp->DASH) + return; + + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) + RTL_CMAC_W8(tp, CMAC_IBCR0, RTL_CMAC_R8(tp, CMAC_IBCR0) | BIT_0); +} + +static void rtl8127_dash2_disable_txrx(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) { + rtl8127_dash2_disable_tx(tp); + rtl8127_dash2_disable_rx(tp); + } +} + +static int rtl8127_wait_dash_fw_ready(struct rtl8127_private *tp) +{ + int rc = -1; + int timeout; + + if (HW_DASH_SUPPORT_TYPE_2(tp) == FALSE && + HW_DASH_SUPPORT_TYPE_3(tp) == FALSE) + goto out; + + if (!tp->DASH) + goto out; + + for (timeout = 0; timeout < 10; timeout++) { + fsleep(10000); + if (rtl8127_ocp_read(tp, 0x124, 1) & BIT_0) { + rc = 1; + goto out; + } + } + + rc = 0; + +out: + return rc; +} + +static void rtl8127_driver_start(struct rtl8127_private *tp) +{ + u32 tmp_value; + + if (HW_DASH_SUPPORT_TYPE_2(tp) == FALSE && + HW_DASH_SUPPORT_TYPE_3(tp) == FALSE) + return; + + if (!tp->AllowAccessDashOcp) + return; + + rtl8127_ocp_write(tp, 0x180, 1, OOB_CMD_DRIVER_START); + tmp_value = rtl8127_ocp_read(tp, 0x30, 1); + tmp_value |= BIT_0; + rtl8127_ocp_write(tp, 0x30, 1, tmp_value); + + rtl8127_wait_dash_fw_ready(tp); +} + +static void rtl8127_driver_stop(struct rtl8127_private *tp) +{ + u32 tmp_value; + struct net_device *dev = tp->dev; + + if (HW_DASH_SUPPORT_TYPE_2(tp) == FALSE && + HW_DASH_SUPPORT_TYPE_3(tp) == FALSE) + return; + + if (!tp->AllowAccessDashOcp) + return; + + rtl8127_dash2_disable_txrx(dev); + + rtl8127_ocp_write(tp, 0x180, 1, OOB_CMD_DRIVER_STOP); + tmp_value = rtl8127_ocp_read(tp, 0x30, 1); + tmp_value |= BIT_0; + rtl8127_ocp_write(tp, 0x30, 1, tmp_value); + + rtl8127_wait_dash_fw_ready(tp); +} + +static void _rtl8127_ephy_write(struct rtl8127_private *tp, int addr, int data) +{ + int i; + + RTL_W32(tp, EPHYAR, + EPHYAR_Write | + (addr & EPHYAR_Reg_Mask_v2) << EPHYAR_Reg_shift | + (data & EPHYAR_Data_Mask)); + + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { + fsleep(R8127_CHANNEL_WAIT_TIME); + + /* Check if the RTL8125 has completed EPHY write */ + if (!(RTL_R32(tp, EPHYAR) & EPHYAR_Flag)) + break; + } + + fsleep(R8127_CHANNEL_EXIT_DELAY_TIME); +} + +static void rtl8127_set_ephy_ext_addr(struct rtl8127_private *tp, int addr) +{ + _rtl8127_ephy_write(tp, EPHYAR_EXT_ADDR, addr); +} + +static int rtl8127_check_ephy_ext_addr(struct rtl8127_private *tp, int addr) +{ + int data; + + data = ((u16)addr >> 12); + + rtl8127_set_ephy_ext_addr(tp, data); + + return (addr & 0xfff); +} + +void rtl8127_ephy_write(struct rtl8127_private *tp, int addr, int data) +{ + _rtl8127_ephy_write(tp, rtl8127_check_ephy_ext_addr(tp, addr), data); +} + +static u16 _rtl8127_ephy_read(struct rtl8127_private *tp, int addr) +{ + int i; + u16 data = 0xffff; + + RTL_W32(tp, EPHYAR, + EPHYAR_Read | (addr & EPHYAR_Reg_Mask_v2) << EPHYAR_Reg_shift); + + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { + fsleep(R8127_CHANNEL_WAIT_TIME); + + /* Check if the RTL8125 has completed EPHY read */ + if (RTL_R32(tp, EPHYAR) & EPHYAR_Flag) { + data = (u16) (RTL_R32(tp, EPHYAR) & EPHYAR_Data_Mask); + break; + } + } + + fsleep(R8127_CHANNEL_EXIT_DELAY_TIME); + + return data; +} + +u16 rtl8127_ephy_read(struct rtl8127_private *tp, int addr) +{ + return _rtl8127_ephy_read(tp, rtl8127_check_ephy_ext_addr(tp, addr)); +} + +/* +static void ClearAndSetPCIePhyBit(struct rtl8127_private *tp, u8 addr, u16 clearmask, u16 setmask) +{ + u16 EphyValue; + + EphyValue = rtl8127_ephy_read(tp, addr); + EphyValue &= ~clearmask; + EphyValue |= setmask; + rtl8127_ephy_write(tp, addr, EphyValue); +} + +static void ClearPCIePhyBit(struct rtl8127_private *tp, u8 addr, u16 mask) +{ + ClearAndSetPCIePhyBit(tp, + addr, + mask, + 0); +} + +static void SetPCIePhyBit(struct rtl8127_private *tp, u8 addr, u16 mask) +{ + ClearAndSetPCIePhyBit(tp, + addr, + 0, + mask); +} +*/ + +static u32 +rtl8127_csi_other_fun_read(struct rtl8127_private *tp, + u8 multi_fun_sel_bit, + u32 addr) +{ + u32 cmd; + int i; + u32 value = 0xffffffff; + + cmd = CSIAR_Read | CSIAR_ByteEn << CSIAR_ByteEn_shift | (addr & CSIAR_Addr_Mask); + + if (tp->mcfg == CFG_METHOD_DEFAULT) + multi_fun_sel_bit = 0; + + if (multi_fun_sel_bit > 7) + goto exit; + + cmd |= multi_fun_sel_bit << 16; + + RTL_W32(tp, CSIAR, cmd); + + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { + fsleep(R8127_CHANNEL_WAIT_TIME); + + /* Check if the RTL8125 has completed CSI read */ + if (RTL_R32(tp, CSIAR) & CSIAR_Flag) { + value = (u32)RTL_R32(tp, CSIDR); + break; + } + } + + fsleep(R8127_CHANNEL_EXIT_DELAY_TIME); + +exit: + return value; +} + +static void +rtl8127_csi_other_fun_write(struct rtl8127_private *tp, + u8 multi_fun_sel_bit, + u32 addr, + u32 value) +{ + u32 cmd; + int i; + + RTL_W32(tp, CSIDR, value); + cmd = CSIAR_Write | CSIAR_ByteEn << CSIAR_ByteEn_shift | (addr & CSIAR_Addr_Mask); + if (tp->mcfg == CFG_METHOD_DEFAULT) + multi_fun_sel_bit = 0; + + if (multi_fun_sel_bit > 7) + return; + + cmd |= multi_fun_sel_bit << 16; + + RTL_W32(tp, CSIAR, cmd); + + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { + fsleep(R8127_CHANNEL_WAIT_TIME); + + /* Check if the RTL8125 has completed CSI write */ + if (!(RTL_R32(tp, CSIAR) & CSIAR_Flag)) + break; + } + + fsleep(R8127_CHANNEL_EXIT_DELAY_TIME); +} + +static u32 +rtl8127_csi_read(struct rtl8127_private *tp, + u32 addr) +{ + u8 multi_fun_sel_bit; + + multi_fun_sel_bit = 0; + + return rtl8127_csi_other_fun_read(tp, multi_fun_sel_bit, addr); +} + +static void +rtl8127_csi_write(struct rtl8127_private *tp, + u32 addr, + u32 value) +{ + u8 multi_fun_sel_bit; + + multi_fun_sel_bit = 0; + + rtl8127_csi_other_fun_write(tp, multi_fun_sel_bit, addr, value); +} + +static u8 +rtl8127_csi_fun0_read_byte(struct rtl8127_private *tp, + u32 addr) +{ + u8 RetVal = 0; + + if (tp->mcfg == CFG_METHOD_DEFAULT) { + struct pci_dev *pdev = tp->pci_dev; + + pci_read_config_byte(pdev, addr, &RetVal); + } else { + u32 TmpUlong; + u16 RegAlignAddr; + u8 ShiftByte; + + RegAlignAddr = addr & ~(0x3); + ShiftByte = addr & (0x3); + TmpUlong = rtl8127_csi_other_fun_read(tp, 0, RegAlignAddr); + TmpUlong >>= (8*ShiftByte); + RetVal = (u8)TmpUlong; + } + + fsleep(R8127_CHANNEL_EXIT_DELAY_TIME); + + return RetVal; +} + +static void +rtl8127_csi_fun0_write_byte(struct rtl8127_private *tp, + u32 addr, + u8 value) +{ + if (tp->mcfg == CFG_METHOD_DEFAULT) { + struct pci_dev *pdev = tp->pci_dev; + + pci_write_config_byte(pdev, addr, value); + } else { + u32 TmpUlong; + u16 RegAlignAddr; + u8 ShiftByte; + + RegAlignAddr = addr & ~(0x3); + ShiftByte = addr & (0x3); + TmpUlong = rtl8127_csi_other_fun_read(tp, 0, RegAlignAddr); + TmpUlong &= ~(0xFF << (8*ShiftByte)); + TmpUlong |= (value << (8*ShiftByte)); + rtl8127_csi_other_fun_write(tp, 0, RegAlignAddr, TmpUlong); + } + + fsleep(R8127_CHANNEL_EXIT_DELAY_TIME); +} + +u32 rtl8127_eri_read_with_oob_base_address(struct rtl8127_private *tp, int addr, int len, int type, const u32 base_address) +{ + int i, val_shift, shift = 0; + u32 value1 = 0, value2 = 0, mask; + u32 eri_cmd; + const u32 transformed_base_address = ((base_address & 0x00FFF000) << 6) | (base_address & 0x000FFF); + + if (len > 4 || len <= 0) + return -1; + + while (len > 0) { + val_shift = addr % ERIAR_Addr_Align; + addr = addr & ~0x3; + + eri_cmd = ERIAR_Read | + transformed_base_address | + type << ERIAR_Type_shift | + ERIAR_ByteEn << ERIAR_ByteEn_shift | + (addr & 0x0FFF); + if (addr & 0xF000) { + u32 tmp; + + tmp = addr & 0xF000; + tmp >>= 12; + eri_cmd |= (tmp << 20) & 0x00F00000; + } + + RTL_W32(tp, ERIAR, eri_cmd); + + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { + fsleep(R8127_CHANNEL_WAIT_TIME); + + /* Check if the RTL8125 has completed ERI read */ + if (RTL_R32(tp, ERIAR) & ERIAR_Flag) + break; + } + + if (len == 1) mask = (0xFF << (val_shift * 8)) & 0xFFFFFFFF; + else if (len == 2) mask = (0xFFFF << (val_shift * 8)) & 0xFFFFFFFF; + else if (len == 3) mask = (0xFFFFFF << (val_shift * 8)) & 0xFFFFFFFF; + else mask = (0xFFFFFFFF << (val_shift * 8)) & 0xFFFFFFFF; + + value1 = RTL_R32(tp, ERIDR) & mask; + value2 |= (value1 >> val_shift * 8) << shift * 8; + + if (len <= 4 - val_shift) { + len = 0; + } else { + len -= (4 - val_shift); + shift = 4 - val_shift; + addr += 4; + } + } + + fsleep(R8127_CHANNEL_EXIT_DELAY_TIME); + + return value2; +} + +u32 rtl8127_eri_read(struct rtl8127_private *tp, int addr, int len, int type) +{ + return rtl8127_eri_read_with_oob_base_address(tp, addr, len, type, 0); +} + +int rtl8127_eri_write_with_oob_base_address(struct rtl8127_private *tp, int addr, int len, u32 value, int type, const u32 base_address) +{ + int i, val_shift, shift = 0; + u32 value1 = 0, mask; + u32 eri_cmd; + const u32 transformed_base_address = ((base_address & 0x00FFF000) << 6) | (base_address & 0x000FFF); + + if (len > 4 || len <= 0) + return -1; + + while (len > 0) { + val_shift = addr % ERIAR_Addr_Align; + addr = addr & ~0x3; + + if (len == 1) mask = (0xFF << (val_shift * 8)) & 0xFFFFFFFF; + else if (len == 2) mask = (0xFFFF << (val_shift * 8)) & 0xFFFFFFFF; + else if (len == 3) mask = (0xFFFFFF << (val_shift * 8)) & 0xFFFFFFFF; + else mask = (0xFFFFFFFF << (val_shift * 8)) & 0xFFFFFFFF; + + value1 = rtl8127_eri_read_with_oob_base_address(tp, addr, 4, type, base_address) & ~mask; + value1 |= ((value << val_shift * 8) >> shift * 8); + + RTL_W32(tp, ERIDR, value1); + + eri_cmd = ERIAR_Write | + transformed_base_address | + type << ERIAR_Type_shift | + ERIAR_ByteEn << ERIAR_ByteEn_shift | + (addr & 0x0FFF); + if (addr & 0xF000) { + u32 tmp; + + tmp = addr & 0xF000; + tmp >>= 12; + eri_cmd |= (tmp << 20) & 0x00F00000; + } + + RTL_W32(tp, ERIAR, eri_cmd); + + for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) { + fsleep(R8127_CHANNEL_WAIT_TIME); + + /* Check if the RTL8125 has completed ERI write */ + if (!(RTL_R32(tp, ERIAR) & ERIAR_Flag)) + break; + } + + if (len <= 4 - val_shift) { + len = 0; + } else { + len -= (4 - val_shift); + shift = 4 - val_shift; + addr += 4; + } + } + + fsleep(R8127_CHANNEL_EXIT_DELAY_TIME); + + return 0; +} + +int rtl8127_eri_write(struct rtl8127_private *tp, int addr, int len, u32 value, int type) +{ + return rtl8127_eri_write_with_oob_base_address(tp, addr, len, value, type, NO_BASE_ADDRESS); +} + +static void +rtl8127_enable_rxdvgate(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + RTL_W8(tp, 0xF2, RTL_R8(tp, 0xF2) | BIT_3); +} + +static void +rtl8127_disable_rxdvgate(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + RTL_W8(tp, 0xF2, RTL_R8(tp, 0xF2) & ~BIT_3); +} + +static u8 +rtl8127_is_gpio_low(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u8 gpio_low = FALSE; + + switch (tp->HwSuppCheckPhyDisableModeVer) { + case 3: + if (!(rtl8127_mac_ocp_read(tp, 0xDC04) & BIT_13)) + gpio_low = TRUE; + break; + } + + if (gpio_low) + dprintk("gpio is low.\n"); + + return gpio_low; +} + +static u8 +rtl8127_is_phy_disable_mode_enabled(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u8 phy_disable_mode_enabled = FALSE; + + switch (tp->HwSuppCheckPhyDisableModeVer) { + case 3: + if (RTL_R8(tp, 0xF2) & BIT_5) + phy_disable_mode_enabled = TRUE; + break; + } + + if (phy_disable_mode_enabled) + dprintk("phy disable mode enabled.\n"); + + return phy_disable_mode_enabled; +} + +static u8 +rtl8127_is_in_phy_disable_mode(struct net_device *dev) +{ + u8 in_phy_disable_mode = FALSE; + + if (rtl8127_is_phy_disable_mode_enabled(dev) && rtl8127_is_gpio_low(dev)) + in_phy_disable_mode = TRUE; + + if (in_phy_disable_mode) + dprintk("Hardware is in phy disable mode.\n"); + + return in_phy_disable_mode; +} + +static void +rtl8127_stop_all_request(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) | StopReq); + fsleep(200); +} + +static void +rtl8127_clear_stop_all_request(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) & (CmdTxEnb | CmdRxEnb)); +} + +void +rtl8127_wait_txrx_fifo_empty(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i; + + /* Txfifo_empty require StopReq been set */ + for (i = 0; i < 3000; i++) { + fsleep(50); + if ((RTL_R8(tp, MCUCmd_reg) & (Txfifo_empty | Rxfifo_empty)) == (Txfifo_empty | Rxfifo_empty)) + break; + } + + for (i = 0; i < 3000; i++) { + fsleep(50); + if ((RTL_R16(tp, IntrMitigate) & (BIT_0 | BIT_1 | BIT_8)) == (BIT_0 | BIT_1 | BIT_8)) + break; + } +} + +#ifdef ENABLE_DASH_SUPPORT + +static inline void +rtl8127_enable_dash2_interrupt(struct rtl8127_private *tp) +{ + if (!tp->DASH) + return; + + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) + RTL_CMAC_W8(tp, CMAC_IBIMR0, (ISRIMR_DASH_TYPE2_ROK | ISRIMR_DASH_TYPE2_TOK | ISRIMR_DASH_TYPE2_TDU | ISRIMR_DASH_TYPE2_RDU | ISRIMR_DASH_TYPE2_RX_DISABLE_IDLE)); +} + +static inline void +rtl8127_disable_dash2_interrupt(struct rtl8127_private *tp) +{ + if (!tp->DASH) + return; + + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) + RTL_CMAC_W8(tp, CMAC_IBIMR0, 0); +} +#endif + +void +rtl8127_enable_hw_linkchg_interrupt(struct rtl8127_private *tp) +{ + switch (tp->HwCurrIsrVer) { + case 6: + RTL_W32(tp, IMR_V2_SET_REG_8125, ISRIMR_V6_LINKCHG); + break; + case 5: + RTL_W32(tp, IMR_V2_SET_REG_8125, ISRIMR_V5_LINKCHG); + break; + case 4: + RTL_W32(tp, IMR_V2_SET_REG_8125, ISRIMR_V4_LINKCHG); + break; + case 2: + case 3: + RTL_W32(tp, IMR_V2_SET_REG_8125, ISRIMR_V2_LINKCHG); + break; + case 1: + RTL_W32(tp, tp->imr_reg[0], LinkChg | RTL_R32(tp, tp->imr_reg[0])); + break; + } + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) + rtl8127_enable_dash2_interrupt(tp); +#endif +} + +static inline void +rtl8127_enable_hw_interrupt(struct rtl8127_private *tp) +{ + switch (tp->HwCurrIsrVer) { + case 2: + case 3: + case 4: + case 5: + case 6: + RTL_W32(tp, IMR_V2_SET_REG_8125, tp->intr_mask); + break; + case 1: + RTL_W32(tp, tp->imr_reg[0], tp->intr_mask); + + if (R8127_MULTI_RX_Q(tp)) { + int i; + for (i=1; inum_rx_rings; i++) + RTL_W16(tp, tp->imr_reg[i], other_q_intr_mask); + } + break; + } + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) + rtl8127_enable_dash2_interrupt(tp); +#endif +} + +static inline void rtl8127_clear_hw_isr_v2(struct rtl8127_private *tp, + u32 message_id) +{ + RTL_W32(tp, ISR_V2_8125, BIT(message_id)); +} + +static inline void +rtl8127_disable_hw_interrupt(struct rtl8127_private *tp) +{ + if (tp->HwCurrIsrVer > 1) { + RTL_W32(tp, IMR_V2_CLEAR_REG_8125, 0xFFFFFFFF); + if (tp->HwCurrIsrVer > 3) + RTL_W32(tp, IMR_V4_L2_CLEAR_REG_8125, 0xFFFFFFFF); + } else { + RTL_W32(tp, tp->imr_reg[0], 0x0000); + + if (R8127_MULTI_RX_Q(tp)) { + int i; + for (i=1; inum_rx_rings; i++) + RTL_W16(tp, tp->imr_reg[i], 0); + } + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) + rtl8127_disable_dash2_interrupt(tp); +#endif + } +} + +static inline void +rtl8127_switch_to_hw_interrupt(struct rtl8127_private *tp) +{ + RTL_W32(tp, TIMER_INT0_8125, 0x0000); + + rtl8127_enable_hw_interrupt(tp); +} + +static inline void +rtl8127_switch_to_timer_interrupt(struct rtl8127_private *tp) +{ + if (tp->use_timer_interrupt) { + RTL_W32(tp, TIMER_INT0_8125, timer_count); + RTL_W32(tp, TCTR0_8125, timer_count); + RTL_W32(tp, tp->imr_reg[0], tp->timer_intr_mask); + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) + rtl8127_enable_dash2_interrupt(tp); +#endif + } else { + rtl8127_switch_to_hw_interrupt(tp); + } +} + +static void +rtl8127_irq_mask_and_ack(struct rtl8127_private *tp) +{ + rtl8127_disable_hw_interrupt(tp); + + if (tp->HwCurrIsrVer > 1) { + RTL_W32(tp, ISR_V2_8125, 0xFFFFFFFF); + if (tp->HwCurrIsrVer > 3) + RTL_W32(tp, ISR_V4_L2_8125, 0xFFFFFFFF); + } else { +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) { + if (tp->dash_printer_enabled) { + RTL_W32(tp, tp->isr_reg[0], RTL_R32(tp, tp->isr_reg[0]) & + ~(ISRIMR_DASH_INTR_EN | ISRIMR_DASH_INTR_CMAC_RESET)); + } else { + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) { + RTL_CMAC_W8(tp, CMAC_IBISR0, RTL_CMAC_R8(tp, CMAC_IBISR0)); + } + } + } else { + RTL_W32(tp, tp->isr_reg[0], RTL_R32(tp, tp->isr_reg[0])); + } +#else + RTL_W32(tp, tp->isr_reg[0], RTL_R32(tp, tp->isr_reg[0])); +#endif + if (R8127_MULTI_RX_Q(tp)) { + int i; + for (i=1; inum_rx_rings; i++) + RTL_W16(tp, tp->isr_reg[i], RTL_R16(tp, tp->isr_reg[i])); + } + } +} + +static void +rtl8127_disable_rx_packet_filter(struct rtl8127_private *tp) +{ + + RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) & + ~(AcceptErr | AcceptRunt |AcceptBroadcast | AcceptMulticast | + AcceptMyPhys | AcceptAllPhys)); +} + +static void +rtl8127_nic_reset(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i; + + rtl8127_disable_rx_packet_filter(tp); + + rtl8127_enable_rxdvgate(dev); + + rtl8127_stop_all_request(dev); + + rtl8127_wait_txrx_fifo_empty(dev); + + rtl8127_clear_stop_all_request(dev); + + /* Soft reset the chip. */ + RTL_W8(tp, ChipCmd, CmdReset); + + /* Check that the chip has finished the reset. */ + for (i = 100; i > 0; i--) { + fsleep(100); + if ((RTL_R8(tp, ChipCmd) & CmdReset) == 0) + break; + } + + /* reset rcr */ + RTL_W32(tp, RxConfig, (RX_DMA_BURST_512 << RxCfgDMAShift)); +} + +static void +rtl8127_hw_set_interrupt_type(struct rtl8127_private *tp, u8 isr_ver) +{ + u8 tmp; + + if (tp->HwSuppIsrVer < 2) + return; + + tmp = RTL_R8(tp, INT_CFG0_8125); + + switch (tp->HwSuppIsrVer) { + case 6: + tmp &= ~INT_CFG0_AVOID_MISS_INTR; + fallthrough; + case 4: + case 5: + if (tp->HwSuppIsrVer == 6) + tmp &= ~INT_CFG0_AUTO_CLEAR_IMR; + else + tmp &= ~INT_CFG0_MSIX_ENTRY_NUM_MODE; + fallthrough; + case 2: + case 3: + tmp &= ~(INT_CFG0_ENABLE_8125); + if (isr_ver > 1) + tmp |= INT_CFG0_ENABLE_8125; + break; + default: + return; + } + + RTL_W8(tp, INT_CFG0_8125, tmp); +} + +static void +rtl8127_hw_clear_timer_int(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + RTL_W32(tp, TIMER_INT0_8125, 0x0000); + RTL_W32(tp, TIMER_INT1_8125, 0x0000); + RTL_W32(tp, TIMER_INT2_8125, 0x0000); + RTL_W32(tp, TIMER_INT3_8125, 0x0000); +} + +static void +rtl8127_hw_clear_int_miti(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i; + + switch (tp->HwSuppIntMitiVer) { + case 3: + case 6: + //IntMITI_0-IntMITI_31 + for (i=0xA00; i<0xB00; i+=4) + RTL_W32(tp, i, 0x0000); + break; + case 4: + case 5: + //IntMITI_0-IntMITI_15 + for (i = 0xA00; i < 0xA80; i += 4) + RTL_W32(tp, i, 0x0000); + + if (tp->HwSuppIntMitiVer == 5) + RTL_W8(tp, INT_CFG0_8125, RTL_R8(tp, INT_CFG0_8125) & + ~(INT_CFG0_TIMEOUT0_BYPASS_8125 | + INT_CFG0_MITIGATION_BYPASS_8125 | + INT_CFG0_RDU_BYPASS_8126)); + else + RTL_W8(tp, INT_CFG0_8125, RTL_R8(tp, INT_CFG0_8125) & + ~(INT_CFG0_TIMEOUT0_BYPASS_8125 | INT_CFG0_MITIGATION_BYPASS_8125)); + + RTL_W16(tp, INT_CFG1_8125, 0x0000); + break; + } +} + +static bool +rtl8127_vec_2_tx_q_num( + struct rtl8127_private *tp, + u32 messageId, + u32 *qnum +) +{ + u32 whichQ = 0xffffffff; + bool rc = false; + + switch (tp->HwSuppIsrVer) { + case 2: + if (messageId == 0x10) + whichQ = 0; + else if (messageId == 0x12 && tp->num_tx_rings > 1) + whichQ = 1; + break; + case 3: + case 4: + if (messageId == 0x00) + whichQ = 0; + else if (messageId == 0x01 && tp->num_tx_rings > 1) + whichQ = 1; + break; + case 5: + if (messageId == 0x10) + whichQ = 0; + else if (messageId == 0x11 && tp->num_tx_rings > 1) + whichQ = 1; + break; + case 6: + if (messageId == 0x08) + whichQ = 0; + else if (messageId == 0x09 && tp->num_tx_rings > 1) + whichQ = 1; + break; + case 7: + if (messageId == 0x1B) + whichQ = 0; + else if (messageId == 0x1C && tp->num_tx_rings > 1) + whichQ = 1; + break; + } + + if (whichQ != 0xffffffff) { + *qnum = whichQ; + rc = true; + } + + return rc; +} + +static bool +rtl8127_vec_2_rx_q_num( + struct rtl8127_private *tp, + u32 messageId, + u32 *qnum +) +{ + u32 whichQ = 0xffffffff; + bool rc = false; + + switch (tp->HwSuppIsrVer) { + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + if (messageId < tp->HwSuppNumRxQueues) + whichQ = messageId; + break; + } + + if (whichQ != 0xffffffff) { + *qnum = whichQ; + rc = true; + } + + return rc; +} + +void +rtl8127_hw_set_timer_int(struct rtl8127_private *tp, + u32 message_id, + u8 timer_intmiti_val) +{ + u32 qnum; + + switch (tp->HwSuppIntMitiVer) { + case 4: + case 5: + case 6: + //ROK + if (rtl8127_vec_2_rx_q_num(tp, message_id, &qnum)) + RTL_W8(tp,INT_MITI_V2_0_RX + 8 * qnum, timer_intmiti_val); + //TOK + if (rtl8127_vec_2_tx_q_num(tp, message_id, &qnum)) + RTL_W8(tp,INT_MITI_V2_0_TX + 8 * qnum, timer_intmiti_val); + break; + } +} + +void +rtl8127_hw_reset(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + rtl8127_lib_reset_prepare(tp); + + /* Disable interrupts */ + rtl8127_irq_mask_and_ack(tp); + + rtl8127_hw_clear_timer_int(dev); + + rtl8127_nic_reset(dev); +} + +static unsigned int +rtl8127_xmii_reset_pending(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned int retval; + unsigned long flags; + + spin_lock_irqsave(&tp->phy_lock, flags); + rtl8127_mdio_write(tp, 0x1f, 0x0000); + retval = rtl8127_mdio_read(tp, MII_BMCR) & BMCR_RESET; + spin_unlock_irqrestore(&tp->phy_lock, flags); + + return retval; +} + +static unsigned int +rtl8127_xmii_link_ok(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u16 status; + + status = RTL_R16(tp, PHYstatus); + if (status == 0xffff) + return 0; + + return (status & LinkStatus) ? 1 : 0; +} + +static int +rtl8127_wait_phy_reset_complete(struct rtl8127_private *tp) +{ + int i, val; + + for (i = 0; i < 2500; i++) { + val = rtl8127_mdio_read(tp, MII_BMCR) & BMCR_RESET; + if (!val) + return 0; + + mdelay(1); + } + + return -1; +} + +static void +rtl8127_xmii_reset_enable(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned long flags; + int ret; + + if (rtl8127_is_in_phy_disable_mode(dev)) + return; + + spin_lock_irqsave(&tp->phy_lock, flags); + + rtl8127_mdio_write(tp, 0x1f, 0x0000); + rtl8127_mdio_write(tp, MII_ADVERTISE, rtl8127_mdio_read(tp, MII_ADVERTISE) & + ~(ADVERTISE_10HALF | ADVERTISE_10FULL | + ADVERTISE_100HALF | ADVERTISE_100FULL)); + rtl8127_mdio_write(tp, MII_CTRL1000, rtl8127_mdio_read(tp, MII_CTRL1000) & + ~(ADVERTISE_1000HALF | ADVERTISE_1000FULL)); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA5D4, rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D4) & + ~(RTK_ADVERTISE_2500FULL | RTK_ADVERTISE_5000FULL | + RTK_ADVERTISE_10000FULL)); + rtl8127_mdio_write(tp, MII_BMCR, BMCR_RESET | BMCR_ANENABLE); + + ret = rtl8127_wait_phy_reset_complete(tp); + + spin_unlock_irqrestore(&tp->phy_lock, flags); + + if (ret != 0 && netif_msg_link(tp)) + printk(KERN_ERR "%s: PHY reset failed.\n", dev->name); +} + +void +rtl8127_init_ring_indexes(struct rtl8127_private *tp) +{ + int i; + + for (i = 0; i < tp->HwSuppNumTxQueues; i++) { + struct rtl8127_tx_ring *ring = &tp->tx_ring[i]; + ring->dirty_tx = ring->cur_tx = 0; + ring->NextHwDesCloPtr = 0; + ring->BeginHwDesCloPtr = 0; + ring->index = i; + ring->priv = tp; + ring->netdev = tp->dev; + + /* reset BQL for queue */ + netdev_tx_reset_queue(txring_txq(ring)); + } + + for (i = 0; i < tp->HwSuppNumRxQueues; i++) { + struct rtl8127_rx_ring *ring = &tp->rx_ring[i]; + ring->dirty_rx = ring->cur_rx = 0; + ring->index = i; + ring->priv = tp; + ring->netdev = tp->dev; + } + +#ifdef ENABLE_LIB_SUPPORT + for (i = 0; i < tp->HwSuppNumTxQueues; i++) { + struct rtl8127_ring *ring = &tp->lib_tx_ring[i]; + ring->direction = RTL8127_CH_DIR_TX; + ring->queue_num = i; + ring->private = tp; + } + + for (i = 0; i < tp->HwSuppNumRxQueues; i++) { + struct rtl8127_ring *ring = &tp->lib_rx_ring[i]; + ring->direction = RTL8127_CH_DIR_RX; + ring->queue_num = i; + ring->private = tp; + } +#endif +} + +static void +rtl8127_issue_offset_99_event(struct rtl8127_private *tp) +{ + rtl8127_mac_ocp_write(tp, 0xE09A, rtl8127_mac_ocp_read(tp, 0xE09A) | BIT_0); +} + +#ifdef ENABLE_DASH_SUPPORT +static void +NICChkTypeEnableDashInterrupt(struct rtl8127_private *tp) +{ + if (tp->DASH) { + // + // even disconnected, enable 3 dash interrupt mask bits for in-band/out-band communication + // + if (HW_DASH_SUPPORT_TYPE_2(tp) || HW_DASH_SUPPORT_TYPE_3(tp)) { + rtl8127_enable_dash2_interrupt(tp); + RTL_W16(tp, IntrMask, (ISRIMR_DASH_INTR_EN | ISRIMR_DASH_INTR_CMAC_RESET)); + } + } +} +#endif + +static int rtl8127_enable_eee_plus(struct rtl8127_private *tp) +{ + rtl8127_mac_ocp_write(tp, 0xE080, rtl8127_mac_ocp_read(tp, 0xE080)|BIT_1); + + return 0; +} + +static int rtl8127_disable_eee_plus(struct rtl8127_private *tp) +{ + rtl8127_mac_ocp_write(tp, 0xE080, rtl8127_mac_ocp_read(tp, 0xE080)&~BIT_1); + + return 0; +} + +static void rtl8127_enable_double_vlan(struct rtl8127_private *tp) +{ + RTL_W16(tp, DOUBLE_VLAN_CONFIG, 0xf002); +} + +static void rtl8127_disable_double_vlan(struct rtl8127_private *tp) +{ + RTL_W16(tp, DOUBLE_VLAN_CONFIG, 0); +} + +static void +rtl8127_link_on_patch(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned long flags; + + rtl8127_hw_config(dev); + + if (RTL_R8(tp, PHYstatus) & _10bps) + rtl8127_enable_eee_plus(tp); + + rtl8127_hw_start(dev); + + netif_carrier_on(dev); + + netif_tx_wake_all_queues(dev); + + spin_lock_irqsave(&tp->phy_lock, flags); + tp->phy_reg_aner = rtl8127_mdio_read(tp, MII_EXPANSION); + tp->phy_reg_anlpar = rtl8127_mdio_read(tp, MII_LPA); + tp->phy_reg_gbsr = rtl8127_mdio_read(tp, MII_STAT1000); + tp->phy_reg_status_2500 = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D6); + spin_unlock_irqrestore(&tp->phy_lock, flags); + +#ifdef ENABLE_PTP_SUPPORT + if (tp->EnablePtp) + rtl8127_set_local_time(tp); +#endif // ENABLE_PTP_SUPPORT +} + +static void +rtl8127_link_down_patch(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + tp->phy_reg_aner = 0; + tp->phy_reg_anlpar = 0; + tp->phy_reg_gbsr = 0; + tp->phy_reg_status_2500 = 0; + + rtl8127_disable_eee_plus(tp); + + netif_carrier_off(dev); + + netif_tx_disable(dev); + + rtl8127_hw_reset(dev); + + rtl8127_tx_clear(tp); + + rtl8127_rx_clear(tp); + + rtl8127_init_ring(dev); + + rtl8127_enable_hw_linkchg_interrupt(tp); + + //rtl8127_set_speed(dev, tp->autoneg, tp->speed, tp->duplex, tp->advertising); + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) + NICChkTypeEnableDashInterrupt(tp); +#endif +} + +static void +_rtl8127_check_link_status(struct net_device *dev, unsigned int link_state) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (link_state != R8127_LINK_STATE_OFF && + link_state != R8127_LINK_STATE_ON) + link_state = tp->link_ok(dev); + + if (link_state == R8127_LINK_STATE_ON) { + rtl8127_link_on_patch(dev); + + if (netif_msg_ifup(tp)) + printk(KERN_INFO PFX "%s: link up\n", dev->name); + } else { + if (netif_msg_ifdown(tp)) + printk(KERN_INFO PFX "%s: link down\n", dev->name); + + rtl8127_link_down_patch(dev); + } +} + +static void +rtl8127_check_link_status(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned int link_status_on; + + tp->resume_not_chg_speed = 0; + + link_status_on = tp->link_ok(dev); + if (netif_carrier_ok(dev) == link_status_on) + return; + + _rtl8127_check_link_status(dev, link_status_on); +} + +static bool +rtl8127_is_autoneg_mode_valid(u32 autoneg) +{ + switch(autoneg) { + case AUTONEG_ENABLE: + case AUTONEG_DISABLE: + return true; + default: + return false; + } +} + +static bool +rtl8127_is_speed_mode_valid(u32 speed) +{ + switch(speed) { + case SPEED_10000: + case SPEED_5000: + case SPEED_2500: + case SPEED_1000: + case SPEED_100: + case SPEED_10: + return true; + default: + return false; + } +} + +static bool +rtl8127_is_duplex_mode_valid(u8 duplex) +{ + switch(duplex) { + case DUPLEX_FULL: + case DUPLEX_HALF: + return true; + default: + return false; + } +} + +static void +rtl8127_set_link_option(struct rtl8127_private *tp, + u8 autoneg, + u32 speed, + u8 duplex, + enum rtl8127_fc_mode fc) +{ + u64 adv; + + if (!rtl8127_is_speed_mode_valid(speed)) + speed = SPEED_10000; + + if (!rtl8127_is_duplex_mode_valid(duplex)) + duplex = DUPLEX_FULL; + + if (!rtl8127_is_autoneg_mode_valid(autoneg)) + autoneg = AUTONEG_ENABLE; + + speed = min(speed, tp->HwSuppMaxPhyLinkSpeed); + + adv = 0; + switch(speed) { + case SPEED_10000: + adv |= ADVERTISED_10000baseT_Full; + fallthrough; + case SPEED_5000: + adv |= RTK_ADVERTISED_5000baseX_Full; + fallthrough; + case SPEED_2500: + adv |= ADVERTISED_2500baseX_Full; + fallthrough; + default: + adv |= (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | + ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | + ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full); + break; + } + + tp->autoneg = autoneg; + tp->speed = speed; + tp->duplex = duplex; + tp->advertising = adv; + tp->fcpause = fc; +} + +static void +rtl8127_wait_ll_share_fifo_ready(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i; + + for (i = 0; i < 10; i++) { + fsleep(100); + if (RTL_R16(tp, 0xD2) & BIT_9) + break; + } +} + +static void +rtl8127_disable_pci_offset_99(struct rtl8127_private *tp) +{ + rtl8127_mac_ocp_write(tp, 0xE032, rtl8127_mac_ocp_read(tp, 0xE032) & ~(BIT_0 | BIT_1)); + + rtl8127_csi_fun0_write_byte(tp, 0x99, 0x00); +} + +static void +rtl8127_enable_pci_offset_99(struct rtl8127_private *tp) +{ + u32 csi_tmp; + + rtl8127_csi_fun0_write_byte(tp, 0x99, tp->org_pci_offset_99); + + csi_tmp = rtl8127_mac_ocp_read(tp, 0xE032); + csi_tmp &= ~(BIT_0 | BIT_1); + if (tp->org_pci_offset_99 & (BIT_5 | BIT_6)) + csi_tmp |= BIT_1; + if (tp->org_pci_offset_99 & BIT_2) + csi_tmp |= BIT_0; + rtl8127_mac_ocp_write(tp, 0xE032, csi_tmp); +} + +static void +rtl8127_init_pci_offset_99(struct rtl8127_private *tp) +{ + rtl8127_mac_ocp_write(tp, 0xCDD0, 0x9003); + rtl8127_set_mac_ocp_bit(tp, 0xE034, (BIT_15 | BIT_14)); + rtl8127_mac_ocp_write(tp, 0xCDD2, 0x8C17); + rtl8127_mac_ocp_write(tp, 0xCDD8, 0x9003); + rtl8127_mac_ocp_write(tp, 0xCDD4, 0x9003); + rtl8127_mac_ocp_write(tp, 0xCDDA, 0x9003); + rtl8127_mac_ocp_write(tp, 0xCDD6, 0x9003); + rtl8127_mac_ocp_write(tp, 0xCDDC, 0x9003); + rtl8127_mac_ocp_write(tp, 0xCDE8, 0x8C08); + rtl8127_mac_ocp_write(tp, 0xCDEA, 0x9003); + rtl8127_mac_ocp_write(tp, 0xCDEC, 0x8C12); + rtl8127_mac_ocp_write(tp, 0xCDEE, 0x9003); + rtl8127_mac_ocp_write(tp, 0xCDF0, 0x8C2E); + rtl8127_mac_ocp_write(tp, 0xCDF2, 0x9003); + rtl8127_mac_ocp_write(tp, 0xCDF4, 0x8892); + rtl8127_mac_ocp_write(tp, 0xCDF6, 0x9003); + rtl8127_mac_ocp_write(tp, 0xCDF4, 0x8849); + rtl8127_mac_ocp_write(tp, 0xCDF6, 0x9003); + rtl8127_set_mac_ocp_bit(tp, 0xE032, BIT_14); + rtl8127_set_mac_ocp_bit(tp, 0xE0A2, BIT_0); + + rtl8127_enable_pci_offset_99(tp); +} + +static void +rtl8127_disable_pci_offset_180(struct rtl8127_private *tp) +{ + rtl8127_clear_mac_ocp_bit(tp, 0xE092, 0x00FF); +} + +static void +rtl8127_enable_pci_offset_180(struct rtl8127_private *tp) +{ + rtl8127_clear_mac_ocp_bit(tp, 0xE094, 0xFF00); + + rtl8127_clear_set_mac_ocp_bit(tp, 0xE092, 0x00FF, BIT_2); +} + +static void +rtl8127_init_pci_offset_180(struct rtl8127_private *tp) +{ + rtl8127_enable_pci_offset_180(tp); +} + +static void +rtl8127_set_pci_99_exit_driver_para(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (tp->org_pci_offset_99 & BIT_2) + rtl8127_issue_offset_99_event(tp); + rtl8127_disable_pci_offset_99(tp); +} + +static void +rtl8127_enable_cfg9346_write(struct rtl8127_private *tp) +{ + RTL_W8(tp, Cfg9346, RTL_R8(tp, Cfg9346) | Cfg9346_Unlock); +} + +static void +rtl8127_disable_cfg9346_write(struct rtl8127_private *tp) +{ + RTL_W8(tp, Cfg9346, RTL_R8(tp, Cfg9346) & ~Cfg9346_Unlock); +} + +static void +rtl8127_enable_exit_l1_mask(struct rtl8127_private *tp) +{ + //(1)ERI(0xD4)(OCP 0xC0AC).bit[7:12]=6'b111111, L1 Mask + rtl8127_set_mac_ocp_bit(tp, 0xC0AC, (BIT_7 | BIT_8 | BIT_9 | BIT_10 | BIT_11 | BIT_12)); +} + +static void +rtl8127_disable_exit_l1_mask(struct rtl8127_private *tp) +{ + //(1)ERI(0xD4)(OCP 0xC0AC).bit[7:12]=6'b000000, L1 Mask + rtl8127_clear_mac_ocp_bit(tp, 0xC0AC, (BIT_7 | BIT_8 | BIT_9 | BIT_10 | BIT_11 | BIT_12)); +} + +static void +rtl8127_enable_extend_tally_couter(struct rtl8127_private *tp) +{ + switch (tp->HwSuppExtendTallyCounterVer) { + case 1: + rtl8127_set_mac_ocp_bit(tp, 0xEA84, (BIT_1 | BIT_0)); + break; + } +} + +static void +rtl8127_disable_extend_tally_couter(struct rtl8127_private *tp) +{ + switch (tp->HwSuppExtendTallyCounterVer) { + case 1: + rtl8127_clear_mac_ocp_bit(tp, 0xEA84, (BIT_1 | BIT_0)); + break; + } +} + +static void +rtl8127_enable_force_clkreq(struct rtl8127_private *tp, bool enable) +{ + if (enable) + RTL_W8(tp, 0xF1, RTL_R8(tp, 0xF1) | BIT_7); + else + RTL_W8(tp, 0xF1, RTL_R8(tp, 0xF1) & ~BIT_7); +} + +static void +rtl8127_enable_aspm_clkreq_lock(struct rtl8127_private *tp, bool enable) +{ + bool unlock_cfg_wr; + + if ((RTL_R8(tp, Cfg9346) & Cfg9346_EEM_MASK) == Cfg9346_Unlock) + unlock_cfg_wr = false; + else + unlock_cfg_wr = true; + + if (unlock_cfg_wr) + rtl8127_enable_cfg9346_write(tp); + + if (enable) { + RTL_W8(tp, INT_CFG0_8125, RTL_R8(tp, INT_CFG0_8125) | BIT_3); + RTL_W8(tp, Config5, RTL_R8(tp, Config5) | BIT_0); + } else { + RTL_W8(tp, INT_CFG0_8125, RTL_R8(tp, INT_CFG0_8125) & ~BIT_3); + RTL_W8(tp, Config5, RTL_R8(tp, Config5) & ~BIT_0); + } + + if (unlock_cfg_wr) + rtl8127_disable_cfg9346_write(tp); +} + +static void +rtl8127_hw_d3_para(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + RTL_W16(tp, RxMaxSize, RX_BUF_SIZE); + + rtl8127_enable_force_clkreq(tp, 0); + rtl8127_enable_aspm_clkreq_lock(tp, 0); + + rtl8127_disable_exit_l1_mask(tp); + +#ifdef ENABLE_REALWOW_SUPPORT + rtl8127_set_realwow_d3_para(dev); +#endif + + rtl8127_set_pci_99_exit_driver_para(dev); + + rtl8127_disable_rxdvgate(dev); + + rtl8127_disable_extend_tally_couter(tp); +} + +static void +rtl8127_enable_magic_packet(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + switch (tp->HwSuppMagicPktVer) { + case WAKEUP_MAGIC_PACKET_V3: + rtl8127_mac_ocp_write(tp, 0xC0B6, rtl8127_mac_ocp_read(tp, 0xC0B6) | BIT_0); + break; + } +} +static void +rtl8127_disable_magic_packet(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + switch (tp->HwSuppMagicPktVer) { + case WAKEUP_MAGIC_PACKET_V3: + rtl8127_mac_ocp_write(tp, 0xC0B6, rtl8127_mac_ocp_read(tp, 0xC0B6) & ~BIT_0); + break; + } +} + +static void +rtl8127_enable_linkchg_wakeup(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + switch (tp->HwSuppLinkChgWakeUpVer) { + case 3: + RTL_W8(tp, Config3, RTL_R8(tp, Config3) | LinkUp); + rtl8127_clear_set_mac_ocp_bit(tp, 0xE0C6, (BIT_5 | BIT_3 | BIT_2), (BIT_4 | BIT_1 | BIT_0)); + break; + } +} + +static void +rtl8127_disable_linkchg_wakeup(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + switch (tp->HwSuppLinkChgWakeUpVer) { + case 3: + RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~LinkUp); + if (!(rtl8127_mac_ocp_read(tp, 0xE0C6) & BIT_0)) + rtl8127_clear_set_mac_ocp_bit(tp, 0xE0C6, (BIT_5 | BIT_3 | BIT_2 | BIT_1), BIT_4); + break; + } +} + +#define WAKE_ANY (WAKE_PHY | WAKE_MAGIC | WAKE_UCAST | WAKE_BCAST | WAKE_MCAST) + +static u32 +rtl8127_get_hw_wol(struct rtl8127_private *tp) +{ + u8 options; + u32 csi_tmp; + u32 wol_opts = 0; + + if (disable_wol_support) + goto out; + + options = RTL_R8(tp, Config1); + if (!(options & PMEnable)) + goto out; + + options = RTL_R8(tp, Config3); + if (options & LinkUp) + wol_opts |= WAKE_PHY; + + switch (tp->HwSuppMagicPktVer) { + case WAKEUP_MAGIC_PACKET_V3: + csi_tmp = rtl8127_mac_ocp_read(tp, 0xC0B6); + if (csi_tmp & BIT_0) + wol_opts |= WAKE_MAGIC; + break; + } + + options = RTL_R8(tp, Config5); + if (options & UWF) + wol_opts |= WAKE_UCAST; + if (options & BWF) + wol_opts |= WAKE_BCAST; + if (options & MWF) + wol_opts |= WAKE_MCAST; + +out: + return wol_opts; +} + +static void +rtl8127_enable_d0_speedup(struct rtl8127_private *tp) +{ + u16 clearmask; + u16 setmask; + + if (FALSE == HW_SUPPORT_D0_SPEED_UP(tp)) + return; + + if (tp->D0SpeedUpSpeed == D0_SPEED_UP_SPEED_DISABLE) + return; + + if (tp->HwSuppD0SpeedUpVer == 1 || tp->HwSuppD0SpeedUpVer == 2) { + //speed up speed + clearmask = (BIT_10 | BIT_9 | BIT_8 | BIT_7); + if (tp->D0SpeedUpSpeed == D0_SPEED_UP_SPEED_2500) + setmask = BIT_7; + else if (tp->D0SpeedUpSpeed == D0_SPEED_UP_SPEED_5000) + setmask = BIT_8; + else if (tp->D0SpeedUpSpeed == D0_SPEED_UP_SPEED_10000) + setmask = BIT_7 | BIT_8; + else + setmask = 0; + rtl8127_clear_set_mac_ocp_bit(tp, 0xE10A, clearmask, setmask); + + //speed up flowcontrol + clearmask = (BIT_15 | BIT_14); + if (tp->HwSuppD0SpeedUpVer == 2) + clearmask |= BIT_13; + + if (tp->fcpause == rtl8127_fc_full) { + setmask = (BIT_15 | BIT_14); + if (tp->HwSuppD0SpeedUpVer == 2) + setmask |= BIT_13; + } else + setmask = 0; + rtl8127_clear_set_mac_ocp_bit(tp, 0xE860, clearmask, setmask); + } + + RTL_W8(tp, 0xD0, RTL_R8(tp, 0xD0) | BIT_3); +} + +static void +rtl8127_disable_d0_speedup(struct rtl8127_private *tp) +{ + if (FALSE == HW_SUPPORT_D0_SPEED_UP(tp)) + return; + + RTL_W8(tp, 0xD0, RTL_R8(tp, 0xD0) & ~BIT_3); +} + +static void +rtl8127_set_hw_wol(struct net_device *dev, u32 wolopts) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i,tmp = 0; + static struct { + u32 opt; + u16 reg; + u8 mask; + } cfg[] = { + { WAKE_PHY, Config3, LinkUp }, + { WAKE_UCAST, Config5, UWF }, + { WAKE_BCAST, Config5, BWF }, + { WAKE_MCAST, Config5, MWF }, + { WAKE_ANY, Config5, LanWake }, + { WAKE_MAGIC, Config3, MagicPacket }, + }; + + switch (tp->HwSuppMagicPktVer) { + case WAKEUP_MAGIC_PACKET_V3: + tmp = ARRAY_SIZE(cfg) - 1; + + if (wolopts & WAKE_MAGIC) + rtl8127_enable_magic_packet(dev); + else + rtl8127_disable_magic_packet(dev); + break; + default: + break; + } + + rtl8127_enable_cfg9346_write(tp); + + for (i = 0; i < tmp; i++) { + u8 options = RTL_R8(tp, cfg[i].reg) & ~cfg[i].mask; + if (wolopts & cfg[i].opt) + options |= cfg[i].mask; + RTL_W8(tp, cfg[i].reg, options); + } + + switch (tp->HwSuppLinkChgWakeUpVer) { + case 3: + if (wolopts & WAKE_PHY) + rtl8127_enable_linkchg_wakeup(dev); + else + rtl8127_disable_linkchg_wakeup(dev); + break; + } + + rtl8127_disable_cfg9346_write(tp); +} + +static void +rtl8127_phy_restart_nway(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (rtl8127_is_in_phy_disable_mode(dev)) + return; + + rtl8127_mdio_write(tp, 0x1F, 0x0000); + rtl8127_mdio_write(tp, MII_BMCR, BMCR_ANENABLE | BMCR_ANRESTART); +} + +static void +rtl8127_phy_setup_force_mode(struct net_device *dev, u32 speed, u8 duplex) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u16 bmcr_true_force = 0; + + if (rtl8127_is_in_phy_disable_mode(dev)) + return; + + if ((speed == SPEED_10) && (duplex == DUPLEX_HALF)) { + bmcr_true_force = BMCR_SPEED10; + } else if ((speed == SPEED_10) && (duplex == DUPLEX_FULL)) { + bmcr_true_force = BMCR_SPEED10 | BMCR_FULLDPLX; + } else if ((speed == SPEED_100) && (duplex == DUPLEX_HALF)) { + bmcr_true_force = BMCR_SPEED100; + } else if ((speed == SPEED_100) && (duplex == DUPLEX_FULL)) { + bmcr_true_force = BMCR_SPEED100 | BMCR_FULLDPLX; + } else { + netif_err(tp, drv, dev, "Failed to set phy force mode!\n"); + return; + } + + rtl8127_mdio_write(tp, 0x1F, 0x0000); + rtl8127_mdio_write(tp, MII_BMCR, bmcr_true_force); +} + +static void +rtl8127_set_pci_pme(struct rtl8127_private *tp, int set) +{ + struct pci_dev *pdev = tp->pci_dev; + u16 pmc; + + if (!pdev->pm_cap) + return; + + pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &pmc); + pmc |= PCI_PM_CTRL_PME_STATUS; + if (set) + pmc |= PCI_PM_CTRL_PME_ENABLE; + else + pmc &= ~PCI_PM_CTRL_PME_ENABLE; + pci_write_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, pmc); +} + +static void +rtl8127_enable_giga_lite(struct rtl8127_private *tp, u64 adv) +{ + if (adv & ADVERTISED_1000baseT_Full) + rtl8127_set_eth_phy_ocp_bit(tp, 0xA428, BIT_9); + else + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA428, BIT_9); + + if (adv & ADVERTISED_2500baseX_Full) + rtl8127_set_eth_phy_ocp_bit(tp, 0xA5EA, BIT_0); + else + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA5EA, BIT_0); + + if (adv & RTK_ADVERTISED_5000baseX_Full) + rtl8127_set_eth_phy_ocp_bit(tp, 0xA5EA, BIT_1); + else + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA5EA, BIT_1); + + if (adv & ADVERTISED_10000baseT_Full) + rtl8127_set_eth_phy_ocp_bit(tp, 0xA5EA, BIT_2); + else + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA5EA, BIT_2); +} + +static void +rtl8127_disable_giga_lite(struct rtl8127_private *tp) +{ + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA428, BIT_9); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA5EA, BIT_0 | BIT_1 | BIT_2); +} + +static void +rtl8127_set_wol_link_speed(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned long flags; + int auto_nego; + int giga_ctrl; + int ctrl_2500; + u64 adv; + u16 anlpar; + u16 gbsr; + u16 status_2500; + u16 aner; + + spin_lock_irqsave(&tp->phy_lock, flags); + + if (tp->autoneg != AUTONEG_ENABLE) + goto exit; + + rtl8127_mdio_write(tp, 0x1F, 0x0000); + + auto_nego = rtl8127_mdio_read(tp, MII_ADVERTISE); + auto_nego &= ~(ADVERTISE_10HALF | ADVERTISE_10FULL + | ADVERTISE_100HALF | ADVERTISE_100FULL); + + giga_ctrl = rtl8127_mdio_read(tp, MII_CTRL1000); + giga_ctrl &= ~(ADVERTISE_1000HALF | ADVERTISE_1000FULL); + + ctrl_2500 = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D4); + ctrl_2500 &= ~(RTK_ADVERTISE_2500FULL | RTK_ADVERTISE_5000FULL | + RTK_ADVERTISE_10000FULL); + + aner = tp->phy_reg_aner; + anlpar = tp->phy_reg_anlpar; + gbsr = tp->phy_reg_gbsr; + status_2500 = tp->phy_reg_status_2500; + if (tp->link_ok(dev)) { + aner = rtl8127_mdio_read(tp, MII_EXPANSION); + anlpar = rtl8127_mdio_read(tp, MII_LPA); + gbsr = rtl8127_mdio_read(tp, MII_STAT1000); + status_2500 = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D6); + } + + adv = tp->advertising; + if ((aner | anlpar | gbsr | status_2500) == 0) { + int auto_nego_tmp = 0; + if (adv & ADVERTISED_10baseT_Half) + auto_nego_tmp |= ADVERTISE_10HALF; + if (adv & ADVERTISED_10baseT_Full) + auto_nego_tmp |= ADVERTISE_10FULL; + if (adv & ADVERTISED_100baseT_Half) + auto_nego_tmp |= ADVERTISE_100HALF; + if (adv & ADVERTISED_100baseT_Full) + auto_nego_tmp |= ADVERTISE_100FULL; + + if (auto_nego_tmp == 0) + goto exit; + + auto_nego |= auto_nego_tmp; + goto skip_check_lpa; + } + if (!(aner & EXPANSION_NWAY)) + goto exit; + + if ((adv & ADVERTISED_10baseT_Half) && (anlpar & LPA_10HALF)) + auto_nego |= ADVERTISE_10HALF; + else if ((adv & ADVERTISED_10baseT_Full) && (anlpar & LPA_10FULL)) + auto_nego |= ADVERTISE_10FULL; + else if ((adv & ADVERTISED_100baseT_Half) && (anlpar & LPA_100HALF)) + auto_nego |= ADVERTISE_100HALF; + else if ((adv & ADVERTISED_100baseT_Full) && (anlpar & LPA_100FULL)) + auto_nego |= ADVERTISE_100FULL; + else if (adv & ADVERTISED_1000baseT_Half && (gbsr & LPA_1000HALF)) + giga_ctrl |= ADVERTISE_1000HALF; + else if (adv & ADVERTISED_1000baseT_Full && (gbsr & LPA_1000FULL)) + giga_ctrl |= ADVERTISE_1000FULL; + else if (adv & ADVERTISED_2500baseX_Full && (status_2500 & RTK_LPA_ADVERTISE_2500FULL)) + ctrl_2500 |= RTK_ADVERTISE_2500FULL; + else if (adv & RTK_ADVERTISED_5000baseX_Full && (status_2500 & RTK_LPA_ADVERTISE_5000FULL)) + ctrl_2500 |= RTK_ADVERTISE_5000FULL; + else if (adv & ADVERTISED_10000baseT_Full && (status_2500 & RTK_LPA_ADVERTISE_10000FULL)) + ctrl_2500 |= RTK_ADVERTISE_10000FULL; + else + goto exit; + +skip_check_lpa: + if (tp->DASH) + auto_nego |= (ADVERTISE_100FULL | ADVERTISE_100HALF | ADVERTISE_10HALF | ADVERTISE_10FULL); + +#ifdef CONFIG_DOWN_SPEED_100 + auto_nego |= (ADVERTISE_100FULL | ADVERTISE_100HALF | ADVERTISE_10HALF | ADVERTISE_10FULL); +#endif + + rtl8127_mdio_write(tp, MII_ADVERTISE, auto_nego); + rtl8127_mdio_write(tp, MII_CTRL1000, giga_ctrl); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA5D4, ctrl_2500); + + rtl8127_disable_giga_lite(tp); + + rtl8127_phy_restart_nway(dev); + +exit: + spin_unlock_irqrestore(&tp->phy_lock, flags); + + return; +} + +static bool +rtl8127_keep_wol_link_speed(struct net_device *dev, u8 from_suspend) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (from_suspend && tp->link_ok(dev) && (tp->wol_opts & WAKE_PHY)) + return 1; + + if (!from_suspend && tp->resume_not_chg_speed) + return 1; + + return 0; +} +static void +rtl8127_powerdown_pll(struct net_device *dev, u8 from_suspend) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + /* Reboot not set wol link speed */ + if (system_state == SYSTEM_RESTART) + return; + + tp->check_keep_link_speed = 0; + if (tp->wol_enabled == WOL_ENABLED || tp->DASH || tp->EnableKCPOffload) { + rtl8127_set_hw_wol(dev, tp->wol_opts); + + rtl8127_enable_cfg9346_write(tp); + RTL_W8(tp, Config2, RTL_R8(tp, Config2) | PMSTS_En); + rtl8127_disable_cfg9346_write(tp); + + /* Enable the PME and clear the status */ + rtl8127_set_pci_pme(tp, 1); + + if (rtl8127_keep_wol_link_speed(dev, from_suspend)) { + tp->check_keep_link_speed = 1; + } else { + if (tp->D0SpeedUpSpeed != D0_SPEED_UP_SPEED_DISABLE) { + rtl8127_enable_d0_speedup(tp); + tp->check_keep_link_speed = 1; + } + + rtl8127_set_wol_link_speed(dev); + } + + RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) | AcceptBroadcast | AcceptMulticast | AcceptMyPhys); + + return; + } + + if (tp->DASH) + return; + + rtl8127_phy_power_down(dev); + + RTL_W8(tp, 0xF2, RTL_R8(tp, 0xF2) & ~BIT_6); +} + +static void rtl8127_powerup_pll(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) | BIT_7 | BIT_6); + + if (tp->resume_not_chg_speed) + return; + + rtl8127_phy_power_up(dev); +} + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) +static void +rtl8127_get_wol(struct net_device *dev, + struct ethtool_wolinfo *wol) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u8 options; + + wol->wolopts = 0; + + if (tp->mcfg == CFG_METHOD_DEFAULT || disable_wol_support) { + wol->supported = 0; + return; + } else { + wol->supported = WAKE_ANY; + } + + options = RTL_R8(tp, Config1); + if (!(options & PMEnable)) + return; + + wol->wolopts = tp->wol_opts; +} + +static int +rtl8127_set_wol(struct net_device *dev, + struct ethtool_wolinfo *wol) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (tp->mcfg == CFG_METHOD_DEFAULT || disable_wol_support) + return -EOPNOTSUPP; + + tp->wol_opts = wol->wolopts; + + tp->wol_enabled = (tp->wol_opts) ? WOL_ENABLED : WOL_DISABLED; + + device_set_wakeup_enable(tp_to_dev(tp), wol->wolopts); + + return 0; +} + +static void +rtl8127_get_drvinfo(struct net_device *dev, + struct ethtool_drvinfo *info) +{ + struct rtl8127_private *tp = netdev_priv(dev); + struct rtl8127_fw *rtl_fw = tp->rtl_fw; + + strscpy(info->driver, MODULENAME, sizeof(info->driver)); + strscpy(info->version, RTL8127_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(tp->pci_dev), sizeof(info->bus_info)); + info->regdump_len = R8127_REGS_DUMP_SIZE; + info->eedump_len = tp->eeprom_len; + BUILD_BUG_ON(sizeof(info->fw_version) < sizeof(rtl_fw->version)); + if (rtl_fw) + strscpy(info->fw_version, rtl_fw->version, + sizeof(info->fw_version)); +} + +static int +rtl8127_get_regs_len(struct net_device *dev) +{ + return R8127_REGS_DUMP_SIZE; +} +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) + +static void +rtl8127_set_d0_speedup_speed(struct rtl8127_private *tp) +{ + if (FALSE == HW_SUPPORT_D0_SPEED_UP(tp)) + return; + + tp->D0SpeedUpSpeed = D0_SPEED_UP_SPEED_DISABLE; + if (tp->autoneg == AUTONEG_ENABLE) { + if (tp->speed == SPEED_10000) + tp->D0SpeedUpSpeed = D0_SPEED_UP_SPEED_10000; + else if (tp->speed == SPEED_5000) + tp->D0SpeedUpSpeed = D0_SPEED_UP_SPEED_5000; + else if (tp->speed == SPEED_2500) + tp->D0SpeedUpSpeed = D0_SPEED_UP_SPEED_2500; + else if (tp->speed == SPEED_1000) + tp->D0SpeedUpSpeed = D0_SPEED_UP_SPEED_1000; + } +} + +static int +rtl8127_set_speed_xmii(struct net_device *dev, + u8 autoneg, + u32 speed, + u8 duplex, + u64 adv) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned long flags; + int auto_nego = 0; + int giga_ctrl = 0; + int ctrl_2500 = 0; + int rc = -EINVAL; + + spin_lock_irqsave(&tp->phy_lock, flags); + + if (!rtl8127_is_speed_mode_valid(speed)) { + speed = SPEED_10000; + duplex = DUPLEX_FULL; + adv |= tp->advertising; + } + + if (eee_giga_lite && (autoneg == AUTONEG_ENABLE)) + rtl8127_enable_giga_lite(tp, adv); + else + rtl8127_disable_giga_lite(tp); + + giga_ctrl = rtl8127_mdio_read(tp, MII_CTRL1000); + giga_ctrl &= ~(ADVERTISE_1000HALF | ADVERTISE_1000FULL); + ctrl_2500 = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D4); + ctrl_2500 &= ~(RTK_ADVERTISE_2500FULL | RTK_ADVERTISE_5000FULL | + RTK_ADVERTISE_10000FULL); + + if (autoneg == AUTONEG_ENABLE) { + /*n-way force*/ + auto_nego = rtl8127_mdio_read(tp, MII_ADVERTISE); + auto_nego &= ~(ADVERTISE_10HALF | ADVERTISE_10FULL | + ADVERTISE_100HALF | ADVERTISE_100FULL | + ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM); + + if (adv & ADVERTISED_10baseT_Half) + auto_nego |= ADVERTISE_10HALF; + if (adv & ADVERTISED_10baseT_Full) + auto_nego |= ADVERTISE_10FULL; + if (adv & ADVERTISED_100baseT_Half) + auto_nego |= ADVERTISE_100HALF; + if (adv & ADVERTISED_100baseT_Full) + auto_nego |= ADVERTISE_100FULL; + if (adv & ADVERTISED_1000baseT_Half) + giga_ctrl |= ADVERTISE_1000HALF; + if (adv & ADVERTISED_1000baseT_Full) + giga_ctrl |= ADVERTISE_1000FULL; + if (adv & ADVERTISED_2500baseX_Full) + ctrl_2500 |= RTK_ADVERTISE_2500FULL; + if (HW_SUPP_PHY_LINK_SPEED_5000M(tp)) { + if (adv & RTK_ADVERTISED_5000baseX_Full) + ctrl_2500 |= RTK_ADVERTISE_5000FULL; + } + if (HW_SUPP_PHY_LINK_SPEED_10000M(tp)) { + if (adv & ADVERTISED_10000baseT_Full) + ctrl_2500 |= RTK_ADVERTISE_10000FULL; + } + + //flow control + if (tp->fcpause == rtl8127_fc_full) + auto_nego |= ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM; + + tp->phy_auto_nego_reg = auto_nego; + tp->phy_1000_ctrl_reg = giga_ctrl; + + tp->phy_2500_ctrl_reg = ctrl_2500; + + rtl8127_mdio_write(tp, 0x1f, 0x0000); + rtl8127_mdio_write(tp, MII_ADVERTISE, auto_nego); + rtl8127_mdio_write(tp, MII_CTRL1000, giga_ctrl); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA5D4, ctrl_2500); + rtl8127_phy_restart_nway(dev); + } else { + /*true force*/ + if (speed == SPEED_10 || speed == SPEED_100) + rtl8127_phy_setup_force_mode(dev, speed, duplex); + else + goto out; + } + + tp->autoneg = autoneg; + tp->speed = speed; + tp->duplex = duplex; + tp->advertising = adv; + + rtl8127_set_d0_speedup_speed(tp); + + rc = 0; +out: + spin_unlock_irqrestore(&tp->phy_lock, flags); + + return rc; +} + +static int +rtl8127_set_speed(struct net_device *dev, + u8 autoneg, + u32 speed, + u8 duplex, + u64 adv) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int ret; + + if (tp->resume_not_chg_speed) + return 0; + + ret = tp->set_speed(dev, autoneg, speed, duplex, adv); + + return ret; +} + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) +static int +rtl8127_set_settings(struct net_device *dev, +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) + struct ethtool_cmd *cmd +#else + const struct ethtool_link_ksettings *cmd +#endif + ) +{ + int ret; + u8 autoneg; + u32 speed; + u8 duplex; + u64 supported = 0, advertising = 0; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) + autoneg = cmd->autoneg; + speed = cmd->speed; + duplex = cmd->duplex; + supported = cmd->supported; + advertising = cmd->advertising; +#else + struct rtl8127_private *tp = netdev_priv(dev); + const struct ethtool_link_settings *base = &cmd->base; + autoneg = base->autoneg; + speed = base->speed; + duplex = base->duplex; + ethtool_convert_link_mode_to_legacy_u32((u32*)&supported, + cmd->link_modes.supported); + ethtool_convert_link_mode_to_legacy_u32((u32*)&advertising, + cmd->link_modes.advertising); + if (test_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, + cmd->link_modes.supported)) + supported |= ADVERTISED_2500baseX_Full; + if (test_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, + cmd->link_modes.advertising)) + advertising |= ADVERTISED_2500baseX_Full; + if (HW_SUPP_PHY_LINK_SPEED_5000M(tp)) { + if (test_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, + cmd->link_modes.supported)) + supported |= RTK_ADVERTISED_5000baseX_Full; + if (test_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, + cmd->link_modes.advertising)) + advertising |= RTK_ADVERTISED_5000baseX_Full; + } + if (HW_SUPP_PHY_LINK_SPEED_10000M(tp)) { + if (test_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, + cmd->link_modes.supported)) + supported |= ADVERTISED_10000baseT_Full; + if (test_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, + cmd->link_modes.advertising)) + advertising |= ADVERTISED_10000baseT_Full; + } +#endif + if (advertising & ~supported) + return -EINVAL; + + ret = rtl8127_set_speed(dev, autoneg, speed, duplex, advertising); + + return ret; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) +static u32 +rtl8127_get_tx_csum(struct net_device *dev) +{ + u32 ret; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + ret = ((dev->features & NETIF_F_IP_CSUM) != 0); +#else + ret = ((dev->features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) != 0); +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + + return ret; +} + +static u32 +rtl8127_get_rx_csum(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u32 ret; + + ret = tp->cp_cmd & RxChkSum; + + return ret; +} + +static int +rtl8127_set_tx_csum(struct net_device *dev, + u32 data) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (tp->mcfg == CFG_METHOD_DEFAULT) + return -EOPNOTSUPP; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + if (data) + dev->features |= NETIF_F_IP_CSUM; + else + dev->features &= ~NETIF_F_IP_CSUM; +#else + if (data) + dev->features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM); + else + dev->features &= ~(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM); +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + + return 0; +} + +static int +rtl8127_set_rx_csum(struct net_device *dev, + u32 data) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (tp->mcfg == CFG_METHOD_DEFAULT) + return -EOPNOTSUPP; + + if (data) + tp->cp_cmd |= RxChkSum; + else + tp->cp_cmd &= ~RxChkSum; + + RTL_W16(tp, CPlusCmd, tp->cp_cmd); + + return 0; +} +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) + +static u32 +rtl8127_rx_desc_opts1(struct rtl8127_private *tp, + struct RxDesc *desc) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + return READ_ONCE(((struct RxDescV3 *)desc)->RxDescNormalDDWord4.opts1); + case RX_DESC_RING_TYPE_4: + return READ_ONCE(((struct RxDescV4 *)desc)->RxDescNormalDDWord2.opts1); + default: + return READ_ONCE(desc->opts1); + } +} + +static u32 +rtl8127_rx_desc_opts2(struct rtl8127_private *tp, + struct RxDesc *desc) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + return ((struct RxDescV3 *)desc)->RxDescNormalDDWord4.opts2; + case RX_DESC_RING_TYPE_4: + return ((struct RxDescV4 *)desc)->RxDescNormalDDWord2.opts2; + default: + return desc->opts2; + } +} + +#ifdef CONFIG_R8127_VLAN + +static void +rtl8127_clear_rx_desc_opts2(struct rtl8127_private *tp, + struct RxDesc *desc) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + ((struct RxDescV3 *)desc)->RxDescNormalDDWord4.opts2 = 0; + break; + case RX_DESC_RING_TYPE_4: + ((struct RxDescV4 *)desc)->RxDescNormalDDWord2.opts2 = 0; + break; + default: + desc->opts2 = 0; + break; + } +} + + +static inline u32 +rtl8127_tx_vlan_tag(struct rtl8127_private *tp, + struct sk_buff *skb) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + return (tp->vlgrp && vlan_tx_tag_present(skb)) ? + TxVlanTag | swab16(vlan_tx_tag_get(skb)) : 0x00; +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,0,0) + return (vlan_tx_tag_present(skb)) ? + TxVlanTag | swab16(vlan_tx_tag_get(skb)) : 0x00; +#else + return (skb_vlan_tag_present(skb)) ? + TxVlanTag | swab16(skb_vlan_tag_get(skb)) : 0x00; +#endif + + return 0; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + +static void +rtl8127_vlan_rx_register(struct net_device *dev, + struct vlan_group *grp) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + tp->vlgrp = grp; + + if (tp->vlgrp) { + tp->rtl8127_rx_config |= (EnableInnerVlan | EnableOuterVlan); + RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) | (EnableInnerVlan | EnableOuterVlan)) + } else { + tp->rtl8127_rx_config &= ~(EnableInnerVlan | EnableOuterVlan); + RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) & ~(EnableInnerVlan | EnableOuterVlan)) + } +} + +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) +static void +rtl8127_vlan_rx_kill_vid(struct net_device *dev, + unsigned short vid) +{ + struct rtl8127_private *tp = netdev_priv(dev); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,21) + if (tp->vlgrp) + tp->vlgrp->vlan_devices[vid] = NULL; +#else + vlan_group_set_device(tp->vlgrp, vid, NULL); +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,21) +} +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) + +static int +rtl8127_rx_vlan_skb(struct rtl8127_private *tp, + struct RxDesc *desc, + struct sk_buff *skb) +{ + u32 opts2 = le32_to_cpu(rtl8127_rx_desc_opts2(tp, desc)); + int ret = -1; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + if (tp->vlgrp && (opts2 & RxVlanTag)) { + rtl8127_rx_hwaccel_skb(skb, tp->vlgrp, + swab16(opts2 & 0xffff)); + ret = 0; + } +#elif LINUX_VERSION_CODE < KERNEL_VERSION(3,10,0) + if (opts2 & RxVlanTag) + __vlan_hwaccel_put_tag(skb, swab16(opts2 & 0xffff)); +#else + if (opts2 & RxVlanTag) + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), swab16(opts2 & 0xffff)); +#endif + + rtl8127_clear_rx_desc_opts2(tp, desc); + return ret; +} + +#else /* !CONFIG_R8127_VLAN */ + +static inline u32 +rtl8127_tx_vlan_tag(struct rtl8127_private *tp, + struct sk_buff *skb) +{ + return 0; +} + +static int +rtl8127_rx_vlan_skb(struct rtl8127_private *tp, + struct RxDesc *desc, + struct sk_buff *skb) +{ + return -1; +} + +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0) + +static netdev_features_t rtl8127_fix_features(struct net_device *dev, + netdev_features_t features) +{ + if (dev->mtu > MSS_MAX) + features &= ~NETIF_F_ALL_TSO; + if (dev->mtu > ETH_DATA_LEN) { + features &= ~NETIF_F_ALL_TSO; + features &= ~NETIF_F_ALL_CSUM; + } +#ifndef CONFIG_R8127_VLAN + features &= ~NETIF_F_ALL_CSUM; +#endif + + return features; +} + +static int rtl8127_hw_set_features(struct net_device *dev, + netdev_features_t features) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u32 rx_config; + + rx_config = RTL_R32(tp, RxConfig); + if (features & NETIF_F_RXALL) { + tp->rtl8127_rx_config |= (AcceptErr | AcceptRunt); + rx_config |= (AcceptErr | AcceptRunt); + } else { + tp->rtl8127_rx_config &= ~(AcceptErr | AcceptRunt); + rx_config &= ~(AcceptErr | AcceptRunt); + } + + if (features & NETIF_F_HW_VLAN_RX) { + tp->rtl8127_rx_config |= (EnableInnerVlan | EnableOuterVlan); + rx_config |= (EnableInnerVlan | EnableOuterVlan); + } else { + tp->rtl8127_rx_config &= ~(EnableInnerVlan | EnableOuterVlan); + rx_config &= ~(EnableInnerVlan | EnableOuterVlan); + } + + RTL_W32(tp, RxConfig, rx_config); + + if (features & NETIF_F_RXCSUM) + tp->cp_cmd |= RxChkSum; + else + tp->cp_cmd &= ~RxChkSum; + + RTL_W16(tp, CPlusCmd, tp->cp_cmd); + RTL_R16(tp, CPlusCmd); + + return 0; +} + +static int rtl8127_set_features(struct net_device *dev, + netdev_features_t features) +{ + features &= NETIF_F_RXALL | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_RX; + + rtl8127_hw_set_features(dev, features); + + return 0; +} + +#endif + +static u8 rtl8127_get_mdi_status(struct rtl8127_private *tp) +{ + if (!tp->link_ok(tp->dev)) + return ETH_TP_MDI_INVALID; + + if (rtl8127_mdio_direct_read_phy_ocp(tp, 0xA444) & BIT_1) + return ETH_TP_MDI; + else + return ETH_TP_MDI_X; +} + +static void rtl8127_gset_xmii(struct net_device *dev, +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) + struct ethtool_cmd *cmd +#else + struct ethtool_link_ksettings *cmd +#endif + ) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u16 aner = tp->phy_reg_aner; + u16 anlpar = tp->phy_reg_anlpar; + u16 gbsr = tp->phy_reg_gbsr; + u16 status_2500 = tp->phy_reg_status_2500; + unsigned long flags; + u64 lpa_adv = 0; + u32 status; + u8 autoneg, duplex; + u32 speed = 0; + u16 bmcr; + u64 supported, advertising; + u8 report_lpa = 0; + + supported = SUPPORTED_10baseT_Half | + SUPPORTED_10baseT_Full | + SUPPORTED_100baseT_Half | + SUPPORTED_100baseT_Full | + SUPPORTED_1000baseT_Full | + SUPPORTED_2500baseX_Full | + RTK_SUPPORTED_5000baseX_Full | + SUPPORTED_10000baseT_Full | + SUPPORTED_Autoneg | + SUPPORTED_TP | + SUPPORTED_Pause | + SUPPORTED_Asym_Pause; + + if (!HW_SUPP_PHY_LINK_SPEED_2500M(tp)) + supported &= ~SUPPORTED_2500baseX_Full; + + if (!HW_SUPP_PHY_LINK_SPEED_5000M(tp)) + supported &= ~RTK_SUPPORTED_5000baseX_Full; + + if (!HW_SUPP_PHY_LINK_SPEED_10000M(tp)) + supported &= ~SUPPORTED_10000baseT_Full; + + advertising = tp->advertising; + if (tp->phy_auto_nego_reg || tp->phy_1000_ctrl_reg || + tp->phy_2500_ctrl_reg) { + advertising = 0; + if (tp->phy_auto_nego_reg & ADVERTISE_10HALF) + advertising |= ADVERTISED_10baseT_Half; + if (tp->phy_auto_nego_reg & ADVERTISE_10FULL) + advertising |= ADVERTISED_10baseT_Full; + if (tp->phy_auto_nego_reg & ADVERTISE_100HALF) + advertising |= ADVERTISED_100baseT_Half; + if (tp->phy_auto_nego_reg & ADVERTISE_100FULL) + advertising |= ADVERTISED_100baseT_Full; + if (tp->phy_1000_ctrl_reg & ADVERTISE_1000FULL) + advertising |= ADVERTISED_1000baseT_Full; + if (tp->phy_2500_ctrl_reg & RTK_ADVERTISE_2500FULL) + advertising |= ADVERTISED_2500baseX_Full; + if (tp->phy_2500_ctrl_reg & RTK_ADVERTISE_5000FULL) + advertising |= RTK_ADVERTISED_5000baseX_Full; + if (tp->phy_2500_ctrl_reg & RTK_ADVERTISE_10000FULL) + advertising |= ADVERTISED_10000baseT_Full; + } + + spin_lock_irqsave(&tp->phy_lock, flags); + rtl8127_mdio_write(tp, 0x1F, 0x0000); + bmcr = rtl8127_mdio_read(tp, MII_BMCR); + spin_unlock_irqrestore(&tp->phy_lock, flags); + if (bmcr & BMCR_ANENABLE) { + autoneg = AUTONEG_ENABLE; + advertising |= ADVERTISED_Autoneg; + } else { + autoneg = AUTONEG_DISABLE; + } + + advertising |= ADVERTISED_TP; + + status = RTL_R32(tp, PHYstatus); + if (netif_running(dev) && (status & LinkStatus)) + report_lpa = 1; + + if (report_lpa) { + /*link on*/ + speed = rtl8127_convert_link_speed(status); + + if (status & TxFlowCtrl) + advertising |= ADVERTISED_Asym_Pause; + + if (status & RxFlowCtrl) + advertising |= ADVERTISED_Pause; + + duplex = ((status & (_1000bpsF | _2500bpsF | _5000bpsF | _10000bpsF)) || + (status & FullDup)) ? + DUPLEX_FULL : DUPLEX_HALF; + + /*link partner*/ + if (aner & EXPANSION_NWAY) + lpa_adv |= ADVERTISED_Autoneg; + if (anlpar & LPA_10HALF) + lpa_adv |= ADVERTISED_10baseT_Half; + if (anlpar & LPA_10FULL) + lpa_adv |= ADVERTISED_10baseT_Full; + if (anlpar & LPA_100HALF) + lpa_adv |= ADVERTISED_100baseT_Half; + if (anlpar & LPA_100FULL) + lpa_adv |= ADVERTISED_100baseT_Full; + if (anlpar & LPA_PAUSE_CAP) + lpa_adv |= ADVERTISED_Pause; + if (anlpar & LPA_PAUSE_ASYM) + lpa_adv |= ADVERTISED_Asym_Pause; + if (gbsr & LPA_1000HALF) + lpa_adv |= ADVERTISED_1000baseT_Half; + if (gbsr & LPA_1000FULL) + lpa_adv |= ADVERTISED_1000baseT_Full; + if (status_2500 & RTK_LPA_ADVERTISE_2500FULL) + lpa_adv |= ADVERTISED_2500baseX_Full; + if (status_2500 & RTK_LPA_ADVERTISE_5000FULL) + lpa_adv |= RTK_ADVERTISED_5000baseX_Full; + if (status_2500 & RTK_LPA_ADVERTISE_10000FULL) + lpa_adv |= ADVERTISED_10000baseT_Full; + } else { + /*link down*/ + speed = SPEED_UNKNOWN; + duplex = DUPLEX_UNKNOWN; + lpa_adv = 0; + } + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) + cmd->supported = (u32)supported; + cmd->advertising = (u32)advertising; + cmd->autoneg = autoneg; + cmd->speed = speed; + cmd->duplex = duplex; + cmd->port = PORT_TP; + cmd->lp_advertising = (u32)lpa_adv; + cmd->eth_tp_mdix = rtl8127_get_mdi_status(tp); +#else + ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, + supported); + ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising, + advertising); + ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.lp_advertising, + lpa_adv); + + if (supported & SUPPORTED_2500baseX_Full) { + linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, + cmd->link_modes.supported, 1); + } + if (advertising & ADVERTISED_2500baseX_Full) { + linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, + cmd->link_modes.advertising, 1); + } + if (supported & RTK_SUPPORTED_5000baseX_Full) { + linkmode_mod_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, + cmd->link_modes.supported, 1); + } + if (advertising & RTK_ADVERTISED_5000baseX_Full) { + linkmode_mod_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, + cmd->link_modes.advertising, 1); + } + if (supported & SUPPORTED_10000baseT_Full) { + linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, + cmd->link_modes.supported, 1); + } + if (advertising & ADVERTISED_10000baseT_Full) { + linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, + cmd->link_modes.advertising, 1); + } + if (report_lpa) { + if (lpa_adv & ADVERTISED_2500baseX_Full) { + linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, + cmd->link_modes.lp_advertising, 1); + } + if (lpa_adv & RTK_ADVERTISED_5000baseX_Full) + linkmode_mod_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, + cmd->link_modes.lp_advertising, 1); + if (lpa_adv & ADVERTISED_10000baseT_Full) + linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, + cmd->link_modes.lp_advertising, 1); + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0) + /* Use ETHTOOL_LINK_MODE_2500baseT_Full_BIT instead of + ETHTOOL_LINK_MODE_2500baseX_Full_BIT. */ + linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseX_Full_BIT, + cmd->link_modes.supported, 0); + + linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseX_Full_BIT, + cmd->link_modes.advertising, 0); + + linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseX_Full_BIT, + cmd->link_modes.lp_advertising, 0); +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0) */ + + cmd->base.autoneg = autoneg; + cmd->base.speed = speed; + cmd->base.duplex = duplex; + cmd->base.port = PORT_TP; + cmd->base.eth_tp_mdix = rtl8127_get_mdi_status(tp); +#endif +} + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) +static int +rtl8127_get_settings(struct net_device *dev, +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) + struct ethtool_cmd *cmd +#else + struct ethtool_link_ksettings *cmd +#endif + ) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + tp->get_settings(dev, cmd); + + return 0; +} + +static void rtl8127_get_regs(struct net_device *dev, struct ethtool_regs *regs, + void *p) +{ + struct rtl8127_private *tp = netdev_priv(dev); + void __iomem *ioaddr = tp->mmio_addr; + unsigned int i; + u8 *data = p; + + if (regs->len < R8127_REGS_DUMP_SIZE) + return /* -EINVAL */; + + memset(p, 0, regs->len); + + for (i = 0; i < R8127_MAC_REGS_SIZE; i++) + *data++ = readb(ioaddr + i); + data = (u8*)p + 256; + + rtl8127_mdio_write(tp, 0x1F, 0x0000); + for (i = 0; i < R8127_PHY_REGS_SIZE/2; i++) { + *(u16*)data = rtl8127_mdio_read(tp, i); + data += 2; + } + data = (u8*)p + 256 * 2; + + for (i = 0; i < R8127_EPHY_REGS_SIZE/2; i++) { + *(u16*)data = rtl8127_ephy_read(tp, i); + data += 2; + } + data = (u8*)p + 256 * 3; + + for (i = 0; i < R8127_ERI_REGS_SIZE; i+=4) { + *(u32*)data = rtl8127_eri_read(tp, i , 4, ERIAR_ExGMAC); + data += 4; + } +} + +static void rtl8127_get_pauseparam(struct net_device *dev, + struct ethtool_pauseparam *pause) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + pause->autoneg = (tp->autoneg ? AUTONEG_ENABLE : AUTONEG_DISABLE); + if (tp->fcpause == rtl8127_fc_rx_pause) + pause->rx_pause = 1; + else if (tp->fcpause == rtl8127_fc_tx_pause) + pause->tx_pause = 1; + else if (tp->fcpause == rtl8127_fc_full) { + pause->rx_pause = 1; + pause->tx_pause = 1; + } +} + +static int rtl8127_set_pauseparam(struct net_device *dev, + struct ethtool_pauseparam *pause) +{ + struct rtl8127_private *tp = netdev_priv(dev); + enum rtl8127_fc_mode newfc; + + if (pause->tx_pause || pause->rx_pause) + newfc = rtl8127_fc_full; + else + newfc = rtl8127_fc_none; + + if (tp->fcpause != newfc) { + tp->fcpause = newfc; + + rtl8127_set_speed(dev, tp->autoneg, tp->speed, tp->duplex, tp->advertising); + } + + return 0; + +} + +static u32 +rtl8127_get_msglevel(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + return tp->msg_enable; +} + +static void +rtl8127_set_msglevel(struct net_device *dev, + u32 value) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + tp->msg_enable = value; +} + +static const char rtl8127_gstrings[][ETH_GSTRING_LEN] = { + /* legacy */ + "tx_packets", + "rx_packets", + "tx_errors", + "rx_errors", + "rx_missed", + "align_errors", + "tx_single_collisions", + "tx_multi_collisions", + "unicast", + "broadcast", + "multicast", + "tx_aborted", + "tx_underrun", + + /* extended */ + "tx_octets", + "rx_octets", + "rx_multicast64", + "tx_unicast64", + "tx_broadcast64", + "tx_multicast64", + "tx_pause_on", + "tx_pause_off", + "tx_pause_all", + "tx_deferred", + "tx_late_collision", + "tx_all_collision", + "tx_aborted32", + "align_errors32", + "rx_frame_too_long", + "rx_runt", + "rx_pause_on", + "rx_pause_off", + "rx_pause_all", + "rx_unknown_opcode", + "rx_mac_error", + "tx_underrun32", + "rx_mac_missed", + "rx_tcam_dropped", + "tdu", + "rdu", +}; +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,33) +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) +static int rtl8127_get_stats_count(struct net_device *dev) +{ + return ARRAY_SIZE(rtl8127_gstrings); +} +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) +#else +static int rtl8127_get_sset_count(struct net_device *dev, int sset) +{ + switch (sset) { + case ETH_SS_STATS: + return ARRAY_SIZE(rtl8127_gstrings); + default: + return -EOPNOTSUPP; + } +} +#endif + +static void +rtl8127_set_ring_size(struct rtl8127_private *tp, u32 rx, u32 tx) +{ + int i; + + for (i = 0; i < R8127_MAX_RX_QUEUES; i++) + tp->rx_ring[i].num_rx_desc = rx; + + for (i = 0; i < R8127_MAX_TX_QUEUES; i++) + tp->tx_ring[i].num_tx_desc = tx; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0) +static void rtl8127_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) +#else +static void rtl8127_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring) +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + ring->rx_max_pending = MAX_NUM_TX_DESC; + ring->tx_max_pending = MAX_NUM_RX_DESC; + ring->rx_pending = tp->rx_ring[0].num_rx_desc; + ring->tx_pending = tp->tx_ring[0].num_tx_desc; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0) +static int rtl8127_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) +#else +static int rtl8127_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring) +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u32 new_rx_count, new_tx_count; + int rc = 0; + + if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) + return -EINVAL; + + new_tx_count = clamp_t(u32, ring->tx_pending, + MIN_NUM_TX_DESC, MAX_NUM_TX_DESC); + + new_rx_count = clamp_t(u32, ring->rx_pending, + MIN_NUM_RX_DESC, MAX_NUM_RX_DESC); + + if ((new_rx_count == tp->rx_ring[0].num_rx_desc) && + (new_tx_count == tp->tx_ring[0].num_tx_desc)) { + /* nothing to do */ + return 0; + } + + if (netif_running(dev)) { + rtl8127_wait_for_quiescence(dev); + rtl8127_close(dev); + } + + rtl8127_set_ring_size(tp, new_rx_count, new_tx_count); + + if (netif_running(dev)) + rc = rtl8127_open(dev); + + return rc; +} +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) +static void +rtl8127_get_ethtool_stats(struct net_device *dev, + struct ethtool_stats *stats, + u64 *data) +{ + struct rtl8127_private *tp = netdev_priv(dev); + struct rtl8127_counters *counters; + dma_addr_t paddr; + + ASSERT_RTNL(); + + counters = tp->tally_vaddr; + paddr = tp->tally_paddr; + if (!counters) + return; + + rtl8127_dump_tally_counter(tp, paddr); + + data[0] = le64_to_cpu(counters->tx_packets); + data[1] = le64_to_cpu(counters->rx_packets); + data[2] = le64_to_cpu(counters->tx_errors); + data[3] = le32_to_cpu(counters->rx_errors); + data[4] = le16_to_cpu(counters->rx_missed); + data[5] = le16_to_cpu(counters->align_errors); + data[6] = le32_to_cpu(counters->tx_one_collision); + data[7] = le32_to_cpu(counters->tx_multi_collision); + data[8] = le64_to_cpu(counters->rx_unicast); + data[9] = le64_to_cpu(counters->rx_broadcast); + data[10] = le32_to_cpu(counters->rx_multicast); + data[11] = le16_to_cpu(counters->tx_aborted); + data[12] = le16_to_cpu(counters->tx_underrun); + + data[13] = le64_to_cpu(counters->tx_octets); + data[14] = le64_to_cpu(counters->rx_octets); + data[15] = le64_to_cpu(counters->rx_multicast64); + data[16] = le64_to_cpu(counters->tx_unicast64); + data[17] = le64_to_cpu(counters->tx_broadcast64); + data[18] = le64_to_cpu(counters->tx_multicast64); + data[19] = le32_to_cpu(counters->tx_pause_on); + data[20] = le32_to_cpu(counters->tx_pause_off); + data[21] = le32_to_cpu(counters->tx_pause_all); + data[22] = le32_to_cpu(counters->tx_deferred); + data[23] = le32_to_cpu(counters->tx_late_collision); + data[24] = le32_to_cpu(counters->tx_all_collision); + data[25] = le32_to_cpu(counters->tx_aborted32); + data[26] = le32_to_cpu(counters->align_errors32); + data[27] = le32_to_cpu(counters->rx_frame_too_long); + data[28] = le32_to_cpu(counters->rx_runt); + data[29] = le32_to_cpu(counters->rx_pause_on); + data[30] = le32_to_cpu(counters->rx_pause_off); + data[31] = le32_to_cpu(counters->rx_pause_all); + data[32] = le32_to_cpu(counters->rx_unknown_opcode); + data[33] = le32_to_cpu(counters->rx_mac_error); + data[34] = le32_to_cpu(counters->tx_underrun32); + data[35] = le32_to_cpu(counters->rx_mac_missed); + data[36] = le32_to_cpu(counters->rx_tcam_dropped); + data[37] = le32_to_cpu(counters->tdu); + data[38] = le32_to_cpu(counters->rdu); +} + +static void +rtl8127_get_strings(struct net_device *dev, + u32 stringset, + u8 *data) +{ + switch (stringset) { + case ETH_SS_STATS: + memcpy(data, rtl8127_gstrings, sizeof(rtl8127_gstrings)); + break; + } +} +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) + +static int rtl_get_eeprom_len(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + return tp->eeprom_len; +} + +static int rtl_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *buf) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i,j,ret; + int start_w, end_w; + int VPD_addr, VPD_data; + u32 *eeprom_buff; + u16 tmp; + + if (tp->eeprom_type == EEPROM_TYPE_NONE) { + dev_printk(KERN_DEBUG, tp_to_dev(tp), "Detect none EEPROM\n"); + return -EOPNOTSUPP; + } else if (eeprom->len == 0 || (eeprom->offset+eeprom->len) > tp->eeprom_len) { + dev_printk(KERN_DEBUG, tp_to_dev(tp), "Invalid parameter\n"); + return -EINVAL; + } + + VPD_addr = 0xD2; + VPD_data = 0xD4; + + start_w = eeprom->offset >> 2; + end_w = (eeprom->offset + eeprom->len - 1) >> 2; + + eeprom_buff = kmalloc(sizeof(u32)*(end_w - start_w + 1), GFP_KERNEL); + if (!eeprom_buff) + return -ENOMEM; + + rtl8127_enable_cfg9346_write(tp); + ret = -EFAULT; + for (i=start_w; i<=end_w; i++) { + pci_write_config_word(tp->pci_dev, VPD_addr, (u16)i*4); + ret = -EFAULT; + for (j = 0; j < 10; j++) { + fsleep(400); + pci_read_config_word(tp->pci_dev, VPD_addr, &tmp); + if (tmp&0x8000) { + ret = 0; + break; + } + } + + if (ret) + break; + + pci_read_config_dword(tp->pci_dev, VPD_data, &eeprom_buff[i-start_w]); + } + rtl8127_disable_cfg9346_write(tp); + + if (!ret) + memcpy(buf, (u8 *)eeprom_buff + (eeprom->offset & 3), eeprom->len); + + kfree(eeprom_buff); + + return ret; +} + +#undef ethtool_op_get_link +#define ethtool_op_get_link _kc_ethtool_op_get_link +static u32 _kc_ethtool_op_get_link(struct net_device *dev) +{ + return netif_carrier_ok(dev) ? 1 : 0; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) +#undef ethtool_op_get_sg +#define ethtool_op_get_sg _kc_ethtool_op_get_sg +static u32 _kc_ethtool_op_get_sg(struct net_device *dev) +{ +#ifdef NETIF_F_SG + return (dev->features & NETIF_F_SG) != 0; +#else + return 0; +#endif +} + +#undef ethtool_op_set_sg +#define ethtool_op_set_sg _kc_ethtool_op_set_sg +static int _kc_ethtool_op_set_sg(struct net_device *dev, u32 data) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (tp->mcfg == CFG_METHOD_DEFAULT) + return -EOPNOTSUPP; + +#ifdef NETIF_F_SG + if (data) + dev->features |= NETIF_F_SG; + else + dev->features &= ~NETIF_F_SG; +#endif + + return 0; +} +#endif + +static void +rtl8127_set_eee_lpi_timer(struct rtl8127_private *tp) +{ + u16 dev_lpi_timer; + + dev_lpi_timer = tp->eee.tx_lpi_timer; + + RTL_W16(tp, EEE_TXIDLE_TIMER_8125, dev_lpi_timer); +} + +static bool rtl8127_is_adv_eee_enabled(struct rtl8127_private *tp) +{ + if (rtl8127_mdio_direct_read_phy_ocp(tp, 0xA430) & BIT_15) + return true; + else + return false; +} + +static void rtl8127_disable_adv_eee(struct rtl8127_private *tp) +{ + bool lock; + + if (rtl8127_is_adv_eee_enabled(tp)) + lock = true; + else + lock = false; + + if (lock) + rtl8127_set_phy_mcu_patch_request(tp); + + rtl8127_clear_mac_ocp_bit(tp, 0xE052, BIT_0); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA442, BIT_12 | BIT_13); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA430, BIT_15); + + if (lock) + rtl8127_clear_phy_mcu_patch_request(tp); +} + +static int rtl8127_enable_eee(struct rtl8127_private *tp) +{ + struct ethtool_keee *eee = &tp->eee; + u16 eee_adv_cap1_t = rtl8127_ethtool_adv_to_mmd_eee_adv_cap1_t(eee->advertised); + u16 eee_adv_cap2_t = rtl8127_ethtool_adv_to_mmd_eee_adv_cap2_t(eee->advertised); + + rtl8127_set_mac_ocp_bit(tp, 0xE040, (BIT_1|BIT_0)); + + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA5D0, + MDIO_EEE_100TX | MDIO_EEE_1000T | MDIO_EEE_10GT, + eee_adv_cap1_t); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA6D4, + MDIO_EEE_2_5GT | MDIO_EEE_5GT, + eee_adv_cap2_t); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA6D8, BIT_4); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA428, BIT_7); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA4A2, BIT_9); + + /*Advanced EEE*/ + rtl8127_disable_adv_eee(tp); + + return 0; +} + +static int rtl8127_disable_eee(struct rtl8127_private *tp) +{ + rtl8127_clear_mac_ocp_bit(tp, 0xE040, (BIT_1|BIT_0)); + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA5D0, + (MDIO_EEE_100TX | MDIO_EEE_1000T | MDIO_EEE_10GT)); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA6D4, + (MDIO_EEE_2_5GT | MDIO_EEE_5GT)); + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA6D8, BIT_4); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA428, BIT_7); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA4A2, BIT_9); + + /*Advanced EEE*/ + rtl8127_disable_adv_eee(tp); + + return 0; +} + +static int rtl_nway_reset(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int ret, bmcr; + + if (unlikely(tp->rtk_enable_diag)) + return -EBUSY; + + /* if autoneg is off, it's an error */ + rtl8127_mdio_write(tp, 0x1F, 0x0000); + bmcr = rtl8127_mdio_read(tp, MII_BMCR); + + if (bmcr & BMCR_ANENABLE) { + bmcr |= BMCR_ANRESTART; + rtl8127_mdio_write(tp, MII_BMCR, bmcr); + ret = 0; + } else { + ret = -EINVAL; + } + + return ret; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0) +static u32 +rtl8127_device_lpi_t_to_ethtool_lpi_t(struct rtl8127_private *tp , u32 lpi_timer) +{ + u32 to_us; + u16 status; + + to_us = lpi_timer * 80; + status = RTL_R16(tp, PHYstatus); + if (status & LinkStatus) { + /*link on*/ + if (HW_SUPP_PHY_LINK_SPEED_10000M(tp)) { + //5G : lpi_timer * 12.8ns + //2.5G : lpi_timer * 25.6ns + //Giga: lpi_timer * 8ns + //100M : lpi_timer * 80ns + if (status & (_10000bpsF)) + to_us = (lpi_timer * 128) / 10; + else if (status & (_5000bpsF)) + to_us = (lpi_timer * 128) / 10; + else if (status & _2500bpsF) + to_us = (lpi_timer * 256) / 10; + else if (status & _1000bpsF) + to_us = lpi_timer * 8; + } else if (HW_SUPP_PHY_LINK_SPEED_5000M(tp)) { + //5G : lpi_timer * 12.8ns + //2.5G : lpi_timer * 25.6ns + //Giga: lpi_timer * 8ns + //100M : lpi_timer * 80ns + if (status & (_5000bpsF)) + to_us = (lpi_timer * 128) / 10; + else if (status & _2500bpsF) + to_us = (lpi_timer * 256) / 10; + else if (status & _1000bpsF) + to_us = lpi_timer * 8; + } else { + //2.5G : lpi_timer * 3.2ns + //Giga: lpi_timer * 8ns + //100M : lpi_timer * 80ns + if (status & _2500bpsF) + to_us = (lpi_timer * 32) / 10; + else if (status & _1000bpsF) + to_us = lpi_timer * 8; + } + } + + //ns to us + to_us /= 1000; + + return to_us; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,9,0) +static void +rtl8127_adv_to_linkmode(unsigned long *mode, u64 adv) +{ + linkmode_zero(mode); + + if (adv & ADVERTISED_10baseT_Half) + linkmode_set_bit(ETHTOOL_LINK_MODE_10baseT_Half_BIT, mode); + if (adv & ADVERTISED_10baseT_Full) + linkmode_set_bit(ETHTOOL_LINK_MODE_10baseT_Full_BIT, mode); + if (adv & ADVERTISED_100baseT_Half) + linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Half_BIT, mode); + if (adv & ADVERTISED_100baseT_Full) + linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, mode); + if (adv & ADVERTISED_1000baseT_Half) + linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseT_Half_BIT, mode); + if (adv & ADVERTISED_1000baseT_Full) + linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT, mode); + if (adv & ADVERTISED_2500baseX_Full) + linkmode_set_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, mode); + if (adv & RTK_ADVERTISED_5000baseX_Full) + linkmode_set_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, mode); + if (adv & ADVERTISED_10000baseT_Full) + linkmode_set_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, mode); +} + +static int +rtl_ethtool_get_eee(struct net_device *net, struct ethtool_keee *edata) +{ + __ETHTOOL_DECLARE_LINK_MODE_MASK(common); + struct rtl8127_private *tp = netdev_priv(net); + struct ethtool_keee *eee = &tp->eee; + u32 tx_lpi_timer; + u16 val; + + if (unlikely(tp->rtk_enable_diag)) + return -EBUSY; + + /* Get LP advertisement EEE */ + val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D2); + mii_eee_cap1_mod_linkmode_t(edata->lp_advertised, val); + val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA6D0); + mii_eee_cap2_mod_linkmode_sup_t(edata->lp_advertised, val); + + /* Get EEE Tx LPI timer*/ + tx_lpi_timer = rtl8127_device_lpi_t_to_ethtool_lpi_t(tp, eee->tx_lpi_timer); + + val = rtl8127_mac_ocp_read(tp, 0xE040); + val &= BIT_1 | BIT_0; + + edata->eee_enabled = !!val; + linkmode_copy(edata->supported, eee->supported); + linkmode_copy(edata->advertised, eee->advertised); + edata->tx_lpi_enabled = edata->eee_enabled; + edata->tx_lpi_timer = tx_lpi_timer; + linkmode_and(common, edata->advertised, edata->lp_advertised); + edata->eee_active = !linkmode_empty(common); + + return 0; +} + +static int +rtl_ethtool_set_eee(struct net_device *net, struct ethtool_keee *edata) +{ + __ETHTOOL_DECLARE_LINK_MODE_MASK(advertising); + __ETHTOOL_DECLARE_LINK_MODE_MASK(tmp); + struct rtl8127_private *tp = netdev_priv(net); + struct ethtool_keee *eee = &tp->eee; + int rc = 0; + + if (!HW_HAS_WRITE_PHY_MCU_RAM_CODE(tp) || + tp->DASH) + return -EOPNOTSUPP; + + if (unlikely(tp->rtk_enable_diag)) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "Diag Enabled\n"); + rc = -EBUSY; + goto out; + } + + if (tp->autoneg != AUTONEG_ENABLE) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "EEE requires autoneg\n"); + rc = -EINVAL; + goto out; + } + + /* + if (edata->tx_lpi_enabled) { + if (edata->tx_lpi_timer > tp->max_jumbo_frame_size || + edata->tx_lpi_timer < ETH_MIN_MTU) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "Valid LPI timer range is %d to %d. \n", + ETH_MIN_MTU, tp->max_jumbo_frame_size); + rc = -EINVAL; + goto out; + } + } + */ + + rtl8127_adv_to_linkmode(advertising, tp->advertising); + if (linkmode_empty(edata->advertised)) { + linkmode_and(edata->advertised, advertising, eee->supported); + } else if (linkmode_andnot(tmp, edata->advertised, advertising)) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "EEE advertised must be a subset of autoneg advertised speeds\n"); + rc = -EINVAL; + goto out; + } + + if (linkmode_andnot(tmp, edata->advertised, eee->supported)) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "EEE advertised must be a subset of support \n"); + rc = -EINVAL; + goto out; + } + + //tp->eee.eee_enabled = edata->eee_enabled; + //tp->eee_adv_t = rtl8127_ethtool_adv_to_mmd_eee_adv_cap1_t(edata->advertised); + + linkmode_copy(eee->advertised, edata->advertised); + //eee->tx_lpi_enabled = edata->tx_lpi_enabled; + //eee->tx_lpi_timer = edata->tx_lpi_timer; + eee->eee_enabled = edata->eee_enabled; + + if (eee->eee_enabled) + rtl8127_enable_eee(tp); + else + rtl8127_disable_eee(tp); + + rtl_nway_reset(net); + +out: + return rc; +} +#else +static int +rtl_ethtool_get_eee(struct net_device *net, struct ethtool_eee *edata) +{ + struct rtl8127_private *tp = netdev_priv(net); + struct ethtool_eee *eee = &tp->eee; + u32 lp, adv, tx_lpi_timer, supported = 0; + u16 val; + + if (unlikely(tp->rtk_enable_diag)) + return -EBUSY; + + /* Get Supported EEE */ + //val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5C4); + //supported = mmd_eee_cap_to_ethtool_sup_t(val); + supported = eee->supported; + + /* Get advertisement EEE */ + adv = eee->advertised; + + /* Get LP advertisement EEE */ + val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D2); + lp = mmd_eee_adv_to_ethtool_adv_t(val); + val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA6D0); + if (val & RTK_LPA_EEE_ADVERTISE_2500FULL) + lp |= ADVERTISED_2500baseX_Full; + + /* Get EEE Tx LPI timer*/ + tx_lpi_timer = rtl8127_device_lpi_t_to_ethtool_lpi_t(tp, eee->tx_lpi_timer); + + val = rtl8127_mac_ocp_read(tp, 0xE040); + val &= BIT_1 | BIT_0; + + edata->eee_enabled = !!val; + edata->eee_active = !!(supported & adv & lp); + edata->supported = supported; + edata->advertised = adv; + edata->lp_advertised = lp; + edata->tx_lpi_enabled = edata->eee_enabled; + edata->tx_lpi_timer = tx_lpi_timer; + + return 0; +} + +static int +rtl_ethtool_set_eee(struct net_device *net, struct ethtool_eee *edata) +{ + struct rtl8127_private *tp = netdev_priv(net); + struct ethtool_eee *eee = &tp->eee; + u32 advertising; + int rc = 0; + + if (!HW_HAS_WRITE_PHY_MCU_RAM_CODE(tp) || + tp->DASH) + return -EOPNOTSUPP; + + if (unlikely(tp->rtk_enable_diag)) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "Diag Enabled\n"); + rc = -EBUSY; + goto out; + } + + if (tp->autoneg != AUTONEG_ENABLE) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "EEE requires autoneg\n"); + rc = -EINVAL; + goto out; + } + + /* + if (edata->tx_lpi_enabled) { + if (edata->tx_lpi_timer > tp->max_jumbo_frame_size || + edata->tx_lpi_timer < ETH_MIN_MTU) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "Valid LPI timer range is %d to %d. \n", + ETH_MIN_MTU, tp->max_jumbo_frame_size); + rc = -EINVAL; + goto out; + } + } + */ + + advertising = tp->advertising; + if (!edata->advertised) { + edata->advertised = advertising & eee->supported; + } else if (edata->advertised & ~advertising) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "EEE advertised %x must be a subset of autoneg advertised speeds %x\n", + edata->advertised, advertising); + rc = -EINVAL; + goto out; + } + + if (edata->advertised & ~eee->supported) { + dev_printk(KERN_WARNING, tp_to_dev(tp), "EEE advertised %x must be a subset of support %x\n", + edata->advertised, eee->supported); + rc = -EINVAL; + goto out; + } + + //tp->eee.eee_enabled = edata->eee_enabled; + //tp->eee_adv_t = ethtool_adv_to_mmd_eee_adv_t(edata->advertised); + + eee->advertised = edata->advertised; + //eee->tx_lpi_enabled = edata->tx_lpi_enabled; + //eee->tx_lpi_timer = edata->tx_lpi_timer; + eee->eee_enabled = edata->eee_enabled; + + if (eee->eee_enabled) + rtl8127_enable_eee(tp); + else + rtl8127_disable_eee(tp); + + rtl_nway_reset(net); + +out: + return rc; +} +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6,9,0) */ +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0) */ + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0) +static void rtl8127_get_channels(struct net_device *dev, + struct ethtool_channels *channel) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + channel->max_rx = tp->HwSuppNumRxQueues; + channel->max_tx = tp->HwSuppNumTxQueues; + channel->rx_count = tp->num_rx_rings; + channel->tx_count = tp->num_tx_rings; +} +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0) */ + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) +static const struct ethtool_ops rtl8127_ethtool_ops = { + .get_drvinfo = rtl8127_get_drvinfo, + .get_regs_len = rtl8127_get_regs_len, + .get_link = ethtool_op_get_link, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) + .get_ringparam = rtl8127_get_ringparam, + .set_ringparam = rtl8127_set_ringparam, +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) + .get_settings = rtl8127_get_settings, + .set_settings = rtl8127_set_settings, +#else + .get_link_ksettings = rtl8127_get_settings, + .set_link_ksettings = rtl8127_set_settings, +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) + .get_pauseparam = rtl8127_get_pauseparam, + .set_pauseparam = rtl8127_set_pauseparam, +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) + .get_msglevel = rtl8127_get_msglevel, + .set_msglevel = rtl8127_set_msglevel, +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) + .get_rx_csum = rtl8127_get_rx_csum, + .set_rx_csum = rtl8127_set_rx_csum, + .get_tx_csum = rtl8127_get_tx_csum, + .set_tx_csum = rtl8127_set_tx_csum, + .get_sg = ethtool_op_get_sg, + .set_sg = ethtool_op_set_sg, +#ifdef NETIF_F_TSO + .get_tso = ethtool_op_get_tso, + .set_tso = ethtool_op_set_tso, +#endif //NETIF_F_TSO +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) + .get_regs = rtl8127_get_regs, + .get_wol = rtl8127_get_wol, + .set_wol = rtl8127_set_wol, + .get_strings = rtl8127_get_strings, +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,33) + .get_stats_count = rtl8127_get_stats_count, +#else + .get_sset_count = rtl8127_get_sset_count, +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,33) + .get_ethtool_stats = rtl8127_get_ethtool_stats, +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,23) +#ifdef ETHTOOL_GPERMADDR + .get_perm_addr = ethtool_op_get_perm_addr, +#endif //ETHTOOL_GPERMADDR +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,23) + .get_eeprom = rtl_get_eeprom, + .get_eeprom_len = rtl_get_eeprom_len, +#ifdef ENABLE_RSS_SUPPORT + .get_rxnfc = rtl8127_get_rxnfc, + .set_rxnfc = rtl8127_set_rxnfc, + .get_rxfh_indir_size = rtl8127_rss_indir_size, + .get_rxfh_key_size = rtl8127_get_rxfh_key_size, + .get_rxfh = rtl8127_get_rxfh, + .set_rxfh = rtl8127_set_rxfh, +#endif //ENABLE_RSS_SUPPORT +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0) +#ifdef ENABLE_PTP_SUPPORT + .get_ts_info = rtl8127_get_ts_info, +#else + .get_ts_info = ethtool_op_get_ts_info, +#endif //ENABLE_PTP_SUPPORT +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0) + .get_eee = rtl_ethtool_get_eee, + .set_eee = rtl_ethtool_set_eee, +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0) */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0) + .get_channels = rtl8127_get_channels, +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0) */ + .nway_reset = rtl_nway_reset, + +}; +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) + +static void rtl8127_get_mac_version(struct rtl8127_private *tp) +{ + u32 reg,val32; + u32 ICVerID; + + val32 = RTL_R32(tp, TxConfig); + reg = val32 & 0x7c800000; + ICVerID = val32 & 0x00700000; + + switch (reg) { + case 0x6C800000: + if (ICVerID == 0x00000000) { + tp->mcfg = CFG_METHOD_1; + } else if (ICVerID == 0x100000) { + tp->mcfg = CFG_METHOD_2; + } else { + tp->mcfg = CFG_METHOD_2; + tp->HwIcVerUnknown = TRUE; + } + + tp->efuse_ver = EFUSE_SUPPORT_V4; + break; + default: + printk("unknown chip version (%x)\n",reg); + tp->mcfg = CFG_METHOD_DEFAULT; + tp->HwIcVerUnknown = TRUE; + tp->efuse_ver = EFUSE_NOT_SUPPORT; + break; + } +} + +static void +rtl8127_print_mac_version(struct rtl8127_private *tp) +{ + int i; + for (i = ARRAY_SIZE(rtl_chip_info) - 1; i >= 0; i--) { + if (tp->mcfg == rtl_chip_info[i].mcfg) { + dprintk("Realtek %s Ethernet controller mcfg = %04d\n", + MODULENAME, rtl_chip_info[i].mcfg); + return; + } + } + + dprintk("mac_version == Unknown\n"); +} + +static void +rtl8127_tally_counter_addr_fill(struct rtl8127_private *tp) +{ + if (!tp->tally_paddr) + return; + + RTL_W32(tp, CounterAddrHigh, (u64)tp->tally_paddr >> 32); + RTL_W32(tp, CounterAddrLow, (u64)tp->tally_paddr & (DMA_BIT_MASK(32))); +} + +static void +rtl8127_tally_counter_clear(struct rtl8127_private *tp) +{ + if (!tp->tally_paddr) + return; + + RTL_W32(tp, CounterAddrHigh, (u64)tp->tally_paddr >> 32); + RTL_W32(tp, CounterAddrLow, ((u64)tp->tally_paddr & (DMA_BIT_MASK(32))) | CounterReset); +} + +static void +rtl8127_clear_phy_ups_reg(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA466, BIT_0); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA468, BIT_3 | BIT_1); +} + +static int +rtl8127_is_ups_resume(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + return (rtl8127_mac_ocp_read(tp, 0xD42C) & BIT_8); +} + +static void +rtl8127_clear_ups_resume_bit(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + rtl8127_clear_mac_ocp_bit(tp, 0xD42C, BIT_8); +} + +static u8 +rtl8127_get_phy_state(struct rtl8127_private *tp) +{ + return (rtl8127_mdio_direct_read_phy_ocp(tp, 0xA420) & 0x7); +} + +static void +rtl8127_wait_phy_ups_resume(struct net_device *dev, u16 PhyState) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i; + + for (i=0; i< 100; i++) { + if (rtl8127_get_phy_state(tp) == PhyState) + break; + else + mdelay(1); + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18) + WARN_ON_ONCE(i == 100); +#endif +} + +void +rtl8127_enable_now_is_oob(struct rtl8127_private *tp) +{ + if (tp->HwSuppNowIsOobVer == 1) + RTL_W8(tp, MCUCmd_reg, RTL_R8(tp, MCUCmd_reg) | Now_is_oob); +} + +void +rtl8127_disable_now_is_oob(struct rtl8127_private *tp) +{ + if (tp->HwSuppNowIsOobVer == 1) + RTL_W8(tp, MCUCmd_reg, RTL_R8(tp, MCUCmd_reg) & ~Now_is_oob); +} + +static void +rtl8127_exit_oob(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u16 data16; + + rtl8127_disable_rx_packet_filter(tp); + + if (HW_DASH_SUPPORT_DASH(tp)) { + rtl8127_driver_start(tp); + rtl8127_dash2_disable_txrx(dev); +#ifdef ENABLE_DASH_SUPPORT + DashHwInit(dev); +#endif + } + +#ifdef ENABLE_REALWOW_SUPPORT + rtl8127_realwow_hw_init(dev); +#else + rtl8127_mac_ocp_write(tp, 0xC0BC, 0x00FF); +#endif //ENABLE_REALWOW_SUPPORT + + rtl8127_nic_reset(dev); + + rtl8127_disable_now_is_oob(tp); + + data16 = rtl8127_mac_ocp_read(tp, 0xE8DE) & ~BIT_14; + rtl8127_mac_ocp_write(tp, 0xE8DE, data16); + rtl8127_wait_ll_share_fifo_ready(dev); + + rtl8127_mac_ocp_write(tp, 0xC0AA, 0x07D0); +#ifdef ENABLE_LIB_SUPPORT + rtl8127_mac_ocp_write(tp, 0xC0A6, 0x04E2); +#else + rtl8127_mac_ocp_write(tp, 0xC0A6, 0x01B5); +#endif + rtl8127_mac_ocp_write(tp, 0xC01E, 0x5555); + + rtl8127_wait_ll_share_fifo_ready(dev); + + //wait ups resume (phy state 2) + if (rtl8127_is_ups_resume(dev)) { + rtl8127_wait_phy_ups_resume(dev, 2); + rtl8127_clear_ups_resume_bit(dev); + rtl8127_clear_phy_ups_reg(dev); + } +} + +void +rtl8127_hw_disable_mac_mcu_bps(struct net_device *dev) +{ + u16 regAddr; + + struct rtl8127_private *tp = netdev_priv(dev); + + rtl8127_enable_aspm_clkreq_lock(tp, 0); + + rtl8127_mac_ocp_write(tp, 0xFC48, 0x0000); + + for (regAddr = 0xFC28; regAddr < 0xFC48; regAddr += 2) { + rtl8127_mac_ocp_write(tp, regAddr, 0x0000); + } + + fsleep(3000); + + rtl8127_mac_ocp_write(tp, 0xFC26, 0x0000); +} + +#ifndef ENABLE_USE_FIRMWARE_FILE +static void +rtl8127_switch_mac_mcu_ram_code_page(struct rtl8127_private *tp, u16 page) +{ + u16 tmpUshort; + + page &= (BIT_1 | BIT_0); + tmpUshort = rtl8127_mac_ocp_read(tp, 0xE446); + tmpUshort &= ~(BIT_1 | BIT_0); + tmpUshort |= page; + rtl8127_mac_ocp_write(tp, 0xE446, tmpUshort); +} + +static void +_rtl8127_set_hw_mcu_patch_code_ver(struct rtl8127_private *tp, u64 ver) +{ + int i; + + /* Switch to page 2 */ + rtl8127_switch_mac_mcu_ram_code_page(tp, 2); + + for (i = 0; i < 8; i += 2) { + rtl8127_mac_ocp_write(tp, 0xF9F8 + 6 - i, (u16)ver); + ver >>= 16; + } + + /* Switch back to page 0 */ + rtl8127_switch_mac_mcu_ram_code_page(tp, 0); +} + +static void +rtl8127_set_hw_mcu_patch_code_ver(struct rtl8127_private *tp, u64 ver) +{ + _rtl8127_set_hw_mcu_patch_code_ver(tp, ver); + + tp->hw_mcu_patch_code_ver = ver; +} + +static u64 +rtl8127_get_hw_mcu_patch_code_ver(struct rtl8127_private *tp) +{ + u64 ver; + int i; + + /* Switch to page 2 */ + rtl8127_switch_mac_mcu_ram_code_page(tp, 2); + + ver = 0; + for (i = 0; i < 8; i += 2) { + ver <<= 16; + ver |= rtl8127_mac_ocp_read(tp, 0xF9F8 + i); + } + + /* Switch back to page 0 */ + rtl8127_switch_mac_mcu_ram_code_page(tp, 0); + + return ver; +} + +static u64 +rtl8127_get_bin_mcu_patch_code_ver(const u16 *entry, u16 entry_cnt) +{ + u64 ver; + int i; + + if (entry == NULL || entry_cnt == 0 || entry_cnt < 4) + return 0; + + ver = 0; + for (i = 0; i < 4; i++) { + ver <<= 16; + ver |= entry[entry_cnt - 4 + i]; + } + + return ver; +} + +static void +_rtl8127_write_mac_mcu_ram_code(struct rtl8127_private *tp, const u16 *entry, u16 entry_cnt) +{ + u16 i; + + for (i = 0; i < entry_cnt; i++) + rtl8127_mac_ocp_write(tp, 0xF800 + i * 2, entry[i]); +} + +static void +_rtl8127_write_mac_mcu_ram_code_with_page(struct rtl8127_private *tp, const u16 *entry, u16 entry_cnt, u16 page_size) +{ + u16 i; + u16 offset; + + if (page_size == 0) + return; + + for (i = 0; i < entry_cnt; i++) { + offset = i % page_size; + if (offset == 0) { + u16 page = (i / page_size); + rtl8127_switch_mac_mcu_ram_code_page(tp, page); + } + rtl8127_mac_ocp_write(tp, 0xF800 + offset * 2, entry[i]); + } +} + +static void +rtl8127_write_mac_mcu_ram_code(struct rtl8127_private *tp, const u16 *entry, u16 entry_cnt) +{ + if (FALSE == HW_SUPPORT_MAC_MCU(tp)) + return; + + if (entry == NULL || entry_cnt == 0) + return; + + if (tp->MacMcuPageSize > 0) + _rtl8127_write_mac_mcu_ram_code_with_page(tp, entry, entry_cnt, tp->MacMcuPageSize); + else + _rtl8127_write_mac_mcu_ram_code(tp, entry, entry_cnt); + + if (tp->bin_mcu_patch_code_ver > 0) + rtl8127_set_hw_mcu_patch_code_ver(tp, tp->bin_mcu_patch_code_ver); +} + +static void +rtl8127_set_mac_mcu_8127a_tc(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + static const u16 mcu_patch_code[] = { + 0xE010, 0xE019, 0xE01B, 0xE01D, 0xE029, 0xE02C, 0xE0E1, 0xE192, 0xE194, + 0xE196, 0xE198, 0xE19A, 0xE19C, 0xE19E, 0xE1A0, 0xE1A2, 0xC008, 0x7100, + 0x4897, 0x9900, 0xC005, 0xC602, 0xBE00, 0x3D8E, 0xD428, 0xD400, 0xC602, + 0xBE00, 0x0000, 0xC502, 0xBD00, 0x0000, 0x48C3, 0x4847, 0x48C1, 0x8CF8, + 0x74F8, 0x74F8, 0x74F8, 0x4842, 0x8CF8, 0x1E10, 0xC502, 0xBD00, 0x14BA, + 0x1E10, 0xC502, 0xBD00, 0x14EE, 0xC643, 0x76C0, 0x49E1, 0xF13F, 0xC140, + 0x7720, 0x49E0, 0xF003, 0x1B00, 0xE00A, 0x49E2, 0xF003, 0x1B04, 0xE006, + 0x49E4, 0xF003, 0x1B08, 0xE002, 0x1B0C, 0x21B8, 0x1A0E, 0x44DA, 0xE893, + 0x481C, 0xE884, 0xE001, 0x49E0, 0xF003, 0x1B00, 0xE00A, 0x49E2, 0xF003, + 0x1B04, 0xE006, 0x49E4, 0xF003, 0x1B08, 0xE002, 0x1B0C, 0x21B8, 0x1A12, + 0x44DA, 0xE87F, 0x481F, 0xE870, 0xE001, 0x49E0, 0xF003, 0x1B00, 0xE00A, + 0x49E2, 0xF003, 0x1B04, 0xE006, 0x49E4, 0xF003, 0x1B08, 0xE002, 0x1B0C, + 0x21B8, 0x1A1C, 0x44DA, 0xE86B, 0x481F, 0xE85C, 0xE004, 0xE04F, 0xDD98, + 0xD450, 0x49E0, 0xF003, 0x1B00, 0xE00A, 0x49E2, 0xF003, 0x1B04, 0xE006, + 0x49E4, 0xF003, 0x1B08, 0xE002, 0x1B0C, 0x21B8, 0x1A0E, 0x44DA, 0xE854, + 0x489E, 0x481F, 0xE844, 0xE001, 0x1908, 0xE83E, 0x49E0, 0xF003, 0x1B00, + 0xE00A, 0x49E2, 0xF003, 0x1B04, 0xE006, 0x49E4, 0xF003, 0x1B08, 0xE002, + 0x1B0C, 0x21B8, 0x1A8A, 0x44DA, 0xE83D, 0x4813, 0xE82E, 0x49F9, 0xF106, + 0x4838, 0xE837, 0x4813, 0xE828, 0xE001, 0x49E0, 0xF003, 0x1B00, 0xE00A, + 0x49E2, 0xF003, 0x1B04, 0xE006, 0x49E4, 0xF003, 0x1B08, 0xE002, 0x1B0C, + 0x21B8, 0x1A84, 0x44DA, 0xE823, 0x4890, 0x4811, 0xE813, 0x49F9, 0xF106, + 0x4838, 0xE81C, 0x4890, 0x4811, 0xE80C, 0xC207, 0x7440, 0xC602, 0xBE00, + 0x14CC, 0x0FFE, 0xDE20, 0xE092, 0xC3FD, 0xE802, 0xFF80, 0xC0FB, 0x7202, + 0x49AE, 0xF1FE, 0x9900, 0x44D3, 0x4413, 0x482F, 0x9A02, 0x7202, 0x49AE, + 0xF1FE, 0xFF80, 0xC0EE, 0x7202, 0x49AE, 0xF1FE, 0x44D3, 0x4413, 0x48AF, + 0x9A02, 0x7202, 0x49AE, 0xF1FE, 0x7100, 0xFF80, 0xB401, 0xB402, 0xB404, + 0xB407, 0xC61F, 0x76C0, 0x49E1, 0xF164, 0xC11C, 0x7720, 0x1906, 0xE88A, + 0x1B0C, 0x21B8, 0x1A40, 0x44DA, 0xE895, 0x4810, 0xE886, 0x190C, 0xE881, + 0x1B08, 0x21B8, 0x1A26, 0x44DA, 0xE88C, 0x4890, 0x4891, 0xE87C, 0x49F9, + 0xF107, 0x4898, 0x4899, 0xE877, 0xE003, 0xDD98, 0xD450, 0x1908, 0xE86F, + 0x49E0, 0xF003, 0x1B00, 0xE00A, 0x49E2, 0xF003, 0x1B04, 0xE006, 0x49E2, + 0xF003, 0x1B08, 0xE002, 0x1B0C, 0x21B8, 0x1A5C, 0x44DA, 0xE86E, 0x4897, + 0x4898, 0x4819, 0x481A, 0xE85C, 0x49F9, 0xF109, 0x4838, 0xE865, 0x4897, + 0x4898, 0x4819, 0x481A, 0xE853, 0xE001, 0x190A, 0xE84D, 0x1B00, 0xE85B, + 0x44E1, 0x4838, 0xE858, 0x44E9, 0x1908, 0xE845, 0x49E0, 0xF003, 0x1B00, + 0xE00A, 0x49E2, 0xF003, 0x1B04, 0xE006, 0x49E4, 0xF003, 0x1B08, 0xE002, + 0x1B0C, 0x21B8, 0x1A86, 0x44DA, 0xE844, 0x44CC, 0xE835, 0x49F9, 0xF108, + 0x4838, 0xE83E, 0x44CD, 0xE82F, 0xE003, 0xE021, 0xFFC0, 0x190A, 0xE827, + 0x1B00, 0x4839, 0xE834, 0x249A, 0x1C00, 0x44E1, 0x1909, 0xE81F, 0x49E0, + 0xF003, 0x1B00, 0xE00A, 0x49E2, 0xF003, 0x1B04, 0xE006, 0x49E4, 0xF003, + 0x1B08, 0xE002, 0x1B0C, 0x21B8, 0x1A1A, 0x44DA, 0xE81E, 0xC5E4, 0x414D, + 0x418C, 0xE80D, 0xB007, 0xB004, 0xB002, 0xB001, 0xC602, 0xBE00, 0x14B2, + 0x0FFE, 0xDE20, 0xC3FE, 0xE802, 0xFF80, 0xC0FC, 0x7202, 0x49AE, 0xF1FE, + 0x9900, 0x44D3, 0x4413, 0x482F, 0x9A02, 0x7202, 0x49AE, 0xF1FE, 0xFF80, + 0xC0EF, 0x7202, 0x49AE, 0xF1FE, 0x44D3, 0x4413, 0x48AF, 0x9A02, 0x7202, + 0x49AE, 0xF1FE, 0x7100, 0xFF80, 0xC502, 0xBD00, 0x0000, 0xC502, 0xBD00, + 0x0000, 0xC502, 0xBD00, 0x0000, 0xC302, 0xBB00, 0x0000, 0xC602, 0xBE00, + 0x0000, 0xC102, 0xB900, 0x0000, 0xC102, 0xB900, 0x0000, 0xC602, 0xBE00, + 0x0000, 0xC602, 0xBE00, 0x0000, 0x1332, 0x0018, 0x0C05, 0x140D + }; + + /* Get BIN mac mcu patch code version */ + tp->bin_mcu_patch_code_ver = rtl8127_get_bin_mcu_patch_code_ver(mcu_patch_code, ARRAY_SIZE(mcu_patch_code)); + + if (tp->hw_mcu_patch_code_ver != tp->bin_mcu_patch_code_ver) + rtl8127_write_mac_mcu_ram_code(tp, mcu_patch_code, ARRAY_SIZE(mcu_patch_code)); + + rtl8127_mac_ocp_write(tp, 0xFC26, 0x8000); + + rtl8127_mac_ocp_write(tp, 0xFC2E, 0x14B8); + rtl8127_mac_ocp_write(tp, 0xFC30, 0x14EC); + + rtl8127_mac_ocp_write(tp, 0xFC48, 0x0018); +} + +static void +_rtl8127_set_mac_mcu_8127a_1(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + static const u16 mcu_patch_code[] = { + 0xE010, 0xE014, 0xE018, 0xE01C, 0xE020, 0xE033, 0xE035, 0xE037, 0xE039, + 0xE03B, 0xE03D, 0xE03F, 0xE041, 0xE043, 0xE045, 0xE047, 0x7020, 0x4809, + 0xC502, 0xBD00, 0x1522, 0x7760, 0x4879, 0xC002, 0xB800, 0x41E2, 0x7160, + 0x4819, 0xC302, 0xBB00, 0x508E, 0x7720, 0x4879, 0xC102, 0xB900, 0x50F8, + 0x9F86, 0xB400, 0xB401, 0xB402, 0xB403, 0xC00D, 0x7100, 0xC20C, 0x7340, + 0x418B, 0x9900, 0xB003, 0xB002, 0xB001, 0xB000, 0xC702, 0xBF00, 0x3550, + 0xFC48, 0xD482, 0xC602, 0xBE00, 0x0000, 0xC602, 0xBE00, 0x0000, 0xC102, + 0xB900, 0x0000, 0xC302, 0xBB00, 0x0000, 0xC002, 0xB800, 0x0000, 0xC002, + 0xB800, 0x0000, 0xC502, 0xBD00, 0x0000, 0xC102, 0xB900, 0x0000, 0xC102, + 0xB900, 0x0000, 0xC602, 0xBE00, 0x0000, 0xC602, 0xBE00, 0x0000, 0x6961, + 0x0019, 0x0311, 0x1431 + }; + + /* Get BIN mac mcu patch code version */ + tp->bin_mcu_patch_code_ver = rtl8127_get_bin_mcu_patch_code_ver(mcu_patch_code, ARRAY_SIZE(mcu_patch_code)); + + if (tp->hw_mcu_patch_code_ver != tp->bin_mcu_patch_code_ver) + rtl8127_write_mac_mcu_ram_code(tp, mcu_patch_code, ARRAY_SIZE(mcu_patch_code)); + + rtl8127_mac_ocp_write(tp, 0xFC26, 0x8000); + + rtl8127_mac_ocp_write(tp, 0xFC28, 0x1520); + rtl8127_mac_ocp_write(tp, 0xFC2A, 0x41E0); + rtl8127_mac_ocp_write(tp, 0xFC2C, 0x508C); + rtl8127_mac_ocp_write(tp, 0xFC2E, 0x50F6); + rtl8127_mac_ocp_write(tp, 0xFC30, 0x354E); + + rtl8127_mac_ocp_write(tp, 0xFC48, 0x001F); +} + +static void +rtl8127_set_mac_mcu_8127a_1(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u8 tmp = (u8)rtl8127_mac_ocp_read(tp, 0xD006); + + if (tmp != 0x04) + return; + + _rtl8127_set_mac_mcu_8127a_1(dev); +} + +static void +rtl8127_hw_mac_mcu_config(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (tp->NotWrMcuPatchCode == TRUE) + return; + + rtl8127_hw_disable_mac_mcu_bps(dev); + + /* Get H/W mac mcu patch code version */ + tp->hw_mcu_patch_code_ver = rtl8127_get_hw_mcu_patch_code_ver(tp); + + switch (tp->mcfg) { + case CFG_METHOD_1: + rtl8127_set_mac_mcu_8127a_tc(dev); + break; + case CFG_METHOD_2: + rtl8127_set_mac_mcu_8127a_1(dev); + break; + default: + break; + } +} +#endif + +#ifdef ENABLE_USE_FIRMWARE_FILE +static void rtl8127_release_firmware(struct rtl8127_private *tp) +{ + if (tp->rtl_fw) { + rtl8127_fw_release_firmware(tp->rtl_fw); + kfree(tp->rtl_fw); + tp->rtl_fw = NULL; + } +} + +static void rtl8127_apply_firmware(struct rtl8127_private *tp) +{ + /* TODO: release firmware if rtl_fw_write_firmware signals failure. */ + if (tp->rtl_fw) { + rtl8127_fw_write_firmware(tp, tp->rtl_fw); + /* At least one firmware doesn't reset tp->ocp_base. */ + tp->ocp_base = OCP_STD_PHY_BASE; + + /* PHY soft reset may still be in progress */ + //phy_read_poll_timeout(tp->phydev, MII_BMCR, val, + // !(val & BMCR_RESET), + // 50000, 600000, true); + rtl8127_wait_phy_reset_complete(tp); + + tp->hw_ram_code_ver = rtl8127_get_hw_phy_mcu_code_ver(tp); + tp->sw_ram_code_ver = tp->hw_ram_code_ver; + tp->HwHasWrRamCodeToMicroP = TRUE; + } +} +#endif + +static void +rtl8127_hw_init(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u32 csi_tmp; + + rtl8127_enable_aspm_clkreq_lock(tp, 0); + rtl8127_enable_force_clkreq(tp, 0); + + //Disable UPS + rtl8127_mac_ocp_write(tp, 0xD40A, rtl8127_mac_ocp_read(tp, 0xD40A) & ~(BIT_4)); + +#ifndef ENABLE_USE_FIRMWARE_FILE + if (!tp->rtl_fw) + rtl8127_hw_mac_mcu_config(dev); +#endif + + //Set PCIE uncorrectable error status mask pcie 0x108 + csi_tmp = rtl8127_csi_read(tp, 0x108); + csi_tmp |= BIT_20; + rtl8127_csi_write(tp, 0x108, csi_tmp); + + rtl8127_enable_cfg9346_write(tp); + rtl8127_disable_linkchg_wakeup(dev); + rtl8127_disable_cfg9346_write(tp); + rtl8127_disable_magic_packet(dev); + rtl8127_disable_d0_speedup(tp); + rtl8127_set_pci_pme(tp, 0); + if (s0_magic_packet == 1) + rtl8127_enable_magic_packet(dev); + +#ifdef ENABLE_USE_FIRMWARE_FILE + if (tp->rtl_fw && + !tp->resume_not_chg_speed && + !(HW_DASH_SUPPORT_TYPE_3(tp) && + tp->HwPkgDet == 0x06)) + rtl8127_apply_firmware(tp); +#endif +} + +static void +rtl8127_clear_ephy_ext_addr(struct rtl8127_private *tp) +{ + rtl8127_set_ephy_ext_addr(tp, 0x0000); +} + +static void +rtl8127_hw_ephy_config_8127_1(struct rtl8127_private *tp) +{ + rtl8127_ephy_write(tp, 0x8088, 0x0064); + rtl8127_ephy_write(tp, 0x8488, 0x0064); + rtl8127_ephy_write(tp, 0x8888, 0x0064); + rtl8127_ephy_write(tp, 0x8C88, 0x0064); + rtl8127_ephy_write(tp, 0x8188, 0x0064); + rtl8127_ephy_write(tp, 0x8588, 0x0064); + rtl8127_ephy_write(tp, 0x8988, 0x0064); + rtl8127_ephy_write(tp, 0x8D88, 0x0064); + rtl8127_ephy_write(tp, 0x808C, 0x09B0); + rtl8127_ephy_write(tp, 0x848C, 0x09B0); + rtl8127_ephy_write(tp, 0x888C, 0x0F90); + rtl8127_ephy_write(tp, 0x8C8C, 0x0F90); + rtl8127_ephy_write(tp, 0x818C, 0x09B0); + rtl8127_ephy_write(tp, 0x858C, 0x09B0); + rtl8127_ephy_write(tp, 0x898C, 0x0F90); + rtl8127_ephy_write(tp, 0x8D8C, 0x0F90); + rtl8127_ephy_write(tp, 0x808A, 0x09B8); + rtl8127_ephy_write(tp, 0x848A, 0x09B8); + rtl8127_ephy_write(tp, 0x888A, 0x0F98); + rtl8127_ephy_write(tp, 0x8C8A, 0x0F98); + rtl8127_ephy_write(tp, 0x818A, 0x09B8); + rtl8127_ephy_write(tp, 0x858A, 0x09B8); + rtl8127_ephy_write(tp, 0x898A, 0x0F98); + rtl8127_ephy_write(tp, 0x8D8A, 0x0F98); + rtl8127_ephy_write(tp, 0x9020, 0x0080); + rtl8127_ephy_write(tp, 0x9420, 0x0080); + rtl8127_ephy_write(tp, 0x9820, 0x0080); + rtl8127_ephy_write(tp, 0x9C20, 0x0080); + rtl8127_ephy_write(tp, 0x901E, 0x0190); + rtl8127_ephy_write(tp, 0x941E, 0x0190); + rtl8127_ephy_write(tp, 0x981E, 0x0140); + rtl8127_ephy_write(tp, 0x9C1E, 0x0140); + rtl8127_ephy_write(tp, 0x901C, 0x0190); + rtl8127_ephy_write(tp, 0x941C, 0x0190); + rtl8127_ephy_write(tp, 0x981C, 0x0140); + rtl8127_ephy_write(tp, 0x9C1C, 0x0140); + + /* Clear extended address */ + rtl8127_clear_ephy_ext_addr(tp); +} + +static void +rtl8127_hw_ephy_config(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + switch (tp->mcfg) { + case CFG_METHOD_2: + rtl8127_hw_ephy_config_8127_1(tp); + break; + default: + /* nothing to do */ + break; + } +} + +static u16 +rtl8127_get_hw_phy_mcu_code_ver(struct rtl8127_private *tp) +{ + u16 hw_ram_code_ver; + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x801E); + hw_ram_code_ver = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA438); + + return hw_ram_code_ver; +} + +static int +rtl8127_check_hw_phy_mcu_code_ver(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int ram_code_ver_match = 0; + + tp->hw_ram_code_ver = rtl8127_get_hw_phy_mcu_code_ver(tp); + + if (tp->hw_ram_code_ver == tp->sw_ram_code_ver) { + ram_code_ver_match = 1; + tp->HwHasWrRamCodeToMicroP = TRUE; + } + + return ram_code_ver_match; +} + +bool +rtl8127_set_phy_mcu_patch_request(struct rtl8127_private *tp) +{ + u16 gphy_val; + u16 WaitCount; + bool bSuccess = TRUE; + + rtl8127_set_eth_phy_ocp_bit(tp, 0xB820, BIT_4); + + WaitCount = 0; + do { + gphy_val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xB800); + udelay(100); + WaitCount++; + } while (!(gphy_val & BIT_6) && (WaitCount < 1000)); + + if (!(gphy_val & BIT_6) && (WaitCount == 1000)) + bSuccess = FALSE; + + if (!bSuccess) + dprintk("rtl8127_set_phy_mcu_patch_request fail.\n"); + + return bSuccess; +} + +bool +rtl8127_clear_phy_mcu_patch_request(struct rtl8127_private *tp) +{ + u16 gphy_val; + u16 WaitCount; + bool bSuccess = TRUE; + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xB820, BIT_4); + + WaitCount = 0; + do { + gphy_val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xB800); + udelay(100); + WaitCount++; + } while ((gphy_val & BIT_6) && (WaitCount < 1000)); + + if ((gphy_val & BIT_6) && (WaitCount == 1000)) + bSuccess = FALSE; + + if (!bSuccess) + dprintk("rtl8127_clear_phy_mcu_patch_request fail.\n"); + + return bSuccess; +} + +#ifndef ENABLE_USE_FIRMWARE_FILE +static void +rtl8127_write_hw_phy_mcu_code_ver(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x801E); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, tp->sw_ram_code_ver); + tp->hw_ram_code_ver = tp->sw_ram_code_ver; +} + +static void +rtl8127_set_phy_mcu_ram_code(struct net_device *dev, const u16 *ramcode, u16 codesize) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u16 i; + u16 addr; + u16 val; + + if (ramcode == NULL || codesize % 2) + goto out; + + for (i = 0; i < codesize; i += 2) { + addr = ramcode[i]; + val = ramcode[i + 1]; + if (addr == 0xFFFF && val == 0xFFFF) + break; + rtl8127_mdio_direct_write_phy_ocp(tp, addr, val); + } + +out: + return; +} + +static void +rtl8127_enable_phy_disable_mode(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + switch (tp->HwSuppCheckPhyDisableModeVer) { + case 3: + RTL_W8(tp, 0xF2, RTL_R8(tp, 0xF2) | BIT_5); + break; + } + + dprintk("enable phy disable mode.\n"); +} + +static void +rtl8127_disable_phy_disable_mode(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + switch (tp->HwSuppCheckPhyDisableModeVer) { + case 3: + RTL_W8(tp, 0xF2, RTL_R8(tp, 0xF2) & ~BIT_5); + break; + } + + mdelay(1); + + dprintk("disable phy disable mode.\n"); +} + +static const u16 phy_mcu_ram_code_8127a_tc_1[] = { + 0xa436, 0x8023, 0xa438, 0x3200, 0xa436, 0xB82E, 0xa438, 0x0001, + 0xBF8E, 0x1410, 0xBF8E, 0x1410, 0xBF90, 0xC20E, 0xBF9E, 0xBFFC, + 0xBFAA, 0x1924, 0xBFB4, 0x0B1E, 0xBFB6, 0x3740, 0xBFB8, 0x460E, + 0xBFBE, 0x000D, 0xBF8A, 0x3FF7, 0xBF9A, 0x0007, 0xBF1E, 0x01FF, + 0xBF1E, 0x01FF, 0xBF1E, 0x01FF, 0xBF2E, 0x454D, 0xbc10, 0xD50C, + 0xbc10, 0x950C, 0xbc10, 0xD50C, 0xbc10, 0x550C, 0xbddE, 0xEF00, + 0xbd32, 0xF000, 0xbd2C, 0x0800, 0xbdc8, 0x04B0, 0xbdc8, 0x0CB0, + 0xBD92, 0x0003, 0xBD94, 0x8000, 0xBD96, 0x000f, 0xBD96, 0x0000, + 0xBD92, 0x0016, 0xBD94, 0x1000, 0xBD96, 0x000f, 0xBD96, 0x0000, + 0xBD92, 0x0016, 0xBD94, 0x1200, 0xBD96, 0x000f, 0xBD96, 0x0000, + 0xBD92, 0x0000, 0xBD94, 0x1F1F, 0xBD96, 0x000f, 0xBD96, 0x0000, + 0xBD92, 0x0001, 0xBD94, 0x1F00, 0xBD96, 0x000f, 0xBD96, 0x0000, + 0xBD92, 0x0003, 0xBD94, 0x0000, 0xBD96, 0x000f, 0xBD96, 0x0000, + 0xBD92, 0x0016, 0xBD94, 0x0000, 0xBD96, 0x000f, 0xBD96, 0x0000, + 0xBD92, 0x0000, 0xBD94, 0x0000, 0xBD96, 0x000f, 0xBD96, 0x0000, + 0xbdc8, 0x08B0, 0xbdc8, 0x00B0, 0xbd32, 0x0000, 0xbd2C, 0x0000, + 0xbc10, 0x750C, 0xbc10, 0x650C, 0xbc10, 0x750C, 0xbc10, 0x550C, + 0xb820, 0x0090, 0xa436, 0xA016, 0xa438, 0x0000, 0xa436, 0xA012, + 0xa438, 0x0000, 0xa436, 0xA014, 0xa438, 0x1800, 0xa438, 0x8010, + 0xa438, 0x1800, 0xa438, 0x809b, 0xa438, 0x1800, 0xa438, 0x8145, + 0xa438, 0x1800, 0xa438, 0x8197, 0xa438, 0x1800, 0xa438, 0x81d4, + 0xa438, 0x1800, 0xa438, 0x8214, 0xa438, 0x1800, 0xa438, 0x8226, + 0xa438, 0x1800, 0xa438, 0x8232, 0xa438, 0xd707, 0xa438, 0x4141, + 0xa438, 0xd70a, 0xa438, 0x4115, 0xa438, 0xd705, 0xa438, 0x40da, + 0xa438, 0xb808, 0xa438, 0xd028, 0xa438, 0xd1c1, 0xa438, 0x1800, + 0xa438, 0x801e, 0xa438, 0x9808, 0xa438, 0xd07b, 0xa438, 0xd1c5, + 0xa438, 0xbe10, 0xa438, 0xd503, 0xa438, 0xa108, 0xa438, 0xd505, + 0xa438, 0x8103, 0xa438, 0xd504, 0xa438, 0xa002, 0xa438, 0xa302, + 0xa438, 0xd707, 0xa438, 0x4061, 0xa438, 0xd503, 0xa438, 0x8b01, + 0xa438, 0xd500, 0xa438, 0xc48a, 0xa438, 0xd503, 0xa438, 0xcc09, + 0xa438, 0xcd58, 0xa438, 0xaf01, 0xa438, 0xd500, 0xa438, 0x1000, + 0xa438, 0x1764, 0xa438, 0xd719, 0xa438, 0x606c, 0xa438, 0xd704, + 0xa438, 0x645c, 0xa438, 0xd75e, 0xa438, 0x604d, 0xa438, 0xfff8, + 0xa438, 0x9e10, 0xa438, 0x1000, 0xa438, 0x1764, 0xa438, 0xd719, + 0xa438, 0x606c, 0xa438, 0xd704, 0xa438, 0x631c, 0xa438, 0xd75e, + 0xa438, 0x404d, 0xa438, 0xfff8, 0xa438, 0xd504, 0xa438, 0xaa18, + 0xa438, 0xa001, 0xa438, 0xa1e0, 0xa438, 0xd500, 0xa438, 0x1000, + 0xa438, 0x1764, 0xa438, 0xd719, 0xa438, 0x7fac, 0xa438, 0xd504, + 0xa438, 0xa001, 0xa438, 0xd500, 0xa438, 0x1000, 0xa438, 0x1764, + 0xa438, 0xd704, 0xa438, 0x5f5c, 0xa438, 0xd719, 0xa438, 0x3aaf, + 0xa438, 0x8058, 0xa438, 0xf016, 0xa438, 0xd707, 0xa438, 0x6121, + 0xa438, 0x1000, 0xa438, 0x16f8, 0xa438, 0xd503, 0xa438, 0xcd59, + 0xa438, 0xaf01, 0xa438, 0xd500, 0xa438, 0x1800, 0xa438, 0x0e49, + 0xa438, 0xd503, 0xa438, 0x8040, 0xa438, 0xd500, 0xa438, 0x1000, + 0xa438, 0x16f8, 0xa438, 0xd503, 0xa438, 0xcd5a, 0xa438, 0xaf01, + 0xa438, 0xd500, 0xa438, 0x1800, 0xa438, 0x0e2f, 0xa438, 0xd504, + 0xa438, 0xa008, 0xa438, 0xa204, 0xa438, 0xd500, 0xa438, 0x1000, + 0xa438, 0x1764, 0xa438, 0xd701, 0xa438, 0x5fa0, 0xa438, 0xd503, + 0xa438, 0xa082, 0xa438, 0xd500, 0xa438, 0xd71e, 0xa438, 0x4097, + 0xa438, 0xd078, 0xa438, 0xd1aa, 0xa438, 0xf003, 0xa438, 0xd078, + 0xa438, 0xd1aa, 0xa438, 0xd707, 0xa438, 0x4081, 0xa438, 0xd70a, + 0xa438, 0x4055, 0xa438, 0xf014, 0xa438, 0xd706, 0xa438, 0x6065, + 0xa438, 0xcc89, 0xa438, 0xf002, 0xa438, 0xcc8b, 0xa438, 0x1000, + 0xa438, 0x0bb2, 0xa438, 0xd705, 0xa438, 0x2ad0, 0xa438, 0x808f, + 0xa438, 0xf003, 0xa438, 0x1000, 0xa438, 0x0bb8, 0xa438, 0x1000, + 0xa438, 0x0bbe, 0xa438, 0x607a, 0xa438, 0x9c01, 0xa438, 0xf002, + 0xa438, 0xbc01, 0xa438, 0x1000, 0xa438, 0x0cc9, 0xa438, 0x1800, + 0xa438, 0x132f, 0xa438, 0x9a10, 0xa438, 0x9d02, 0xa438, 0xd706, + 0xa438, 0x629a, 0xa438, 0x61bb, 0xa438, 0xd707, 0xa438, 0x60d7, + 0xa438, 0xd70a, 0xa438, 0x6196, 0xa438, 0xd0e5, 0xa438, 0xd1e7, + 0xa438, 0xf012, 0xa438, 0xd70a, 0xa438, 0x5f55, 0xa438, 0xd060, + 0xa438, 0xd1e8, 0xa438, 0xf00d, 0xa438, 0xd056, 0xa438, 0xd1e8, + 0xa438, 0xf00a, 0xa438, 0xd043, 0xa438, 0xd1e8, 0xa438, 0xf007, + 0xa438, 0x609b, 0xa438, 0xd078, 0xa438, 0xd1e9, 0xa438, 0xf003, + 0xa438, 0xd07f, 0xa438, 0xd1e9, 0xa438, 0xd503, 0xa438, 0xab01, + 0xa438, 0xd500, 0xa438, 0xd706, 0xa438, 0x6139, 0xa438, 0xd503, + 0xa438, 0x6065, 0xa438, 0xa7f0, 0xa438, 0xf003, 0xa438, 0x0cf0, + 0xa438, 0x0750, 0xa438, 0x8908, 0xa438, 0xf004, 0xa438, 0xd503, + 0xa438, 0xa908, 0xa438, 0x87f0, 0xa438, 0xd503, 0xa438, 0xa040, + 0xa438, 0xd500, 0xa438, 0xd705, 0xa438, 0x407b, 0xa438, 0x1000, + 0xa438, 0x1ac8, 0xa438, 0xd503, 0xa438, 0x0c07, 0xa438, 0x0902, + 0xa438, 0xa008, 0xa438, 0xd500, 0xa438, 0x9d80, 0xa438, 0xc48c, + 0xa438, 0xd73e, 0xa438, 0x6000, 0xa438, 0xd706, 0xa438, 0x419b, + 0xa438, 0xd503, 0xa438, 0x0c87, 0xa438, 0x0981, 0xa438, 0xd500, + 0xa438, 0xd073, 0xa438, 0xd1b7, 0xa438, 0xc490, 0xa438, 0x1000, + 0xa438, 0x1764, 0xa438, 0xd704, 0xa438, 0x5fbb, 0xa438, 0xd503, + 0xa438, 0xd706, 0xa438, 0x607a, 0xa438, 0x617c, 0xa438, 0xf013, + 0xa438, 0xd70b, 0xa438, 0x40b4, 0xa438, 0xd706, 0xa438, 0x6309, + 0xa438, 0x6348, 0xa438, 0xf025, 0xa438, 0xd706, 0xa438, 0x6408, + 0xa438, 0xf022, 0xa438, 0xd70b, 0xa438, 0x40b4, 0xa438, 0xd706, + 0xa438, 0x6429, 0xa438, 0x6468, 0xa438, 0xf028, 0xa438, 0xd706, + 0xa438, 0x6468, 0xa438, 0xf025, 0xa438, 0xd70b, 0xa438, 0x67d4, + 0xa438, 0xd706, 0xa438, 0x6488, 0xa438, 0x6589, 0xa438, 0x2c69, + 0xa438, 0x8135, 0xa438, 0x66ab, 0xa438, 0xf037, 0xa438, 0xc320, + 0xa438, 0xc420, 0xa438, 0xf03c, 0xa438, 0xd707, 0xa438, 0x6077, + 0xa438, 0xc310, 0xa438, 0xf004, 0xa438, 0xd70a, 0xa438, 0x5fb5, + 0xa438, 0xc340, 0xa438, 0xc410, 0xa438, 0xf033, 0xa438, 0xc340, + 0xa438, 0xc420, 0xa438, 0xf030, 0xa438, 0xc340, 0xa438, 0xc440, + 0xa438, 0xf02d, 0xa438, 0xc304, 0xa438, 0xc408, 0xa438, 0xf02a, + 0xa438, 0xc302, 0xa438, 0xc404, 0xa438, 0xf027, 0xa438, 0xc308, + 0xa438, 0xc408, 0xa438, 0xf024, 0xa438, 0xc308, 0xa438, 0xc410, + 0xa438, 0xf021, 0xa438, 0xd707, 0xa438, 0x6077, 0xa438, 0xc304, + 0xa438, 0xf004, 0xa438, 0xd70a, 0xa438, 0x5fb5, 0xa438, 0xc308, + 0xa438, 0xc408, 0xa438, 0xf018, 0xa438, 0xd707, 0xa438, 0x6077, + 0xa438, 0xc308, 0xa438, 0xf004, 0xa438, 0xd70a, 0xa438, 0x5fb5, + 0xa438, 0xc310, 0xa438, 0xc410, 0xa438, 0xf00f, 0xa438, 0xc304, + 0xa438, 0xc408, 0xa438, 0xf00c, 0xa438, 0xc302, 0xa438, 0xc404, + 0xa438, 0xf009, 0xa438, 0xd707, 0xa438, 0x6077, 0xa438, 0xc308, + 0xa438, 0xf004, 0xa438, 0xd70a, 0xa438, 0x5fb5, 0xa438, 0xc310, + 0xa438, 0xc410, 0xa438, 0x1800, 0xa438, 0x14e7, 0xa438, 0xd07b, + 0xa438, 0xd1c5, 0xa438, 0xd503, 0xa438, 0xa028, 0xa438, 0x8970, + 0xa438, 0x880f, 0xa438, 0x0c0f, 0xa438, 0x0909, 0xa438, 0x87f0, + 0xa438, 0xc600, 0xa438, 0xa521, 0xa438, 0xd501, 0xa438, 0xce01, + 0xa438, 0xa202, 0xa438, 0xa201, 0xa438, 0x8201, 0xa438, 0xce00, + 0xa438, 0xd500, 0xa438, 0xd706, 0xa438, 0x4425, 0xa438, 0xd503, + 0xa438, 0xab80, 0xa438, 0xd500, 0xa438, 0x1000, 0xa438, 0x0c1d, + 0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0xaac0, 0xa438, 0xd00d, + 0xa438, 0xd1a2, 0xa438, 0xd700, 0xa438, 0x401a, 0xa438, 0xa60c, + 0xa438, 0xd010, 0xa438, 0xd1a2, 0xa438, 0xd700, 0xa438, 0x401a, + 0xa438, 0xa70c, 0xa438, 0xd09e, 0xa438, 0xd1a2, 0xa438, 0xd700, + 0xa438, 0x401a, 0xa438, 0xce00, 0xa438, 0xd500, 0xa438, 0xd505, + 0xa438, 0xab01, 0xa438, 0xd500, 0xa438, 0x1000, 0xa438, 0x0c30, + 0xa438, 0xd505, 0xa438, 0x8b01, 0xa438, 0xd500, 0xa438, 0xbc02, + 0xa438, 0xc482, 0xa438, 0xd503, 0xa438, 0xcc01, 0xa438, 0xcd0c, + 0xa438, 0xaf01, 0xa438, 0xd500, 0xa438, 0xd75e, 0xa438, 0x4000, + 0xa438, 0xd706, 0xa438, 0x4245, 0xa438, 0x1000, 0xa438, 0x0c1d, + 0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0x8ac0, 0xa438, 0x860c, + 0xa438, 0x870c, 0xa438, 0xce00, 0xa438, 0xd500, 0xa438, 0xd505, + 0xa438, 0xab01, 0xa438, 0xd500, 0xa438, 0x1000, 0xa438, 0x0c30, + 0xa438, 0xd505, 0xa438, 0x8b01, 0xa438, 0xd500, 0xa438, 0x1800, + 0xa438, 0x0e74, 0xa438, 0xd707, 0xa438, 0x43cf, 0xa438, 0x1000, + 0xa438, 0x0c1d, 0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0xaac0, + 0xa438, 0xd00d, 0xa438, 0xd1a2, 0xa438, 0xd700, 0xa438, 0x401a, + 0xa438, 0xa60c, 0xa438, 0xd010, 0xa438, 0xd1a2, 0xa438, 0xd700, + 0xa438, 0x401a, 0xa438, 0xa70c, 0xa438, 0xd09e, 0xa438, 0xd1a2, + 0xa438, 0xd700, 0xa438, 0x401a, 0xa438, 0xce00, 0xa438, 0xd500, + 0xa438, 0xd505, 0xa438, 0xab01, 0xa438, 0xd500, 0xa438, 0x1000, + 0xa438, 0x0c30, 0xa438, 0xd505, 0xa438, 0x8b01, 0xa438, 0xd500, + 0xa438, 0xd501, 0xa438, 0xce00, 0xa438, 0xab10, 0xa438, 0xbb10, + 0xa438, 0x9c02, 0xa438, 0xd75e, 0xa438, 0x6000, 0xa438, 0xd707, + 0xa438, 0x424f, 0xa438, 0x1000, 0xa438, 0x0c1d, 0xa438, 0xd501, + 0xa438, 0xce01, 0xa438, 0x8ac0, 0xa438, 0x860c, 0xa438, 0x870c, + 0xa438, 0xce00, 0xa438, 0xd500, 0xa438, 0xd505, 0xa438, 0xab01, + 0xa438, 0xd500, 0xa438, 0x1000, 0xa438, 0x0c30, 0xa438, 0xd505, + 0xa438, 0x8b01, 0xa438, 0xd500, 0xa438, 0x9808, 0xa438, 0x9c01, + 0xa438, 0x1800, 0xa438, 0x172a, 0xa438, 0xd707, 0xa438, 0x43cf, + 0xa438, 0x1000, 0xa438, 0x0c1d, 0xa438, 0xd501, 0xa438, 0xce01, + 0xa438, 0xaac0, 0xa438, 0xd00d, 0xa438, 0xd1a2, 0xa438, 0xd700, + 0xa438, 0x401a, 0xa438, 0xa60c, 0xa438, 0xd010, 0xa438, 0xd1a2, + 0xa438, 0xd700, 0xa438, 0x401a, 0xa438, 0xa70c, 0xa438, 0xd09e, + 0xa438, 0xd1a2, 0xa438, 0xd700, 0xa438, 0x401a, 0xa438, 0xce00, + 0xa438, 0xd500, 0xa438, 0xd505, 0xa438, 0xab01, 0xa438, 0xd500, + 0xa438, 0x1000, 0xa438, 0x0c30, 0xa438, 0xd505, 0xa438, 0x8b01, + 0xa438, 0xd500, 0xa438, 0x1000, 0xa438, 0x1a6b, 0xa438, 0x9808, + 0xa438, 0x9c01, 0xa438, 0xd707, 0xa438, 0x424f, 0xa438, 0x1000, + 0xa438, 0x0c1d, 0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0x8ac0, + 0xa438, 0x860c, 0xa438, 0x870c, 0xa438, 0xce00, 0xa438, 0xd500, + 0xa438, 0xd505, 0xa438, 0xab01, 0xa438, 0xd500, 0xa438, 0x1000, + 0xa438, 0x0c30, 0xa438, 0xd505, 0xa438, 0x8b01, 0xa438, 0xd500, + 0xa438, 0xd503, 0xa438, 0x8b80, 0xa438, 0xd500, 0xa438, 0x1000, + 0xa438, 0x1aa5, 0xa438, 0xd503, 0xa438, 0xcda3, 0xa438, 0xaf01, + 0xa438, 0x1800, 0xa438, 0x181c, 0xa438, 0xd700, 0xa438, 0x41f6, + 0xa438, 0xd703, 0xa438, 0x41a3, 0xa438, 0x1000, 0xa438, 0x0c1d, + 0xa438, 0xd501, 0xa438, 0xa580, 0xa438, 0xa701, 0xa438, 0xd500, + 0xa438, 0xd014, 0xa438, 0xd1c3, 0xa438, 0xd703, 0xa438, 0x401c, + 0xa438, 0x1000, 0xa438, 0x0c30, 0xa438, 0x1800, 0xa438, 0x069b, + 0xa438, 0xba10, 0xa438, 0xd70c, 0xa438, 0x4107, 0xa438, 0xd702, + 0xa438, 0x40d0, 0xa438, 0xd504, 0xa438, 0xa110, 0xa438, 0xd500, + 0xa438, 0x1800, 0xa438, 0x1447, 0xa438, 0x1800, 0xa438, 0x1420, + 0xa438, 0xd70c, 0xa438, 0x60a6, 0xa438, 0xd501, 0xa438, 0xce01, + 0xa438, 0x840f, 0xa438, 0xce00, 0xa438, 0xd503, 0xa438, 0x8008, + 0xa438, 0xd500, 0xa438, 0x1800, 0xa438, 0x1360, 0xa436, 0xA026, + 0xa438, 0x1359, 0xa436, 0xA024, 0xa438, 0x141f, 0xa436, 0xA022, + 0xa438, 0x068f, 0xa436, 0xA020, 0xa438, 0x1815, 0xa436, 0xA006, + 0xa438, 0x1723, 0xa436, 0xA004, 0xa438, 0x0e59, 0xa436, 0xA002, + 0xa438, 0x1452, 0xa436, 0xA000, 0xa438, 0x12bb, 0xa436, 0xA008, + 0xa438, 0xff00, 0xa436, 0xA016, 0xa438, 0x0010, 0xa436, 0xA012, + 0xa438, 0x0000, 0xa436, 0xA014, 0xa438, 0x1800, 0xa438, 0x8010, + 0xa438, 0x1800, 0xa438, 0x8010, 0xa438, 0x1800, 0xa438, 0x8019, + 0xa438, 0x1800, 0xa438, 0x8019, 0xa438, 0x1800, 0xa438, 0x8019, + 0xa438, 0x1800, 0xa438, 0x8019, 0xa438, 0x1800, 0xa438, 0x8019, + 0xa438, 0x1800, 0xa438, 0x8019, 0xa438, 0xd700, 0xa438, 0x2a59, + 0xa438, 0x0101, 0xa438, 0x2841, 0xa438, 0x0122, 0xa438, 0x2d69, + 0xa438, 0x00e6, 0xa438, 0x1800, 0xa438, 0x00e0, 0xa436, 0xA08E, + 0xa438, 0x0000, 0xa436, 0xA08C, 0xa438, 0x0000, 0xa436, 0xA08A, + 0xa438, 0x0000, 0xa436, 0xA088, 0xa438, 0x0000, 0xa436, 0xA086, + 0xa438, 0x0000, 0xa436, 0xA084, 0xa438, 0x0000, 0xa436, 0xA082, + 0xa438, 0x00e2, 0xa436, 0xA080, 0xa438, 0x0000, 0xa436, 0xA090, + 0xa438, 0x0002, 0xa436, 0xA016, 0xa438, 0x0020, 0xa436, 0xA012, + 0xa438, 0x0000, 0xa436, 0xA014, 0xa438, 0x1800, 0xa438, 0x8010, + 0xa438, 0x1800, 0xa438, 0x9dfb, 0xa438, 0x1800, 0xa438, 0x9e02, + 0xa438, 0x1800, 0xa438, 0x9e02, 0xa438, 0x1800, 0xa438, 0x9e02, + 0xa438, 0x1800, 0xa438, 0x9e02, 0xa438, 0x1800, 0xa438, 0x9e02, + 0xa438, 0x1800, 0xa438, 0x9e02, 0xa438, 0xd71f, 0xa438, 0x626d, + 0xa438, 0xd71e, 0xa438, 0x4103, 0xa438, 0xa70c, 0xa438, 0xb801, + 0xa438, 0xba04, 0xa438, 0xd71f, 0xa438, 0x6001, 0xa438, 0x1800, + 0xa438, 0x001c, 0xa438, 0xb801, 0xa438, 0xd704, 0xa438, 0x2211, + 0xa438, 0x9d54, 0xa438, 0xd700, 0xa438, 0x2739, 0xa438, 0x837a, + 0xa438, 0x1800, 0xa438, 0x814b, 0xa438, 0xcd70, 0xa438, 0xb801, + 0xa438, 0xa708, 0xa438, 0xd700, 0xa438, 0x60d7, 0xa438, 0x6073, + 0xa438, 0xce02, 0xa438, 0xf004, 0xa438, 0xce01, 0xa438, 0xf002, + 0xa438, 0xce01, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0ccf, 0xa438, 0x0b04, 0xa438, 0xcc22, + 0xa438, 0xcd01, 0xa438, 0xa702, 0xa438, 0x9503, 0xa438, 0xa501, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fbd, + 0xa438, 0x8501, 0xa438, 0x1000, 0xa438, 0x9c02, 0xa438, 0xa610, + 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xcd71, 0xa438, 0xa501, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fbd, + 0xa438, 0x8501, 0xa438, 0xd707, 0xa438, 0x61cf, 0xa438, 0xa502, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd702, 0xa438, 0x5fbe, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, + 0xa438, 0xd403, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x8502, + 0xa438, 0xa340, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108, + 0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000, + 0xa438, 0x9c5b, 0xa438, 0xa304, 0xa438, 0xa440, 0xa438, 0xa8c0, + 0xa438, 0xa2fc, 0xa438, 0xa120, 0xa438, 0x0ca0, 0xa438, 0x0480, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa370, 0xa438, 0x9503, + 0xa438, 0xcd72, 0xa438, 0xd1f5, 0xa438, 0xd057, 0xa438, 0xd1c4, + 0xa438, 0xd066, 0xa438, 0xd1c4, 0xa438, 0xd077, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x607c, 0xa438, 0x613d, + 0xa438, 0xfffb, 0xa438, 0xa310, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fbd, 0xa438, 0xa607, 0xa438, 0xf007, + 0xa438, 0xa607, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fbc, 0xa438, 0xa310, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fbb, 0xa438, 0x8840, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0cf8, 0xa438, 0x0d48, 0xa438, 0x8320, + 0xa438, 0xa180, 0xa438, 0x9503, 0xa438, 0xd1c4, 0xa438, 0xd055, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fbb, + 0xa438, 0xd706, 0xa438, 0x6227, 0xa438, 0xd700, 0xa438, 0x5f3a, + 0xa438, 0x88c0, 0xa438, 0x82fc, 0xa438, 0x8120, 0xa438, 0x8350, + 0xa438, 0x84a0, 0xa438, 0x8607, 0xa438, 0xa510, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8df8, 0xa438, 0x8370, 0xa438, 0x8180, + 0xa438, 0x9503, 0xa438, 0xff96, 0xa438, 0x8510, 0xa438, 0xa508, + 0xa438, 0x8508, 0xa438, 0xcd73, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x9920, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x6065, + 0xa438, 0x5f94, 0xa438, 0xffdf, 0xa438, 0xb820, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, + 0xa438, 0x8120, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108, + 0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000, + 0xa438, 0x9c5b, 0xa438, 0xa304, 0xa438, 0x8880, 0xa438, 0x8480, + 0xa438, 0x8606, 0xa438, 0xcd74, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8b0f, 0xa438, 0x8c3f, 0xa438, 0x9503, 0xa438, 0xa810, + 0xa438, 0xa120, 0xa438, 0xa310, 0xa438, 0xa4a0, 0xa438, 0xa606, + 0xa438, 0xd700, 0xa438, 0x37cd, 0xa438, 0x80e5, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8310, 0xa438, 0x9503, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x80f0, 0xa438, 0x33a9, 0xa438, 0x80ed, + 0xa438, 0xd17a, 0xa438, 0xd04c, 0xa438, 0xf006, 0xa438, 0xd199, + 0xa438, 0xd04c, 0xa438, 0xf003, 0xa438, 0xd1d6, 0xa438, 0xd04c, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, + 0xa438, 0xd706, 0xa438, 0x5f69, 0xa438, 0xd700, 0xa438, 0x60d7, + 0xa438, 0x6073, 0xa438, 0xce05, 0xa438, 0xf004, 0xa438, 0xce04, + 0xa438, 0xf002, 0xa438, 0xce04, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, 0xa438, 0x0b02, + 0xa438, 0x0c38, 0xa438, 0x0c10, 0xa438, 0x9503, 0xa438, 0xa180, + 0xa438, 0xa680, 0xa438, 0xcd75, 0xa438, 0xd199, 0xa438, 0xd04b, + 0xa438, 0xd13b, 0xa438, 0xd055, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fbb, 0xa438, 0xa302, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd706, + 0xa438, 0x5f6a, 0xa438, 0xbb50, 0xa438, 0xcd76, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb5, 0xa438, 0x9b10, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb5, + 0xa438, 0xa120, 0xa438, 0x0c12, 0xa438, 0x0310, 0xa438, 0x8480, + 0xa438, 0x0c84, 0xa438, 0x0604, 0xa438, 0xcd77, 0xa438, 0xd1a0, + 0xa438, 0xd04b, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x422a, 0xa438, 0xa1a0, + 0xa438, 0xa312, 0xa438, 0xa480, 0xa438, 0xa684, 0xa438, 0xcd78, + 0xa438, 0xd148, 0xa438, 0xd048, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xcd79, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x7f8a, 0xa438, 0x9b40, + 0xa438, 0xcd7f, 0xa438, 0xd71f, 0xa438, 0x7fe1, 0xa438, 0x1800, + 0xa438, 0x001c, 0xa438, 0xa708, 0xa438, 0xd700, 0xa438, 0x60cf, + 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, + 0xa438, 0xf00b, 0xa438, 0xce02, 0xa438, 0xf00a, 0xa438, 0xce02, + 0xa438, 0xf008, 0xa438, 0xce01, 0xa438, 0xf006, 0xa438, 0xce01, + 0xa438, 0xf004, 0xa438, 0xce01, 0xa438, 0xf002, 0xa438, 0xce01, + 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0ccf, 0xa438, 0x0b04, 0xa438, 0xcc22, 0xa438, 0xcd01, + 0xa438, 0xa702, 0xa438, 0x9503, 0xa438, 0xcd15, 0xa438, 0xa501, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fbd, + 0xa438, 0x8501, 0xa438, 0xba20, 0xa438, 0xd701, 0xa438, 0x4154, + 0xa438, 0xd115, 0xa438, 0xd04f, 0xa438, 0x1000, 0xa438, 0x9c02, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, + 0xa438, 0xf003, 0xa438, 0x1000, 0xa438, 0x9c02, 0xa438, 0xa610, + 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xcd16, 0xa438, 0xa501, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fbd, + 0xa438, 0x8501, 0xa438, 0xd707, 0xa438, 0x61cf, 0xa438, 0xa502, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd702, 0xa438, 0x5fbe, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, + 0xa438, 0xd403, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x8502, + 0xa438, 0xa340, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108, + 0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000, + 0xa438, 0x9c5b, 0xa438, 0xa304, 0xa438, 0xa440, 0xa438, 0xa8c0, + 0xa438, 0xd705, 0xa438, 0x40d3, 0xa438, 0xd707, 0xa438, 0x4082, + 0xa438, 0x0cfc, 0xa438, 0x02bc, 0xa438, 0xf002, 0xa438, 0xa2fc, + 0xa438, 0xa120, 0xa438, 0x0ca0, 0xa438, 0x0480, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa370, 0xa438, 0x9503, 0xa438, 0xcd17, + 0xa438, 0xd1f5, 0xa438, 0xd057, 0xa438, 0xd1c4, 0xa438, 0xd066, + 0xa438, 0xd1c4, 0xa438, 0xd077, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x607c, 0xa438, 0x613d, 0xa438, 0xfffb, + 0xa438, 0xa310, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fbd, 0xa438, 0xa607, 0xa438, 0xf007, 0xa438, 0xa607, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fbc, + 0xa438, 0xa310, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fbb, 0xa438, 0x8840, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0cf8, 0xa438, 0x0d48, 0xa438, 0x8320, 0xa438, 0xa180, + 0xa438, 0x9503, 0xa438, 0xd1c4, 0xa438, 0xd055, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fbb, 0xa438, 0xd706, + 0xa438, 0x6227, 0xa438, 0xd700, 0xa438, 0x5f3a, 0xa438, 0x88c0, + 0xa438, 0x82fc, 0xa438, 0x8120, 0xa438, 0x8350, 0xa438, 0x84a0, + 0xa438, 0x8607, 0xa438, 0xa510, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8df8, 0xa438, 0x8370, 0xa438, 0x8180, 0xa438, 0x9503, + 0xa438, 0xff8f, 0xa438, 0x8510, 0xa438, 0xa508, 0xa438, 0x8508, + 0xa438, 0xcd18, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x5fb4, 0xa438, 0xb920, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x9920, 0xa438, 0x9a20, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x6065, + 0xa438, 0x5f94, 0xa438, 0xffde, 0xa438, 0xb820, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, + 0xa438, 0x8120, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108, + 0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000, + 0xa438, 0x9c5b, 0xa438, 0xa304, 0xa438, 0x8880, 0xa438, 0x8480, + 0xa438, 0x8606, 0xa438, 0xcd19, 0xa438, 0xd705, 0xa438, 0x40d3, + 0xa438, 0xd707, 0xa438, 0x4082, 0xa438, 0x8310, 0xa438, 0x1000, + 0xa438, 0x9c0c, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8b0f, + 0xa438, 0x8c3f, 0xa438, 0x9503, 0xa438, 0xa810, 0xa438, 0xa120, + 0xa438, 0xd705, 0xa438, 0x40b3, 0xa438, 0xd707, 0xa438, 0x4062, + 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa4a0, + 0xa438, 0xa606, 0xa438, 0xd700, 0xa438, 0x37cd, 0xa438, 0x8236, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8310, 0xa438, 0x9503, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8241, 0xa438, 0x33a9, + 0xa438, 0x823e, 0xa438, 0xd17a, 0xa438, 0xd04c, 0xa438, 0xf006, + 0xa438, 0xd199, 0xa438, 0xd04c, 0xa438, 0xf003, 0xa438, 0xd1d6, + 0xa438, 0xd04c, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706, + 0xa438, 0x5f29, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce05, 0xa438, 0xf00a, 0xa438, 0xce05, 0xa438, 0xf008, + 0xa438, 0xce04, 0xa438, 0xf006, 0xa438, 0xce04, 0xa438, 0xf004, + 0xa438, 0xce04, 0xa438, 0xf002, 0xa438, 0xce04, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, + 0xa438, 0x0b02, 0xa438, 0x0c38, 0xa438, 0x0c10, 0xa438, 0x9503, + 0xa438, 0xa180, 0xa438, 0xa680, 0xa438, 0xcd1a, 0xa438, 0xd199, + 0xa438, 0xd04b, 0xa438, 0xd13b, 0xa438, 0xd055, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7b, 0xa438, 0xa302, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, + 0xa438, 0xd706, 0xa438, 0x5f2a, 0xa438, 0xbb10, 0xa438, 0xcd1b, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd71f, 0xa438, 0x5f75, 0xa438, 0xa704, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce03, 0xa438, 0xf00a, + 0xa438, 0xce03, 0xa438, 0xf008, 0xa438, 0xce03, 0xa438, 0xf006, + 0xa438, 0xce03, 0xa438, 0xf004, 0xa438, 0xce03, 0xa438, 0xf002, + 0xa438, 0xce03, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xa00a, + 0xa438, 0x81a0, 0xa438, 0x8312, 0xa438, 0x8480, 0xa438, 0xa686, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503, + 0xa438, 0x9b10, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x7fb5, 0xa438, 0xcd2a, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x5fa4, 0xa438, 0xd17b, 0xa438, 0xd04a, + 0xa438, 0xa980, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f68, 0xa438, 0x800a, + 0xa438, 0x8604, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8801, + 0xa438, 0x9503, 0xa438, 0xd40c, 0xa438, 0x1000, 0xa438, 0x9bbf, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, + 0xa438, 0xd417, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xa00a, + 0xa438, 0xa604, 0xa438, 0xd700, 0xa438, 0x39b9, 0xa438, 0x82e4, + 0xa438, 0xd707, 0xa438, 0x432f, 0xa438, 0xd700, 0xa438, 0x608f, + 0xa438, 0x60b1, 0xa438, 0x60d3, 0xa438, 0x60f5, 0xa438, 0xce08, + 0xa438, 0xf007, 0xa438, 0xce08, 0xa438, 0xf005, 0xa438, 0xce08, + 0xa438, 0xf003, 0xa438, 0xce08, 0xa438, 0xf001, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xa1a0, 0xa438, 0xa302, 0xa438, 0xa480, + 0xa438, 0xd707, 0xa438, 0x409f, 0xa438, 0x4062, 0xa438, 0x8310, + 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xd17b, 0xa438, 0xd049, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, + 0xa438, 0xd706, 0xa438, 0x5f68, 0xa438, 0xd700, 0xa438, 0x37cd, + 0xa438, 0x8302, 0xa438, 0x800a, 0xa438, 0x8604, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd417, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xa00a, 0xa438, 0xa604, + 0xa438, 0xd17b, 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f68, + 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, + 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, + 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, 0xa438, 0xce08, + 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, 0xa438, 0xce08, + 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0xa1a0, 0xa438, 0xd707, 0xa438, 0x40df, 0xa438, 0x40a2, + 0xa438, 0x0cfc, 0xa438, 0x02bc, 0xa438, 0x8310, 0xa438, 0xf004, + 0xa438, 0x0cfc, 0xa438, 0x02fc, 0xa438, 0xa310, 0xa438, 0xa302, + 0xa438, 0xa480, 0xa438, 0xa686, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa340, 0xa438, 0x9503, 0xa438, 0xcd2b, 0xa438, 0xd199, + 0xa438, 0xd04a, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f68, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d02, 0xa438, 0x9503, + 0xa438, 0xcd2c, 0xa438, 0xd199, 0xa438, 0xd04b, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd706, + 0xa438, 0x5f6a, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8312, + 0xa438, 0x8604, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, + 0xa438, 0x9503, 0xa438, 0xd417, 0xa438, 0x1000, 0xa438, 0x9bbf, + 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xd707, 0xa438, 0x409f, + 0xa438, 0x4062, 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, + 0xa438, 0xa302, 0xa438, 0xa604, 0xa438, 0xd409, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x5fb4, 0xa438, 0xb920, 0xa438, 0xcd2d, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x9920, + 0xa438, 0xbb10, 0xa438, 0xcd2e, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x5fb5, 0xa438, 0x800a, 0xa438, 0x81a0, + 0xa438, 0x8312, 0xa438, 0x8480, 0xa438, 0x0c86, 0xa438, 0x0680, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8340, 0xa438, 0xa140, + 0xa438, 0x9503, 0xa438, 0x9b10, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fb5, 0xa438, 0x1800, 0xa438, 0x8740, + 0xa438, 0xa70c, 0xa438, 0x8510, 0xa438, 0xd700, 0xa438, 0x60cf, + 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, + 0xa438, 0xf00b, 0xa438, 0xce03, 0xa438, 0xf00a, 0xa438, 0xce03, + 0xa438, 0xf008, 0xa438, 0xce03, 0xa438, 0xf006, 0xa438, 0xce03, + 0xa438, 0xf004, 0xa438, 0xce03, 0xa438, 0xf002, 0xa438, 0xce03, + 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0ccf, 0xa438, 0x0b04, 0xa438, 0xcc21, 0xa438, 0xcd10, + 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x406d, 0xa438, 0x0cc0, + 0xa438, 0x0080, 0xa438, 0xd198, 0xa438, 0xd07f, 0xa438, 0xcd11, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fa4, + 0xa438, 0xd102, 0xa438, 0xd040, 0xa438, 0xcd12, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0x1000, + 0xa438, 0x9c02, 0xa438, 0xa340, 0xa438, 0x1000, 0xa438, 0x9c52, + 0xa438, 0xa110, 0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8110, + 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xa304, 0xa438, 0xa224, + 0xa438, 0xa00a, 0xa438, 0xa802, 0xa438, 0xa980, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa370, 0xa438, 0x9503, 0xa438, 0xcd13, + 0xa438, 0xd700, 0xa438, 0x2469, 0xa438, 0x83db, 0xa438, 0x634b, + 0xa438, 0x39b9, 0xa438, 0x83c5, 0xa438, 0xf017, 0xa438, 0xd17a, + 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f64, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d02, 0xa438, 0x9503, + 0xa438, 0xcd14, 0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd706, + 0xa438, 0x5f65, 0xa438, 0x800a, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8340, 0xa438, 0xd700, 0xa438, 0x37cd, 0xa438, 0x83e3, + 0xa438, 0xa180, 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x2469, + 0xa438, 0x8409, 0xa438, 0xd700, 0xa438, 0x65eb, 0xa438, 0xd700, + 0xa438, 0x6609, 0xa438, 0xd705, 0xa438, 0x6051, 0xa438, 0xf009, + 0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xa00a, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd705, 0xa438, 0x7fb1, + 0xa438, 0xd700, 0xa438, 0x39b9, 0xa438, 0x83fa, 0xa438, 0xf01e, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa801, 0xa438, 0x9503, + 0xa438, 0xd40c, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd417, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xf00f, 0xa438, 0xd700, + 0xa438, 0x39b9, 0xa438, 0x840d, 0xa438, 0xf00e, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa801, 0xa438, 0x9503, 0xa438, 0xd40c, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd700, 0xa438, 0x6044, + 0xa438, 0xf004, 0xa438, 0xd407, 0xa438, 0x1000, 0xa438, 0x9bbf, + 0xa438, 0xa508, 0xa438, 0x8508, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8d07, 0xa438, 0x9503, 0xa438, 0xa00a, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503, 0xa438, 0xd704, + 0xa438, 0x6091, 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf003, + 0xa438, 0xd1c4, 0xa438, 0xd045, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x6a8b, 0xa438, 0xd700, 0xa438, 0x5f7a, + 0xa438, 0xd706, 0xa438, 0x5f24, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c07, 0xa438, 0x0d02, 0xa438, 0x9503, 0xa438, 0xd704, + 0xa438, 0x6091, 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf003, + 0xa438, 0xd1c4, 0xa438, 0xd045, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f65, + 0xa438, 0x800a, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8340, + 0xa438, 0xa180, 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x2d49, + 0xa438, 0x848d, 0xa438, 0x37c9, 0xa438, 0x8459, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd417, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xf023, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd417, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8d07, 0xa438, 0x9503, 0xa438, 0xa00a, 0xa438, 0xd17a, + 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f64, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d02, 0xa438, 0x9503, + 0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f65, + 0xa438, 0x800a, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, + 0xa438, 0x9503, 0xa438, 0xd417, 0xa438, 0x1000, 0xa438, 0x9bbf, + 0xa438, 0xf00b, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x5fab, 0xa438, 0xba08, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7f8b, 0xa438, 0x9a08, 0xa438, 0xa00a, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb4, + 0xa438, 0x800a, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8340, + 0xa438, 0xa180, 0xa438, 0x9503, 0xa438, 0xb920, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x9920, + 0xa438, 0xd700, 0xa438, 0x296d, 0xa438, 0x84bc, 0xa438, 0x1000, + 0xa438, 0x9c52, 0xa438, 0xa004, 0xa438, 0x1000, 0xa438, 0x9be6, + 0xa438, 0x8004, 0xa438, 0xa001, 0xa438, 0x1000, 0xa438, 0x9be6, + 0xa438, 0x8001, 0xa438, 0xa020, 0xa438, 0x1000, 0xa438, 0x9be6, + 0xa438, 0x8020, 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xd120, + 0xa438, 0xd040, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0x8704, 0xa438, 0xcd21, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fad, 0xa438, 0xa501, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fbd, + 0xa438, 0x8501, 0xa438, 0xba20, 0xa438, 0xd700, 0xa438, 0x2969, + 0xa438, 0x84d5, 0xa438, 0xd700, 0xa438, 0x612b, 0xa438, 0xd701, + 0xa438, 0x40f4, 0xa438, 0xd196, 0xa438, 0xd04d, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0x8802, + 0xa438, 0xcd22, 0xa438, 0xd17a, 0xa438, 0xd05a, 0xa438, 0xa501, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fbd, + 0xa438, 0x8501, 0xa438, 0xa502, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd702, 0xa438, 0x5fbe, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd403, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0x8502, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8801, 0xa438, 0x9503, 0xa438, 0xd40c, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0xd707, 0xa438, 0x428f, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce06, 0xa438, 0xf01d, + 0xa438, 0xce06, 0xa438, 0xf01b, 0xa438, 0xce06, 0xa438, 0xf019, + 0xa438, 0xce06, 0xa438, 0xf017, 0xa438, 0xce06, 0xa438, 0xf015, + 0xa438, 0xce06, 0xa438, 0xf013, 0xa438, 0xd700, 0xa438, 0x60cf, + 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, + 0xa438, 0xf00b, 0xa438, 0xce02, 0xa438, 0xf00a, 0xa438, 0xce02, + 0xa438, 0xf008, 0xa438, 0xce01, 0xa438, 0xf006, 0xa438, 0xce01, + 0xa438, 0xf004, 0xa438, 0xce01, 0xa438, 0xf002, 0xa438, 0xce01, + 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xa340, 0xa438, 0x1000, + 0xa438, 0x9c52, 0xa438, 0xa110, 0xa438, 0x1000, 0xa438, 0x9be6, + 0xa438, 0x8110, 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xa304, + 0xa438, 0xa440, 0xa438, 0xa8c0, 0xa438, 0xd707, 0xa438, 0x40cf, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8d07, 0xa438, 0x9503, + 0xa438, 0xa00a, 0xa438, 0xa120, 0xa438, 0xa310, 0xa438, 0x0ca0, + 0xa438, 0x0480, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x853b, + 0xa438, 0x60a4, 0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0xa308, + 0xa438, 0xf002, 0xa438, 0x8308, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c60, 0xa438, 0x0340, 0xa438, 0x9503, 0xa438, 0xcd23, + 0xa438, 0xd162, 0xa438, 0xd048, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0x8840, 0xa438, 0xd1c4, + 0xa438, 0xd045, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x6327, 0xa438, 0xd700, + 0xa438, 0x5f3b, 0xa438, 0x88c0, 0xa438, 0x800a, 0xa438, 0x8120, + 0xa438, 0x8358, 0xa438, 0x8308, 0xa438, 0x84a0, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8360, 0xa438, 0x9503, 0xa438, 0xd707, + 0xa438, 0x2f7d, 0xa438, 0x84d7, 0xa438, 0xd17a, 0xa438, 0xd05a, + 0xa438, 0xa501, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, + 0xa438, 0x5fbd, 0xa438, 0x8501, 0xa438, 0xff8b, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920, + 0xa438, 0xcd24, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x7fb4, 0xa438, 0x9920, 0xa438, 0x9a20, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x6065, 0xa438, 0x5f94, + 0xa438, 0xffd9, 0xa438, 0xb820, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0x800a, + 0xa438, 0x8120, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108, + 0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000, + 0xa438, 0x9c5b, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8598, + 0xa438, 0x6144, 0xa438, 0xd704, 0xa438, 0x60bd, 0xa438, 0xd707, + 0xa438, 0x417f, 0xa438, 0x4140, 0xa438, 0xf006, 0xa438, 0x82fc, + 0xa438, 0xa201, 0xa438, 0xf007, 0xa438, 0xa2fc, 0xa438, 0xf005, + 0xa438, 0x0cfc, 0xa438, 0x02bc, 0xa438, 0xf002, 0xa438, 0xa2fc, + 0xa438, 0xa304, 0xa438, 0x8880, 0xa438, 0x0cc0, 0xa438, 0x0440, + 0xa438, 0xcd25, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x85b7, + 0xa438, 0x6224, 0xa438, 0xd704, 0xa438, 0x605d, 0xa438, 0xf004, + 0xa438, 0xd704, 0xa438, 0x613b, 0xa438, 0xf00b, 0xa438, 0xd700, + 0xa438, 0x2969, 0xa438, 0x85b7, 0xa438, 0x60eb, 0xa438, 0xd707, + 0xa438, 0x40bf, 0xa438, 0x4080, 0xa438, 0x8310, 0xa438, 0x1000, + 0xa438, 0x9c0c, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, 0xa438, 0xf008, + 0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, 0xa438, 0xf004, + 0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8c07, + 0xa438, 0x0c07, 0xa438, 0x0d03, 0xa438, 0xd700, 0xa438, 0x2969, + 0xa438, 0x85d9, 0xa438, 0x60c4, 0xa438, 0xd704, 0xa438, 0x409d, + 0xa438, 0x8b0f, 0xa438, 0x8c38, 0xa438, 0xf003, 0xa438, 0xab07, + 0xa438, 0xac38, 0xa438, 0x0c07, 0xa438, 0x0d03, 0xa438, 0x9503, + 0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0xa120, 0xa438, 0xd700, + 0xa438, 0x2969, 0xa438, 0x85e7, 0xa438, 0x6064, 0xa438, 0xd704, + 0xa438, 0x607d, 0xa438, 0xa4a0, 0xa438, 0xa605, 0xa438, 0xd17a, + 0xa438, 0xd049, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x85f9, + 0xa438, 0x6164, 0xa438, 0xd704, 0xa438, 0x413d, 0xa438, 0xa00a, + 0xa438, 0xa120, 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, + 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa403, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x409a, 0xa438, 0xd706, 0xa438, 0x4046, 0xa438, 0xf014, + 0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff5, 0xa438, 0x1000, + 0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x8120, 0xa438, 0x8310, + 0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77, 0xa438, 0x8380, + 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdd, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8b0f, 0xa438, 0x8c38, 0xa438, 0x8d38, + 0xa438, 0x9503, 0xa438, 0xcd26, 0xa438, 0xd704, 0xa438, 0x6291, + 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x862e, 0xa438, 0x37c9, + 0xa438, 0x862b, 0xa438, 0x33a9, 0xa438, 0x8628, 0xa438, 0xd18a, + 0xa438, 0xd04b, 0xa438, 0xf00c, 0xa438, 0xd17a, 0xa438, 0xd04b, + 0xa438, 0xf009, 0xa438, 0xd1c6, 0xa438, 0xd04b, 0xa438, 0xf006, + 0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd1c4, + 0xa438, 0xd046, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8641, + 0xa438, 0x6164, 0xa438, 0xd704, 0xa438, 0x413d, 0xa438, 0xa00a, + 0xa438, 0xa120, 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, + 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa403, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x409a, 0xa438, 0xd706, 0xa438, 0x4048, 0xa438, 0xf014, + 0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff5, 0xa438, 0x1000, + 0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x8120, 0xa438, 0x8310, + 0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77, 0xa438, 0x8380, + 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdd, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d01, 0xa438, 0x9503, + 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, + 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, + 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, 0xa438, 0xce08, + 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, 0xa438, 0xce08, + 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0xa180, 0xa438, 0xcd27, 0xa438, 0xd704, 0xa438, 0x61f1, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8685, 0xa438, 0x33a9, + 0xa438, 0x8682, 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf009, + 0xa438, 0xd1b7, 0xa438, 0xd049, 0xa438, 0xf006, 0xa438, 0xd17a, + 0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd128, 0xa438, 0xd044, + 0xa438, 0xd13b, 0xa438, 0xd055, 0xa438, 0xd700, 0xa438, 0x2969, + 0xa438, 0x869a, 0xa438, 0x6164, 0xa438, 0xd704, 0xa438, 0x413d, + 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xd704, 0xa438, 0x407b, + 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa403, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd700, 0xa438, 0x405b, 0xa438, 0xf014, 0xa438, 0xd704, + 0xa438, 0x6055, 0xa438, 0xfff7, 0xa438, 0x1000, 0xa438, 0x9c52, + 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8310, 0xa438, 0xa380, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd705, 0xa438, 0x5f77, 0xa438, 0x8380, 0xa438, 0x1000, + 0xa438, 0x9c5b, 0xa438, 0xffdf, 0xa438, 0xa302, 0xa438, 0xd700, + 0xa438, 0x2969, 0xa438, 0x86c4, 0xa438, 0x6184, 0xa438, 0xd704, + 0xa438, 0x415d, 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xd704, + 0xa438, 0x407b, 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, + 0xa438, 0xa302, 0xa438, 0xa403, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x409a, + 0xa438, 0xd706, 0xa438, 0x4048, 0xa438, 0xf014, 0xa438, 0xd704, + 0xa438, 0x6055, 0xa438, 0xfff5, 0xa438, 0x1000, 0xa438, 0x9c52, + 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8312, 0xa438, 0xa380, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd705, 0xa438, 0x5f77, 0xa438, 0x8380, 0xa438, 0x1000, + 0xa438, 0x9c5b, 0xa438, 0xffdc, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c0f, 0xa438, 0x0b02, 0xa438, 0x0c38, 0xa438, 0x0c10, + 0xa438, 0x9503, 0xa438, 0xcd28, 0xa438, 0xd704, 0xa438, 0x61f1, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x86f5, 0xa438, 0x33a9, + 0xa438, 0x86f2, 0xa438, 0xd199, 0xa438, 0xd04a, 0xa438, 0xf009, + 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf006, 0xa438, 0xd199, + 0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd128, 0xa438, 0xd044, + 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8709, 0xa438, 0x6184, + 0xa438, 0xd704, 0xa438, 0x415d, 0xa438, 0xa00a, 0xa438, 0xa1a0, + 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, 0xa438, 0xf002, + 0xa438, 0xa310, 0xa438, 0xa302, 0xa438, 0xa403, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x409a, 0xa438, 0xd706, 0xa438, 0x404a, 0xa438, 0xf014, + 0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff5, 0xa438, 0x1000, + 0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8312, + 0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77, 0xa438, 0x8380, + 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdc, 0xa438, 0x8403, + 0xa438, 0xd409, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xbb10, + 0xa438, 0xcd29, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd71f, 0xa438, 0x5f75, 0xa438, 0x800a, + 0xa438, 0x81a0, 0xa438, 0x8312, 0xa438, 0x8480, 0xa438, 0x8604, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8340, 0xa438, 0xa140, + 0xa438, 0x9503, 0xa438, 0x9b10, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fb5, 0xa438, 0xcd30, 0xa438, 0xaa80, + 0xa438, 0xd704, 0xa438, 0x2319, 0xa438, 0x9d9a, 0xa438, 0xd700, + 0xa438, 0x40bd, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xaf01, + 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x5fb4, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8798, + 0xa438, 0xd700, 0xa438, 0x273d, 0xa438, 0x8794, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x876a, 0xa438, 0x33a9, 0xa438, 0x8762, + 0xa438, 0xd702, 0xa438, 0x6099, 0xa438, 0xd1b7, 0xa438, 0xd05c, + 0xa438, 0xf013, 0xa438, 0xd1b7, 0xa438, 0xd05c, 0xa438, 0xf010, + 0xa438, 0xd702, 0xa438, 0x6099, 0xa438, 0xd1b7, 0xa438, 0xd05c, + 0xa438, 0xf00b, 0xa438, 0xd1b7, 0xa438, 0xd05c, 0xa438, 0xf008, + 0xa438, 0xd702, 0xa438, 0x6099, 0xa438, 0xd199, 0xa438, 0xd05d, + 0xa438, 0xf003, 0xa438, 0xd199, 0xa438, 0xd05d, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x8786, 0xa438, 0x33a9, 0xa438, 0x877e, + 0xa438, 0xd702, 0xa438, 0x6099, 0xa438, 0xd1bf, 0xa438, 0xd06d, + 0xa438, 0xf013, 0xa438, 0xd1de, 0xa438, 0xd06d, 0xa438, 0xf010, + 0xa438, 0xd702, 0xa438, 0x6099, 0xa438, 0xd1bf, 0xa438, 0xd06d, + 0xa438, 0xf00b, 0xa438, 0xd1bf, 0xa438, 0xd06d, 0xa438, 0xf008, + 0xa438, 0xd702, 0xa438, 0x6099, 0xa438, 0xd199, 0xa438, 0xd06e, + 0xa438, 0xf003, 0xa438, 0xd199, 0xa438, 0xd06e, 0xa438, 0xd703, + 0xa438, 0x60d0, 0xa438, 0x1000, 0xa438, 0x9c20, 0xa438, 0xd41a, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd408, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0xcd31, 0xa438, 0xd700, 0xa438, 0x2fa9, + 0xa438, 0x879f, 0xa438, 0x33c9, 0xa438, 0x87a2, 0xa438, 0x6117, + 0xa438, 0xf00a, 0xa438, 0xd141, 0xa438, 0xd043, 0xa438, 0xf009, + 0xa438, 0xd121, 0xa438, 0xd043, 0xa438, 0xf006, 0xa438, 0xd122, + 0xa438, 0xd042, 0xa438, 0xf003, 0xa438, 0xd181, 0xa438, 0xd043, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, + 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8817, 0xa438, 0xd700, + 0xa438, 0x6cab, 0xa438, 0xd700, 0xa438, 0x6d47, 0xa438, 0x800a, + 0xa438, 0x81a0, 0xa438, 0x8312, 0xa438, 0x8480, 0xa438, 0x0c86, + 0xa438, 0x0680, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8702, + 0xa438, 0x9503, 0xa438, 0xd407, 0xa438, 0x1000, 0xa438, 0x9bbf, + 0xa438, 0xd703, 0xa438, 0x60d0, 0xa438, 0x1000, 0xa438, 0x9c20, + 0xa438, 0xd41a, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd406, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xcd39, 0xa438, 0xd404, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd700, 0xa438, 0x60cf, + 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, + 0xa438, 0xf00b, 0xa438, 0xce03, 0xa438, 0xf00a, 0xa438, 0xce03, + 0xa438, 0xf008, 0xa438, 0xce03, 0xa438, 0xf006, 0xa438, 0xce03, + 0xa438, 0xf004, 0xa438, 0xce03, 0xa438, 0xf002, 0xa438, 0xce03, + 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c07, 0xa438, 0x0c01, 0xa438, 0x8d07, 0xa438, 0x9503, + 0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0xa302, 0xa438, 0x0ca0, + 0xa438, 0x0480, 0xa438, 0xa684, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xb920, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x8810, + 0xa438, 0x9920, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x60e5, 0xa438, 0x5f94, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xfff0, 0xa438, 0xb820, + 0xa438, 0xa810, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd706, 0xa438, 0x5f69, + 0xa438, 0xf010, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xae80, + 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x2b6d, 0xa438, 0x88c0, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, + 0xa438, 0xd406, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xcd32, + 0xa438, 0xd701, 0xa438, 0x6191, 0xa438, 0xa504, 0xa438, 0xcd3a, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x6067, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd701, 0xa438, 0x5f3a, + 0xa438, 0x8504, 0xa438, 0xd700, 0xa438, 0x2739, 0xa438, 0x88b6, + 0xa438, 0xd707, 0xa438, 0x6061, 0xa438, 0x1800, 0xa438, 0x88a8, + 0xa438, 0xd193, 0xa438, 0xd047, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, + 0xa438, 0xd706, 0xa438, 0x5f29, 0xa438, 0xd700, 0xa438, 0x60cf, + 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, + 0xa438, 0xf00b, 0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, + 0xa438, 0xf008, 0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, + 0xa438, 0xf004, 0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, + 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c0f, 0xa438, 0x0b01, 0xa438, 0x0c3f, 0xa438, 0x0c08, + 0xa438, 0x9503, 0xa438, 0xd707, 0xa438, 0x409f, 0xa438, 0x4062, + 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa120, + 0xa438, 0xa420, 0xa438, 0xd193, 0xa438, 0xd048, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f29, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce03, 0xa438, 0xf00a, + 0xa438, 0xce03, 0xa438, 0xf008, 0xa438, 0xce03, 0xa438, 0xf006, + 0xa438, 0xce03, 0xa438, 0xf004, 0xa438, 0xce03, 0xa438, 0xf002, + 0xa438, 0xce03, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0c01, 0xa438, 0x9503, + 0xa438, 0x8420, 0xa438, 0x800a, 0xa438, 0x8120, 0xa438, 0x8312, + 0xa438, 0x8480, 0xa438, 0x8604, 0xa438, 0xd419, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0xd702, 0xa438, 0x4080, 0xa438, 0xbb20, + 0xa438, 0x1800, 0xa438, 0x8c32, 0xa438, 0xa00a, 0xa438, 0xa302, + 0xa438, 0xa480, 0xa438, 0xa604, 0xa438, 0xd193, 0xa438, 0xd047, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f29, + 0xa438, 0x1800, 0xa438, 0x888b, 0xa438, 0x800a, 0xa438, 0x8302, + 0xa438, 0x8480, 0xa438, 0x8604, 0xa438, 0xd405, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0xbb20, 0xa438, 0xa00a, 0xa438, 0xa302, + 0xa438, 0xa480, 0xa438, 0xa604, 0xa438, 0x1800, 0xa438, 0x8c32, + 0xa438, 0xd405, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd404, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd700, 0xa438, 0x2b69, + 0xa438, 0x88c0, 0xa438, 0xf06d, 0xa438, 0xd700, 0xa438, 0x60cf, + 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, + 0xa438, 0xf00b, 0xa438, 0xce03, 0xa438, 0xf00a, 0xa438, 0xce03, + 0xa438, 0xf008, 0xa438, 0xce03, 0xa438, 0xf006, 0xa438, 0xce03, + 0xa438, 0xf004, 0xa438, 0xce03, 0xa438, 0xf002, 0xa438, 0xce03, + 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c07, 0xa438, 0x0c01, 0xa438, 0x0c3f, 0xa438, 0x0d08, + 0xa438, 0x9503, 0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0xa302, + 0xa438, 0x0ca0, 0xa438, 0x0480, 0xa438, 0xd700, 0xa438, 0x2969, + 0xa438, 0x88e8, 0xa438, 0x60a4, 0xa438, 0xd704, 0xa438, 0x407d, + 0xa438, 0x8604, 0xa438, 0xf002, 0xa438, 0xa604, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0xd700, 0xa438, 0x37cd, + 0xa438, 0x88f0, 0xa438, 0x8310, 0xa438, 0x9503, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xb920, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, + 0xa438, 0x8810, 0xa438, 0x9920, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x60e5, 0xa438, 0x5f94, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xfff0, + 0xa438, 0xb820, 0xa438, 0xa810, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0xd700, + 0xa438, 0x2d59, 0xa438, 0x8c32, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8c07, 0xa438, 0x9503, 0xa438, 0xa120, 0xa438, 0xd700, + 0xa438, 0x2969, 0xa438, 0x891f, 0xa438, 0x60e4, 0xa438, 0xd704, + 0xa438, 0x40bd, 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, + 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xd700, 0xa438, 0x2969, + 0xa438, 0x8928, 0xa438, 0x60a4, 0xa438, 0xd704, 0xa438, 0x407d, + 0xa438, 0xa202, 0xa438, 0xf002, 0xa438, 0x8202, 0xa438, 0xa420, + 0xa438, 0x1800, 0xa438, 0x8d48, 0xa438, 0x8810, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fbb, 0xa438, 0xd17a, + 0xa438, 0xd05a, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, 0xa438, 0xf008, + 0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, 0xa438, 0xf004, + 0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, + 0xa438, 0x0b04, 0xa438, 0x0c3f, 0xa438, 0x0c21, 0xa438, 0x8d07, + 0xa438, 0x9503, 0xa438, 0xa340, 0xa438, 0x1000, 0xa438, 0x9c52, + 0xa438, 0xa110, 0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8110, + 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xa304, 0xa438, 0xa440, + 0xa438, 0xa8c0, 0xa438, 0x8810, 0xa438, 0xa00a, 0xa438, 0xa120, + 0xa438, 0xa310, 0xa438, 0xd704, 0xa438, 0x405d, 0xa438, 0xa308, + 0xa438, 0x0cfc, 0xa438, 0x0224, 0xa438, 0x0ca0, 0xa438, 0x0480, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0xd700, + 0xa438, 0x37cd, 0xa438, 0x896d, 0xa438, 0x8310, 0xa438, 0x9503, + 0xa438, 0xd162, 0xa438, 0xd048, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0x8840, 0xa438, 0xd1c4, + 0xa438, 0xd045, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x6127, 0xa438, 0xd700, + 0xa438, 0x5f3b, 0xa438, 0x88c0, 0xa438, 0x800a, 0xa438, 0x8120, + 0xa438, 0x8350, 0xa438, 0x84a0, 0xa438, 0xffad, 0xa438, 0xb920, + 0xa438, 0xcd33, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x7fb4, 0xa438, 0x9920, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x6065, 0xa438, 0x5f94, 0xa438, 0xffee, + 0xa438, 0xb820, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0x800a, 0xa438, 0x8120, + 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108, 0xa438, 0x1000, + 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000, 0xa438, 0x9c5b, + 0xa438, 0xd704, 0xa438, 0x60bd, 0xa438, 0xd707, 0xa438, 0x413f, + 0xa438, 0x4100, 0xa438, 0xf004, 0xa438, 0x0cfd, 0xa438, 0x0201, + 0xa438, 0xf005, 0xa438, 0x0cfc, 0xa438, 0x02bc, 0xa438, 0xf002, + 0xa438, 0xa2fc, 0xa438, 0xa304, 0xa438, 0x8880, 0xa438, 0x0cc0, + 0xa438, 0x0440, 0xa438, 0xcd34, 0xa438, 0xd704, 0xa438, 0x407d, + 0xa438, 0x405b, 0xa438, 0xf006, 0xa438, 0xd704, 0xa438, 0x60fd, + 0xa438, 0xd707, 0xa438, 0x40bf, 0xa438, 0x4080, 0xa438, 0x8310, + 0xa438, 0x1000, 0xa438, 0x9c0c, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8c07, 0xa438, 0xd704, 0xa438, 0x409d, 0xa438, 0x8b0f, + 0xa438, 0x8c38, 0xa438, 0xf004, 0xa438, 0x0c0f, 0xa438, 0x0b07, + 0xa438, 0xac38, 0xa438, 0x0c38, 0xa438, 0x0d10, 0xa438, 0x9503, + 0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0xa120, 0xa438, 0xd704, + 0xa438, 0x607d, 0xa438, 0xa4a0, 0xa438, 0xa604, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x89db, 0xa438, 0xd17a, 0xa438, 0xd049, + 0xa438, 0xf003, 0xa438, 0xd19f, 0xa438, 0xd049, 0xa438, 0xd704, + 0xa438, 0x413d, 0xa438, 0xa00a, 0xa438, 0xa120, 0xa438, 0xd704, + 0xa438, 0x407b, 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, + 0xa438, 0xa403, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x6a14, 0xa438, 0xd700, 0xa438, 0x409a, 0xa438, 0xd706, + 0xa438, 0x4988, 0xa438, 0xf014, 0xa438, 0xd704, 0xa438, 0x6055, + 0xa438, 0xfff5, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0x800a, + 0xa438, 0x8120, 0xa438, 0x8310, 0xa438, 0xa380, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd705, + 0xa438, 0x5f77, 0xa438, 0x8380, 0xa438, 0x1000, 0xa438, 0x9c5b, + 0xa438, 0xffdd, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8b0f, + 0xa438, 0x8c38, 0xa438, 0x8d38, 0xa438, 0x9503, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x8a14, 0xa438, 0x33a9, 0xa438, 0x8a11, + 0xa438, 0xd1f4, 0xa438, 0xd04b, 0xa438, 0xf006, 0xa438, 0xd1b7, + 0xa438, 0xd04b, 0xa438, 0xf003, 0xa438, 0xd1c6, 0xa438, 0xd04c, + 0xa438, 0xd704, 0xa438, 0x413d, 0xa438, 0xa00a, 0xa438, 0xa120, + 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, 0xa438, 0xf002, + 0xa438, 0xa310, 0xa438, 0xa403, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x62f4, 0xa438, 0xd700, 0xa438, 0x405a, + 0xa438, 0xf01a, 0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff7, + 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x8120, + 0xa438, 0x8310, 0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77, + 0xa438, 0x8380, 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdf, + 0xa438, 0x8403, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x629c, 0xa438, 0xfffb, 0xa438, 0x8403, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x6096, 0xa438, 0xd700, + 0xa438, 0x619c, 0xa438, 0xfffa, 0xa438, 0xd706, 0xa438, 0x4128, + 0xa438, 0xd702, 0xa438, 0x60b0, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xae80, 0xa438, 0x9503, 0xa438, 0x1800, 0xa438, 0x8b62, + 0xa438, 0xd17a, 0xa438, 0xd05a, 0xa438, 0xd700, 0xa438, 0x60cf, + 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, + 0xa438, 0xf00b, 0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, + 0xa438, 0xf008, 0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, + 0xa438, 0xf004, 0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, + 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c0f, 0xa438, 0x0b04, 0xa438, 0x0c3f, 0xa438, 0x0c21, + 0xa438, 0x8d07, 0xa438, 0x9503, 0xa438, 0xa340, 0xa438, 0x1000, + 0xa438, 0x9c52, 0xa438, 0xa110, 0xa438, 0x1000, 0xa438, 0x9be6, + 0xa438, 0x8110, 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xa304, + 0xa438, 0xa440, 0xa438, 0xa8c0, 0xa438, 0x8810, 0xa438, 0xa00a, + 0xa438, 0xa120, 0xa438, 0xa310, 0xa438, 0xd704, 0xa438, 0x405d, + 0xa438, 0xa308, 0xa438, 0x0cfc, 0xa438, 0x0224, 0xa438, 0x0ca0, + 0xa438, 0x0480, 0xa438, 0x8604, 0xa438, 0xcd35, 0xa438, 0xd162, + 0xa438, 0xd048, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0x8840, 0xa438, 0xd1c4, 0xa438, 0xd045, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, + 0xa438, 0xd706, 0xa438, 0x6127, 0xa438, 0xd700, 0xa438, 0x5f3b, + 0xa438, 0x88c0, 0xa438, 0x800a, 0xa438, 0x8120, 0xa438, 0x8350, + 0xa438, 0x84a0, 0xa438, 0xffb3, 0xa438, 0xbb80, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920, + 0xa438, 0xcd36, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x7fb4, 0xa438, 0x9920, 0xa438, 0x9b80, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x6065, 0xa438, 0x5f94, + 0xa438, 0xffe8, 0xa438, 0xb820, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0x800a, + 0xa438, 0x8120, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108, + 0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000, + 0xa438, 0x9c5b, 0xa438, 0xd704, 0xa438, 0x60bd, 0xa438, 0xd707, + 0xa438, 0x413f, 0xa438, 0x4100, 0xa438, 0xf004, 0xa438, 0x0cfd, + 0xa438, 0x0201, 0xa438, 0xf005, 0xa438, 0x0cfc, 0xa438, 0x02bc, + 0xa438, 0xf002, 0xa438, 0xa2fc, 0xa438, 0xa304, 0xa438, 0x8880, + 0xa438, 0x0cc0, 0xa438, 0x0440, 0xa438, 0xcd37, 0xa438, 0xd704, + 0xa438, 0x407d, 0xa438, 0x405b, 0xa438, 0xf006, 0xa438, 0xd704, + 0xa438, 0x60fd, 0xa438, 0xd707, 0xa438, 0x40bf, 0xa438, 0x4080, + 0xa438, 0x8310, 0xa438, 0x1000, 0xa438, 0x9c0c, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8c07, 0xa438, 0xd704, 0xa438, 0x409d, + 0xa438, 0x8b0f, 0xa438, 0x8c38, 0xa438, 0xf004, 0xa438, 0x0c0f, + 0xa438, 0x0b07, 0xa438, 0xac38, 0xa438, 0x0c38, 0xa438, 0x0d10, + 0xa438, 0x9503, 0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0xa120, + 0xa438, 0xd704, 0xa438, 0x607d, 0xa438, 0xa4a0, 0xa438, 0xa604, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8b01, 0xa438, 0x33a9, + 0xa438, 0x8afe, 0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0xf006, + 0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0xf003, 0xa438, 0xd17a, + 0xa438, 0xd048, 0xa438, 0xd704, 0xa438, 0x413d, 0xa438, 0xa00a, + 0xa438, 0xa120, 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, + 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa403, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x409a, 0xa438, 0xd706, 0xa438, 0x4048, 0xa438, 0xf014, + 0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff5, 0xa438, 0x1000, + 0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x8120, 0xa438, 0x8310, + 0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77, 0xa438, 0x8380, + 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdd, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8b0f, 0xa438, 0x8c38, 0xa438, 0x8d38, + 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8b3a, + 0xa438, 0x33a9, 0xa438, 0x8b37, 0xa438, 0xd199, 0xa438, 0xd04b, + 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf003, + 0xa438, 0xd189, 0xa438, 0xd04c, 0xa438, 0xd704, 0xa438, 0x413d, + 0xa438, 0xa00a, 0xa438, 0xa120, 0xa438, 0xd704, 0xa438, 0x407b, + 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa403, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd700, 0xa438, 0x409a, 0xa438, 0xd706, 0xa438, 0x4048, + 0xa438, 0xf014, 0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff5, + 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x8120, + 0xa438, 0x8310, 0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77, + 0xa438, 0x8380, 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdd, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d01, + 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce08, 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, + 0xa438, 0xce08, 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, + 0xa438, 0xce08, 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xa180, 0xa438, 0xcd38, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x8b92, 0xa438, 0x33a9, 0xa438, 0x8b8a, + 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd17a, 0xa438, 0xd04a, + 0xa438, 0xf013, 0xa438, 0xd100, 0xa438, 0xd049, 0xa438, 0xf010, + 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd1b7, 0xa438, 0xd049, + 0xa438, 0xf00b, 0xa438, 0xd100, 0xa438, 0xd048, 0xa438, 0xf008, + 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd17a, 0xa438, 0xd04b, + 0xa438, 0xf003, 0xa438, 0xd199, 0xa438, 0xd04a, 0xa438, 0xd13b, + 0xa438, 0xd055, 0xa438, 0xd704, 0xa438, 0x413d, 0xa438, 0xa00a, + 0xa438, 0xa1a0, 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, + 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa403, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x405b, 0xa438, 0xf014, 0xa438, 0xd704, 0xa438, 0x6055, + 0xa438, 0xfff7, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0x800a, + 0xa438, 0x81a0, 0xa438, 0x8310, 0xa438, 0xa380, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd705, + 0xa438, 0x5f77, 0xa438, 0x8380, 0xa438, 0x1000, 0xa438, 0x9c5b, + 0xa438, 0xffdf, 0xa438, 0xa302, 0xa438, 0xd704, 0xa438, 0x415d, + 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xd704, 0xa438, 0x407b, + 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa302, + 0xa438, 0xa403, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x409a, 0xa438, 0xd706, + 0xa438, 0x4048, 0xa438, 0xf014, 0xa438, 0xd704, 0xa438, 0x6055, + 0xa438, 0xfff5, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0x800a, + 0xa438, 0x81a0, 0xa438, 0x8312, 0xa438, 0xa380, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd705, + 0xa438, 0x5f77, 0xa438, 0x8380, 0xa438, 0x1000, 0xa438, 0x9c5b, + 0xa438, 0xffdc, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, + 0xa438, 0x0b02, 0xa438, 0x0c38, 0xa438, 0x0c10, 0xa438, 0x9503, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8c03, 0xa438, 0x33a9, + 0xa438, 0x8bfb, 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd199, + 0xa438, 0xd04a, 0xa438, 0xf013, 0xa438, 0xd100, 0xa438, 0xd04a, + 0xa438, 0xf010, 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd17a, + 0xa438, 0xd04a, 0xa438, 0xf00b, 0xa438, 0xd100, 0xa438, 0xd048, + 0xa438, 0xf008, 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd199, + 0xa438, 0xd04b, 0xa438, 0xf003, 0xa438, 0xd16b, 0xa438, 0xd04b, + 0xa438, 0xd704, 0xa438, 0x415d, 0xa438, 0xa00a, 0xa438, 0xa1a0, + 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, 0xa438, 0xf002, + 0xa438, 0xa310, 0xa438, 0xa302, 0xa438, 0xa403, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x40ba, 0xa438, 0xd706, 0xa438, 0x406a, 0xa438, 0x8403, + 0xa438, 0xf014, 0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff4, + 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x81a0, + 0xa438, 0x8312, 0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77, + 0xa438, 0x8380, 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdb, + 0xa438, 0x81a0, 0xa438, 0x8310, 0xa438, 0xa302, 0xa438, 0xa00a, + 0xa438, 0xa480, 0xa438, 0x8420, 0xa438, 0xd700, 0xa438, 0x2969, + 0xa438, 0x8c42, 0xa438, 0x60e4, 0xa438, 0xd700, 0xa438, 0x40a7, + 0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0x8604, 0xa438, 0xf002, + 0xa438, 0xa604, 0xa438, 0xd700, 0xa438, 0x60c7, 0xa438, 0xa602, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa702, 0xa438, 0x9503, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503, + 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, + 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce03, + 0xa438, 0xf00a, 0xa438, 0xce03, 0xa438, 0xf008, 0xa438, 0xce03, + 0xa438, 0xf006, 0xa438, 0xce03, 0xa438, 0xf004, 0xa438, 0xce03, + 0xa438, 0xf002, 0xa438, 0xce03, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0c01, + 0xa438, 0x0c3f, 0xa438, 0x0d08, 0xa438, 0x9503, 0xa438, 0xd700, + 0xa438, 0x646d, 0xa438, 0x37c9, 0xa438, 0x8c83, 0xa438, 0x33a9, + 0xa438, 0x8c79, 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, + 0xa438, 0x6098, 0xa438, 0xd100, 0xa438, 0xd048, 0xa438, 0xf01a, + 0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0xf017, 0xa438, 0xd700, + 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd100, + 0xa438, 0xd049, 0xa438, 0xf010, 0xa438, 0xd17a, 0xa438, 0xd04c, + 0xa438, 0xf00d, 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, + 0xa438, 0x6098, 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf006, + 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd17a, + 0xa438, 0xd048, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706, + 0xa438, 0x5f29, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, 0xa438, 0xf008, + 0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, 0xa438, 0xf004, + 0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, + 0xa438, 0x0b01, 0xa438, 0x0c3f, 0xa438, 0x0c08, 0xa438, 0x9503, + 0xa438, 0xa120, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8cc7, + 0xa438, 0x6224, 0xa438, 0xd700, 0xa438, 0x4147, 0xa438, 0xd704, + 0xa438, 0x409d, 0xa438, 0xd704, 0xa438, 0x613b, 0xa438, 0xf00a, + 0xa438, 0xd707, 0xa438, 0x411f, 0xa438, 0x40e0, 0xa438, 0xf004, + 0xa438, 0xd707, 0xa438, 0x409f, 0xa438, 0x4062, 0xa438, 0x8310, + 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xd700, 0xa438, 0x2969, + 0xa438, 0x8cd2, 0xa438, 0x60e4, 0xa438, 0xd700, 0xa438, 0x40a7, + 0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0xa202, 0xa438, 0xf002, + 0xa438, 0x8202, 0xa438, 0xa420, 0xa438, 0xcd3b, 0xa438, 0xd700, + 0xa438, 0x65ad, 0xa438, 0x43c7, 0xa438, 0xd700, 0xa438, 0x37c9, + 0xa438, 0x8ced, 0xa438, 0x33a9, 0xa438, 0x8ce5, 0xa438, 0xd702, + 0xa438, 0x4098, 0xa438, 0xd199, 0xa438, 0xd04c, 0xa438, 0xf024, + 0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0xf021, 0xa438, 0xd702, + 0xa438, 0x4098, 0xa438, 0xd1c6, 0xa438, 0xd04c, 0xa438, 0xf01c, + 0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0xf019, 0xa438, 0xd702, + 0xa438, 0x4098, 0xa438, 0xd199, 0xa438, 0xd04c, 0xa438, 0xf014, + 0xa438, 0xd17a, 0xa438, 0xd04c, 0xa438, 0xf011, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x8d00, 0xa438, 0x33a9, 0xa438, 0x8cfd, + 0xa438, 0xd1e5, 0xa438, 0xd04c, 0xa438, 0xf009, 0xa438, 0xd191, + 0xa438, 0xd04d, 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd04d, + 0xa438, 0xf003, 0xa438, 0xd16b, 0xa438, 0xd04c, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2c, 0xa438, 0xd700, + 0xa438, 0x40e7, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c07, + 0xa438, 0x0d01, 0xa438, 0x9503, 0xa438, 0xf006, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d02, 0xa438, 0x9503, + 0xa438, 0xcd3c, 0xa438, 0xd700, 0xa438, 0x644d, 0xa438, 0x43c7, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8d33, 0xa438, 0x33a9, + 0xa438, 0x8d2b, 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd17a, + 0xa438, 0xd04a, 0xa438, 0xf019, 0xa438, 0xd17a, 0xa438, 0xd048, + 0xa438, 0xf016, 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd17a, + 0xa438, 0xd04b, 0xa438, 0xf011, 0xa438, 0xd17a, 0xa438, 0xd048, + 0xa438, 0xf00e, 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd17a, + 0xa438, 0xd04a, 0xa438, 0xf009, 0xa438, 0xd17a, 0xa438, 0xd049, + 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd04b, 0xa438, 0xf003, + 0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, + 0xa438, 0xd706, 0xa438, 0x5f2d, 0xa438, 0xd700, 0xa438, 0x60cf, + 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, + 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf00a, 0xa438, 0xce08, + 0xa438, 0xf008, 0xa438, 0xce08, 0xa438, 0xf006, 0xa438, 0xce08, + 0xa438, 0xf004, 0xa438, 0xce08, 0xa438, 0xf002, 0xa438, 0xce08, + 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c0f, 0xa438, 0x0b02, 0xa438, 0x0c38, 0xa438, 0x0c10, + 0xa438, 0x9503, 0xa438, 0xa180, 0xa438, 0xcd3d, 0xa438, 0xd700, + 0xa438, 0x2969, 0xa438, 0x8d8a, 0xa438, 0x37c9, 0xa438, 0x8d80, + 0xa438, 0x33a9, 0xa438, 0x8d76, 0xa438, 0xd700, 0xa438, 0x40c7, + 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd15c, 0xa438, 0xd04a, + 0xa438, 0xf01a, 0xa438, 0xd199, 0xa438, 0xd04b, 0xa438, 0xf017, + 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, + 0xa438, 0xd13e, 0xa438, 0xd04a, 0xa438, 0xf010, 0xa438, 0xd199, + 0xa438, 0xd04b, 0xa438, 0xf00d, 0xa438, 0xd700, 0xa438, 0x40c7, + 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd17a, 0xa438, 0xd04c, + 0xa438, 0xf006, 0xa438, 0xd199, 0xa438, 0xd04c, 0xa438, 0xf003, + 0xa438, 0xd17a, 0xa438, 0xd04b, 0xa438, 0xd700, 0xa438, 0x37c9, + 0xa438, 0x8da5, 0xa438, 0x33a9, 0xa438, 0x8d9b, 0xa438, 0xd700, + 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd100, + 0xa438, 0xd050, 0xa438, 0xf017, 0xa438, 0xd17a, 0xa438, 0xd05a, + 0xa438, 0xf014, 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, + 0xa438, 0x6098, 0xa438, 0xd100, 0xa438, 0xd050, 0xa438, 0xf00d, + 0xa438, 0xd17a, 0xa438, 0xd05a, 0xa438, 0xf00a, 0xa438, 0xd700, + 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd17a, + 0xa438, 0xd05a, 0xa438, 0xf003, 0xa438, 0xd17a, 0xa438, 0xd05a, + 0xa438, 0xd707, 0xa438, 0x3ad1, 0xa438, 0x8e73, 0xa438, 0xd705, + 0xa438, 0x36b1, 0xa438, 0x8ddd, 0xa438, 0xd700, 0xa438, 0x2969, + 0xa438, 0x8dd5, 0xa438, 0x63c4, 0xa438, 0xd700, 0xa438, 0x40e7, + 0xa438, 0xd704, 0xa438, 0x40bd, 0xa438, 0xd704, 0xa438, 0x60dc, + 0xa438, 0x63fb, 0xa438, 0xf016, 0xa438, 0xd707, 0xa438, 0x429f, + 0xa438, 0xf001, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7b, 0xa438, 0xd700, + 0xa438, 0x40e7, 0xa438, 0xd704, 0xa438, 0x40bd, 0xa438, 0xd704, + 0xa438, 0x407c, 0xa438, 0x1800, 0xa438, 0x8e73, 0xa438, 0xd706, + 0xa438, 0x2b59, 0xa438, 0x8e73, 0xa438, 0xf009, 0xa438, 0xaa20, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd701, 0xa438, 0x5f72, 0xa438, 0x8a20, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd700, + 0xa438, 0x4287, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce08, 0xa438, 0xf01e, 0xa438, 0xce08, 0xa438, 0xf01c, + 0xa438, 0xce08, 0xa438, 0xf01a, 0xa438, 0xce08, 0xa438, 0xf018, + 0xa438, 0xce08, 0xa438, 0xf016, 0xa438, 0xce08, 0xa438, 0xf014, + 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, + 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, + 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf009, 0xa438, 0xce08, + 0xa438, 0xf007, 0xa438, 0xce08, 0xa438, 0xf005, 0xa438, 0xce08, + 0xa438, 0xf003, 0xa438, 0xce08, 0xa438, 0xf001, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8e23, + 0xa438, 0x6224, 0xa438, 0xd700, 0xa438, 0x4147, 0xa438, 0xd704, + 0xa438, 0x409d, 0xa438, 0xd704, 0xa438, 0x417b, 0xa438, 0xf008, + 0xa438, 0xd707, 0xa438, 0x411f, 0xa438, 0x40e0, 0xa438, 0xf004, + 0xa438, 0xd707, 0xa438, 0x409f, 0xa438, 0x4062, 0xa438, 0x8320, + 0xa438, 0xf002, 0xa438, 0xa320, 0xa438, 0x8310, 0xa438, 0xcd3e, + 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8e4b, 0xa438, 0x37c9, + 0xa438, 0x8e41, 0xa438, 0x33a9, 0xa438, 0x8e37, 0xa438, 0xd700, + 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd15d, + 0xa438, 0xd04b, 0xa438, 0xf01a, 0xa438, 0xd17a, 0xa438, 0xd04b, + 0xa438, 0xf017, 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, + 0xa438, 0x6098, 0xa438, 0xd16b, 0xa438, 0xd04b, 0xa438, 0xf010, + 0xa438, 0xd17a, 0xa438, 0xd04b, 0xa438, 0xf00d, 0xa438, 0xd700, + 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd17a, + 0xa438, 0xd04b, 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd04b, + 0xa438, 0xf003, 0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd700, + 0xa438, 0x2969, 0xa438, 0x8e6b, 0xa438, 0x6264, 0xa438, 0xd700, + 0xa438, 0x4187, 0xa438, 0xd704, 0xa438, 0x40bd, 0xa438, 0xd704, + 0xa438, 0x41bb, 0xa438, 0x1800, 0xa438, 0x8f25, 0xa438, 0xd707, + 0xa438, 0x413f, 0xa438, 0x4100, 0xa438, 0x1800, 0xa438, 0x8f25, + 0xa438, 0xd707, 0xa438, 0x409f, 0xa438, 0x4060, 0xa438, 0x1800, + 0xa438, 0x8f25, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd701, 0xa438, 0x5f77, 0xa438, 0x1800, + 0xa438, 0x8f25, 0xa438, 0xd700, 0xa438, 0x4287, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf01e, + 0xa438, 0xce08, 0xa438, 0xf01c, 0xa438, 0xce08, 0xa438, 0xf01a, + 0xa438, 0xce08, 0xa438, 0xf018, 0xa438, 0xce08, 0xa438, 0xf016, + 0xa438, 0xce08, 0xa438, 0xf014, 0xa438, 0xd700, 0xa438, 0x60cf, + 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, + 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf00b, 0xa438, 0xce08, + 0xa438, 0xf009, 0xa438, 0xce08, 0xa438, 0xf007, 0xa438, 0xce08, + 0xa438, 0xf005, 0xa438, 0xce08, 0xa438, 0xf003, 0xa438, 0xce08, + 0xa438, 0xf001, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa102, 0xa438, 0xa802, 0xa438, 0x9503, + 0xa438, 0xaa02, 0xa438, 0xd700, 0xa438, 0x40e7, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c0f, 0xa438, 0x0b05, 0xa438, 0xac38, + 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd704, + 0xa438, 0x60fd, 0xa438, 0xd707, 0xa438, 0x40a0, 0xa438, 0xf003, + 0xa438, 0xd707, 0xa438, 0x4042, 0xa438, 0xa308, 0xa438, 0xa310, + 0xa438, 0xcd3f, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8ed0, + 0xa438, 0x33a9, 0xa438, 0x8ec6, 0xa438, 0xd700, 0xa438, 0x40c7, + 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd1d6, 0xa438, 0xd04a, + 0xa438, 0xf017, 0xa438, 0xd1d6, 0xa438, 0xd04a, 0xa438, 0xf014, + 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, + 0xa438, 0xd1d6, 0xa438, 0xd04a, 0xa438, 0xf00d, 0xa438, 0xd1d6, + 0xa438, 0xd04a, 0xa438, 0xf00a, 0xa438, 0xd700, 0xa438, 0x40c7, + 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd199, 0xa438, 0xd04a, + 0xa438, 0xf003, 0xa438, 0xd199, 0xa438, 0xd04a, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0x0c30, 0xa438, 0x0320, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x8efa, 0xa438, 0x33a9, 0xa438, 0x8ef0, + 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, + 0xa438, 0xd199, 0xa438, 0xd04a, 0xa438, 0xf017, 0xa438, 0xd17a, + 0xa438, 0xd04b, 0xa438, 0xf014, 0xa438, 0xd700, 0xa438, 0x40c7, + 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd199, 0xa438, 0xd04a, + 0xa438, 0xf00d, 0xa438, 0xd17a, 0xa438, 0xd04b, 0xa438, 0xf00a, + 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, + 0xa438, 0xd17a, 0xa438, 0xd04b, 0xa438, 0xf003, 0xa438, 0xd17a, + 0xa438, 0xd04b, 0xa438, 0xd700, 0xa438, 0x4287, 0xa438, 0xd704, + 0xa438, 0x425d, 0xa438, 0xd704, 0xa438, 0x421c, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd701, + 0xa438, 0x5f77, 0xa438, 0x1000, 0xa438, 0x9c16, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xf009, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, + 0xa438, 0xd701, 0xa438, 0x5f37, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8102, 0xa438, 0x9503, 0xa438, 0x8a02, 0xa438, 0xcd40, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, + 0xa438, 0xd40a, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xbb10, + 0xa438, 0xcd41, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd71f, 0xa438, 0x5f75, 0xa438, 0x800a, + 0xa438, 0x81a0, 0xa438, 0x8302, 0xa438, 0x8480, 0xa438, 0xd700, + 0xa438, 0x6047, 0xa438, 0xa680, 0xa438, 0x8606, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8340, 0xa438, 0xd700, 0xa438, 0x6060, + 0xa438, 0xa120, 0xa438, 0xa302, 0xa438, 0x9503, 0xa438, 0x8c40, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8750, 0xa438, 0x8702, + 0xa438, 0x9503, 0xa438, 0x9b30, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8011, 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fb5, 0xa438, 0xcd42, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5f94, 0xa438, 0xd700, + 0xa438, 0x2969, 0xa438, 0x8f67, 0xa438, 0xd700, 0xa438, 0x6140, + 0xa438, 0xd705, 0xa438, 0x611e, 0xa438, 0xd703, 0xa438, 0x40d0, + 0xa438, 0x1000, 0xa438, 0x9c20, 0xa438, 0xd41a, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0x8340, 0xa438, 0xa801, 0xa438, 0xd700, + 0xa438, 0x2fa9, 0xa438, 0x8f70, 0xa438, 0x33c9, 0xa438, 0x8f73, + 0xa438, 0x6117, 0xa438, 0xf00a, 0xa438, 0xd141, 0xa438, 0xd043, + 0xa438, 0xf009, 0xa438, 0xd121, 0xa438, 0xd043, 0xa438, 0xf006, + 0xa438, 0xd122, 0xa438, 0xd042, 0xa438, 0xf003, 0xa438, 0xd181, + 0xa438, 0xd043, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce08, 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, + 0xa438, 0xce08, 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, + 0xa438, 0xce08, 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd705, 0xa438, 0x611e, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0ccf, 0xa438, 0x0b02, 0xa438, 0x8cc7, + 0xa438, 0x9503, 0xa438, 0xf008, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0ccf, 0xa438, 0x0b46, 0xa438, 0x0cc7, 0xa438, 0x0c03, + 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x40e7, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c3f, 0xa438, 0x0d09, 0xa438, 0x9503, + 0xa438, 0xf006, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c3f, + 0xa438, 0x0d0a, 0xa438, 0x9503, 0xa438, 0xd705, 0xa438, 0x607e, + 0xa438, 0xa302, 0xa438, 0xf00b, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa011, 0xa438, 0x9503, 0xa438, 0xd14f, 0xa438, 0xd043, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, + 0xa438, 0xa810, 0xa438, 0x8240, 0xa438, 0xa00a, 0xa438, 0xa1a0, + 0xa438, 0xa480, 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, + 0xa438, 0x407d, 0xa438, 0x8604, 0xa438, 0xf002, 0xa438, 0xa604, + 0xa438, 0xd700, 0xa438, 0x60c7, 0xa438, 0xa682, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa702, 0xa438, 0x9503, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xb920, + 0xa438, 0xcd43, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x7fb4, 0xa438, 0x8810, 0xa438, 0x9920, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x60e5, 0xa438, 0x5f94, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, + 0xa438, 0xffef, 0xa438, 0xb820, 0xa438, 0xa810, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, + 0xa438, 0xbb20, 0xa438, 0xcd44, 0xa438, 0xd700, 0xa438, 0x4060, + 0xa438, 0x1800, 0xa438, 0x9072, 0xa438, 0xd700, 0xa438, 0x37c9, + 0xa438, 0x9003, 0xa438, 0x33a9, 0xa438, 0x9000, 0xa438, 0xd17a, + 0xa438, 0xd047, 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd048, + 0xa438, 0xf003, 0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd700, + 0xa438, 0x2d49, 0xa438, 0x9014, 0xa438, 0xd701, 0xa438, 0x60b0, + 0xa438, 0x1800, 0xa438, 0x90c4, 0xa438, 0x1800, 0xa438, 0x911b, + 0xa438, 0x0c06, 0xa438, 0x0a06, 0xa438, 0x0cc0, 0xa438, 0x0cc0, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa7a0, 0xa438, 0x9503, + 0xa438, 0xcd45, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x902a, + 0xa438, 0x33a9, 0xa438, 0x9027, 0xa438, 0xd17a, 0xa438, 0xd047, + 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0xf003, + 0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, + 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0x8c40, 0xa438, 0xd418, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xac40, 0xa438, 0xaa01, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd706, 0xa438, 0x5f6f, 0xa438, 0xd700, 0xa438, 0x37c9, + 0xa438, 0x904b, 0xa438, 0x33a9, 0xa438, 0x9048, 0xa438, 0xd17a, + 0xa438, 0xd047, 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd048, + 0xa438, 0xf003, 0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xab20, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa750, 0xa438, 0x9503, + 0xa438, 0xcd46, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x9066, + 0xa438, 0x33a9, 0xa438, 0x9063, 0xa438, 0xd1b7, 0xa438, 0xd04d, + 0xa438, 0xf006, 0xa438, 0xd1b7, 0xa438, 0xd04d, 0xa438, 0xf003, + 0xa438, 0xd1b7, 0xa438, 0xd04d, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, + 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0x1800, 0xa438, 0x90c4, + 0xa438, 0xd701, 0xa438, 0x4830, 0xa438, 0xd700, 0xa438, 0x47e7, + 0xa438, 0xd704, 0xa438, 0x47bd, 0xa438, 0xd704, 0xa438, 0x477c, + 0xa438, 0x0c06, 0xa438, 0x0a06, 0xa438, 0x0cc0, 0xa438, 0x0cc0, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x9089, 0xa438, 0x33a9, + 0xa438, 0x9086, 0xa438, 0xd17a, 0xa438, 0xd047, 0xa438, 0xf006, + 0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0xf003, 0xa438, 0xd17a, + 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706, + 0xa438, 0x5f2f, 0xa438, 0x8c40, 0xa438, 0xd418, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0xac40, 0xa438, 0xaa01, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd706, + 0xa438, 0x5f6f, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x90aa, + 0xa438, 0x33a9, 0xa438, 0x90a7, 0xa438, 0xd17a, 0xa438, 0xd047, + 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0xf003, + 0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, + 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd701, 0xa438, 0x40f0, + 0xa438, 0xac40, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa750, + 0xa438, 0x9503, 0xa438, 0xab20, 0xa438, 0xd1c4, 0xa438, 0xd046, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x424f, + 0xa438, 0x8c40, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8750, + 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd417, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0xd701, 0xa438, 0x40d0, 0xa438, 0xac40, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa750, 0xa438, 0x9503, + 0xa438, 0xcd47, 0xa438, 0xd700, 0xa438, 0x686b, 0xa438, 0x6060, + 0xa438, 0x1800, 0xa438, 0x90f3, 0xa438, 0xd700, 0xa438, 0x37c9, + 0xa438, 0x90e7, 0xa438, 0x33a9, 0xa438, 0x90e4, 0xa438, 0xd15c, + 0xa438, 0xd04c, 0xa438, 0xf006, 0xa438, 0xd182, 0xa438, 0xd04c, + 0xa438, 0xf003, 0xa438, 0xd191, 0xa438, 0xd04c, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0x1800, + 0xa438, 0x911b, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0ccf, + 0xa438, 0x0b03, 0xa438, 0x8cc7, 0xa438, 0x9503, 0xa438, 0xd700, + 0xa438, 0x60e7, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c3f, + 0xa438, 0x0d1a, 0xa438, 0x9503, 0xa438, 0xf006, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c3f, 0xa438, 0x0d19, 0xa438, 0x9503, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x9111, 0xa438, 0x33a9, + 0xa438, 0x910e, 0xa438, 0xd19f, 0xa438, 0xd049, 0xa438, 0xf006, + 0xa438, 0xd199, 0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd199, + 0xa438, 0xd04b, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706, + 0xa438, 0x5f2f, 0xa438, 0xd416, 0xa438, 0x1000, 0xa438, 0x9bbf, + 0xa438, 0xbb10, 0xa438, 0xcd4f, 0xa438, 0xcd50, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd71f, + 0xa438, 0x5f75, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8302, + 0xa438, 0x8480, 0xa438, 0xd700, 0xa438, 0x6047, 0xa438, 0xa682, + 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, 0xa438, 0x407d, + 0xa438, 0x8604, 0xa438, 0xf002, 0xa438, 0xa604, 0xa438, 0x8818, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8340, 0xa438, 0xa120, + 0xa438, 0xa302, 0xa438, 0x9503, 0xa438, 0x8c40, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8750, 0xa438, 0x9503, 0xa438, 0x9b30, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8011, 0xa438, 0x9503, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb5, + 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, + 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, + 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, 0xa438, 0xce08, + 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, 0xa438, 0xce08, + 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0xcd51, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x5f94, 0xa438, 0xd71e, 0xa438, 0x6103, 0xa438, 0xd700, + 0xa438, 0x2fa9, 0xa438, 0x916f, 0xa438, 0x33c9, 0xa438, 0x9172, + 0xa438, 0x6177, 0xa438, 0x61b1, 0xa438, 0xd101, 0xa438, 0xd040, + 0xa438, 0xf00c, 0xa438, 0xd141, 0xa438, 0xd043, 0xa438, 0xf009, + 0xa438, 0xd121, 0xa438, 0xd043, 0xa438, 0xf006, 0xa438, 0xd122, + 0xa438, 0xd042, 0xa438, 0xf003, 0xa438, 0xd181, 0xa438, 0xd043, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, + 0xa438, 0xd705, 0xa438, 0x60fe, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0ccf, 0xa438, 0x0b03, 0xa438, 0x8c07, 0xa438, 0x9503, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8cc0, 0xa438, 0x9503, + 0xa438, 0x8106, 0xa438, 0xd700, 0xa438, 0x60e7, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c3f, 0xa438, 0x0d1a, 0xa438, 0x9503, + 0xa438, 0xf006, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c3f, + 0xa438, 0x0d19, 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x2f99, + 0xa438, 0x919c, 0xa438, 0xa804, 0xa438, 0xd705, 0xa438, 0x607e, + 0xa438, 0xa302, 0xa438, 0xf00b, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa017, 0xa438, 0x9503, 0xa438, 0xd14f, 0xa438, 0xd043, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, + 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xa480, 0xa438, 0xd700, + 0xa438, 0x40a7, 0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0x8644, + 0xa438, 0xf003, 0xa438, 0x0c44, 0xa438, 0x0604, 0xa438, 0xd701, + 0xa438, 0x40d0, 0xa438, 0xac40, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa750, 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x6047, + 0xa438, 0xa682, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, + 0xa438, 0xc5aa, 0xa438, 0x9503, 0xa438, 0xab80, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5faf, 0xa438, 0xb920, + 0xa438, 0xcd52, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x7fb4, 0xa438, 0x9920, 0xa438, 0xa00a, 0xa438, 0xa1a0, + 0xa438, 0xd705, 0xa438, 0x605e, 0xa438, 0xa302, 0xa438, 0xa480, + 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, 0xa438, 0x407d, + 0xa438, 0x8644, 0xa438, 0xf003, 0xa438, 0x0c44, 0xa438, 0x0604, + 0xa438, 0xa902, 0xa438, 0x8920, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa480, 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9c2a, + 0xa438, 0xd707, 0xa438, 0x40ae, 0xa438, 0x0c18, 0xa438, 0x0a08, + 0xa438, 0x1000, 0xa438, 0x9c30, 0xa438, 0xcd60, 0xa438, 0xd101, + 0xa438, 0xd040, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa230, + 0xa438, 0x9503, 0xa438, 0xd703, 0xa438, 0x68d1, 0xa438, 0xcd62, + 0xa438, 0x1000, 0xa438, 0x9a89, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd707, 0xa438, 0x4058, 0xa438, 0xa901, 0xa438, 0xd700, + 0xa438, 0x62dc, 0xa438, 0xd71f, 0xa438, 0x628e, 0xa438, 0xd704, + 0xa438, 0x4067, 0xa438, 0x1800, 0xa438, 0x96ac, 0xa438, 0xd704, + 0xa438, 0x40ab, 0xa438, 0xd705, 0xa438, 0x607f, 0xa438, 0x1800, + 0xa438, 0x974b, 0xa438, 0xd704, 0xa438, 0x609f, 0xa438, 0xd705, + 0xa438, 0x405d, 0xa438, 0xf005, 0xa438, 0xd704, 0xa438, 0x5c96, + 0xa438, 0xd75f, 0xa438, 0x5c40, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xc5aa, 0xa438, 0x9503, 0xa438, 0xd705, 0xa438, 0x60de, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c38, 0xa438, 0x0d18, + 0xa438, 0x9503, 0xa438, 0xd702, 0xa438, 0x4357, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8f80, 0xa438, 0x9503, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf00b, + 0xa438, 0xce08, 0xa438, 0xf009, 0xa438, 0xce08, 0xa438, 0xf007, + 0xa438, 0xce08, 0xa438, 0xf005, 0xa438, 0xce08, 0xa438, 0xf003, + 0xa438, 0xce08, 0xa438, 0xf001, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0xcd61, 0xa438, 0x1000, 0xa438, 0x9a89, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd707, 0xa438, 0x4058, 0xa438, 0xa901, + 0xa438, 0xd71f, 0xa438, 0x2e71, 0xa438, 0x925e, 0xa438, 0xd704, + 0xa438, 0x2739, 0xa438, 0x96ac, 0xa438, 0xd704, 0xa438, 0x40ab, + 0xa438, 0xd705, 0xa438, 0x607f, 0xa438, 0x1800, 0xa438, 0x974b, + 0xa438, 0xd704, 0xa438, 0x60bf, 0xa438, 0xd705, 0xa438, 0x407d, + 0xa438, 0x1800, 0xa438, 0x97da, 0xa438, 0xd704, 0xa438, 0x5cb6, + 0xa438, 0xd75f, 0xa438, 0x5c60, 0xa438, 0x1800, 0xa438, 0x997c, + 0xa438, 0xaa10, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8480, + 0xa438, 0x9503, 0xa438, 0x9904, 0xa438, 0xcd80, 0xa438, 0x1000, + 0xa438, 0x9c99, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8302, + 0xa438, 0x8480, 0xa438, 0x8646, 0xa438, 0x1000, 0xa438, 0x9d44, + 0xa438, 0xd707, 0xa438, 0x605d, 0xa438, 0x8320, 0xa438, 0x8c40, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8750, 0xa438, 0x8340, + 0xa438, 0x8120, 0xa438, 0x8302, 0xa438, 0xa61c, 0xa438, 0x9503, + 0xa438, 0xd701, 0xa438, 0x4050, 0xa438, 0xab20, 0xa438, 0x8b80, + 0xa438, 0x0c0c, 0xa438, 0x0808, 0xa438, 0xd702, 0xa438, 0x4191, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fa1, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8f02, 0xa438, 0x9503, + 0xa438, 0xd400, 0xa438, 0xb302, 0xa438, 0xd200, 0xa438, 0xb910, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fae, + 0xa438, 0x9910, 0xa438, 0xd71f, 0xa438, 0x409e, 0xa438, 0xd1b8, + 0xa438, 0xd049, 0xa438, 0xf012, 0xa438, 0xd700, 0xa438, 0x2fa9, + 0xa438, 0x92a1, 0xa438, 0x6131, 0xa438, 0x33c9, 0xa438, 0x92a7, + 0xa438, 0xd15d, 0xa438, 0xd040, 0xa438, 0xf009, 0xa438, 0xd193, + 0xa438, 0xd040, 0xa438, 0xf006, 0xa438, 0xd1db, 0xa438, 0xd040, + 0xa438, 0xf003, 0xa438, 0xd16f, 0xa438, 0xd040, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa380, 0xa438, 0x9503, 0xa438, 0xd417, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd700, 0xa438, 0x4127, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8bc0, 0xa438, 0x8cc7, + 0xa438, 0x0c3f, 0xa438, 0x0d08, 0xa438, 0x9503, 0xa438, 0xf007, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8bc0, 0xa438, 0x8cc7, + 0xa438, 0xcd48, 0xa438, 0x9503, 0xa438, 0x0c06, 0xa438, 0x0102, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xc555, 0xa438, 0x9503, + 0xa438, 0xcd81, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd707, 0xa438, 0x40fd, 0xa438, 0xd193, + 0xa438, 0xd047, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xa810, 0xa438, 0x8902, 0xa438, 0xd707, + 0xa438, 0x407b, 0xa438, 0x8801, 0xa438, 0xa340, 0xa438, 0xd71f, + 0xa438, 0x3ffd, 0xa438, 0x9521, 0xa438, 0xcd82, 0xa438, 0xd704, + 0xa438, 0x407f, 0xa438, 0x1800, 0xa438, 0x9390, 0xa438, 0x8810, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa204, 0xa438, 0xa108, + 0xa438, 0x9503, 0xa438, 0xa604, 0xa438, 0xd131, 0xa438, 0xd046, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, + 0xa438, 0xd707, 0xa438, 0x5f75, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8108, 0xa438, 0x9503, 0xa438, 0x0c07, 0xa438, 0x0b02, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c03, 0xa438, 0x0200, + 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce0b, 0xa438, 0xf00a, 0xa438, 0xce0b, 0xa438, 0xf008, + 0xa438, 0xce0a, 0xa438, 0xf006, 0xa438, 0xce0a, 0xa438, 0xf004, + 0xa438, 0xce09, 0xa438, 0xf002, 0xa438, 0xce09, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xab08, 0xa438, 0xcdc0, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd704, 0xa438, 0x5fb7, 0xa438, 0x8b08, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8204, 0xa438, 0xa380, + 0xa438, 0x9503, 0xa438, 0xd414, 0xa438, 0xb308, 0xa438, 0xd202, + 0xa438, 0xb302, 0xa438, 0xb301, 0xa438, 0xcdc1, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fa1, 0xa438, 0xd400, + 0xa438, 0xb302, 0xa438, 0xd200, 0xa438, 0xd71f, 0xa438, 0x6060, + 0xa438, 0xd704, 0xa438, 0x6074, 0xa438, 0x1800, 0xa438, 0x9390, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c91, 0xa438, 0x0080, + 0xa438, 0xa060, 0xa438, 0xa101, 0xa438, 0xa408, 0xa438, 0xa240, + 0xa438, 0x9503, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c26, + 0xa438, 0x0022, 0xa438, 0xa110, 0xa438, 0xa011, 0xa438, 0x9503, + 0xa438, 0xd17a, 0xa438, 0xd046, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c38, 0xa438, 0x0c18, 0xa438, 0x9503, 0xa438, 0xa318, + 0xa438, 0xd131, 0xa438, 0xd046, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c83, 0xa438, 0x0282, 0xa438, 0x9503, 0xa438, 0x0c07, + 0xa438, 0x0b02, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce11, 0xa438, 0xf00a, 0xa438, 0xce11, 0xa438, 0xf008, + 0xa438, 0xce10, 0xa438, 0xf006, 0xa438, 0xce10, 0xa438, 0xf004, + 0xa438, 0xce0f, 0xa438, 0xf002, 0xa438, 0xce0f, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa204, 0xa438, 0x9503, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xab08, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd704, 0xa438, 0x3ebc, 0xa438, 0x9370, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8284, 0xa438, 0x9503, 0xa438, 0x8b08, + 0xa438, 0xd704, 0xa438, 0x61b9, 0xa438, 0xd705, 0xa438, 0x407c, + 0xa438, 0x1800, 0xa438, 0x938e, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa402, 0xa438, 0x9503, 0xa438, 0x8310, 0xa438, 0xcdc3, + 0xa438, 0x1800, 0xa438, 0x9337, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8402, 0xa438, 0x9503, 0xa438, 0xcdc2, 0xa438, 0xf003, + 0xa438, 0x1000, 0xa438, 0x9c99, 0xa438, 0x8320, 0xa438, 0xd705, + 0xa438, 0x429e, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce02, 0xa438, 0xf01d, 0xa438, 0xce02, 0xa438, 0xf01b, + 0xa438, 0xce01, 0xa438, 0xf019, 0xa438, 0xce01, 0xa438, 0xf017, + 0xa438, 0xce01, 0xa438, 0xf015, 0xa438, 0xce01, 0xa438, 0xf013, + 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, + 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce06, + 0xa438, 0xf00a, 0xa438, 0xce06, 0xa438, 0xf008, 0xa438, 0xce06, + 0xa438, 0xf006, 0xa438, 0xce06, 0xa438, 0xf004, 0xa438, 0xce06, + 0xa438, 0xf002, 0xa438, 0xce06, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0xd705, 0xa438, 0x61be, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c0f, 0xa438, 0x0b05, 0xa438, 0x0c38, 0xa438, 0x0c28, + 0xa438, 0x9503, 0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0xa302, + 0xa438, 0xa4a0, 0xa438, 0xf012, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c0f, 0xa438, 0x0b06, 0xa438, 0x0c3f, 0xa438, 0x0c1b, + 0xa438, 0xd700, 0xa438, 0x6067, 0xa438, 0x0cc0, 0xa438, 0x0d80, + 0xa438, 0x9503, 0xa438, 0x8810, 0xa438, 0x800a, 0xa438, 0x8302, + 0xa438, 0xa8c0, 0xa438, 0x8420, 0xa438, 0xa480, 0xa438, 0xd700, + 0xa438, 0x40c7, 0xa438, 0xd704, 0xa438, 0x409d, 0xa438, 0x0ce3, + 0xa438, 0x0203, 0xa438, 0xf003, 0xa438, 0x0ce0, 0xa438, 0x02a0, + 0xa438, 0xa120, 0xa438, 0xd700, 0xa438, 0x4187, 0xa438, 0xd704, + 0xa438, 0x40dd, 0xa438, 0xd704, 0xa438, 0x617b, 0xa438, 0xd704, + 0xa438, 0x613c, 0xa438, 0xf00a, 0xa438, 0xd707, 0xa438, 0x411f, + 0xa438, 0x40e0, 0xa438, 0xf004, 0xa438, 0xd707, 0xa438, 0x409f, + 0xa438, 0x4062, 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa318, + 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, 0xa438, 0x407d, + 0xa438, 0x8604, 0xa438, 0xf002, 0xa438, 0xa604, 0xa438, 0xd700, + 0xa438, 0x6047, 0xa438, 0xa682, 0xa438, 0x1000, 0xa438, 0x9cc6, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb4, + 0xa438, 0xb920, 0xa438, 0xcd83, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x8810, 0xa438, 0x9920, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x60e5, + 0xa438, 0x5f94, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd706, + 0xa438, 0x5fa7, 0xa438, 0xffef, 0xa438, 0xb820, 0xa438, 0xa810, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fa5, + 0xa438, 0x9820, 0xa438, 0xbb20, 0xa438, 0xd705, 0xa438, 0x605e, + 0xa438, 0xf018, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, 0xa438, 0xf008, + 0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, 0xa438, 0xf004, + 0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xa00a, 0xa438, 0xa420, 0xa438, 0x88c0, + 0xa438, 0xd701, 0xa438, 0x40d0, 0xa438, 0xac40, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa750, 0xa438, 0x9503, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x9450, 0xa438, 0x33a9, 0xa438, 0x944d, + 0xa438, 0xd1b7, 0xa438, 0xd04b, 0xa438, 0xf006, 0xa438, 0xd1b7, + 0xa438, 0xd04b, 0xa438, 0xf003, 0xa438, 0xd1b7, 0xa438, 0xd04b, + 0xa438, 0xd707, 0xa438, 0x3ad1, 0xa438, 0x94d8, 0xa438, 0xd700, + 0xa438, 0x40c7, 0xa438, 0xd704, 0xa438, 0x409d, 0xa438, 0xd704, + 0xa438, 0x465c, 0xa438, 0xf004, 0xa438, 0xd707, 0xa438, 0x605f, + 0xa438, 0xf02e, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x946a, + 0xa438, 0x33a9, 0xa438, 0x9467, 0xa438, 0xd199, 0xa438, 0xd05a, + 0xa438, 0xf006, 0xa438, 0xd199, 0xa438, 0xd05a, 0xa438, 0xf003, + 0xa438, 0xd1d6, 0xa438, 0xd05a, 0xa438, 0xcd84, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7b, 0xa438, 0xd700, 0xa438, 0x42c7, 0xa438, 0xd704, + 0xa438, 0x429d, 0xa438, 0xd704, 0xa438, 0x425c, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd706, + 0xa438, 0x5f6f, 0xa438, 0x8a04, 0xa438, 0x8b20, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa380, 0xa438, 0x9503, 0xa438, 0xd401, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x1800, 0xa438, 0x94d8, + 0xa438, 0xd706, 0xa438, 0x69ab, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, + 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd700, 0xa438, 0x4167, + 0xa438, 0xd704, 0xa438, 0x40bd, 0xa438, 0xd704, 0xa438, 0x615b, + 0xa438, 0x1800, 0xa438, 0x94a5, 0xa438, 0xd707, 0xa438, 0x411f, + 0xa438, 0x40e0, 0xa438, 0xf004, 0xa438, 0xd707, 0xa438, 0x409f, + 0xa438, 0x4062, 0xa438, 0x8320, 0xa438, 0xf002, 0xa438, 0xa320, + 0xa438, 0x8310, 0xa438, 0xcd85, 0xa438, 0xd700, 0xa438, 0x37c9, + 0xa438, 0x94b3, 0xa438, 0x33a9, 0xa438, 0x94b0, 0xa438, 0xd1b7, + 0xa438, 0xd04a, 0xa438, 0xf006, 0xa438, 0xd1b7, 0xa438, 0xd04a, + 0xa438, 0xf003, 0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd700, + 0xa438, 0x41a7, 0xa438, 0xd704, 0xa438, 0x40dd, 0xa438, 0xd704, + 0xa438, 0x605b, 0xa438, 0xf00d, 0xa438, 0x1800, 0xa438, 0x8f25, + 0xa438, 0xd707, 0xa438, 0x413f, 0xa438, 0x4100, 0xa438, 0x1800, + 0xa438, 0x8f25, 0xa438, 0xd707, 0xa438, 0x409f, 0xa438, 0x4062, + 0xa438, 0x1800, 0xa438, 0x8f25, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd701, 0xa438, 0x5f77, + 0xa438, 0x1800, 0xa438, 0x8f25, 0xa438, 0xd705, 0xa438, 0x413e, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, 0xa438, 0x0b03, + 0xa438, 0x0c38, 0xa438, 0x0c00, 0xa438, 0xa104, 0xa438, 0x9503, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa102, 0xa438, 0x9503, + 0xa438, 0xaa02, 0xa438, 0xa310, 0xa438, 0xcd86, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x94f4, 0xa438, 0x33a9, 0xa438, 0x94f1, + 0xa438, 0xd1d6, 0xa438, 0xd04a, 0xa438, 0xf006, 0xa438, 0xd1d6, + 0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd199, 0xa438, 0xd04a, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, + 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0x0c30, 0xa438, 0x0320, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x9509, 0xa438, 0x33a9, + 0xa438, 0x9506, 0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0xf006, + 0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd1b7, + 0xa438, 0xd04a, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd701, + 0xa438, 0x5f37, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8102, + 0xa438, 0x8104, 0xa438, 0x9503, 0xa438, 0xd701, 0xa438, 0x40b0, + 0xa438, 0xaa02, 0xa438, 0xac80, 0xa438, 0x1800, 0xa438, 0x8f25, + 0xa438, 0x8a02, 0xa438, 0x1800, 0xa438, 0x8f25, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa708, 0xa438, 0x9503, 0xa438, 0xcd87, + 0xa438, 0xd704, 0xa438, 0x615f, 0xa438, 0x8810, 0xa438, 0xd702, + 0xa438, 0x40e6, 0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce06, 0xa438, 0xf00a, + 0xa438, 0xce06, 0xa438, 0xf008, 0xa438, 0xce06, 0xa438, 0xf006, + 0xa438, 0xce06, 0xa438, 0xf004, 0xa438, 0xce06, 0xa438, 0xf002, + 0xa438, 0xce06, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xd707, + 0xa438, 0x413b, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, + 0xa438, 0x0b03, 0xa438, 0x0c38, 0xa438, 0x0c18, 0xa438, 0x9503, + 0xa438, 0xf006, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, + 0xa438, 0x0b07, 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x40c7, + 0xa438, 0xd704, 0xa438, 0x409d, 0xa438, 0x0ce3, 0xa438, 0x0203, + 0xa438, 0xf003, 0xa438, 0x0ce0, 0xa438, 0x02a0, 0xa438, 0xa120, + 0xa438, 0xd707, 0xa438, 0x425b, 0xa438, 0xd700, 0xa438, 0x4167, + 0xa438, 0xd704, 0xa438, 0x40dd, 0xa438, 0xd704, 0xa438, 0x613b, + 0xa438, 0xd704, 0xa438, 0x60fc, 0xa438, 0xf008, 0xa438, 0xd707, + 0xa438, 0x40df, 0xa438, 0xf003, 0xa438, 0xd707, 0xa438, 0x407f, + 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa318, 0xa438, 0xa00a, + 0xa438, 0xa302, 0xa438, 0xa4a0, 0xa438, 0xd700, 0xa438, 0x40a7, + 0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0x8604, 0xa438, 0xf002, + 0xa438, 0xa604, 0xa438, 0xd700, 0xa438, 0x6047, 0xa438, 0xa682, + 0xa438, 0x1000, 0xa438, 0x9cc6, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920, 0xa438, 0xcd88, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, + 0xa438, 0x8810, 0xa438, 0x9920, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x60e5, 0xa438, 0x5f94, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xffef, + 0xa438, 0xb820, 0xa438, 0xa810, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0xbb20, + 0xa438, 0xd701, 0xa438, 0x40d0, 0xa438, 0xac40, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa750, 0xa438, 0x9503, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x95be, 0xa438, 0x33a9, 0xa438, 0x95b6, + 0xa438, 0xd702, 0xa438, 0x4086, 0xa438, 0xd161, 0xa438, 0xd049, + 0xa438, 0xf00e, 0xa438, 0xd19e, 0xa438, 0xd049, 0xa438, 0xf00b, + 0xa438, 0xd702, 0xa438, 0x4086, 0xa438, 0xd186, 0xa438, 0xd049, + 0xa438, 0xf006, 0xa438, 0xd1c3, 0xa438, 0xd049, 0xa438, 0xf003, + 0xa438, 0xd1cf, 0xa438, 0xd049, 0xa438, 0xcd89, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd707, + 0xa438, 0x40fb, 0xa438, 0x8310, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c0f, 0xa438, 0x0b05, 0xa438, 0x9503, 0xa438, 0xcd8a, + 0xa438, 0xd707, 0xa438, 0x419b, 0xa438, 0xd700, 0xa438, 0x33ad, + 0xa438, 0x95df, 0xa438, 0xd189, 0xa438, 0xd045, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xcd8b, 0xa438, 0xd40a, 0xa438, 0x1000, + 0xa438, 0x9bbf, 0xa438, 0xbb10, 0xa438, 0xcd8c, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd71f, + 0xa438, 0x5f75, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8302, + 0xa438, 0x8480, 0xa438, 0xd700, 0xa438, 0x6047, 0xa438, 0xa680, + 0xa438, 0x8606, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8340, + 0xa438, 0x8750, 0xa438, 0x9503, 0xa438, 0x8c40, 0xa438, 0x9b30, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb5, + 0xa438, 0xcd8d, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x5f94, 0xa438, 0x8340, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8708, 0xa438, 0x9503, 0xa438, 0xa801, 0xa438, 0xd700, + 0xa438, 0x2fa9, 0xa438, 0x9610, 0xa438, 0x33c9, 0xa438, 0x9613, + 0xa438, 0x6117, 0xa438, 0xf00a, 0xa438, 0xd141, 0xa438, 0xd043, + 0xa438, 0xf009, 0xa438, 0xd121, 0xa438, 0xd043, 0xa438, 0xf006, + 0xa438, 0xd122, 0xa438, 0xd042, 0xa438, 0xf003, 0xa438, 0xd181, + 0xa438, 0xd043, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, 0xa438, 0xf008, + 0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, 0xa438, 0xf004, + 0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, + 0xa438, 0x0b02, 0xa438, 0x8cc7, 0xa438, 0x9503, 0xa438, 0xd700, + 0xa438, 0x40e7, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c3f, + 0xa438, 0x0d09, 0xa438, 0x9503, 0xa438, 0xf006, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c3f, 0xa438, 0x0d08, 0xa438, 0x9503, + 0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0x0ca0, 0xa438, 0x0120, + 0xa438, 0xa302, 0xa438, 0xa480, 0xa438, 0xd700, 0xa438, 0x40a7, + 0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0x8604, 0xa438, 0xf002, + 0xa438, 0xa604, 0xa438, 0xd700, 0xa438, 0x6047, 0xa438, 0xa682, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, + 0xa438, 0xb920, 0xa438, 0xcd8e, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x8810, 0xa438, 0x9920, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x60e5, + 0xa438, 0x5f94, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd706, + 0xa438, 0x5fa7, 0xa438, 0xffef, 0xa438, 0xb820, 0xa438, 0xa810, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fa5, + 0xa438, 0x9820, 0xa438, 0xbb20, 0xa438, 0xd701, 0xa438, 0x40d0, + 0xa438, 0xac40, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa750, + 0xa438, 0x9503, 0xa438, 0xd1c4, 0xa438, 0xd046, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, + 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x422f, 0xa438, 0x8c40, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8750, 0xa438, 0x8380, + 0xa438, 0x9503, 0xa438, 0xd417, 0xa438, 0x1000, 0xa438, 0x9bbf, + 0xa438, 0xd701, 0xa438, 0x40d0, 0xa438, 0xac40, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa750, 0xa438, 0x9503, 0xa438, 0xd192, + 0xa438, 0xd047, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706, + 0xa438, 0x5f2f, 0xa438, 0xd707, 0xa438, 0x409b, 0xa438, 0xd416, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xbb10, 0xa438, 0xcd8f, + 0xa438, 0x1800, 0xa438, 0x9120, 0xa438, 0x1000, 0xa438, 0x9c99, + 0xa438, 0x8302, 0xa438, 0x8646, 0xa438, 0xd700, 0xa438, 0x60c7, + 0xa438, 0xa680, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8702, + 0xa438, 0x9503, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa680, + 0xa438, 0x850f, 0xa438, 0x9503, 0xa438, 0xd701, 0xa438, 0x4050, + 0xa438, 0x8b20, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0ccf, + 0xa438, 0x0bc8, 0xa438, 0x0cc7, 0xa438, 0x0c44, 0xa438, 0x9503, + 0xa438, 0x0c06, 0xa438, 0x0102, 0xa438, 0xd700, 0xa438, 0x40e7, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c3f, 0xa438, 0x0d24, + 0xa438, 0x9503, 0xa438, 0xf005, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xcde4, 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9cdd, + 0xa438, 0xcd90, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa601, + 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x2fa9, 0xa438, 0x96e1, + 0xa438, 0x60d1, 0xa438, 0x6113, 0xa438, 0x6157, 0xa438, 0xd150, + 0xa438, 0xd040, 0xa438, 0xf009, 0xa438, 0xd1a0, 0xa438, 0xd040, + 0xa438, 0xf006, 0xa438, 0xd128, 0xa438, 0xd040, 0xa438, 0xf003, + 0xa438, 0xd114, 0xa438, 0xd040, 0xa438, 0x1000, 0xa438, 0x99a8, + 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd704, 0xa438, 0x7f6b, + 0xa438, 0xa81a, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8603, + 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9d31, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8480, + 0xa438, 0x1000, 0xa438, 0x99a8, 0xa438, 0xd707, 0xa438, 0x4058, + 0xa438, 0xa901, 0xa438, 0xd704, 0xa438, 0x2b59, 0xa438, 0x974b, + 0xa438, 0xd704, 0xa438, 0x604c, 0xa438, 0xfff6, 0xa438, 0xd701, + 0xa438, 0x6056, 0xa438, 0xa704, 0xa438, 0xa302, 0xa438, 0xd700, + 0xa438, 0x40a7, 0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0x8604, + 0xa438, 0xf002, 0xa438, 0xa604, 0xa438, 0xd700, 0xa438, 0x60c7, + 0xa438, 0xa602, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa702, + 0xa438, 0x9503, 0xa438, 0xd701, 0xa438, 0x4055, 0xa438, 0xa940, + 0xa438, 0xd701, 0xa438, 0x4050, 0xa438, 0xab20, 0xa438, 0xcd91, + 0xa438, 0x1000, 0xa438, 0x9cf4, 0xa438, 0x1000, 0xa438, 0x99a8, + 0xa438, 0xd704, 0xa438, 0x7f6c, 0xa438, 0xa120, 0xa438, 0xd707, + 0xa438, 0x4051, 0xa438, 0xa180, 0xa438, 0xa480, 0xa438, 0x8302, + 0xa438, 0x8606, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8702, + 0xa438, 0x9503, 0xa438, 0xd701, 0xa438, 0x6056, 0xa438, 0x8704, + 0xa438, 0x1000, 0xa438, 0x9cf4, 0xa438, 0xce93, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xd701, 0xa438, 0x4050, 0xa438, 0x8b20, + 0xa438, 0xcd92, 0xa438, 0x1000, 0xa438, 0x99a8, 0xa438, 0xd707, + 0xa438, 0x7fb7, 0xa438, 0x81a0, 0xa438, 0x8480, 0xa438, 0x1000, + 0xa438, 0x9d31, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x1800, + 0xa438, 0x96fe, 0xa438, 0xd705, 0xa438, 0x40d9, 0xa438, 0xd701, + 0xa438, 0x6056, 0xa438, 0xa704, 0xa438, 0xa010, 0xa438, 0xf002, + 0xa438, 0x8010, 0xa438, 0xd701, 0xa438, 0x4095, 0xa438, 0xd705, + 0xa438, 0x4059, 0xa438, 0xa940, 0xa438, 0xd705, 0xa438, 0x6119, + 0xa438, 0x0c06, 0xa438, 0x0102, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c07, 0xa438, 0x0d04, 0xa438, 0x9503, 0xa438, 0xd701, + 0xa438, 0x4050, 0xa438, 0xab20, 0xa438, 0xcd93, 0xa438, 0x1000, + 0xa438, 0x99a8, 0xa438, 0xd704, 0xa438, 0x464b, 0xa438, 0xd707, + 0xa438, 0x5f78, 0xa438, 0x1000, 0xa438, 0x9d13, 0xa438, 0x800a, + 0xa438, 0x81a0, 0xa438, 0xd700, 0xa438, 0x6117, 0xa438, 0x6093, + 0xa438, 0xd15e, 0xa438, 0xd040, 0xa438, 0xf006, 0xa438, 0xd132, + 0xa438, 0xd040, 0xa438, 0xf003, 0xa438, 0xd119, 0xa438, 0xd040, + 0xa438, 0x1000, 0xa438, 0x9d08, 0xa438, 0x1000, 0xa438, 0x99a8, + 0xa438, 0xd73e, 0xa438, 0x6065, 0xa438, 0xd700, 0xa438, 0x5f3a, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x85f0, 0xa438, 0x9503, + 0xa438, 0xa008, 0xa438, 0xd707, 0xa438, 0x4052, 0xa438, 0xa002, + 0xa438, 0x8010, 0xa438, 0xd705, 0xa438, 0x4099, 0xa438, 0xd701, + 0xa438, 0x6056, 0xa438, 0x8704, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0xd701, 0xa438, 0x4050, 0xa438, 0x8b20, 0xa438, 0xcd94, + 0xa438, 0x1000, 0xa438, 0x99a8, 0xa438, 0xd707, 0xa438, 0x7fb8, + 0xa438, 0x8010, 0xa438, 0xd705, 0xa438, 0x4099, 0xa438, 0xd701, + 0xa438, 0x6056, 0xa438, 0x8704, 0xa438, 0xd705, 0xa438, 0x4099, + 0xa438, 0xd701, 0xa438, 0x4050, 0xa438, 0x8b20, 0xa438, 0xd705, + 0xa438, 0x61f9, 0xa438, 0x8106, 0xa438, 0xd700, 0xa438, 0x60e7, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d02, + 0xa438, 0x9503, 0xa438, 0xf006, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c07, 0xa438, 0x0d01, 0xa438, 0x9503, 0xa438, 0x800a, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0cf0, 0xa438, 0x05a0, + 0xa438, 0x9503, 0xa438, 0xd705, 0xa438, 0x4099, 0xa438, 0x1000, + 0xa438, 0x9d31, 0xa438, 0xf014, 0xa438, 0xa1a0, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf00a, + 0xa438, 0xce08, 0xa438, 0xf008, 0xa438, 0xce08, 0xa438, 0xf006, + 0xa438, 0xce08, 0xa438, 0xf004, 0xa438, 0xce08, 0xa438, 0xf002, + 0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xd705, + 0xa438, 0x39cc, 0xa438, 0x91f3, 0xa438, 0x1800, 0xa438, 0x96fe, + 0xa438, 0xd75e, 0xa438, 0x60f4, 0xa438, 0x9b08, 0xa438, 0x9920, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8403, 0xa438, 0x9503, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8208, 0xa438, 0x8404, + 0xa438, 0x9503, 0xa438, 0xcda1, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, + 0xa438, 0xd704, 0xa438, 0x5f37, 0xa438, 0x8b08, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8204, 0xa438, 0x9503, 0xa438, 0xd707, + 0xa438, 0x6536, 0xa438, 0x0c07, 0xa438, 0x0b00, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c03, 0xa438, 0x0200, 0xa438, 0x9503, + 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, + 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce0b, + 0xa438, 0xf00a, 0xa438, 0xce0b, 0xa438, 0xf008, 0xa438, 0xce0a, + 0xa438, 0xf006, 0xa438, 0xce0a, 0xa438, 0xf004, 0xa438, 0xce09, + 0xa438, 0xf002, 0xa438, 0xce09, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa204, 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0xab08, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, 0xa438, 0xd704, + 0xa438, 0x5f37, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8204, + 0xa438, 0x9503, 0xa438, 0x8b08, 0xa438, 0xd414, 0xa438, 0xd202, + 0xa438, 0xb308, 0xa438, 0xb302, 0xa438, 0xb301, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xaf02, 0xa438, 0x9503, 0xa438, 0xcdaf, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x99a0, + 0xa438, 0x1000, 0xa438, 0x99a4, 0xa438, 0xd701, 0xa438, 0x5f21, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8f02, 0xa438, 0x9503, + 0xa438, 0xd400, 0xa438, 0xb302, 0xa438, 0xd200, 0xa438, 0xd704, + 0xa438, 0x67b4, 0xa438, 0x1000, 0xa438, 0x9cc6, 0xa438, 0xbb08, + 0xa438, 0x9a10, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce08, 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, + 0xa438, 0xce08, 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, + 0xa438, 0xce08, 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xa302, + 0xa438, 0xa480, 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, + 0xa438, 0x407d, 0xa438, 0x8644, 0xa438, 0xf003, 0xa438, 0x0c44, + 0xa438, 0x0604, 0xa438, 0xd700, 0xa438, 0x6047, 0xa438, 0xa682, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8230, 0xa438, 0x8410, + 0xa438, 0x9503, 0xa438, 0xd707, 0xa438, 0x404e, 0xa438, 0x8a10, + 0xa438, 0xcda9, 0xa438, 0xd110, 0xa438, 0xd040, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x99a0, 0xa438, 0x1000, + 0xa438, 0x99a4, 0xa438, 0xd700, 0xa438, 0x5f3a, 0xa438, 0x1800, + 0xa438, 0x91f3, 0xa438, 0xba10, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c97, 0xa438, 0x0086, 0xa438, 0xa060, 0xa438, 0xa101, + 0xa438, 0xa408, 0xa438, 0xa240, 0xa438, 0x9503, 0xa438, 0xcda2, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa020, 0xa438, 0xa110, + 0xa438, 0xa011, 0xa438, 0x9503, 0xa438, 0xd17a, 0xa438, 0xd046, + 0xa438, 0xcda3, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, 0xa438, 0xd700, + 0xa438, 0x5f3a, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c83, + 0xa438, 0x0282, 0xa438, 0x9503, 0xa438, 0x0c07, 0xa438, 0x0b02, + 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, + 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce11, + 0xa438, 0xf00a, 0xa438, 0xce11, 0xa438, 0xf008, 0xa438, 0xce10, + 0xa438, 0xf006, 0xa438, 0xce10, 0xa438, 0xf004, 0xa438, 0xce0f, + 0xa438, 0xf002, 0xa438, 0xce0f, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa204, 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0xab08, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, 0xa438, 0xd704, + 0xa438, 0x3ebc, 0xa438, 0x98b7, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8284, 0xa438, 0x9503, 0xa438, 0x8b08, 0xa438, 0xcda4, + 0xa438, 0xd704, 0xa438, 0x6419, 0xa438, 0x3ad1, 0xa438, 0x9969, + 0xa438, 0xd705, 0xa438, 0x60dc, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa402, 0xa438, 0x9503, 0xa438, 0xf012, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8402, 0xa438, 0x9503, 0xa438, 0xd705, + 0xa438, 0x60db, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa401, + 0xa438, 0x9503, 0xa438, 0xf007, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c21, 0xa438, 0x0420, 0xa438, 0x9503, 0xa438, 0x9a10, + 0xa438, 0xd705, 0xa438, 0x34a4, 0xa438, 0x9886, 0xa438, 0x1800, + 0xa438, 0x9969, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c10, + 0xa438, 0x0010, 0xa438, 0x8410, 0xa438, 0xa250, 0xa438, 0x9503, + 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, 0xa438, 0x407d, + 0xa438, 0x8644, 0xa438, 0xf003, 0xa438, 0x0c44, 0xa438, 0x0604, + 0xa438, 0xd700, 0xa438, 0x40e7, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c38, 0xa438, 0x0d28, 0xa438, 0x9503, 0xa438, 0xf007, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0cf8, 0xa438, 0x0d90, + 0xa438, 0x9503, 0xa438, 0xa682, 0xa438, 0xd1f1, 0xa438, 0xd046, + 0xa438, 0xcda5, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, 0xa438, 0xd704, + 0xa438, 0x60ef, 0xa438, 0xd706, 0xa438, 0x40b1, 0xa438, 0xd700, + 0xa438, 0x5eba, 0xa438, 0x1800, 0xa438, 0x9925, 0xa438, 0xd162, + 0xa438, 0xd055, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, + 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, 0xa438, 0xd700, + 0xa438, 0x5f3b, 0xa438, 0xd704, 0xa438, 0x692f, 0xa438, 0xd706, + 0xa438, 0x48f1, 0xa438, 0x1800, 0xa438, 0x9907, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf00a, + 0xa438, 0xce08, 0xa438, 0xf008, 0xa438, 0xce08, 0xa438, 0xf006, + 0xa438, 0xce08, 0xa438, 0xf004, 0xa438, 0xce08, 0xa438, 0xf002, + 0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xa00a, + 0xa438, 0xd179, 0xa438, 0xd047, 0xa438, 0xcda6, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x99a0, 0xa438, 0x1000, + 0xa438, 0x99a4, 0xa438, 0xd704, 0xa438, 0x60ef, 0xa438, 0xd706, + 0xa438, 0x40b1, 0xa438, 0xd700, 0xa438, 0x5eba, 0xa438, 0x1800, + 0xa438, 0x995b, 0xa438, 0xd162, 0xa438, 0xd055, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x99a0, 0xa438, 0x1000, + 0xa438, 0x99a4, 0xa438, 0xd700, 0xa438, 0x5f3b, 0xa438, 0xd704, + 0xa438, 0x626f, 0xa438, 0xd706, 0xa438, 0x4231, 0xa438, 0x1800, + 0xa438, 0x993d, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0ccf, + 0xa438, 0x0b46, 0xa438, 0x0c07, 0xa438, 0x0c03, 0xa438, 0x9503, + 0xa438, 0xa1a0, 0xa438, 0xa480, 0xa438, 0xb920, 0xa438, 0x9a10, + 0xa438, 0xcda7, 0xa438, 0x1800, 0xa438, 0x91f3, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8091, 0xa438, 0x8101, 0xa438, 0x8250, + 0xa438, 0x0c28, 0xa438, 0x0420, 0xa438, 0xd700, 0xa438, 0x4087, + 0xa438, 0x0c38, 0xa438, 0x0d18, 0xa438, 0xf003, 0xa438, 0x0cf8, + 0xa438, 0x0d58, 0xa438, 0x9503, 0xa438, 0x8106, 0xa438, 0xcda8, + 0xa438, 0x1800, 0xa438, 0x91f3, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8090, 0xa438, 0x8101, 0xa438, 0x8248, 0xa438, 0x9503, + 0xa438, 0x1000, 0xa438, 0x9cb7, 0xa438, 0x1000, 0xa438, 0x9c90, + 0xa438, 0x8106, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0ccf, + 0xa438, 0x0b03, 0xa438, 0x8c07, 0xa438, 0xd700, 0xa438, 0x4087, + 0xa438, 0x0c38, 0xa438, 0x0d18, 0xa438, 0xf003, 0xa438, 0x0cf8, + 0xa438, 0x0d98, 0xa438, 0x8408, 0xa438, 0x9503, 0xa438, 0x9c01, + 0xa438, 0xa1a0, 0xa438, 0xa302, 0xa438, 0xd707, 0xa438, 0x404e, + 0xa438, 0x8a10, 0xa438, 0x1000, 0xa438, 0x9cc6, 0xa438, 0xcdab, + 0xa438, 0x1800, 0xa438, 0x923e, 0xa438, 0xd71f, 0xa438, 0x2e71, + 0xa438, 0x925e, 0xa438, 0x0800, 0xa438, 0xd704, 0xa438, 0x2739, + 0xa438, 0x96ac, 0xa438, 0x0800, 0xa438, 0xd71f, 0xa438, 0x62ae, + 0xa438, 0xd706, 0xa438, 0x6400, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd705, 0xa438, 0x61d9, 0xa438, 0xd704, 0xa438, 0x4187, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa601, 0xa438, 0x9503, + 0xa438, 0x800a, 0xa438, 0x1000, 0xa438, 0x9d31, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0x1800, 0xa438, 0x96ac, 0xa438, 0x0800, + 0xa438, 0xcd99, 0xa438, 0xa70c, 0xa438, 0x881a, 0xa438, 0x8010, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8683, 0xa438, 0x9503, + 0xa438, 0xd701, 0xa438, 0x4050, 0xa438, 0xab20, 0xa438, 0x1800, + 0xa438, 0x925e, 0xa438, 0x8010, 0xa438, 0xa704, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa60c, 0xa438, 0xa603, 0xa438, 0x9503, + 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8302, 0xa438, 0x8480, + 0xa438, 0x8604, 0xa438, 0x8602, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8702, 0xa438, 0x9503, 0xa438, 0xd701, 0xa438, 0x4050, + 0xa438, 0xab20, 0xa438, 0xcd96, 0xa438, 0xa901, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x7ce0, 0xa438, 0xd706, + 0xa438, 0x2109, 0xa438, 0x9a75, 0xa438, 0xd71f, 0xa438, 0x7aae, + 0xa438, 0xd704, 0xa438, 0x7ea7, 0xa438, 0xd706, 0xa438, 0x5e62, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8f40, 0xa438, 0x9503, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa610, 0xa438, 0x9503, + 0xa438, 0xa708, 0xa438, 0x881a, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0ccf, 0xa438, 0x0b03, 0xa438, 0x8cc7, 0xa438, 0x9503, + 0xa438, 0x8106, 0xa438, 0xd700, 0xa438, 0x60c7, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xcd5a, 0xa438, 0x9503, 0xa438, 0xf006, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c3f, 0xa438, 0x0d19, + 0xa438, 0x9503, 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xa302, + 0xa438, 0xa480, 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, + 0xa438, 0x407d, 0xa438, 0x8604, 0xa438, 0xf002, 0xa438, 0xa604, + 0xa438, 0xd700, 0xa438, 0x60c7, 0xa438, 0xa602, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa702, 0xa438, 0x9503, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8680, 0xa438, 0x9503, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xc500, 0xa438, 0x9503, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x72ce, 0xa438, 0xd704, + 0xa438, 0x5f65, 0xa438, 0xd705, 0xa438, 0x4378, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce1d, 0xa438, 0xf00b, + 0xa438, 0xce1d, 0xa438, 0xf009, 0xa438, 0xce1d, 0xa438, 0xf007, + 0xa438, 0xce1d, 0xa438, 0xf005, 0xa438, 0xce1c, 0xa438, 0xf003, + 0xa438, 0xce1c, 0xa438, 0xf001, 0xa438, 0x1000, 0xa438, 0x9bde, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8640, 0xa438, 0x9503, + 0xa438, 0xf01a, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce05, 0xa438, 0xf00b, 0xa438, 0xce05, 0xa438, 0xf009, + 0xa438, 0xce04, 0xa438, 0xf007, 0xa438, 0xce04, 0xa438, 0xf005, + 0xa438, 0xce04, 0xa438, 0xf003, 0xa438, 0xce04, 0xa438, 0xf001, + 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa640, 0xa438, 0x9503, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8602, 0xa438, 0x8601, 0xa438, 0x9503, 0xa438, 0x8640, + 0xa438, 0x1000, 0xa438, 0x9cc6, 0xa438, 0xd703, 0xa438, 0x6131, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xaf80, 0xa438, 0x0c38, + 0xa438, 0x0d30, 0xa438, 0x9503, 0xa438, 0xd193, 0xa438, 0xd067, + 0xa438, 0xcd98, 0xa438, 0x1800, 0xa438, 0x91f3, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa608, 0xa438, 0x9503, 0xa438, 0xa708, + 0xa438, 0xd701, 0xa438, 0x6056, 0xa438, 0x8704, 0xa438, 0x1000, + 0xa438, 0x9d31, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xa81a, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8603, 0xa438, 0x9503, + 0xa438, 0xcd97, 0xa438, 0x1800, 0xa438, 0x96fe, 0xa438, 0xd706, + 0xa438, 0x4103, 0xa438, 0xd705, 0xa438, 0x407f, 0xa438, 0x1800, + 0xa438, 0x9b77, 0xa438, 0xd105, 0xa438, 0xd056, 0xa438, 0xf03f, + 0xa438, 0xd705, 0xa438, 0x3fa7, 0xa438, 0x9a9a, 0xa438, 0xd705, + 0xa438, 0x605a, 0xa438, 0xf092, 0xa438, 0x1800, 0xa438, 0x9b1e, + 0xa438, 0xd704, 0xa438, 0x4146, 0xa438, 0x800a, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d04, 0xa438, 0x9503, + 0xa438, 0x0c06, 0xa438, 0x0102, 0xa438, 0xf010, 0xa438, 0xa00a, + 0xa438, 0x8106, 0xa438, 0xd700, 0xa438, 0x60e7, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d02, 0xa438, 0x9503, + 0xa438, 0xf006, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c07, + 0xa438, 0x0d01, 0xa438, 0x9503, 0xa438, 0xd705, 0xa438, 0x61d4, + 0xa438, 0xd704, 0xa438, 0x609f, 0xa438, 0x6170, 0xa438, 0x2d71, + 0xa438, 0x9ac3, 0xa438, 0xd702, 0xa438, 0x60d7, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0cf0, 0xa438, 0x05a0, 0xa438, 0x9503, + 0xa438, 0xf0fc, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa108, + 0xa438, 0xa204, 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, + 0xa438, 0xd707, 0xa438, 0x5f35, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa410, 0xa438, 0x9503, 0xa438, 0x800a, 0xa438, 0x81a0, + 0xa438, 0x8302, 0xa438, 0x8480, 0xa438, 0x8642, 0xa438, 0x1000, + 0xa438, 0x9d44, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8108, + 0xa438, 0x9503, 0xa438, 0x8c40, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8750, 0xa438, 0x9503, 0xa438, 0xaa10, 0xa438, 0xd706, + 0xa438, 0x4063, 0xa438, 0x8604, 0xa438, 0xf022, 0xa438, 0xa604, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8220, 0xa438, 0x8301, + 0xa438, 0x8420, 0xa438, 0x8108, 0xa438, 0x0c03, 0xa438, 0x0201, + 0xa438, 0x9503, 0xa438, 0x0c07, 0xa438, 0x0b01, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce0e, 0xa438, 0xf00a, + 0xa438, 0xce0e, 0xa438, 0xf008, 0xa438, 0xce0d, 0xa438, 0xf006, + 0xa438, 0xce0d, 0xa438, 0xf004, 0xa438, 0xce0c, 0xa438, 0xf002, + 0xa438, 0xce0c, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xab08, + 0xa438, 0xcda0, 0xa438, 0xd706, 0xa438, 0x4043, 0xa438, 0xf0b1, + 0xa438, 0xd705, 0xa438, 0x411e, 0xa438, 0xd162, 0xa438, 0xd045, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa008, 0xa438, 0x9503, + 0xa438, 0xf0a8, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8008, + 0xa438, 0x9503, 0xa438, 0xd1c4, 0xa438, 0xd055, 0xa438, 0xf0a1, + 0xa438, 0xd700, 0xa438, 0x605a, 0xa438, 0xf09e, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8008, 0xa438, 0x8210, 0xa438, 0xa210, + 0xa438, 0x9503, 0xa438, 0xd162, 0xa438, 0xd055, 0xa438, 0xd704, + 0xa438, 0x608f, 0xa438, 0xd706, 0xa438, 0x4051, 0xa438, 0xf04a, + 0xa438, 0xd705, 0xa438, 0x415e, 0xa438, 0xd705, 0xa438, 0x6112, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c06, 0xa438, 0x0002, + 0xa438, 0xa110, 0xa438, 0xa305, 0xa438, 0x9503, 0xa438, 0xd705, + 0xa438, 0x40de, 0xa438, 0xd706, 0xa438, 0x6771, 0xa438, 0xd700, + 0xa438, 0x60bb, 0xa438, 0xf07f, 0xa438, 0xd700, 0xa438, 0x661b, + 0xa438, 0xf07c, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8090, + 0xa438, 0x8101, 0xa438, 0x0ccf, 0xa438, 0x0b03, 0xa438, 0x8c07, + 0xa438, 0xd700, 0xa438, 0x4087, 0xa438, 0x0c38, 0xa438, 0x0d18, + 0xa438, 0xf003, 0xa438, 0x0cf8, 0xa438, 0x0d58, 0xa438, 0x8270, + 0xa438, 0x8408, 0xa438, 0x8301, 0xa438, 0x9503, 0xa438, 0x8106, + 0xa438, 0x9c01, 0xa438, 0xd705, 0xa438, 0x40b5, 0xa438, 0x1000, + 0xa438, 0x9cb7, 0xa438, 0xd601, 0xa438, 0xd628, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8208, 0xa438, 0x9503, 0xa438, 0xcdac, + 0xa438, 0xd1c4, 0xa438, 0xd054, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0x1000, 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, + 0xa438, 0xd700, 0xa438, 0x5f3b, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0xa230, 0xa438, 0x9503, 0xa438, 0xf04e, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa404, 0xa438, 0x8230, 0xa438, 0x9503, + 0xa438, 0xf048, 0xa438, 0xd700, 0xa438, 0x48db, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8204, 0xa438, 0x8410, 0xa438, 0x9503, + 0xa438, 0x8b08, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce08, 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, + 0xa438, 0xce08, 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, + 0xa438, 0xce08, 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xa480, + 0xa438, 0xd705, 0xa438, 0x611e, 0xa438, 0xa302, 0xa438, 0x1000, + 0xa438, 0x9cc6, 0xa438, 0xd707, 0xa438, 0x404e, 0xa438, 0x8a10, + 0xa438, 0xf006, 0xa438, 0x8302, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8301, 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x6047, + 0xa438, 0xa682, 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, + 0xa438, 0x407d, 0xa438, 0x8644, 0xa438, 0xf003, 0xa438, 0x0c44, + 0xa438, 0x0604, 0xa438, 0xd701, 0xa438, 0x40d0, 0xa438, 0xac40, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa750, 0xa438, 0x9503, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa220, 0xa438, 0x8210, + 0xa438, 0xa210, 0xa438, 0xa620, 0xa438, 0x9503, 0xa438, 0xcdaa, + 0xa438, 0x0800, 0xa438, 0xd202, 0xa438, 0xb309, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fa1, 0xa438, 0xd400, + 0xa438, 0xb302, 0xa438, 0xd200, 0xa438, 0x0800, 0xa438, 0xd71f, + 0xa438, 0x6261, 0xa438, 0xcdff, 0xa438, 0xd705, 0xa438, 0x41d5, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c1f, 0xa438, 0x0e12, + 0xa438, 0x9503, 0xa438, 0xd70c, 0xa438, 0x5ff3, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x8e1f, 0xa438, 0x9503, 0xa438, 0xd70c, + 0xa438, 0x7f73, 0xa438, 0x1800, 0xa438, 0x001c, 0xa438, 0x0800, + 0xa438, 0xab10, 0xa438, 0xd71f, 0xa438, 0x210c, 0xa438, 0x001c, + 0xa438, 0xd701, 0xa438, 0x5f98, 0xa438, 0x8b10, 0xa438, 0x0800, + 0xa438, 0xd71f, 0xa438, 0x210c, 0xa438, 0x001c, 0xa438, 0xd701, + 0xa438, 0x5f99, 0xa438, 0x0800, 0xa438, 0xa110, 0xa438, 0x1000, + 0xa438, 0x9be6, 0xa438, 0x8110, 0xa438, 0xa140, 0xa438, 0x1000, + 0xa438, 0x9be6, 0xa438, 0x8140, 0xa438, 0xa004, 0xa438, 0x1000, + 0xa438, 0x9be6, 0xa438, 0x8004, 0xa438, 0xa001, 0xa438, 0x1000, + 0xa438, 0x9be6, 0xa438, 0x8001, 0xa438, 0xa020, 0xa438, 0x1000, + 0xa438, 0x9be6, 0xa438, 0x8020, 0xa438, 0xa101, 0xa438, 0x0800, + 0xa438, 0xd202, 0xa438, 0x0c09, 0xa438, 0x1301, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fa1, 0xa438, 0xb302, + 0xa438, 0xd200, 0xa438, 0x0800, 0xa438, 0xd204, 0xa438, 0x0c09, + 0xa438, 0x1301, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, + 0xa438, 0x5fa2, 0xa438, 0xb302, 0xa438, 0xd200, 0xa438, 0x0800, + 0xa438, 0xd208, 0xa438, 0x0c09, 0xa438, 0x1301, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fa3, 0xa438, 0xb302, + 0xa438, 0xd200, 0xa438, 0x0800, 0xa438, 0xd210, 0xa438, 0x0c09, + 0xa438, 0x1301, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, + 0xa438, 0x5fa4, 0xa438, 0xb302, 0xa438, 0xd200, 0xa438, 0x0800, + 0xa438, 0xac01, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd707, + 0xa438, 0x5fad, 0xa438, 0x0800, 0xa438, 0xac04, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd707, 0xa438, 0x5fab, 0xa438, 0x0800, + 0xa438, 0xd71f, 0xa438, 0x4354, 0xa438, 0x8810, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xb920, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, + 0xa438, 0x9920, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x6085, 0xa438, 0xd71f, 0xa438, 0x7e34, 0xa438, 0xfffa, + 0xa438, 0xb820, 0xa438, 0xa810, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0x0800, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa308, 0xa438, 0x9503, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fb3, + 0xa438, 0x0800, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8308, + 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, + 0xa438, 0x7fb3, 0xa438, 0x0800, 0xa438, 0xc000, 0xa438, 0xc100, + 0xa438, 0xc200, 0xa438, 0xc300, 0xa438, 0xc400, 0xa438, 0xc500, + 0xa438, 0xc600, 0xa438, 0xc700, 0xa438, 0xc828, 0xa438, 0xc904, + 0xa438, 0xca00, 0xa438, 0xcb00, 0xa438, 0xcc00, 0xa438, 0xce00, + 0xa438, 0xcf00, 0xa438, 0xd000, 0xa438, 0xd100, 0xa438, 0xd200, + 0xa438, 0xd300, 0xa438, 0xd400, 0xa438, 0xd500, 0xa438, 0xd700, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xc000, 0xa438, 0xc100, + 0xa438, 0xc200, 0xa438, 0xc300, 0xa438, 0xc400, 0xa438, 0xc500, + 0xa438, 0xc600, 0xa438, 0xa63c, 0xa438, 0xc700, 0xa438, 0xc800, + 0xa438, 0xc900, 0xa438, 0xca00, 0xa438, 0xcb00, 0xa438, 0xcc00, + 0xa438, 0xcd00, 0xa438, 0xce00, 0xa438, 0xcf00, 0xa438, 0x9503, + 0xa438, 0xcd00, 0xa438, 0x0800, 0xa438, 0xd601, 0xa438, 0xd608, + 0xa438, 0xd610, 0xa438, 0xd618, 0xa438, 0xd620, 0xa438, 0xd628, + 0xa438, 0xd630, 0xa438, 0xd638, 0xa438, 0x0800, 0xa438, 0x8b0f, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8097, 0xa438, 0x8109, + 0xa438, 0x82ff, 0xa438, 0x843f, 0xa438, 0x9503, 0xa438, 0x9920, + 0xa438, 0x9b08, 0xa438, 0x9a10, 0xa438, 0xd705, 0xa438, 0x4235, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c1f, 0xa438, 0x0e12, + 0xa438, 0x8440, 0xa438, 0x9503, 0xa438, 0xd70c, 0xa438, 0x5ff3, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8e1f, 0xa438, 0x9503, + 0xa438, 0xd70c, 0xa438, 0x7f73, 0xa438, 0xd601, 0xa438, 0xd628, + 0xa438, 0x0800, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c1f, + 0xa438, 0x0e12, 0xa438, 0x8440, 0xa438, 0x9503, 0xa438, 0xd70c, + 0xa438, 0x5ff3, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8e1f, + 0xa438, 0x9503, 0xa438, 0xd70c, 0xa438, 0x7f73, 0xa438, 0x0800, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x9ccf, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c70, 0xa438, 0x0350, 0xa438, 0x9503, + 0xa438, 0xf00e, 0xa438, 0xd702, 0xa438, 0x40f9, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0x0c70, 0xa438, 0x0340, 0xa438, 0x9503, + 0xa438, 0xf006, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c70, + 0xa438, 0x0340, 0xa438, 0x9503, 0xa438, 0x0800, 0xa438, 0xd700, + 0xa438, 0x37c9, 0xa438, 0x9ce6, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c70, 0xa438, 0x0370, 0xa438, 0x9503, 0xa438, 0xf00e, + 0xa438, 0xd702, 0xa438, 0x40f9, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x0c70, 0xa438, 0x0350, 0xa438, 0x9503, 0xa438, 0xf006, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c70, 0xa438, 0x0350, + 0xa438, 0x9503, 0xa438, 0x0800, 0xa438, 0x8910, 0xa438, 0xd704, + 0xa438, 0x61a8, 0xa438, 0xd702, 0xa438, 0x60d6, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xaf40, 0xa438, 0x9503, 0xa438, 0xf00a, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8f40, 0xa438, 0x9503, + 0xa438, 0xf005, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8f40, + 0xa438, 0x9503, 0xa438, 0x0800, 0xa438, 0x8910, 0xa438, 0xd705, + 0xa438, 0x4059, 0xa438, 0xa910, 0xa438, 0xd704, 0xa438, 0x6068, + 0xa438, 0x0000, 0xa438, 0xf002, 0xa438, 0x0000, 0xa438, 0x8910, + 0xa438, 0x0800, 0xa438, 0xd703, 0xa438, 0x6080, 0xa438, 0x6121, + 0xa438, 0x61c2, 0xa438, 0x6263, 0xa438, 0xd707, 0xa438, 0x4070, + 0xa438, 0xce98, 0xa438, 0xf015, 0xa438, 0xce94, 0xa438, 0xf013, + 0xa438, 0xd707, 0xa438, 0x4070, 0xa438, 0xce99, 0xa438, 0xf00f, + 0xa438, 0xce95, 0xa438, 0xf00d, 0xa438, 0xd707, 0xa438, 0x4070, + 0xa438, 0xce9a, 0xa438, 0xf009, 0xa438, 0xce96, 0xa438, 0xf007, + 0xa438, 0xd707, 0xa438, 0x4070, 0xa438, 0xce9b, 0xa438, 0xf003, + 0xa438, 0xce97, 0xa438, 0xf001, 0xa438, 0x0800, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce00, 0xa438, 0xf00a, + 0xa438, 0xce00, 0xa438, 0xf008, 0xa438, 0xce00, 0xa438, 0xf006, + 0xa438, 0xce00, 0xa438, 0xf004, 0xa438, 0xce00, 0xa438, 0xf002, + 0xa438, 0xce00, 0xa438, 0x0800, 0xa438, 0x0c03, 0xa438, 0x1502, + 0xa438, 0x8702, 0xa438, 0x9503, 0xa438, 0xd101, 0xa438, 0xd040, + 0xa438, 0xd700, 0xa438, 0x5ffa, 0xa438, 0x8680, 0xa438, 0xd700, + 0xa438, 0x60a7, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa702, + 0xa438, 0x9503, 0xa438, 0x0800, 0xa438, 0xcdfe, 0xa438, 0xa708, + 0xa438, 0xa2fc, 0xa438, 0xba20, 0xa438, 0xa980, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, + 0xa438, 0x9920, 0xa438, 0x0ca0, 0xa438, 0x0480, 0xa438, 0xd706, + 0xa438, 0x5fe7, 0xa438, 0x84a0, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x9920, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fa5, + 0xa438, 0xb820, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, + 0xa438, 0x7f85, 0xa438, 0x9820, 0xa438, 0xa810, 0xa438, 0xd700, + 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, + 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf00a, + 0xa438, 0xce08, 0xa438, 0xf008, 0xa438, 0xce08, 0xa438, 0xf006, + 0xa438, 0xce08, 0xa438, 0xf004, 0xa438, 0xce08, 0xa438, 0xf002, + 0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xa120, + 0xa438, 0xa4a0, 0xa438, 0xa980, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd706, 0xa438, 0x5faf, 0xa438, 0x1800, 0xa438, 0x9de8, + 0xa438, 0xcdfd, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, + 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, + 0xa438, 0xce08, 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, + 0xa438, 0xce08, 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, + 0xa438, 0xce08, 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, + 0xa438, 0x9bde, 0xa438, 0xa980, 0xa438, 0xa810, 0xa438, 0xa00a, + 0xa438, 0xa1a0, 0xa438, 0xa312, 0xa438, 0xa4a0, 0xa438, 0xa604, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5faf, + 0xa438, 0x0c30, 0xa438, 0x0320, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd704, 0xa438, 0x5fa4, 0xa438, 0x800a, 0xa438, 0x81a0, + 0xa438, 0x8312, 0xa438, 0x84a0, 0xa438, 0x8604, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920, + 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, + 0xa438, 0x9920, 0xa438, 0x8340, 0xa438, 0xa801, 0xa438, 0xa980, + 0xa438, 0x8240, 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xa302, + 0xa438, 0xa4a0, 0xa438, 0xa604, 0xa438, 0xd1c8, 0xa438, 0xd045, + 0xa438, 0xd700, 0xa438, 0x5ffa, 0xa438, 0xd706, 0xa438, 0x5faf, + 0xa438, 0xd40a, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd416, + 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x800a, 0xa438, 0x81a0, + 0xa438, 0x8302, 0xa438, 0x8480, 0xa438, 0xa604, 0xa438, 0xa801, + 0xa438, 0x8818, 0xa438, 0xbb10, 0xa438, 0x1000, 0xa438, 0x9bc9, + 0xa438, 0xd71f, 0xa438, 0x5fb5, 0xa438, 0x9b10, 0xa438, 0x1000, + 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb5, 0xa438, 0x1800, + 0xa438, 0x915e, 0xa438, 0x1000, 0xa438, 0x1c45, 0xa438, 0x1000, + 0xa438, 0x9c64, 0xa438, 0xac10, 0xa438, 0x1800, 0xa438, 0x0021, + 0xa436, 0xA10E, 0xa438, 0xffff, 0xa436, 0xA10C, 0xa438, 0xffff, + 0xa436, 0xA10A, 0xa438, 0xffff, 0xa436, 0xA108, 0xa438, 0xffff, + 0xa436, 0xA106, 0xa438, 0xffff, 0xa436, 0xA104, 0xa438, 0xffff, + 0xa436, 0xA102, 0xa438, 0x001c, 0xa436, 0xA100, 0xa438, 0x0073, + 0xa436, 0xA110, 0xa438, 0x0003, 0xa436, 0xA016, 0xa438, 0x0020, + 0xa436, 0xA012, 0xa438, 0x1ff8, 0xa436, 0xA014, 0xa438, 0xb904, + 0xa438, 0xd18a, 0xa438, 0xd17a, 0xa438, 0x9c10, 0xa438, 0x0000, + 0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000, 0xa436, 0xA164, + 0xa438, 0x1CAC, 0xa436, 0xA166, 0xa438, 0x0666, 0xa436, 0xA168, + 0xa438, 0x0669, 0xa436, 0xA16A, 0xa438, 0x001f, 0xa436, 0xA16C, + 0xa438, 0x3fff, 0xa436, 0xA16E, 0xa438, 0x3fff, 0xa436, 0xA170, + 0xa438, 0x3fff, 0xa436, 0xA172, 0xa438, 0x3fff, 0xa436, 0xA162, + 0xa438, 0x000f, 0xa436, 0xb87c, 0xa438, 0x8a23, 0xa436, 0xb87e, + 0xa438, 0xaf8a, 0xa438, 0x3baf, 0xa438, 0x8a4c, 0xa438, 0xaf8a, + 0xa438, 0x5caf, 0xa438, 0x8a62, 0xa438, 0xaf8a, 0xa438, 0x68af, + 0xa438, 0x8a9e, 0xa438, 0xaf8b, 0xa438, 0x4daf, 0xa438, 0x8ba8, + 0xa438, 0xe080, 0xa438, 0x15ad, 0xa438, 0x2505, 0xa438, 0xd101, + 0xa438, 0xaf58, 0xa438, 0xcc02, 0xa438, 0x1e44, 0xa438, 0xaf59, + 0xa438, 0x05ee, 0xa438, 0x88af, 0xa438, 0x0102, 0xa438, 0x7eb6, + 0xa438, 0xad2a, 0xa438, 0x03af, 0xa438, 0x5b41, 0xa438, 0xaf5b, + 0xa438, 0x4a02, 0xa438, 0x8c4c, 0xa438, 0xaf69, 0xa438, 0x4f02, + 0xa438, 0x8d51, 0xa438, 0xaf65, 0xa438, 0xaf02, 0xa438, 0x7b73, + 0xa438, 0xbf7a, 0xa438, 0xa302, 0xa438, 0x7b73, 0xa438, 0xbf8e, + 0xa438, 0x5b02, 0xa438, 0x7b73, 0xa438, 0xbf8e, 0xa438, 0x5e02, + 0xa438, 0x7b73, 0xa438, 0xbf8e, 0xa438, 0x6102, 0xa438, 0x7b73, + 0xa438, 0xbf7a, 0xa438, 0xa602, 0xa438, 0x7b73, 0xa438, 0xbf8e, + 0xa438, 0x6402, 0xa438, 0x7b73, 0xa438, 0xbf8e, 0xa438, 0x6702, + 0xa438, 0x7b73, 0xa438, 0xbf8e, 0xa438, 0x6a02, 0xa438, 0x7b73, + 0xa438, 0xaf64, 0xa438, 0xcfe0, 0xa438, 0x898a, 0xa438, 0xe189, + 0xa438, 0x8bc4, 0xa438, 0xef67, 0xa438, 0xe089, 0xa438, 0x8ce1, + 0xa438, 0x898e, 0xa438, 0x1b01, 0xa438, 0xef20, 0xa438, 0xad27, + 0xa438, 0x04c7, 0xa438, 0x78ff, 0xa438, 0x109e, 0xa438, 0x050d, + 0xa438, 0x8180, 0xa438, 0xaef9, 0xa438, 0xac37, 0xa438, 0x09cf, + 0xa438, 0x027a, 0xa438, 0xcdac, 0xa438, 0x5030, 0xa438, 0xae13, + 0xa438, 0xe089, 0xa438, 0x8ae1, 0xa438, 0x898b, 0xa438, 0xef74, + 0xa438, 0xef46, 0xa438, 0xce02, 0xa438, 0x7acd, 0xa438, 0xef64, + 0xa438, 0xac50, 0xa438, 0x1be0, 0xa438, 0x8991, 0xa438, 0x3802, + 0xa438, 0xac27, 0xa438, 0x13ef, 0xa438, 0x46e4, 0xa438, 0x898a, + 0xa438, 0xe589, 0xa438, 0x8b3b, 0xa438, 0x02e7, 0xa438, 0x898d, + 0xa438, 0xe289, 0xa438, 0x8ee6, 0xa438, 0x898c, 0xa438, 0x1f44, + 0xa438, 0xe189, 0xa438, 0x68e2, 0xa438, 0x898e, 0xa438, 0x3a09, + 0xa438, 0xef32, 0xa438, 0xad37, 0xa438, 0x06c6, 0xa438, 0xef64, + 0xa438, 0x7aff, 0xa438, 0x129e, 0xa438, 0x050d, 0xa438, 0x6182, + 0xa438, 0xaef9, 0xa438, 0xac3f, 0xa438, 0x04ef, 0xa438, 0x74ae, + 0xa438, 0x03ef, 0xa438, 0x76ce, 0xa438, 0x027a, 0xa438, 0xcdad, + 0xa438, 0x501e, 0xa438, 0xe089, 0xa438, 0x9138, 0xa438, 0x02ac, + 0xa438, 0x2709, 0xa438, 0xe089, 0xa438, 0x8f10, 0xa438, 0xe489, + 0xa438, 0x8fae, 0xa438, 0x0de0, 0xa438, 0x8991, 0xa438, 0x2801, + 0xa438, 0xe189, 0xa438, 0x921e, 0xa438, 0x10e5, 0xa438, 0x8992, + 0xa438, 0xe089, 0xa438, 0x9110, 0xa438, 0xe489, 0xa438, 0x91af, + 0xa438, 0x68c5, 0xa438, 0xcffb, 0xa438, 0xee89, 0xa438, 0x6700, + 0xa438, 0xe389, 0xa438, 0x671f, 0xa438, 0x77d8, 0xa438, 0xa000, + 0xa438, 0x1213, 0xa438, 0xe789, 0xa438, 0x67d2, 0xa438, 0x031a, + 0xa438, 0x92e1, 0xa438, 0x8966, 0xa438, 0x1b13, 0xa438, 0x9fed, + 0xa438, 0xaf8b, 0xa438, 0x87a0, 0xa438, 0x040b, 0xa438, 0xbcbf, + 0xa438, 0x7ab2, 0xa438, 0x027d, 0xa438, 0x5abd, 0xa438, 0xac28, + 0xa438, 0xe0d8, 0xa438, 0x1f11, 0xa438, 0x0267, 0xa438, 0xc3d7, + 0xa438, 0x0001, 0xa438, 0xaed5, 0xa438, 0xad50, 0xa438, 0x02ae, + 0xa438, 0x12e0, 0xa438, 0x8987, 0xa438, 0xad20, 0xa438, 0x06ee, + 0xa438, 0x8987, 0xa438, 0x00ae, 0xa438, 0x06ee, 0xa438, 0x8989, + 0xa438, 0x00ae, 0xa438, 0x06e0, 0xa438, 0x895c, 0xa438, 0xe489, + 0xa438, 0x89ff, 0xa438, 0xaf6b, 0xa438, 0x3cd4, 0xa438, 0x000f, + 0xa438, 0xbf79, 0xa438, 0x3202, 0xa438, 0x7b9f, 0xa438, 0xd400, + 0xa438, 0x04bf, 0xa438, 0x5075, 0xa438, 0xd700, 0xa438, 0x0802, + 0xa438, 0x7da0, 0xa438, 0xbf8e, 0xa438, 0x6d02, 0xa438, 0x7b73, + 0xa438, 0xd402, 0xa438, 0x00bf, 0xa438, 0x8e70, 0xa438, 0x027b, + 0xa438, 0x9fd4, 0xa438, 0x000f, 0xa438, 0xbf8e, 0xa438, 0x7302, + 0xa438, 0x7b9f, 0xa438, 0xbf8e, 0xa438, 0x7302, 0xa438, 0x7b73, + 0xa438, 0xd41b, 0xa438, 0x3ebf, 0xa438, 0x8e6d, 0xa438, 0x027b, + 0xa438, 0x9fd4, 0xa438, 0x00dc, 0xa438, 0xbf8e, 0xa438, 0x7002, + 0xa438, 0x7b9f, 0xa438, 0xd400, 0xa438, 0x0fbf, 0xa438, 0x8e73, + 0xa438, 0x027b, 0xa438, 0x9fbf, 0xa438, 0x8e73, 0xa438, 0x027b, + 0xa438, 0x73d4, 0xa438, 0x1bbe, 0xa438, 0xbf8e, 0xa438, 0x6d02, + 0xa438, 0x7b9f, 0xa438, 0xd400, 0xa438, 0xdcbf, 0xa438, 0x8e70, + 0xa438, 0x027b, 0xa438, 0x9fd4, 0xa438, 0x000f, 0xa438, 0xbf8e, + 0xa438, 0x7302, 0xa438, 0x7b9f, 0xa438, 0xbf8e, 0xa438, 0x7302, + 0xa438, 0x7b73, 0xa438, 0xd700, 0xa438, 0x86b7, 0xa438, 0xfed4, + 0xa438, 0x003e, 0xa438, 0xbf8e, 0xa438, 0x6d02, 0xa438, 0x7b9f, + 0xa438, 0xd400, 0xa438, 0xdcbf, 0xa438, 0x8e70, 0xa438, 0x027b, + 0xa438, 0x9fd4, 0xa438, 0x000f, 0xa438, 0xbf8e, 0xa438, 0x7302, + 0xa438, 0x7b9f, 0xa438, 0xbf8e, 0xa438, 0x7302, 0xa438, 0x7b73, + 0xa438, 0xbf79, 0xa438, 0x3202, 0xa438, 0x7b73, 0xa438, 0xaf51, + 0xa438, 0x4ff8, 0xa438, 0xf9fa, 0xa438, 0xcefb, 0xa438, 0xef79, + 0xa438, 0xfbe0, 0xa438, 0x8967, 0xa438, 0xef10, 0xa438, 0x4803, + 0xa438, 0xbf89, 0xa438, 0x721a, 0xa438, 0x90ef, 0xa438, 0x79bf, + 0xa438, 0x8982, 0xa438, 0x1a91, 0xa438, 0xec01, 0xa438, 0x0702, + 0xa438, 0x7eb6, 0xa438, 0xad29, 0xa438, 0x10e0, 0xa438, 0x898f, + 0xa438, 0x3804, 0xa438, 0x9f06, 0xa438, 0xef97, 0xa438, 0x1919, + 0xa438, 0xec02, 0xa438, 0xaf8d, 0xa438, 0x48ef, 0xa438, 0x97ec, + 0xa438, 0x04e0, 0xa438, 0x898f, 0xa438, 0x380c, 0xa438, 0x9e3d, + 0xa438, 0xe089, 0xa438, 0x8d38, 0xa438, 0x03ac, 0xa438, 0x273d, + 0xa438, 0xe089, 0xa438, 0x8d38, 0xa438, 0x039e, 0xa438, 0x3f07, + 0xa438, 0xec03, 0xa438, 0x07e0, 0xa438, 0x898d, 0xa438, 0x3804, + 0xa438, 0x9e53, 0xa438, 0x19ec, 0xa438, 0x02e0, 0xa438, 0x898d, + 0xa438, 0x3807, 0xa438, 0xac27, 0xa438, 0x6089, 0xa438, 0xec03, + 0xa438, 0xe089, 0xa438, 0x8d38, 0xa438, 0x079e, 0xa438, 0x5807, + 0xa438, 0xec04, 0xa438, 0x07e0, 0xa438, 0x898d, 0xa438, 0x3808, + 0xa438, 0x9e61, 0xa438, 0xaf8d, 0xa438, 0x4819, 0xa438, 0xec03, + 0xa438, 0x19ec, 0xa438, 0x02ae, 0xa438, 0x7819, 0xa438, 0xec03, + 0xa438, 0x07ec, 0xa438, 0x0207, 0xa438, 0xae6f, 0xa438, 0xe089, + 0xa438, 0x5da0, 0xa438, 0x010e, 0xa438, 0xe089, 0xa438, 0x92a0, + 0xa438, 0x0308, 0xa438, 0x1f55, 0xa438, 0x07ec, 0xa438, 0x0307, + 0xa438, 0xae55, 0xa438, 0x07ec, 0xa438, 0x0207, 0xa438, 0xd103, + 0xa438, 0x026b, 0xa438, 0x74ae, 0xa438, 0x50e0, 0xa438, 0x895d, + 0xa438, 0xa001, 0xa438, 0x0be0, 0xa438, 0x8992, 0xa438, 0xad21, + 0xa438, 0x05d5, 0xa438, 0x010f, 0xa438, 0xae39, 0xa438, 0xd102, + 0xa438, 0x026b, 0xa438, 0x74ae, 0xa438, 0x38ae, 0xa438, 0x36e0, + 0xa438, 0x895d, 0xa438, 0xa001, 0xa438, 0x1fe0, 0xa438, 0x8992, + 0xa438, 0xa003, 0xa438, 0x191f, 0xa438, 0x5507, 0xa438, 0xec04, + 0xa438, 0x07ae, 0xa438, 0x17e0, 0xa438, 0x895d, 0xa438, 0xa001, + 0xa438, 0x0be0, 0xa438, 0x8992, 0xa438, 0xad21, 0xa438, 0x05d5, + 0xa438, 0x010f, 0xa438, 0xae06, 0xa438, 0xef97, 0xa438, 0xec00, + 0xa438, 0xae0b, 0xa438, 0x0266, 0xa438, 0xa7ae, 0xa438, 0x0602, + 0xa438, 0x66a7, 0xa438, 0x19ec, 0xa438, 0x02ff, 0xa438, 0xef97, + 0xa438, 0xffc6, 0xa438, 0xfefd, 0xa438, 0xfc04, 0xa438, 0xf8f9, + 0xa438, 0xfaef, 0xa438, 0x69fb, 0xa438, 0xcfd3, 0xa438, 0x00e7, + 0xa438, 0x8967, 0xa438, 0xbf89, 0xa438, 0x6e1a, 0xa438, 0x93bc, + 0xa438, 0xdaad, 0xa438, 0x302d, 0xa438, 0xbf76, 0xa438, 0x111f, + 0xa438, 0x4402, 0xa438, 0x7d66, 0xa438, 0x0266, 0xa438, 0xfcbf, + 0xa438, 0x8982, 0xa438, 0xe189, 0xa438, 0x67a1, 0xa438, 0x0001, + 0xa438, 0x191f, 0xa438, 0x44d9, 0xa438, 0x81bf, 0xa438, 0x7aa0, + 0xa438, 0x027d, 0xa438, 0x66bf, 0xa438, 0x7611, 0xa438, 0xd400, + 0xa438, 0x0102, 0xa438, 0x7d66, 0xa438, 0xbdae, 0xa438, 0x02ae, + 0xa438, 0xc9da, 0xa438, 0xad31, 0xa438, 0x31bf, 0xa438, 0x761d, + 0xa438, 0x1f44, 0xa438, 0x027d, 0xa438, 0x6602, 0xa438, 0x66fc, + 0xa438, 0xbf89, 0xa438, 0x8219, 0xa438, 0xe189, 0xa438, 0x67a1, + 0xa438, 0x0202, 0xa438, 0xae06, 0xa438, 0xa103, 0xa438, 0x02ae, + 0xa438, 0x0119, 0xa438, 0x1f44, 0xa438, 0xd981, 0xa438, 0xbf7a, + 0xa438, 0xa302, 0xa438, 0x7d66, 0xa438, 0xbf76, 0xa438, 0x1dd4, + 0xa438, 0x0001, 0xa438, 0x027d, 0xa438, 0x66bd, 0xa438, 0xdaad, + 0xa438, 0x322c, 0xa438, 0xbf76, 0xa438, 0x291f, 0xa438, 0x4402, + 0xa438, 0x7d66, 0xa438, 0x0266, 0xa438, 0xfcbf, 0xa438, 0x8982, + 0xa438, 0x1919, 0xa438, 0xe189, 0xa438, 0x67a1, 0xa438, 0x0302, + 0xa438, 0xae01, 0xa438, 0x191f, 0xa438, 0x44d9, 0xa438, 0x81bf, + 0xa438, 0x7aa6, 0xa438, 0x027d, 0xa438, 0x66bf, 0xa438, 0x7629, + 0xa438, 0xd400, 0xa438, 0x0102, 0xa438, 0x7d66, 0xa438, 0x028e, + 0xa438, 0x0f13, 0xa438, 0xe789, 0xa438, 0x67e0, 0xa438, 0x8966, + 0xa438, 0x1b03, 0xa438, 0x9f8b, 0xa438, 0xc7ff, 0xa438, 0xef96, + 0xa438, 0xfefd, 0xa438, 0xfc04, 0xa438, 0xf8f9, 0xa438, 0xfaef, + 0xa438, 0x69fb, 0xa438, 0xcfe3, 0xa438, 0x8967, 0xa438, 0xbf89, + 0xa438, 0x7e1a, 0xa438, 0x93bc, 0xa438, 0xda6a, 0xa438, 0x009e, + 0xa438, 0x2fbf, 0xa438, 0x7653, 0xa438, 0xe389, 0xa438, 0x671f, + 0xa438, 0x4402, 0xa438, 0x7d66, 0xa438, 0x0267, 0xa438, 0x1abd, + 0xa438, 0x1f00, 0xa438, 0xd959, 0xa438, 0x010c, 0xa438, 0x1269, + 0xa438, 0x08a3, 0xa438, 0x0104, 0xa438, 0x6902, 0xa438, 0xae02, + 0xa438, 0x6901, 0xa438, 0xbf7a, 0xa438, 0xb202, 0xa438, 0x7d66, + 0xa438, 0xd101, 0xa438, 0xbf76, 0xa438, 0x5302, 0xa438, 0x7d66, + 0xa438, 0xc7ff, 0xa438, 0xef96, 0xa438, 0xfefd, 0xa438, 0xfc04, + 0xa438, 0x32b1, 0xa438, 0x4a32, 0xa438, 0xb24a, 0xa438, 0x32b3, + 0xa438, 0x4a10, 0xa438, 0xb14a, 0xa438, 0x10b2, 0xa438, 0x4a10, + 0xa438, 0xb34a, 0xa438, 0xf0bd, 0xa438, 0x94f0, 0xa438, 0xbd92, + 0xa438, 0x74bd, 0xa438, 0x9600, 0xa436, 0xb85e, 0xa438, 0x58ca, + 0xa436, 0xb860, 0xa438, 0x5b3d, 0xa436, 0xb862, 0xa438, 0x694c, + 0xa436, 0xb864, 0xa438, 0x65ac, 0xa436, 0xb886, 0xa438, 0x64cc, + 0xa436, 0xb888, 0xa438, 0x683e, 0xa436, 0xb88a, 0xa438, 0x6ae6, + 0xa436, 0xb88c, 0xa438, 0x5119, 0xa436, 0xb838, 0xa438, 0x00ff, + 0xb820, 0x0010, 0xa464, 0x0001, 0xa436, 0x8474, 0xa438, 0x0000, + 0xa436, 0x8608, 0xa438, 0xaf86, 0xa438, 0x20af, 0xa438, 0x865e, + 0xa438, 0xaf86, 0xa438, 0xacaf, 0xa438, 0x86b7, 0xa438, 0xaf88, + 0xa438, 0xbdaf, 0xa438, 0x88d4, 0xa438, 0xaf88, 0xa438, 0xe3af, + 0xa438, 0x893f, 0xa438, 0xbf8a, 0xa438, 0x1102, 0xa438, 0x7589, + 0xa438, 0xbf8a, 0xa438, 0x1702, 0xa438, 0x756a, 0xa438, 0xbf8a, + 0xa438, 0x1d02, 0xa438, 0x756a, 0xa438, 0xbf8a, 0xa438, 0x1402, + 0xa438, 0x7589, 0xa438, 0xbf8a, 0xa438, 0x1a02, 0xa438, 0x756a, + 0xa438, 0xbf8a, 0xa438, 0x2002, 0xa438, 0x756a, 0xa438, 0xa200, + 0xa438, 0x08bf, 0xa438, 0x6e23, 0xa438, 0x0275, 0xa438, 0x3eae, + 0xa438, 0x0ca2, 0xa438, 0x0609, 0xa438, 0xe08f, 0xa438, 0x7dad, + 0xa438, 0x2003, 0xa438, 0xaf0f, 0xa438, 0xc9af, 0xa438, 0x0fd5, + 0xa438, 0xe084, 0xa438, 0x68a0, 0xa438, 0x0005, 0xa438, 0x0264, + 0xa438, 0xe6ae, 0xa438, 0x0ba0, 0xa438, 0x0105, 0xa438, 0x0265, + 0xa438, 0xb4ae, 0xa438, 0x0302, 0xa438, 0x8677, 0xa438, 0xaf64, + 0xa438, 0xe1f8, 0xa438, 0xf9e0, 0xa438, 0x8469, 0xa438, 0xe184, + 0xa438, 0x6a14, 0xa438, 0xe484, 0xa438, 0x69e5, 0xa438, 0x846a, + 0xa438, 0xe283, 0xa438, 0xade3, 0xa438, 0x83ae, 0xa438, 0x1b45, + 0xa438, 0x9f11, 0xa438, 0xee84, 0xa438, 0x6900, 0xa438, 0xee84, + 0xa438, 0x6a00, 0xa438, 0xee84, 0xa438, 0x6800, 0xa438, 0x0264, + 0xa438, 0xe6ae, 0xa438, 0x08e0, 0xa438, 0x8043, 0xa438, 0xf626, + 0xa438, 0xe480, 0xa438, 0x43fd, 0xa438, 0xfc04, 0xa438, 0xe080, + 0xa438, 0x43f6, 0xa438, 0x26e4, 0xa438, 0x8043, 0xa438, 0xaf65, + 0xa438, 0xa8ee, 0xa438, 0x8468, 0xa438, 0x0202, 0xa438, 0x86c9, + 0xa438, 0xe080, 0xa438, 0x43f6, 0xa438, 0x26e4, 0xa438, 0x8043, + 0xa438, 0xaf66, 0xa438, 0x0bf8, 0xa438, 0xf9ef, 0xa438, 0x59f9, + 0xa438, 0xfafb, 0xa438, 0xe18f, 0xa438, 0x7ea1, 0xa438, 0x0003, + 0xa438, 0xaf88, 0xa438, 0x64bf, 0xa438, 0x6c43, 0xa438, 0x0275, + 0xa438, 0x89d0, 0xa438, 0x003c, 0xa438, 0x008a, 0xa438, 0xad27, + 0xa438, 0x03af, 0xa438, 0x86fe, 0xa438, 0xee8f, 0xa438, 0x7e00, + 0xa438, 0xee84, 0xa438, 0x6d0f, 0xa438, 0xd401, 0xa438, 0x28d6, + 0xa438, 0x0010, 0xa438, 0x0288, 0xa438, 0x6caf, 0xa438, 0x8864, + 0xa438, 0xee84, 0xa438, 0x6d00, 0xa438, 0xd600, 0xa438, 0x8f02, + 0xa438, 0x889b, 0xa438, 0xee84, 0xa438, 0x6d01, 0xa438, 0xd600, + 0xa438, 0xc002, 0xa438, 0x886c, 0xa438, 0xee84, 0xa438, 0x6d01, + 0xa438, 0xd600, 0xa438, 0x8f02, 0xa438, 0x889b, 0xa438, 0xee84, + 0xa438, 0x6d02, 0xa438, 0xd600, 0xa438, 0xc002, 0xa438, 0x886c, + 0xa438, 0xee84, 0xa438, 0x6d02, 0xa438, 0xd600, 0xa438, 0x8f02, + 0xa438, 0x889b, 0xa438, 0xee84, 0xa438, 0x6d04, 0xa438, 0xd600, + 0xa438, 0xc002, 0xa438, 0x886c, 0xa438, 0xee84, 0xa438, 0x6d03, + 0xa438, 0xd600, 0xa438, 0x8f02, 0xa438, 0x889b, 0xa438, 0xee84, + 0xa438, 0x6d08, 0xa438, 0xd600, 0xa438, 0xc002, 0xa438, 0x886c, + 0xa438, 0xee84, 0xa438, 0x6d00, 0xa438, 0xd600, 0xa438, 0x9002, + 0xa438, 0x889b, 0xa438, 0xee84, 0xa438, 0x6d01, 0xa438, 0xd600, + 0xa438, 0xc102, 0xa438, 0x886c, 0xa438, 0xee84, 0xa438, 0x6d01, + 0xa438, 0xd600, 0xa438, 0x9002, 0xa438, 0x889b, 0xa438, 0xee84, + 0xa438, 0x6d02, 0xa438, 0xd600, 0xa438, 0xc102, 0xa438, 0x886c, + 0xa438, 0xee84, 0xa438, 0x6d02, 0xa438, 0xd600, 0xa438, 0x9002, + 0xa438, 0x889b, 0xa438, 0xee84, 0xa438, 0x6d04, 0xa438, 0xd600, + 0xa438, 0xc102, 0xa438, 0x886c, 0xa438, 0xee84, 0xa438, 0x6d03, + 0xa438, 0xd600, 0xa438, 0x9002, 0xa438, 0x889b, 0xa438, 0xee84, + 0xa438, 0x6d08, 0xa438, 0xd600, 0xa438, 0xc102, 0xa438, 0x886c, + 0xa438, 0xee84, 0xa438, 0x6d00, 0xa438, 0xd600, 0xa438, 0x9102, + 0xa438, 0x889b, 0xa438, 0xee84, 0xa438, 0x6d01, 0xa438, 0xd600, + 0xa438, 0xc202, 0xa438, 0x886c, 0xa438, 0xee84, 0xa438, 0x6d01, + 0xa438, 0xd600, 0xa438, 0x9102, 0xa438, 0x889b, 0xa438, 0xee84, + 0xa438, 0x6d02, 0xa438, 0xd600, 0xa438, 0xc202, 0xa438, 0x886c, + 0xa438, 0xee84, 0xa438, 0x6d02, 0xa438, 0xd600, 0xa438, 0x9102, + 0xa438, 0x889b, 0xa438, 0xee84, 0xa438, 0x6d04, 0xa438, 0xd600, + 0xa438, 0xc202, 0xa438, 0x886c, 0xa438, 0xee84, 0xa438, 0x6d03, + 0xa438, 0xd600, 0xa438, 0x9102, 0xa438, 0x889b, 0xa438, 0xee84, + 0xa438, 0x6d08, 0xa438, 0xd600, 0xa438, 0xc202, 0xa438, 0x886c, + 0xa438, 0xee84, 0xa438, 0x6d0f, 0xa438, 0xd414, 0xa438, 0x00d6, + 0xa438, 0x000d, 0xa438, 0x0288, 0xa438, 0x6cbf, 0xa438, 0x8a2c, + 0xa438, 0x0275, 0xa438, 0x3ebf, 0xa438, 0x8a2f, 0xa438, 0x0275, + 0xa438, 0x3ebf, 0xa438, 0x8a32, 0xa438, 0x0275, 0xa438, 0x3ebf, + 0xa438, 0x8a35, 0xa438, 0x0275, 0xa438, 0x3ebf, 0xa438, 0x8a38, + 0xa438, 0x0275, 0xa438, 0x3ed4, 0xa438, 0x000f, 0xa438, 0xbf71, + 0xa438, 0x5c02, 0xa438, 0x756a, 0xa438, 0xbf71, 0xa438, 0x5f02, + 0xa438, 0x7589, 0xa438, 0xef31, 0xa438, 0xe783, 0xa438, 0x9fa3, + 0xa438, 0x0ff2, 0xa438, 0xd400, 0xa438, 0x00bf, 0xa438, 0x715c, + 0xa438, 0x0275, 0xa438, 0x6abf, 0xa438, 0x8a2c, 0xa438, 0x0275, + 0xa438, 0x47bf, 0xa438, 0x8a2f, 0xa438, 0x0275, 0xa438, 0x47bf, + 0xa438, 0x8a32, 0xa438, 0x0275, 0xa438, 0x47bf, 0xa438, 0x8a35, + 0xa438, 0x0275, 0xa438, 0x47bf, 0xa438, 0x8a38, 0xa438, 0x0275, + 0xa438, 0x47ee, 0xa438, 0x846d, 0xa438, 0x0fd4, 0xa438, 0x1000, + 0xa438, 0xd600, 0xa438, 0x0d02, 0xa438, 0x886c, 0xa438, 0xfffe, + 0xa438, 0xfdef, 0xa438, 0x95fd, 0xa438, 0xfc04, 0xa438, 0xf8f9, + 0xa438, 0xef59, 0xa438, 0xf9fa, 0xa438, 0xef44, 0xa438, 0xbf73, + 0xa438, 0xbf02, 0xa438, 0x756a, 0xa438, 0xef46, 0xa438, 0xbf73, + 0xa438, 0xc202, 0xa438, 0x756a, 0xa438, 0xe184, 0xa438, 0x6dbf, + 0xa438, 0x8a29, 0xa438, 0x0275, 0xa438, 0x6ad4, 0xa438, 0x0000, + 0xa438, 0xbf8a, 0xa438, 0x2902, 0xa438, 0x756a, 0xa438, 0xfefd, + 0xa438, 0xef95, 0xa438, 0xfdfc, 0xa438, 0x04f9, 0xa438, 0xef59, + 0xa438, 0xf9fa, 0xa438, 0xef46, 0xa438, 0xbf73, 0xa438, 0xc202, + 0xa438, 0x756a, 0xa438, 0xe184, 0xa438, 0x6dbf, 0xa438, 0x8a23, + 0xa438, 0x0275, 0xa438, 0x6abf, 0xa438, 0x8a26, 0xa438, 0x0275, + 0xa438, 0x89fe, 0xa438, 0xfdef, 0xa438, 0x95fd, 0xa438, 0x04e4, + 0xa438, 0x8044, 0xa438, 0xee84, 0xa438, 0x6d0f, 0xa438, 0xd401, + 0xa438, 0x20d6, 0xa438, 0x0010, 0xa438, 0x0288, 0xa438, 0x6cee, + 0xa438, 0x8f7e, 0xa438, 0x01af, 0xa438, 0x6879, 0xa438, 0xee84, + 0xa438, 0x6900, 0xa438, 0xee84, 0xa438, 0x6a00, 0xa438, 0xee8f, + 0xa438, 0x7e00, 0xa438, 0xaf64, 0xa438, 0x8702, 0xa438, 0x6a2a, + 0xa438, 0xe384, 0xa438, 0xf302, 0xa438, 0x6a4c, 0xa438, 0xac28, + 0xa438, 0x08e0, 0xa438, 0x84f6, 0xa438, 0xf722, 0xa438, 0xe484, + 0xa438, 0xf602, 0xa438, 0x6af9, 0xa438, 0xbf73, 0xa438, 0x4d02, + 0xa438, 0x753e, 0xa438, 0xbf73, 0xa438, 0x5002, 0xa438, 0x753e, + 0xa438, 0xbf73, 0xa438, 0x5302, 0xa438, 0x753e, 0xa438, 0xbf73, + 0xa438, 0x5602, 0xa438, 0x753e, 0xa438, 0xd500, 0xa438, 0x0002, + 0xa438, 0x6b1d, 0xa438, 0xbf73, 0xa438, 0x4402, 0xa438, 0x7547, + 0xa438, 0xbf73, 0xa438, 0x4702, 0xa438, 0x7547, 0xa438, 0xbf73, + 0xa438, 0x7702, 0xa438, 0x753e, 0xa438, 0xbf73, 0xa438, 0x4a02, + 0xa438, 0x7547, 0xa438, 0xbf73, 0xa438, 0x5302, 0xa438, 0x753e, + 0xa438, 0xbf73, 0xa438, 0x5602, 0xa438, 0x7547, 0xa438, 0xaf69, + 0xa438, 0x4d02, 0xa438, 0x8948, 0xa438, 0x021e, 0xa438, 0x40af, + 0xa438, 0x1e3f, 0xa438, 0xf8fa, 0xa438, 0xef69, 0xa438, 0xe080, + 0xa438, 0x4fac, 0xa438, 0x2417, 0xa438, 0xe080, 0xa438, 0x44ad, + 0xa438, 0x2417, 0xa438, 0x0289, 0xa438, 0x74e0, 0xa438, 0x8044, + 0xa438, 0xac24, 0xa438, 0x0ebf, 0xa438, 0x8a3b, 0xa438, 0x0275, + 0xa438, 0x47ae, 0xa438, 0x0602, 0xa438, 0x8a06, 0xa438, 0x0289, + 0xa438, 0xfbef, 0xa438, 0x96fe, 0xa438, 0xfc04, 0xa438, 0xf8f9, + 0xa438, 0xfaef, 0xa438, 0x69fa, 0xa438, 0xfbd2, 0xa438, 0x00a2, + 0xa438, 0x0403, 0xa438, 0xaf89, 0xa438, 0xeabf, 0xa438, 0x6af1, + 0xa438, 0x0277, 0xa438, 0x24ef, 0xa438, 0x010d, 0xa438, 0x11d0, + 0xa438, 0x00ef, 0xa438, 0x640c, 0xa438, 0x66ef, 0xa438, 0x12bf, + 0xa438, 0x8a23, 0xa438, 0x0275, 0xa438, 0x6ad3, 0xa438, 0x01a3, + 0xa438, 0x4302, 0xa438, 0xae44, 0xa438, 0x1f00, 0xa438, 0xef13, + 0xa438, 0xbf73, 0xa438, 0xc202, 0xa438, 0x756a, 0xa438, 0xbf8a, + 0xa438, 0x3e02, 0xa438, 0x7589, 0xa438, 0xd100, 0xa438, 0x0d01, + 0xa438, 0x0c01, 0xa438, 0x1a46, 0xa438, 0xbf73, 0xa438, 0xbf02, + 0xa438, 0x756a, 0xa438, 0x1f00, 0xa438, 0xef13, 0xa438, 0xbf73, + 0xa438, 0xc202, 0xa438, 0x756a, 0xa438, 0xd101, 0xa438, 0xef02, + 0xa438, 0x10b0, 0xa438, 0x02ae, 0xa438, 0x0449, 0xa438, 0x02ae, + 0xa438, 0xf8bf, 0xa438, 0x73c5, 0xa438, 0x0275, 0xa438, 0x6abf, + 0xa438, 0x73c5, 0xa438, 0x0275, 0xa438, 0x3e13, 0xa438, 0xaeb7, + 0xa438, 0x12af, 0xa438, 0x897d, 0xa438, 0xd500, 0xa438, 0x0102, + 0xa438, 0x6b1d, 0xa438, 0x0289, 0xa438, 0xfbff, 0xa438, 0xfeef, + 0xa438, 0x96fe, 0xa438, 0xfdfc, 0xa438, 0x04f8, 0xa438, 0xe080, + 0xa438, 0x44f6, 0xa438, 0x24e4, 0xa438, 0x8044, 0xa438, 0xfc04, + 0xa438, 0xf8e0, 0xa438, 0x804f, 0xa438, 0xf624, 0xa438, 0xe480, + 0xa438, 0x4ffc, 0xa438, 0x0455, 0xa438, 0xa6fe, 0xa438, 0x44a6, + 0xa438, 0xfe66, 0xa438, 0xa4b6, 0xa438, 0x55a4, 0xa438, 0xb666, + 0xa438, 0xac0e, 0xa438, 0x55ac, 0xa438, 0x0efe, 0xa438, 0xbda4, + 0xa438, 0xf0bd, 0xa438, 0x9830, 0xa438, 0xbd96, 0xa438, 0xffbd, + 0xa438, 0xdeee, 0xa438, 0xbdde, 0xa438, 0xddbd, 0xa438, 0xdebb, + 0xa438, 0xbdde, 0xa438, 0xaabd, 0xa438, 0xde44, 0xa438, 0xac00, + 0xa438, 0xf0bd, 0xa438, 0x9a00, 0xa436, 0xb818, 0xa438, 0x0fb9, + 0xa436, 0xb81a, 0xa438, 0x64c3, 0xa436, 0xb81c, 0xa438, 0x64f1, + 0xa436, 0xb81e, 0xa438, 0x6607, 0xa436, 0xb850, 0xa438, 0x6876, + 0xa436, 0xb852, 0xa438, 0x647f, 0xa436, 0xb878, 0xa438, 0x68e8, + 0xa436, 0xb884, 0xa438, 0x1e3c, 0xa436, 0xb832, 0xa438, 0x00df, + 0xB82E, 0x0000, 0xa436, 0x8023, 0xa438, 0x0000, 0xB820, 0x0000, + 0xFFFF, 0xFFFF +}; + +static const u16 phy_mcu_ram_code_8127a_1[] = { + 0xa436, 0x8023, 0xa438, 0x6100, 0xa436, 0xB82E, 0xa438, 0x0001, + 0xb820, 0x0090, 0xa436, 0xA016, 0xa438, 0x0000, 0xa436, 0xA012, + 0xa438, 0x0000, 0xa436, 0xA014, 0xa438, 0x1800, 0xa438, 0x8010, + 0xa438, 0x1800, 0xa438, 0x801a, 0xa438, 0x1800, 0xa438, 0x801a, + 0xa438, 0x1800, 0xa438, 0x801a, 0xa438, 0x1800, 0xa438, 0x801a, + 0xa438, 0x1800, 0xa438, 0x801a, 0xa438, 0x1800, 0xa438, 0x801a, + 0xa438, 0x1800, 0xa438, 0x801a, 0xa438, 0xce00, 0xa438, 0x2941, + 0xa438, 0x8017, 0xa438, 0x2c59, 0xa438, 0x8017, 0xa438, 0x1800, + 0xa438, 0x0e11, 0xa438, 0x8aff, 0xa438, 0x1800, 0xa438, 0x0e11, + 0xa436, 0xA026, 0xa438, 0xffff, 0xa436, 0xA024, 0xa438, 0xffff, + 0xa436, 0xA022, 0xa438, 0xffff, 0xa436, 0xA020, 0xa438, 0xffff, + 0xa436, 0xA006, 0xa438, 0xffff, 0xa436, 0xA004, 0xa438, 0xffff, + 0xa436, 0xA002, 0xa438, 0xffff, 0xa436, 0xA000, 0xa438, 0x0e10, + 0xa436, 0xA008, 0xa438, 0x0100, 0xa436, 0xA016, 0xa438, 0x0000, + 0xa436, 0xA012, 0xa438, 0x0ff8, 0xa436, 0xA014, 0xa438, 0x219a, + 0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000, + 0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000, 0xa436, 0xA152, + 0xa438, 0x21a4, 0xa436, 0xA154, 0xa438, 0x3fff, 0xa436, 0xA156, + 0xa438, 0x3fff, 0xa436, 0xA158, 0xa438, 0x3fff, 0xa436, 0xA15A, + 0xa438, 0x3fff, 0xa436, 0xA15C, 0xa438, 0x3fff, 0xa436, 0xA15E, + 0xa438, 0x3fff, 0xa436, 0xA160, 0xa438, 0x3fff, 0xa436, 0xA150, + 0xa438, 0x0001, 0xa436, 0xA016, 0xa438, 0x0010, 0xa436, 0xA012, + 0xa438, 0x0000, 0xa436, 0xA014, 0xa438, 0x1800, 0xa438, 0x8010, + 0xa438, 0x1800, 0xa438, 0x8014, 0xa438, 0x1800, 0xa438, 0x801a, + 0xa438, 0x1800, 0xa438, 0x801e, 0xa438, 0x1800, 0xa438, 0x8026, + 0xa438, 0x1800, 0xa438, 0x802e, 0xa438, 0x1800, 0xa438, 0x8036, + 0xa438, 0x1800, 0xa438, 0x803a, 0xa438, 0xce01, 0xa438, 0x8208, + 0xa438, 0x1800, 0xa438, 0x0028, 0xa438, 0x1000, 0xa438, 0x02c5, + 0xa438, 0x1000, 0xa438, 0x0304, 0xa438, 0x1800, 0xa438, 0x0119, + 0xa438, 0xce01, 0xa438, 0x8208, 0xa438, 0x1800, 0xa438, 0x009e, + 0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0xa50f, 0xa438, 0x8208, + 0xa438, 0xd500, 0xa438, 0xaa0f, 0xa438, 0x1800, 0xa438, 0x015b, + 0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0xa50f, 0xa438, 0x8208, + 0xa438, 0xd500, 0xa438, 0xaa0f, 0xa438, 0x1800, 0xa438, 0x01a9, + 0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0xa50f, 0xa438, 0x8208, + 0xa438, 0xd500, 0xa438, 0xaa0f, 0xa438, 0x1800, 0xa438, 0x01f4, + 0xa438, 0x8208, 0xa438, 0xd500, 0xa438, 0x1800, 0xa438, 0x02a5, + 0xa438, 0xa208, 0xa438, 0xd500, 0xa438, 0x1800, 0xa438, 0x02b8, + 0xa436, 0xA08E, 0xa438, 0x02b7, 0xa436, 0xA08C, 0xa438, 0x02a4, + 0xa436, 0xA08A, 0xa438, 0x01e7, 0xa436, 0xA088, 0xa438, 0x019c, + 0xa436, 0xA086, 0xa438, 0x014e, 0xa436, 0xA084, 0xa438, 0x009d, + 0xa436, 0xA082, 0xa438, 0x0117, 0xa436, 0xA080, 0xa438, 0x0027, + 0xa436, 0xA090, 0xa438, 0x00ff, 0xa436, 0xA016, 0xa438, 0x0020, + 0xa436, 0xA012, 0xa438, 0x0000, 0xa436, 0xA014, 0xa438, 0x1800, + 0xa438, 0x8010, 0xa438, 0x1800, 0xa438, 0x801d, 0xa438, 0x1800, + 0xa438, 0x803b, 0xa438, 0x1800, 0xa438, 0x8087, 0xa438, 0x1800, + 0xa438, 0x808e, 0xa438, 0x1800, 0xa438, 0x809d, 0xa438, 0x1800, + 0xa438, 0x80b7, 0xa438, 0x1800, 0xa438, 0x80c4, 0xa438, 0xd1bc, + 0xa438, 0xd040, 0xa438, 0x1000, 0xa438, 0x1cd2, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd700, 0xa438, 0x273d, 0xa438, 0x801b, + 0xa438, 0x1800, 0xa438, 0x07d1, 0xa438, 0x1800, 0xa438, 0x080e, + 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8032, 0xa438, 0x33a9, + 0xa438, 0x802a, 0xa438, 0xd705, 0xa438, 0x4084, 0xa438, 0xd1f4, + 0xa438, 0xd048, 0xa438, 0xf013, 0xa438, 0xd1b7, 0xa438, 0xd04b, + 0xa438, 0xf010, 0xa438, 0xd705, 0xa438, 0x4084, 0xa438, 0xd1f4, + 0xa438, 0xd048, 0xa438, 0xf00b, 0xa438, 0xd1b7, 0xa438, 0xd04b, + 0xa438, 0xf008, 0xa438, 0xd705, 0xa438, 0x4084, 0xa438, 0xd1f4, + 0xa438, 0xd048, 0xa438, 0xf003, 0xa438, 0xd1b7, 0xa438, 0xd04b, + 0xa438, 0x1800, 0xa438, 0x14cc, 0xa438, 0xd700, 0xa438, 0x2b59, + 0xa438, 0x803f, 0xa438, 0xf003, 0xa438, 0x1800, 0xa438, 0x118f, + 0xa438, 0x6060, 0xa438, 0x1800, 0xa438, 0x1167, 0xa438, 0xd700, + 0xa438, 0x60c7, 0xa438, 0xd704, 0xa438, 0x609f, 0xa438, 0xd705, + 0xa438, 0x4043, 0xa438, 0xf003, 0xa438, 0x1800, 0xa438, 0x1150, + 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8702, 0xa438, 0x8011, + 0xa438, 0x9503, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8302, + 0xa438, 0x8480, 0xa438, 0x8686, 0xa438, 0xcde0, 0xa438, 0xd1ff, + 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x1cd2, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0xd705, 0xa438, 0x417e, 0xa438, 0x0c03, + 0xa438, 0x1502, 0xa438, 0xa011, 0xa438, 0x9503, 0xa438, 0xd1c8, + 0xa438, 0xd045, 0xa438, 0x1000, 0xa438, 0x1cd2, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa702, + 0xa438, 0x9503, 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xa480, + 0xa438, 0xa686, 0xa438, 0xd705, 0xa438, 0x605e, 0xa438, 0xa302, + 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8083, + 0xa438, 0x33a9, 0xa438, 0x807f, 0xa438, 0xd178, 0xa438, 0xd04b, + 0xa438, 0x1800, 0xa438, 0x115d, 0xa438, 0xd1c8, 0xa438, 0xd04b, + 0xa438, 0x1800, 0xa438, 0x115d, 0xa438, 0xd1e6, 0xa438, 0xd04b, + 0xa438, 0x1800, 0xa438, 0x115d, 0xa438, 0xd71f, 0xa438, 0x6080, + 0xa438, 0xd704, 0xa438, 0x1800, 0xa438, 0x1bc0, 0xa438, 0x1800, + 0xa438, 0x1bc4, 0xa438, 0x4134, 0xa438, 0xd115, 0xa438, 0xd04f, + 0xa438, 0x1000, 0xa438, 0x1d0b, 0xa438, 0x1000, 0xa438, 0x80ad, + 0xa438, 0x1800, 0xa438, 0x01f2, 0xa438, 0x1000, 0xa438, 0x1d0b, + 0xa438, 0x1000, 0xa438, 0x80ad, 0xa438, 0x1800, 0xa438, 0x01f9, + 0xa438, 0x2969, 0xa438, 0x80a3, 0xa438, 0xd700, 0xa438, 0x606b, + 0xa438, 0xd701, 0xa438, 0x60b4, 0xa438, 0x1000, 0xa438, 0x80ad, + 0xa438, 0x1800, 0xa438, 0x0551, 0xa438, 0xd196, 0xa438, 0xd04d, + 0xa438, 0x1000, 0xa438, 0x80ad, 0xa438, 0x1800, 0xa438, 0x054d, + 0xa438, 0xd208, 0xa438, 0x0c09, 0xa438, 0x1301, 0xa438, 0x1000, + 0xa438, 0x1cd2, 0xa438, 0xd701, 0xa438, 0x5fa3, 0xa438, 0xb302, + 0xa438, 0xd200, 0xa438, 0x0800, 0xa438, 0xd705, 0xa438, 0x6064, + 0xa438, 0x1800, 0xa438, 0x140a, 0xa438, 0x8810, 0xa438, 0xd199, + 0xa438, 0xd04b, 0xa438, 0x1000, 0xa438, 0x1cd2, 0xa438, 0xd700, + 0xa438, 0x5fba, 0xa438, 0x1800, 0xa438, 0x140a, 0xa436, 0xA10E, + 0xa438, 0xffff, 0xa436, 0xA10C, 0xa438, 0x1352, 0xa436, 0xA10A, + 0xa438, 0x0545, 0xa436, 0xA108, 0xa438, 0x01ed, 0xa436, 0xA106, + 0xa438, 0x1bbf, 0xa436, 0xA104, 0xa438, 0x114b, 0xa436, 0xA102, + 0xa438, 0x14bf, 0xa436, 0xA100, 0xa438, 0x07ce, 0xa436, 0xA110, + 0xa438, 0x007f, 0xa436, 0xA016, 0xa438, 0x0020, 0xa436, 0xA012, + 0xa438, 0x1ff8, 0xa436, 0xA014, 0xa438, 0xd1ce, 0xa438, 0x0000, + 0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000, + 0xa438, 0x0000, 0xa438, 0x0000, 0xa436, 0xA164, 0xa438, 0x07fc, + 0xa436, 0xA166, 0xa438, 0x143d, 0xa436, 0xA168, 0xa438, 0x3fff, + 0xa436, 0xA16A, 0xa438, 0x3fff, 0xa436, 0xA16C, 0xa438, 0x3fff, + 0xa436, 0xA16E, 0xa438, 0x3fff, 0xa436, 0xA170, 0xa438, 0x3fff, + 0xa436, 0xA172, 0xa438, 0x3fff, 0xa436, 0xA162, 0xa438, 0x0003, + 0xa436, 0xb87c, 0xa438, 0x8994, 0xa436, 0xb87e, 0xa438, 0xaf89, + 0xa438, 0xacaf, 0xa438, 0x89e4, 0xa438, 0xaf89, 0xa438, 0xecaf, + 0xa438, 0x8a04, 0xa438, 0xaf8a, 0xa438, 0x2eaf, 0xa438, 0x8a4a, + 0xa438, 0xaf8d, 0xa438, 0x31af, 0xa438, 0x8dc6, 0xa438, 0x1f55, + 0xa438, 0xe18f, 0xa438, 0xe3a1, 0xa438, 0x0007, 0xa438, 0xee86, + 0xa438, 0xe900, 0xa438, 0xaf4f, 0xa438, 0x9ead, 0xa438, 0x281b, + 0xa438, 0xe18f, 0xa438, 0xfcef, 0xa438, 0x71bf, 0xa438, 0x74f6, + 0xa438, 0x027e, 0xa438, 0xd2ef, 0xa438, 0x641c, 0xa438, 0x670d, + 0xa438, 0x67ef, 0xa438, 0x461f, 0xa438, 0x00bf, 0xa438, 0x74f6, + 0xa438, 0x027e, 0xa438, 0xdee1, 0xa438, 0x8fe3, 0xa438, 0x0d11, + 0xa438, 0xe58f, 0xa438, 0xe313, 0xa438, 0xaeca, 0xa438, 0x028d, + 0xa438, 0xd1d3, 0xa438, 0x01af, 0xa438, 0x40d1, 0xa438, 0xbf7a, + 0xa438, 0x6102, 0xa438, 0x7d44, 0xa438, 0xa100, 0xa438, 0x09e0, + 0xa438, 0x8ffa, 0xa438, 0xe18f, 0xa438, 0xfbaf, 0xa438, 0x683d, + 0xa438, 0x027f, 0xa438, 0xa9af, 0xa438, 0x682c, 0xa438, 0xbf8e, + 0xa438, 0x4102, 0xa438, 0x7d44, 0xa438, 0xe58f, 0xa438, 0xecbf, + 0xa438, 0x74cc, 0xa438, 0x027d, 0xa438, 0x44e3, 0xa438, 0x8fed, + 0xa438, 0x0d31, 0xa438, 0xf63f, 0xa438, 0x0d11, 0xa438, 0xf62f, + 0xa438, 0x1b13, 0xa438, 0xad2f, 0xa438, 0x06bf, 0xa438, 0x8e41, + 0xa438, 0x027c, 0xa438, 0xf9d1, 0xa438, 0x01af, 0xa438, 0x5974, + 0xa438, 0xee88, 0xa438, 0x8600, 0xa438, 0xe08f, 0xa438, 0xebad, + 0xa438, 0x200b, 0xa438, 0xe18f, 0xa438, 0xecbf, 0xa438, 0x8e41, + 0xa438, 0x027d, 0xa438, 0x25ae, 0xa438, 0x04ee, 0xa438, 0x8feb, + 0xa438, 0x01af, 0xa438, 0x5945, 0xa438, 0xad28, 0xa438, 0x2ce0, + 0xa438, 0x8fea, 0xa438, 0xa000, 0xa438, 0x0502, 0xa438, 0x8af0, + 0xa438, 0xae1e, 0xa438, 0xa001, 0xa438, 0x0502, 0xa438, 0x8b9f, + 0xa438, 0xae16, 0xa438, 0xa002, 0xa438, 0x0502, 0xa438, 0x8c0f, + 0xa438, 0xae0e, 0xa438, 0xa003, 0xa438, 0x0502, 0xa438, 0x8c95, + 0xa438, 0xae06, 0xa438, 0xa004, 0xa438, 0x0302, 0xa438, 0x8d08, + 0xa438, 0xaf63, 0xa438, 0x8902, 0xa438, 0x8a7f, 0xa438, 0xaf63, + 0xa438, 0x81f8, 0xa438, 0xef49, 0xa438, 0xf8e0, 0xa438, 0x8015, + 0xa438, 0xad21, 0xa438, 0x19bf, 0xa438, 0x7bd8, 0xa438, 0x027c, + 0xa438, 0xf9bf, 0xa438, 0x7bf3, 0xa438, 0x027d, 0xa438, 0x44bf, + 0xa438, 0x7bf6, 0xa438, 0x027c, 0xa438, 0xf902, 0xa438, 0x638e, + 0xa438, 0xee8f, 0xa438, 0xea00, 0xa438, 0xe080, 0xa438, 0x16ad, + 0xa438, 0x233d, 0xa438, 0xbf7b, 0xa438, 0xf302, 0xa438, 0x7d44, + 0xa438, 0xbf7a, 0xa438, 0x9402, 0xa438, 0x7cf9, 0xa438, 0xbf8e, + 0xa438, 0x4402, 0xa438, 0x7cf9, 0xa438, 0xbf7a, 0xa438, 0xa602, + 0xa438, 0x7cf9, 0xa438, 0xbf7a, 0xa438, 0xa302, 0xa438, 0x7cf9, + 0xa438, 0xbf7a, 0xa438, 0xa902, 0xa438, 0x7cf9, 0xa438, 0xbf7a, + 0xa438, 0xac02, 0xa438, 0x7cf9, 0xa438, 0xbf8e, 0xa438, 0x4702, + 0xa438, 0x7cf9, 0xa438, 0xbf8e, 0xa438, 0x4a02, 0xa438, 0x7cf9, + 0xa438, 0x0263, 0xa438, 0x8eee, 0xa438, 0x8fea, 0xa438, 0x00bf, + 0xa438, 0x7c02, 0xa438, 0x027c, 0xa438, 0xf9fc, 0xa438, 0xef94, + 0xa438, 0xfc04, 0xa438, 0xf8f9, 0xa438, 0xfbef, 0xa438, 0x79fb, + 0xa438, 0xe080, 0xa438, 0x15ac, 0xa438, 0x2103, 0xa438, 0xaf8b, + 0xa438, 0x70ee, 0xa438, 0x8888, 0xa438, 0x00ee, 0xa438, 0x888a, + 0xa438, 0x00ee, 0xa438, 0x888b, 0xa438, 0x00bf, 0xa438, 0x7bd8, + 0xa438, 0x027d, 0xa438, 0x02bf, 0xa438, 0x6000, 0xa438, 0xd788, + 0xa438, 0x881f, 0xa438, 0x44d4, 0xa438, 0x000c, 0xa438, 0x0273, + 0xa438, 0x3b02, 0xa438, 0x7fa9, 0xa438, 0xac28, 0xa438, 0x05ac, + 0xa438, 0x290d, 0xa438, 0xae18, 0xa438, 0xe188, 0xa438, 0x98bf, + 0xa438, 0x7be1, 0xa438, 0x027d, 0xa438, 0x25ae, 0xa438, 0x18e1, + 0xa438, 0x8898, 0xa438, 0x0d11, 0xa438, 0xbf7b, 0xa438, 0xe102, + 0xa438, 0x7d25, 0xa438, 0xae0b, 0xa438, 0xe188, 0xa438, 0x980d, + 0xa438, 0x12bf, 0xa438, 0x7be1, 0xa438, 0x027d, 0xa438, 0x25bf, + 0xa438, 0x88a0, 0xa438, 0xda19, 0xa438, 0xdb19, 0xa438, 0xd819, + 0xa438, 0xd91f, 0xa438, 0x77bf, 0xa438, 0x88b1, 0xa438, 0xde19, + 0xa438, 0xdf19, 0xa438, 0xdc19, 0xa438, 0xdd19, 0xa438, 0x17a7, + 0xa438, 0x0004, 0xa438, 0xf302, 0xa438, 0x63cd, 0xa438, 0xee8f, + 0xa438, 0xea01, 0xa438, 0xe080, 0xa438, 0x16ad, 0xa438, 0x2319, + 0xa438, 0xee88, 0xa438, 0x8800, 0xa438, 0xee88, 0xa438, 0x8a00, + 0xa438, 0xee88, 0xa438, 0x8b00, 0xa438, 0xbf8e, 0xa438, 0x4402, + 0xa438, 0x7d02, 0xa438, 0x0263, 0xa438, 0xcdee, 0xa438, 0x8fea, + 0xa438, 0x0102, 0xa438, 0x70de, 0xa438, 0xbf7c, 0xa438, 0x0202, + 0xa438, 0x7d02, 0xa438, 0xffef, 0xa438, 0x97ff, 0xa438, 0xfdfc, + 0xa438, 0x04f8, 0xa438, 0xf9fa, 0xa438, 0xef69, 0xa438, 0xfae0, + 0xa438, 0x888a, 0xa438, 0xe188, 0xa438, 0x8b14, 0xa438, 0xe488, + 0xa438, 0x8ae5, 0xa438, 0x888b, 0xa438, 0xbf88, 0xa438, 0x94d8, + 0xa438, 0x19d9, 0xa438, 0xef64, 0xa438, 0xe088, 0xa438, 0x8ae1, + 0xa438, 0x888b, 0xa438, 0x1b46, 0xa438, 0x9f30, 0xa438, 0x1f44, + 0xa438, 0xe488, 0xa438, 0x8ae5, 0xa438, 0x888b, 0xa438, 0xe080, + 0xa438, 0x15ad, 0xa438, 0x211a, 0xa438, 0x0260, 0xa438, 0xece0, + 0xa438, 0x8016, 0xa438, 0xad23, 0xa438, 0x1602, 0xa438, 0x7c86, + 0xa438, 0xef47, 0xa438, 0xe48f, 0xa438, 0xe9e5, 0xa438, 0x8fe8, + 0xa438, 0xee8f, 0xa438, 0xea02, 0xa438, 0xae0b, 0xa438, 0x028c, + 0xa438, 0x2eae, 0xa438, 0x0602, 0xa438, 0x8bfe, 0xa438, 0x0270, + 0xa438, 0xdefe, 0xa438, 0xef96, 0xa438, 0xfefd, 0xa438, 0xfc04, + 0xa438, 0xf8e1, 0xa438, 0x8888, 0xa438, 0x11e5, 0xa438, 0x8888, + 0xa438, 0xad2a, 0xa438, 0x04ee, 0xa438, 0x8888, 0xa438, 0x00fc, + 0xa438, 0x04f8, 0xa438, 0xfafb, 0xa438, 0xe08f, 0xa438, 0xe9e1, + 0xa438, 0x8fe8, 0xa438, 0xef64, 0xa438, 0x1f00, 0xa438, 0xe18f, + 0xa438, 0xe6ef, 0xa438, 0x7402, 0xa438, 0x7ca1, 0xa438, 0xad50, + 0xa438, 0x0302, 0xa438, 0x8c2e, 0xa438, 0xfffe, 0xa438, 0xfc04, + 0xa438, 0xf8fa, 0xa438, 0xef69, 0xa438, 0xfbbf, 0xa438, 0x7bf3, + 0xa438, 0x027d, 0xa438, 0x44ac, 0xa438, 0x284c, 0xa438, 0x0264, + 0xa438, 0x1cbf, 0xa438, 0x8e47, 0xa438, 0x027d, 0xa438, 0x02bf, + 0xa438, 0x8e4a, 0xa438, 0x027d, 0xa438, 0x02d1, 0xa438, 0x43b1, + 0xa438, 0xfebf, 0xa438, 0x7aa6, 0xa438, 0x027c, 0xa438, 0xf9bf, + 0xa438, 0x7aa3, 0xa438, 0x027c, 0xa438, 0xf9bf, 0xa438, 0x7aa9, + 0xa438, 0x027c, 0xa438, 0xf9bf, 0xa438, 0x7aac, 0xa438, 0x027d, + 0xa438, 0x02d1, 0xa438, 0x80e0, 0xa438, 0x8888, 0xa438, 0x100e, + 0xa438, 0x11b0, 0xa438, 0xfcbf, 0xa438, 0x7a94, 0xa438, 0x027d, + 0xa438, 0x2502, 0xa438, 0x7c86, 0xa438, 0xef47, 0xa438, 0xe48f, + 0xa438, 0xe9e5, 0xa438, 0x8fe8, 0xa438, 0xee8f, 0xa438, 0xea03, + 0xa438, 0xae07, 0xa438, 0xee8f, 0xa438, 0xea01, 0xa438, 0x0270, + 0xa438, 0xdeff, 0xa438, 0xef96, 0xa438, 0xfefc, 0xa438, 0x04f8, + 0xa438, 0xf9fa, 0xa438, 0xfbef, 0xa438, 0x79fb, 0xa438, 0xbf7a, + 0xa438, 0x9402, 0xa438, 0x7d44, 0xa438, 0xef21, 0xa438, 0xbf7a, + 0xa438, 0xb802, 0xa438, 0x7d44, 0xa438, 0x1f21, 0xa438, 0x9e19, + 0xa438, 0xe08f, 0xa438, 0xe9e1, 0xa438, 0x8fe8, 0xa438, 0xef64, + 0xa438, 0x1f00, 0xa438, 0xe18f, 0xa438, 0xe4ef, 0xa438, 0x7402, + 0xa438, 0x7ca1, 0xa438, 0xad50, 0xa438, 0x3dee, 0xa438, 0x8fe7, + 0xa438, 0x01bf, 0xa438, 0x7a94, 0xa438, 0x027c, 0xa438, 0xf9bf, + 0xa438, 0x7aa6, 0xa438, 0x027c, 0xa438, 0xf9bf, 0xa438, 0x7aa3, + 0xa438, 0x027c, 0xa438, 0xf9bf, 0xa438, 0x7aa9, 0xa438, 0x027c, + 0xa438, 0xf9bf, 0xa438, 0x7aac, 0xa438, 0x027d, 0xa438, 0x02bf, + 0xa438, 0x8e47, 0xa438, 0x027c, 0xa438, 0xf9bf, 0xa438, 0x8e4a, + 0xa438, 0x027c, 0xa438, 0xf902, 0xa438, 0x7c86, 0xa438, 0xef47, + 0xa438, 0xe48f, 0xa438, 0xe9e5, 0xa438, 0x8fe8, 0xa438, 0xee8f, + 0xa438, 0xea04, 0xa438, 0xffef, 0xa438, 0x97ff, 0xa438, 0xfefd, + 0xa438, 0xfc04, 0xa438, 0xf8fa, 0xa438, 0xfbe0, 0xa438, 0x8fe9, + 0xa438, 0xe18f, 0xa438, 0xe8ef, 0xa438, 0x641f, 0xa438, 0x00e1, + 0xa438, 0x8fe5, 0xa438, 0xef74, 0xa438, 0x027c, 0xa438, 0xa1ad, + 0xa438, 0x500d, 0xa438, 0x0263, 0xa438, 0x8e02, 0xa438, 0x8bfe, + 0xa438, 0xee8f, 0xa438, 0xea01, 0xa438, 0x0270, 0xa438, 0xdeff, + 0xa438, 0xfefc, 0xa438, 0x04e3, 0xa438, 0x8fd8, 0xa438, 0xe787, + 0xa438, 0x75e4, 0xa438, 0x8fe1, 0xa438, 0xe58f, 0xa438, 0xe2bf, + 0xa438, 0x8fd9, 0xa438, 0xef32, 0xa438, 0x0c31, 0xa438, 0x1a93, + 0xa438, 0xdc19, 0xa438, 0xdd02, 0xa438, 0x7fa9, 0xa438, 0xac2a, + 0xa438, 0x18e0, 0xa438, 0x8fe1, 0xa438, 0xe18f, 0xa438, 0xe2ef, + 0xa438, 0x74e1, 0xa438, 0x8775, 0xa438, 0x1f00, 0xa438, 0xef64, + 0xa438, 0xe18f, 0xa438, 0xd8e5, 0xa438, 0x8775, 0xa438, 0xaf4d, + 0xa438, 0x72bf, 0xa438, 0x7b3c, 0xa438, 0xef32, 0xa438, 0x4b03, + 0xa438, 0x1a93, 0xa438, 0x027d, 0xa438, 0x44ef, 0xa438, 0x64e1, + 0xa438, 0x8fff, 0xa438, 0x1f00, 0xa438, 0xef74, 0xa438, 0x1b67, + 0xa438, 0xac4f, 0xa438, 0xcee0, 0xa438, 0x8ffd, 0xa438, 0xe18f, + 0xa438, 0xfeef, 0xa438, 0x64e0, 0xa438, 0x8fe1, 0xa438, 0xe18f, + 0xa438, 0xe2ef, 0xa438, 0x7402, 0xa438, 0x7c53, 0xa438, 0xac50, + 0xa438, 0x02ae, 0xa438, 0xb6e1, 0xa438, 0x8775, 0xa438, 0x1f00, + 0xa438, 0xef64, 0xa438, 0xe18f, 0xa438, 0xfcef, 0xa438, 0x711c, + 0xa438, 0x670d, 0xa438, 0x67ef, 0xa438, 0x46e5, 0xa438, 0x8775, + 0xa438, 0xef32, 0xa438, 0xd101, 0xa438, 0xa300, 0xa438, 0x02ae, + 0xa438, 0x050c, 0xa438, 0x1183, 0xa438, 0xaef6, 0xa438, 0xe08f, + 0xa438, 0xe31e, 0xa438, 0x10e5, 0xa438, 0x8fe3, 0xa438, 0xae89, + 0xa438, 0xe287, 0xa438, 0x75e6, 0xa438, 0x8fd8, 0xa438, 0x1f22, + 0xa438, 0xaf4d, 0xa438, 0x42f8, 0xa438, 0xf9ef, 0xa438, 0x59fa, + 0xa438, 0xfbbf, 0xa438, 0x8fee, 0xa438, 0x027f, 0xa438, 0xa90d, + 0xa438, 0x1149, 0xa438, 0x041a, 0xa438, 0x91d7, 0xa438, 0x8df3, + 0xa438, 0xd68e, 0xa438, 0x2302, 0xa438, 0x72aa, 0xa438, 0xfffe, + 0xa438, 0xef95, 0xa438, 0xfdfc, 0xa438, 0x0400, 0xa438, 0x7591, + 0xa438, 0x0275, 0xa438, 0x4404, 0xa438, 0x758e, 0xa438, 0x2675, + 0xa438, 0x4100, 0xa438, 0x8e26, 0xa438, 0x028e, 0xa438, 0x2304, + 0xa438, 0x759d, 0xa438, 0x2675, 0xa438, 0x4700, 0xa438, 0x8e32, + 0xa438, 0x028e, 0xa438, 0x2f04, 0xa438, 0x8e2c, 0xa438, 0x268e, + 0xa438, 0x2900, 0xa438, 0x8e3e, 0xa438, 0x028e, 0xa438, 0x3b04, + 0xa438, 0x8e38, 0xa438, 0x268e, 0xa438, 0x35fe, 0xa438, 0xad96, + 0xa438, 0xdcad, 0xa438, 0x96ba, 0xa438, 0xad96, 0xa438, 0x98ad, + 0xa438, 0x9676, 0xa438, 0xad98, 0xa438, 0x54ad, 0xa438, 0x9876, + 0xa438, 0xae38, 0xa438, 0x54ae, 0xa438, 0x38fe, 0xa438, 0xae3a, + 0xa438, 0xdcae, 0xa438, 0x3abb, 0xa438, 0xbf14, 0xa438, 0x99bd, + 0xa438, 0xe0cc, 0xa438, 0xbdc8, 0xa438, 0xddbd, 0xa438, 0xc800, + 0xa436, 0xb85e, 0xa438, 0x4f9a, 0xa436, 0xb860, 0xa438, 0x40cf, + 0xa436, 0xb862, 0xa438, 0x6829, 0xa436, 0xb864, 0xa438, 0x5972, + 0xa436, 0xb886, 0xa438, 0x5941, 0xa436, 0xb888, 0xa438, 0x636b, + 0xa436, 0xb88a, 0xa438, 0x4d6b, 0xa436, 0xb88c, 0xa438, 0x4d40, + 0xa436, 0xb838, 0xa438, 0x00ff, 0xb820, 0x0010, 0xa436, 0x8608, + 0xa438, 0xaf86, 0xa438, 0xdaaf, 0xa438, 0x894c, 0xa438, 0xaf8a, + 0xa438, 0xf8af, 0xa438, 0x8bf3, 0xa438, 0xaf8b, 0xa438, 0xf3af, + 0xa438, 0x8bf3, 0xa438, 0xaf8b, 0xa438, 0xf3af, 0xa438, 0x8bf3, + 0xa438, 0x006f, 0xa438, 0x4a03, 0xa438, 0x6f47, 0xa438, 0x266f, + 0xa438, 0x5900, 0xa438, 0x6f4d, 0xa438, 0x016f, 0xa438, 0x5004, + 0xa438, 0x6f56, 0xa438, 0x056f, 0xa438, 0x5f06, 0xa438, 0x6f5c, + 0xa438, 0x2774, 0xa438, 0x7800, 0xa438, 0x6f68, 0xa438, 0x246f, + 0xa438, 0x6b20, 0xa438, 0x6f6e, 0xa438, 0x206f, 0xa438, 0x7410, + 0xa438, 0x7469, 0xa438, 0x1074, 0xa438, 0x6c10, 0xa438, 0x746f, + 0xa438, 0x1074, 0xa438, 0x7225, 0xa438, 0x8bfc, 0xa438, 0x008c, + 0xa438, 0x0802, 0xa438, 0x8c02, 0xa438, 0x038b, 0xa438, 0xff04, + 0xa438, 0x6eed, 0xa438, 0x278c, 0xa438, 0x0520, 0xa438, 0x74da, + 0xa438, 0x2074, 0xa438, 0xdd20, 0xa438, 0x74e0, 0xa438, 0x0074, + 0xa438, 0xe300, 0xa438, 0x6ef3, 0xa438, 0x006e, 0xa438, 0xf600, + 0xa438, 0x6ef9, 0xa438, 0x006e, 0xa438, 0xfc00, 0xa438, 0x6eff, + 0xa438, 0x006f, 0xa438, 0x0200, 0xa438, 0x6f05, 0xa438, 0x026f, + 0xa438, 0x0802, 0xa438, 0x6f0b, 0xa438, 0x026f, 0xa438, 0x0e02, + 0xa438, 0x6f11, 0xa438, 0x026f, 0xa438, 0x1402, 0xa438, 0x6f17, + 0xa438, 0x226f, 0xa438, 0x1a00, 0xa438, 0x723e, 0xa438, 0x016e, + 0xa438, 0xed24, 0xa438, 0x6f50, 0xa438, 0x0072, 0xa438, 0x4701, + 0xa438, 0x724a, 0xa438, 0x0272, 0xa438, 0x4d23, 0xa438, 0x7250, + 0xa438, 0x1074, 0xa438, 0x6910, 0xa438, 0x746c, 0xa438, 0x1074, + 0xa438, 0x6f00, 0xa438, 0x7472, 0xa438, 0x158c, 0xa438, 0x0b15, + 0xa438, 0x8c0e, 0xa438, 0x158c, 0xa438, 0x1105, 0xa438, 0x8c14, + 0xa438, 0x006f, 0xa438, 0x4a03, 0xa438, 0x6f47, 0xa438, 0x266f, + 0xa438, 0x5900, 0xa438, 0x731f, 0xa438, 0x0273, 0xa438, 0x2203, + 0xa438, 0x8c08, 0xa438, 0xee84, 0xa438, 0x7100, 0xa438, 0x0286, + 0xa438, 0xece0, 0xa438, 0x8043, 0xa438, 0xf626, 0xa438, 0xe480, + 0xa438, 0x43af, 0xa438, 0x6611, 0xa438, 0xf8e0, 0xa438, 0x8012, + 0xa438, 0xac26, 0xa438, 0x03af, 0xa438, 0x86ff, 0xa438, 0x0287, + 0xa438, 0x0102, 0xa438, 0x8906, 0xa438, 0x0289, 0xa438, 0x29fc, + 0xa438, 0x04f8, 0xa438, 0xf9ef, 0xa438, 0x59f9, 0xa438, 0xfaee, + 0xa438, 0x8476, 0xa438, 0x00d6, 0xa438, 0x008f, 0xa438, 0x0266, + 0xa438, 0x53ef, 0xa438, 0x643e, 0xa438, 0x1200, 0xa438, 0xac4f, + 0xa438, 0x08e4, 0xa438, 0x8fe7, 0xa438, 0xe58f, 0xa438, 0xe8ae, + 0xa438, 0x06e0, 0xa438, 0x8fe7, 0xa438, 0xe18f, 0xa438, 0xe8ee, + 0xa438, 0x8476, 0xa438, 0x01d6, 0xa438, 0x00c0, 0xa438, 0x0266, + 0xa438, 0x71ee, 0xa438, 0x8476, 0xa438, 0x00d6, 0xa438, 0x0090, + 0xa438, 0x0266, 0xa438, 0x53ef, 0xa438, 0x643e, 0xa438, 0x1200, + 0xa438, 0xac4f, 0xa438, 0x08e4, 0xa438, 0x8fe9, 0xa438, 0xe58f, + 0xa438, 0xeaae, 0xa438, 0x06e0, 0xa438, 0x8fe9, 0xa438, 0xe18f, + 0xa438, 0xeaee, 0xa438, 0x8476, 0xa438, 0x01d6, 0xa438, 0x00c1, + 0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x8476, 0xa438, 0x00d6, + 0xa438, 0x0091, 0xa438, 0x0266, 0xa438, 0x53ef, 0xa438, 0x643e, + 0xa438, 0x1200, 0xa438, 0xac4f, 0xa438, 0x08e4, 0xa438, 0x8feb, + 0xa438, 0xe58f, 0xa438, 0xecae, 0xa438, 0x06e0, 0xa438, 0x8feb, + 0xa438, 0xe18f, 0xa438, 0xecee, 0xa438, 0x8476, 0xa438, 0x01d6, + 0xa438, 0x00c2, 0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x8476, + 0xa438, 0x01d6, 0xa438, 0x008f, 0xa438, 0x0266, 0xa438, 0x53ef, + 0xa438, 0x643e, 0xa438, 0x1200, 0xa438, 0xac4f, 0xa438, 0x08e4, + 0xa438, 0x8fed, 0xa438, 0xe58f, 0xa438, 0xeeae, 0xa438, 0x06e0, + 0xa438, 0x8fed, 0xa438, 0xe18f, 0xa438, 0xeeee, 0xa438, 0x8476, + 0xa438, 0x02d6, 0xa438, 0x00c0, 0xa438, 0x0266, 0xa438, 0x71ee, + 0xa438, 0x8476, 0xa438, 0x01d6, 0xa438, 0x0090, 0xa438, 0x0266, + 0xa438, 0x53ef, 0xa438, 0x643e, 0xa438, 0x1200, 0xa438, 0xac4f, + 0xa438, 0x08e4, 0xa438, 0x8fef, 0xa438, 0xe58f, 0xa438, 0xf0ae, + 0xa438, 0x06e0, 0xa438, 0x8fef, 0xa438, 0xe18f, 0xa438, 0xf0ee, + 0xa438, 0x8476, 0xa438, 0x02d6, 0xa438, 0x00c1, 0xa438, 0x0266, + 0xa438, 0x71ee, 0xa438, 0x8476, 0xa438, 0x01d6, 0xa438, 0x0091, + 0xa438, 0x0266, 0xa438, 0x53ef, 0xa438, 0x643e, 0xa438, 0x1200, + 0xa438, 0xac4f, 0xa438, 0x08e4, 0xa438, 0x8ff1, 0xa438, 0xe58f, + 0xa438, 0xf2ae, 0xa438, 0x06e0, 0xa438, 0x8ff1, 0xa438, 0xe18f, + 0xa438, 0xf2ee, 0xa438, 0x8476, 0xa438, 0x02d6, 0xa438, 0x00c2, + 0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x8476, 0xa438, 0x02d6, + 0xa438, 0x008f, 0xa438, 0x0266, 0xa438, 0x53ef, 0xa438, 0x643e, + 0xa438, 0x1200, 0xa438, 0xac4f, 0xa438, 0x08e4, 0xa438, 0x8ff3, + 0xa438, 0xe58f, 0xa438, 0xf4ae, 0xa438, 0x06e0, 0xa438, 0x8ff3, + 0xa438, 0xe18f, 0xa438, 0xf4ee, 0xa438, 0x8476, 0xa438, 0x04d6, + 0xa438, 0x00c0, 0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x8476, + 0xa438, 0x02d6, 0xa438, 0x0090, 0xa438, 0x0266, 0xa438, 0x53ef, + 0xa438, 0x643e, 0xa438, 0x1200, 0xa438, 0xac4f, 0xa438, 0x08e4, + 0xa438, 0x8ff5, 0xa438, 0xe58f, 0xa438, 0xf6ae, 0xa438, 0x06e0, + 0xa438, 0x8ff5, 0xa438, 0xe18f, 0xa438, 0xf6ee, 0xa438, 0x8476, + 0xa438, 0x04d6, 0xa438, 0x00c1, 0xa438, 0x0266, 0xa438, 0x71ee, + 0xa438, 0x8476, 0xa438, 0x02d6, 0xa438, 0x0091, 0xa438, 0x0266, + 0xa438, 0x53ef, 0xa438, 0x643e, 0xa438, 0x1200, 0xa438, 0xac4f, + 0xa438, 0x08e4, 0xa438, 0x8ff7, 0xa438, 0xe58f, 0xa438, 0xf8ae, + 0xa438, 0x06e0, 0xa438, 0x8ff7, 0xa438, 0xe18f, 0xa438, 0xf8ee, + 0xa438, 0x8476, 0xa438, 0x04d6, 0xa438, 0x00c2, 0xa438, 0x0266, + 0xa438, 0x71ee, 0xa438, 0x8476, 0xa438, 0x03d6, 0xa438, 0x008f, + 0xa438, 0x0266, 0xa438, 0x53ef, 0xa438, 0x643e, 0xa438, 0x1200, + 0xa438, 0xac4f, 0xa438, 0x08e4, 0xa438, 0x8ff9, 0xa438, 0xe58f, + 0xa438, 0xfaae, 0xa438, 0x06e0, 0xa438, 0x8ff9, 0xa438, 0xe18f, + 0xa438, 0xfaee, 0xa438, 0x8476, 0xa438, 0x08d6, 0xa438, 0x00c0, + 0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x8476, 0xa438, 0x03d6, + 0xa438, 0x0090, 0xa438, 0x0266, 0xa438, 0x53ef, 0xa438, 0x643e, + 0xa438, 0x1200, 0xa438, 0xac4f, 0xa438, 0x08e4, 0xa438, 0x8ffb, + 0xa438, 0xe58f, 0xa438, 0xfcae, 0xa438, 0x06e0, 0xa438, 0x8ffb, + 0xa438, 0xe18f, 0xa438, 0xfcee, 0xa438, 0x8476, 0xa438, 0x08d6, + 0xa438, 0x00c1, 0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x8476, + 0xa438, 0x03d6, 0xa438, 0x0091, 0xa438, 0x0266, 0xa438, 0x53ef, + 0xa438, 0x643e, 0xa438, 0x1200, 0xa438, 0xac4f, 0xa438, 0x08e4, + 0xa438, 0x8ffd, 0xa438, 0xe58f, 0xa438, 0xfeae, 0xa438, 0x06e0, + 0xa438, 0x8ffd, 0xa438, 0xe18f, 0xa438, 0xfeee, 0xa438, 0x8476, + 0xa438, 0x08d6, 0xa438, 0x00c2, 0xa438, 0x0266, 0xa438, 0x71fe, + 0xa438, 0xfdef, 0xa438, 0x95fd, 0xa438, 0xfc04, 0xa438, 0xf8f9, + 0xa438, 0xfad4, 0xa438, 0x0400, 0xa438, 0xd600, 0xa438, 0x0dd3, + 0xa438, 0x0fe7, 0xa438, 0x8476, 0xa438, 0x0266, 0xa438, 0x71d4, + 0xa438, 0x1400, 0xa438, 0xd600, 0xa438, 0x0dd3, 0xa438, 0x0fe7, + 0xa438, 0x8476, 0xa438, 0x0266, 0xa438, 0x71fe, 0xa438, 0xfdfc, + 0xa438, 0x04f8, 0xa438, 0xf9fa, 0xa438, 0xd410, 0xa438, 0x00d6, + 0xa438, 0x000d, 0xa438, 0xd30f, 0xa438, 0xe784, 0xa438, 0x7602, + 0xa438, 0x6671, 0xa438, 0xd400, 0xa438, 0x00d6, 0xa438, 0x000d, + 0xa438, 0xd30f, 0xa438, 0xe784, 0xa438, 0x7602, 0xa438, 0x6671, + 0xa438, 0xfefd, 0xa438, 0xfc04, 0xa438, 0xe080, 0xa438, 0x4fac, + 0xa438, 0x2317, 0xa438, 0xe080, 0xa438, 0x44ad, 0xa438, 0x231a, + 0xa438, 0x0289, 0xa438, 0x75e0, 0xa438, 0x8044, 0xa438, 0xac23, + 0xa438, 0x11bf, 0xa438, 0x6ecf, 0xa438, 0x0276, 0xa438, 0x74ae, + 0xa438, 0x0902, 0xa438, 0x8adb, 0xa438, 0x021f, 0xa438, 0xe702, + 0xa438, 0x1fbb, 0xa438, 0xaf1f, 0xa438, 0x95f8, 0xa438, 0xf9ef, + 0xa438, 0x59f9, 0xa438, 0xfafb, 0xa438, 0xe080, 0xa438, 0x12ac, + 0xa438, 0x2303, 0xa438, 0xaf8a, 0xa438, 0xd0d4, 0xa438, 0x0120, + 0xa438, 0xd600, 0xa438, 0x10d2, 0xa438, 0x0fe6, 0xa438, 0x8476, + 0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x846f, 0xa438, 0x00d4, + 0xa438, 0x000f, 0xa438, 0xbf72, 0xa438, 0x9e02, 0xa438, 0x7697, + 0xa438, 0x0275, 0xa438, 0xbeef, 0xa438, 0x47e4, 0xa438, 0x8474, + 0xa438, 0xe584, 0xa438, 0x75bf, 0xa438, 0x729b, 0xa438, 0x0276, + 0xa438, 0xb6e5, 0xa438, 0x846f, 0xa438, 0xef31, 0xa438, 0xbf6e, + 0xa438, 0x0602, 0xa438, 0x76b6, 0xa438, 0xef64, 0xa438, 0xbf6e, + 0xa438, 0x0902, 0xa438, 0x76b6, 0xa438, 0x1e64, 0xa438, 0xbf6e, + 0xa438, 0x0f02, 0xa438, 0x76b6, 0xa438, 0x1e64, 0xa438, 0xac40, + 0xa438, 0x05a3, 0xa438, 0x0f0c, 0xa438, 0xae26, 0xa438, 0xa303, + 0xa438, 0x02ae, 0xa438, 0x21a3, 0xa438, 0x0c02, 0xa438, 0xae1c, + 0xa438, 0xe084, 0xa438, 0x74e1, 0xa438, 0x8475, 0xa438, 0xef64, + 0xa438, 0xd000, 0xa438, 0xd196, 0xa438, 0xef74, 0xa438, 0x0275, + 0xa438, 0xd9ad, 0xa438, 0x50b7, 0xa438, 0xe083, 0xa438, 0xecf7, + 0xa438, 0x23e4, 0xa438, 0x83ec, 0xa438, 0xbf72, 0xa438, 0x9e02, + 0xa438, 0x766b, 0xa438, 0x0287, 0xa438, 0x0102, 0xa438, 0x8906, + 0xa438, 0xee83, 0xa438, 0xe800, 0xa438, 0xbf72, 0xa438, 0x6b02, + 0xa438, 0x766b, 0xa438, 0xbf72, 0xa438, 0x6e02, 0xa438, 0x766b, + 0xa438, 0xbf72, 0xa438, 0x7102, 0xa438, 0x766b, 0xa438, 0xbf72, + 0xa438, 0x7402, 0xa438, 0x766b, 0xa438, 0xbf72, 0xa438, 0x7702, + 0xa438, 0x766b, 0xa438, 0xbf72, 0xa438, 0x7a02, 0xa438, 0x766b, + 0xa438, 0xd400, 0xa438, 0x0fbf, 0xa438, 0x7295, 0xa438, 0x0276, + 0xa438, 0x97d7, 0xa438, 0x0400, 0xa438, 0xbf6e, 0xa438, 0x0602, + 0xa438, 0x76b6, 0xa438, 0xef64, 0xa438, 0xbf6e, 0xa438, 0x0902, + 0xa438, 0x76b6, 0xa438, 0x1e64, 0xa438, 0xbf6e, 0xa438, 0x0f02, + 0xa438, 0x76b6, 0xa438, 0x1e64, 0xa438, 0xac40, 0xa438, 0x0fbf, + 0xa438, 0x7298, 0xa438, 0x0276, 0xa438, 0xb6e5, 0xa438, 0x83e8, + 0xa438, 0xa10f, 0xa438, 0x28af, 0xa438, 0x8a95, 0xa438, 0xbf8b, + 0xa438, 0xf302, 0xa438, 0x76b6, 0xa438, 0xac28, 0xa438, 0x02ae, + 0xa438, 0x0bbf, 0xa438, 0x8bf9, 0xa438, 0x0276, 0xa438, 0xb6e5, + 0xa438, 0x83e8, 0xa438, 0xae09, 0xa438, 0xbf8b, 0xa438, 0xf602, + 0xa438, 0x76b6, 0xa438, 0xe583, 0xa438, 0xe8a1, 0xa438, 0x0303, + 0xa438, 0xaf8a, 0xa438, 0x95b7, 0xa438, 0xafe2, 0xa438, 0x83ec, + 0xa438, 0xf735, 0xa438, 0xe683, 0xa438, 0xecbf, 0xa438, 0x7295, + 0xa438, 0x0276, 0xa438, 0x6bbf, 0xa438, 0x726b, 0xa438, 0x0276, + 0xa438, 0x74bf, 0xa438, 0x726e, 0xa438, 0x0276, 0xa438, 0x74bf, + 0xa438, 0x7271, 0xa438, 0x0276, 0xa438, 0x74bf, 0xa438, 0x7274, + 0xa438, 0x0276, 0xa438, 0x74bf, 0xa438, 0x7277, 0xa438, 0x0276, + 0xa438, 0x74bf, 0xa438, 0x727a, 0xa438, 0x0276, 0xa438, 0x7402, + 0xa438, 0x8929, 0xa438, 0xd401, 0xa438, 0x28d6, 0xa438, 0x0010, + 0xa438, 0xd20f, 0xa438, 0xe684, 0xa438, 0x7602, 0xa438, 0x6671, + 0xa438, 0x021f, 0xa438, 0xbbff, 0xa438, 0xfefd, 0xa438, 0xef95, + 0xa438, 0xfdfc, 0xa438, 0x04f8, 0xa438, 0xf9ef, 0xa438, 0x59f9, + 0xa438, 0xe080, 0xa438, 0x12ad, 0xa438, 0x230c, 0xa438, 0xbf72, + 0xa438, 0x9e02, 0xa438, 0x766b, 0xa438, 0xbf72, 0xa438, 0x9502, + 0xa438, 0x766b, 0xa438, 0xfdef, 0xa438, 0x95fd, 0xa438, 0xfc04, + 0xa438, 0xbf6e, 0xa438, 0x0602, 0xa438, 0x76b6, 0xa438, 0xef64, + 0xa438, 0xbf6e, 0xa438, 0x0902, 0xa438, 0x76b6, 0xa438, 0x1e64, + 0xa438, 0xbf6e, 0xa438, 0x0f02, 0xa438, 0x76b6, 0xa438, 0x1e64, + 0xa438, 0xac40, 0xa438, 0x0ebf, 0xa438, 0x7298, 0xa438, 0x0276, + 0xa438, 0xb6e5, 0xa438, 0x8478, 0xa438, 0xa10f, 0xa438, 0x26ae, + 0xa438, 0x47bf, 0xa438, 0x8bf3, 0xa438, 0x0276, 0xa438, 0xb6ac, + 0xa438, 0x2802, 0xa438, 0xae0b, 0xa438, 0xbf8b, 0xa438, 0xf902, + 0xa438, 0x76b6, 0xa438, 0xe584, 0xa438, 0x78ae, 0xa438, 0x09bf, + 0xa438, 0x8bf6, 0xa438, 0x0276, 0xa438, 0xb6e5, 0xa438, 0x8478, + 0xa438, 0xa103, 0xa438, 0x02ae, 0xa438, 0x23e0, 0xa438, 0x8474, + 0xa438, 0xe184, 0xa438, 0x75ef, 0xa438, 0x64e0, 0xa438, 0x83fc, + 0xa438, 0xe183, 0xa438, 0xfdef, 0xa438, 0x7402, 0xa438, 0x75d9, + 0xa438, 0xad50, 0xa438, 0x0ae0, 0xa438, 0x83ec, 0xa438, 0xf721, + 0xa438, 0xe483, 0xa438, 0xecae, 0xa438, 0x03af, 0xa438, 0x68e4, + 0xa438, 0xbf72, 0xa438, 0x9502, 0xa438, 0x766b, 0xa438, 0xe083, + 0xa438, 0xebad, 0xa438, 0x2170, 0xa438, 0xbf73, 0xa438, 0x7f02, + 0xa438, 0x766b, 0xa438, 0xd700, 0xa438, 0x64bf, 0xa438, 0x73c4, + 0xa438, 0x0276, 0xa438, 0xb6a4, 0xa438, 0x0000, 0xa438, 0x02ae, + 0xa438, 0x0d87, 0xa438, 0xa700, 0xa438, 0x00ef, 0xa438, 0xe183, + 0xa438, 0xecf7, 0xa438, 0x2ae5, 0xa438, 0x83ec, 0xa438, 0xbf73, + 0xa438, 0xbe02, 0xa438, 0x766b, 0xa438, 0xbf73, 0xa438, 0xb802, + 0xa438, 0x766b, 0xa438, 0xbf73, 0xa438, 0xc102, 0xa438, 0x766b, + 0xa438, 0xbf73, 0xa438, 0xbb02, 0xa438, 0x766b, 0xa438, 0xe084, + 0xa438, 0x9ee1, 0xa438, 0x849f, 0xa438, 0xbf72, 0xa438, 0x7d02, + 0xa438, 0x7697, 0xa438, 0xbf72, 0xa438, 0x8002, 0xa438, 0x7697, + 0xa438, 0xbf72, 0xa438, 0x8302, 0xa438, 0x7697, 0xa438, 0xbf72, + 0xa438, 0x8602, 0xa438, 0x7697, 0xa438, 0xbf72, 0xa438, 0x8902, + 0xa438, 0x7674, 0xa438, 0xbf72, 0xa438, 0x8c02, 0xa438, 0x7674, + 0xa438, 0xbf72, 0xa438, 0x8f02, 0xa438, 0x7674, 0xa438, 0xbf72, + 0xa438, 0x9202, 0xa438, 0x7674, 0xa438, 0xee84, 0xa438, 0x7700, + 0xa438, 0xe080, 0xa438, 0x44f6, 0xa438, 0x21e4, 0xa438, 0x8044, + 0xa438, 0xaf68, 0xa438, 0xe411, 0xa438, 0xd1a4, 0xa438, 0x10bc, + 0xa438, 0x7432, 0xa438, 0xbc74, 0xa438, 0xbbbf, 0xa438, 0x14cc, + 0xa438, 0xbfaa, 0xa438, 0x00bf, 0xa438, 0x9055, 0xa438, 0xbf06, + 0xa438, 0x10bf, 0xa438, 0xb876, 0xa438, 0xbe02, 0xa438, 0x54be, + 0xa438, 0x0232, 0xa438, 0xbe02, 0xa438, 0x10be, 0xa438, 0x0200, + 0xa436, 0x8fe7, 0xa438, 0x1200, 0xa436, 0x8fe9, 0xa438, 0x1200, + 0xa436, 0x8feb, 0xa438, 0x1200, 0xa436, 0x8fed, 0xa438, 0x1200, + 0xa436, 0x8fef, 0xa438, 0x1200, 0xa436, 0x8ff1, 0xa438, 0x1200, + 0xa436, 0x8ff3, 0xa438, 0x1200, 0xa436, 0x8ff5, 0xa438, 0x1200, + 0xa436, 0x8ff7, 0xa438, 0x1200, 0xa436, 0x8ff9, 0xa438, 0x1200, + 0xa436, 0x8ffb, 0xa438, 0x1200, 0xa436, 0x8ffd, 0xa438, 0x1200, + 0xa436, 0xb818, 0xa438, 0x6602, 0xa436, 0xb81a, 0xa438, 0x1f75, + 0xa436, 0xb81c, 0xa438, 0x67eb, 0xa436, 0xb81e, 0xa438, 0xffff, + 0xa436, 0xb850, 0xa438, 0xffff, 0xa436, 0xb852, 0xa438, 0xffff, + 0xa436, 0xb878, 0xa438, 0xffff, 0xa436, 0xb884, 0xa438, 0xffff, + 0xa436, 0xb832, 0xa438, 0x0007, 0xB82E, 0x0000, 0xa436, 0x8023, + 0xa438, 0x0000, 0xB820, 0x0000, 0xFFFF, 0xFFFF +}; + +static const u16 phy_mcu_ram_code_8127a_2[] = { + 0xb892, 0x0000, 0xB88E, 0xc07c, 0xB890, 0x0203, 0xB890, 0x0304, + 0xB890, 0x0405, 0xB890, 0x0607, 0xB890, 0x0809, 0xB890, 0x0B0D, + 0xB890, 0x0F11, 0xB890, 0x1418, 0xB890, 0x1B20, 0xB890, 0x252B, + 0xB890, 0x343E, 0xB890, 0x4854, 0xB890, 0x6203, 0xB890, 0x0304, + 0xB890, 0x0506, 0xB890, 0x080A, 0xB890, 0x0C0E, 0xB890, 0x1216, + 0xB890, 0x1B22, 0xB890, 0x2A34, 0xB890, 0x404F, 0xB890, 0x6171, + 0xB890, 0x7884, 0xB890, 0x9097, 0xB890, 0x0203, 0xB890, 0x0406, + 0xB890, 0x080B, 0xB890, 0x0E13, 0xB890, 0x1820, 0xB890, 0x2A39, + 0xB890, 0x4856, 0xB890, 0xE060, 0xB890, 0xE050, 0xB890, 0xD080, + 0xB890, 0x8070, 0xB890, 0x70A0, 0xB890, 0x1000, 0xB890, 0x60D0, + 0xB890, 0xB010, 0xB890, 0xE0B0, 0xB890, 0x80C0, 0xB890, 0xE000, + 0xB890, 0x2020, 0xB890, 0x1020, 0xB890, 0xE090, 0xB890, 0x80C0, + 0xB890, 0x3020, 0xB890, 0x00E0, 0xB890, 0x40A0, 0xB890, 0xE020, + 0xB890, 0x5060, 0xB890, 0xE0D0, 0xB890, 0xA000, 0xB890, 0x3030, + 0xB890, 0x4070, 0xB890, 0xE0E0, 0xB890, 0xD080, 0xB890, 0xA010, + 0xB890, 0xE040, 0xB890, 0x80B0, 0xB890, 0x50B0, 0xB890, 0x2090, + 0xB820, 0x0000, 0xFFFF, 0xFFFF +}; + +static void +rtl8127_real_set_phy_mcu_8127a_tc_1(struct net_device *dev) +{ + rtl8127_set_phy_mcu_ram_code(dev, + phy_mcu_ram_code_8127a_tc_1, + ARRAY_SIZE(phy_mcu_ram_code_8127a_tc_1)); +} + +static void +rtl8127_set_phy_mcu_8127a_tc(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + rtl8127_set_phy_mcu_patch_request(tp); + + rtl8127_real_set_phy_mcu_8127a_tc_1(dev); + + rtl8127_clear_phy_mcu_patch_request(tp); +} + +static void +rtl8127_real_set_phy_mcu_8127a_1(struct net_device *dev) +{ + rtl8127_set_phy_mcu_ram_code(dev, + phy_mcu_ram_code_8127a_1, + ARRAY_SIZE(phy_mcu_ram_code_8127a_1)); +} + +static void +rtl8127_real_set_phy_mcu_8127a_2(struct net_device *dev) +{ + rtl8127_set_phy_mcu_ram_code(dev, + phy_mcu_ram_code_8127a_2, + ARRAY_SIZE(phy_mcu_ram_code_8127a_2)); +} + +static void +rtl8127_set_phy_mcu_8127a_1(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + rtl8127_set_phy_mcu_patch_request(tp); + + rtl8127_real_set_phy_mcu_8127a_1(dev); + + rtl8127_clear_phy_mcu_patch_request(tp); + + rtl8127_set_phy_mcu_patch_request(tp); + + rtl8127_real_set_phy_mcu_8127a_2(dev); + + rtl8127_clear_phy_mcu_patch_request(tp); +} + +static void +rtl8127_init_hw_phy_mcu(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u8 require_disable_phy_disable_mode = FALSE; + + if (tp->NotWrRamCodeToMicroP == TRUE) + return; + + if (rtl8127_check_hw_phy_mcu_code_ver(dev)) + return; + + if (HW_SUPPORT_CHECK_PHY_DISABLE_MODE(tp) && rtl8127_is_in_phy_disable_mode(dev)) + require_disable_phy_disable_mode = TRUE; + + if (require_disable_phy_disable_mode) + rtl8127_disable_phy_disable_mode(dev); + + switch (tp->mcfg) { + case CFG_METHOD_1: + rtl8127_set_phy_mcu_8127a_tc(dev); + break; + case CFG_METHOD_2: + rtl8127_set_phy_mcu_8127a_1(dev); + break; + default: + break; + } + + if (require_disable_phy_disable_mode) + rtl8127_enable_phy_disable_mode(dev); + + rtl8127_write_hw_phy_mcu_code_ver(dev); + + rtl8127_mdio_write(tp,0x1F, 0x0000); + + tp->HwHasWrRamCodeToMicroP = TRUE; +} +#endif + +static void +rtl8127_enable_phy_aldps(struct rtl8127_private *tp) +{ + //enable aldps + //GPHY OCP 0xA430 bit[2] = 0x1 (en_aldps) + rtl8127_set_eth_phy_ocp_bit(tp, 0xA430, BIT_2); +} + +static void +rtl8127_tgphy_irq_mask_and_ack(struct rtl8127_private *tp) +{ + switch (tp->mcfg) { + case CFG_METHOD_2: + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA4D2, 0x0000); + (void)rtl8127_mdio_direct_read_phy_ocp(tp, 0xA4D4); + break; + default: + break; + } +} + +static void +rtl8127_hw_phy_config_8127a_tc_1(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + rtl8127_set_eth_phy_ocp_bit(tp, 0xA442, BIT_11); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x815E); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xa438, + 0xFF00, + 0x8600); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x8169); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xa438, + 0xFF00, + 0x8600); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x8174); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xa438, + 0xFF00, + 0xA100); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x83BF); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xa438, + 0xFF00, + 0x5A00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x83C5); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xa438, + 0xFF00, + 0x5A00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x83CB); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xa438, + 0xFF00, + 0x8B00); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8238); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xC000, + 0x4000); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x823A); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xA000); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x8148); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xa438, + 0xFF00, + 0x0100); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84AD); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x0C00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84B2); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x0800); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84B7); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x1400); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84BC); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0040); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84C0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00D6); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84BE); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00A0); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84AE); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0C0C); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84B0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0C0C); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xBD7A, 0xAAAA); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xBCE0, 0x6666); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x85FC); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0AAA); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x85FF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0AAA); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xAC32, BIT_3); + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xAC32, BIT_11); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xADDC, + 0x3FFF, + 0x2000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8111); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0F00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80E9); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0F00); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xAEC4, + 0xFF00, + 0x4600); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xAC56, + 0x0007, + 0x0005); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x825B); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0D00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8283); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0334); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8289); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x5600); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xADB8, 0x0190); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xAE3A, + 0x00FF, + 0x0026); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xAE4A, + 0x0FF0, + 0x0150); + + rtl8127_set_eth_phy_ocp_bit(tp, 0xAEC2, BIT_12); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xAE22, 0x0352); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xAEC0, 0x00FA); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8188); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xF500); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8203); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xF500); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x827E); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xF500); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81CB); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8246); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x818A); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xF500); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8205); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xF500); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8280); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xF500); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x83DD); + rtl8127_set_eth_phy_ocp_bit(tp, 0xB87E, BIT_9); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x840B); + rtl8127_set_eth_phy_ocp_bit(tp, 0xB87E, BIT_9); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x83BC); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_10); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x83BE); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_10); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x83C0); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x0700, + 0x0400); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x83C2); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_10); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x83C4); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_10); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x83C6); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x0700, + 0x0400); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84C2); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0xFEFF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84C4); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0003); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84C6); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0116); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84C8); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x6300); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x822A); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x4FFF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81AF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x4067); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8134); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x5069); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x822C); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x1A00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81B1); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x3A00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8136); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x5000); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x810E); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0x9000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8114); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0x9000); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xB63C, BIT_9); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80B4); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xB63B); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80E4); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0700); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80E5); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0x7000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80EA); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0500); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80EB); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0x5000); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8291); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0416); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8015); + rtl8127_set_eth_phy_ocp_bit(tp, 0xB87E, BIT_11); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x895E); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x01A0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8960); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x01A0); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x826B); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xF0AF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81F0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xF0AF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8175); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xF0AF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x826D); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0100); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81F2); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0100); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8177); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0100); + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xAC1C, BIT_8 | BIT_7); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xAC1E, BIT_13 | BIT_12); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xAD96, 0xAAFF); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xAD98, + 0x00FF, + 0x00AA); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xAE38, + 0x3FFF, + 0x2554); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xAE3A, + 0xF000, + 0xA000); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8932); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0900); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x892F); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0900); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x892C); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0900); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x85B6); + rtl8127_set_eth_phy_ocp_bit(tp, 0xB87E, 0xFF00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x85B4); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xFFFF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8905); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0xFF00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8853); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x2800); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x884B); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x3F00); + + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xBDE6, 0x3FFF); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xBDE8, 0x3FFF); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xBF0E, 0x0003); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8156); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x1600); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x80AB); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x7500); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x80C3); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x090D); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x80C6); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0xC600); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x80BF); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x5500); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8096); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x4500); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x809D); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x0200); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x809B); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0xE50A); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8099); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x9906); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x831F); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x5000); + + + rtl8127_set_eth_phy_ocp_bit(tp, 0xB648, BIT_14); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA4E0, BIT_15); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x849A); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0004); + + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xAC1C, 0x0C00); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA42C, BIT_6); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xACBA, 0xFC00); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8122); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0xC000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8123); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0xC000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80FA); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0x2000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x825B); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0xB000); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80D2); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0xD000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80D3); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0xD000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80C8); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0200); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80CA); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0300); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80E2); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x2300); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80A9); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x0F00, + 0x0A00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80AA); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0x5000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80AB); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0xA000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x805A); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0x2000); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8106); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x40CC); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x812C); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x40CC); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8096); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x7500); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x809C); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x6300); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x859E); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x1F00); + + + if (aspm && HW_HAS_WRITE_PHY_MCU_RAM_CODE(tp)) + rtl8127_enable_phy_aldps(tp); +} + +static void +rtl8127_hw_phy_config_8127a_1(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + rtl8127_tgphy_irq_mask_and_ack(tp); + + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA442, BIT_11); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8415); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x9300); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81A3); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x0F00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81AE); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x0F00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81B9); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0xB900); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x83B0); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0x0E00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x83C5); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0x0E00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x83DA); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0x0E00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x83EF); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0x0E00); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8173); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x8620); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8175); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x8671); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x817C); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_13); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8187); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_13); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8192); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_13); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x819D); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_13); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81A8); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA438, BIT_13); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81B3); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA438, BIT_13); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81BE); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_13); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x817D); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0xA600); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8188); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0xA600); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8193); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0xA600); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x819E); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0xA600); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81A9); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x1400); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81B4); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x1400); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81BF); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0xA600); + + + rtl8127_clear_eth_phy_ocp_bit(tp, 0xAEAA, (BIT_5 | BIT_3)); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84F0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x201C); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84F2); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x3117); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xAEC6, 0x0000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xAE20, 0xFFFF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xAECE, 0xFFFF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xAED2, 0xFFFF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xAEC8, 0x0000); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xAED0, BIT_0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xADB8, 0x0150); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8197); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x5000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8231); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x5000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x82CB); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x5000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x82CD); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x5700); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8233); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x5700); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8199); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x5700); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x815A); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0150); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81F4); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0150); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x828E); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0150); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81B1); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x824B); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x82E5); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0000); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84F7); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x2800); + rtl8127_set_eth_phy_ocp_bit(tp, 0xAEC2, BIT_12); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81B3); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xAD00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x824D); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xAD00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x82E7); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xAD00); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xAE4E, + 0x000F, + 0x0001); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x82CE); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xF000, + 0x4000); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84AC); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84AE); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84B0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xF818); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84B2); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x6000); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FFC); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x6008); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FFE); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xF450); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8015); + rtl8127_set_eth_phy_ocp_bit(tp, 0xB87E, BIT_9); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8016); + rtl8127_set_eth_phy_ocp_bit(tp, 0xB87E, BIT_11); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FE6); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x0800); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FE4); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x2114); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8647); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xA7B1); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8649); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xBBCA); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x864B); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0xDC00); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8154); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xC000, + 0x4000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8158); + rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0xC000); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x826C); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xFFFF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x826E); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xFFFF); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8872); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x0E00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8012); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_11); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8012); + rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_14); + rtl8127_set_eth_phy_ocp_bit(tp, 0xB576, BIT_0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x834A); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x0700); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8217); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0x3F00, + 0x2A00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81B1); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0xFF00, + 0x0B00); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FED); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x4E00); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8370); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x8671); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8372); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x86C8); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8401); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x86C8); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8403); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x86DA); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8406); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x1800, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8408); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x1800, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x840A); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x1800, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x840C); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x1800, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x840E); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x1800, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8410); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x1800, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8412); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x1800, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8414); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x1800, + 0x1000); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8416); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xA438, + 0x1800, + 0x1000); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x82BD); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x1F40); + + + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xBFB4, + 0x07FF, + 0x0328); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xBFB6, 0x3E14); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81C4); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x003B); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0086); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00B7); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00DB); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00FE); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00FE); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00FE); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00FE); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00C3); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0078); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0047); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0023); + + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x88D7); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x01A0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x88D9); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x01A0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FFA); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x002A); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FEE); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xFFDF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FF0); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xFFFF); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FF2); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0A4A); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FF4); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xAA5A); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FF6); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0A4A); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FF8); + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xAA5A); + + rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x88D5); + rtl8127_clear_and_set_eth_phy_ocp_bit(tp, + 0xB87E, + 0xFF00, + 0x0200); + + + rtl8127_set_eth_phy_ocp_bit(tp, 0xA430, BIT_1 | BIT_0); + + + if (aspm && HW_HAS_WRITE_PHY_MCU_RAM_CODE(tp)) + rtl8127_enable_phy_aldps(tp); +} + +static void +rtl8127_hw_phy_config(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned long flags; + + if (tp->resume_not_chg_speed) + return; + + tp->phy_reset_enable(dev); + + if (HW_DASH_SUPPORT_TYPE_3(tp) && tp->HwPkgDet == 0x06) + return; + + spin_lock_irqsave(&tp->phy_lock, flags); + +#ifndef ENABLE_USE_FIRMWARE_FILE + if (!tp->rtl_fw) + rtl8127_init_hw_phy_mcu(dev); +#endif + + switch (tp->mcfg) { + case CFG_METHOD_1: + rtl8127_hw_phy_config_8127a_tc_1(dev); + break; + case CFG_METHOD_2: + rtl8127_hw_phy_config_8127a_1(dev); + break; + default: + break; + } + + //legacy force mode(Chap 22) + rtl8127_clear_eth_phy_ocp_bit(tp, 0xA5B4, BIT_15); + + rtl8127_mdio_write(tp, 0x1F, 0x0000); + + if (HW_HAS_WRITE_PHY_MCU_RAM_CODE(tp)) { + if (tp->eee.eee_enabled) + rtl8127_enable_eee(tp); + else + rtl8127_disable_eee(tp); + } + + spin_unlock_irqrestore(&tp->phy_lock, flags); +} + +static void +rtl8127_up(struct net_device *dev) +{ + rtl8127_hw_init(dev); + rtl8127_hw_reset(dev); + rtl8127_powerup_pll(dev); + rtl8127_hw_ephy_config(dev); + rtl8127_hw_phy_config(dev); + rtl8127_hw_config(dev); +} + +/* +static inline void rtl8127_delete_esd_timer(struct net_device *dev, struct timer_list *timer) +{ + del_timer_sync(timer); +} + +static inline void rtl8127_request_esd_timer(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + struct timer_list *timer = &tp->esd_timer; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) + setup_timer(timer, rtl8127_esd_timer, (unsigned long)dev); +#else + timer_setup(timer, rtl8127_esd_timer, 0); +#endif + mod_timer(timer, jiffies + RTL8127_ESD_TIMEOUT); +} +*/ + +/* +static inline void rtl8127_delete_link_timer(struct net_device *dev, struct timer_list *timer) +{ + del_timer_sync(timer); +} + +static inline void rtl8127_request_link_timer(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + struct timer_list *timer = &tp->link_timer; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) + setup_timer(timer, rtl8127_link_timer, (unsigned long)dev); +#else + timer_setup(timer, rtl8127_link_timer, 0); +#endif + mod_timer(timer, jiffies + RTL8127_LINK_TIMEOUT); +} +*/ + +#ifdef CONFIG_NET_POLL_CONTROLLER +/* + * Polling 'interrupt' - used by things like netconsole to send skbs + * without having to re-enable interrupts. It's not called while + * the interrupt routine is executing. + */ +static void +rtl8127_netpoll(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i; + for (i = 0; i < tp->irq_nvecs; i++) { + struct r8127_irq *irq = &tp->irq_tbl[i]; + struct r8127_napi *r8127napi = &tp->r8127napi[i]; + + disable_irq(irq->vector); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + irq->handler(irq->vector, r8127napi); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) + irq->handler(irq->vector, r8127napi, NULL); +#else + irq->handler(irq->vector, r8127napi); +#endif + + enable_irq(irq->vector); + } +} +#endif //CONFIG_NET_POLL_CONTROLLER + +static void +rtl8127_setup_interrupt_mask(struct rtl8127_private *tp) +{ + int i; + + if (tp->HwCurrIsrVer == 6) { + tp->intr_mask = ISRIMR_V6_LINKCHG | ISRIMR_V6_TOK_Q0; + if (tp->num_tx_rings > 1) + tp->intr_mask |= ISRIMR_V6_TOK_Q1; + for (i = 0; i < tp->num_rx_rings; i++) + tp->intr_mask |= ISRIMR_V6_ROK_Q0 << i; + } else if (tp->HwCurrIsrVer == 5) { + tp->intr_mask = ISRIMR_V5_LINKCHG | ISRIMR_V5_TOK_Q0; + if (tp->num_tx_rings > 1) + tp->intr_mask |= ISRIMR_V5_TOK_Q1; + for (i = 0; i < tp->num_rx_rings; i++) + tp->intr_mask |= ISRIMR_V5_ROK_Q0 << i; + } else if (tp->HwCurrIsrVer == 4) { + tp->intr_mask = ISRIMR_V4_LINKCHG; + for (i = 0; i < tp->num_rx_rings; i++) + tp->intr_mask |= ISRIMR_V4_ROK_Q0 << i; + } else if (tp->HwCurrIsrVer == 3) { + tp->intr_mask = ISRIMR_V2_LINKCHG; + for (i = 0; i < max(tp->num_tx_rings, tp->num_rx_rings); i++) + tp->intr_mask |= ISRIMR_V2_ROK_Q0 << i; + } else if (tp->HwCurrIsrVer == 2) { + tp->intr_mask = ISRIMR_V2_LINKCHG | ISRIMR_TOK_Q0; + if (tp->num_tx_rings > 1) + tp->intr_mask |= ISRIMR_TOK_Q1; + + for (i = 0; i < tp->num_rx_rings; i++) + tp->intr_mask |= ISRIMR_V2_ROK_Q0 << i; + } else { + tp->intr_mask = LinkChg | RxDescUnavail | TxOK | RxOK | SWInt; + tp->timer_intr_mask = LinkChg | PCSTimeout; + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) { + if (HW_DASH_SUPPORT_TYPE_3(tp)) { + tp->timer_intr_mask |= (ISRIMR_DASH_INTR_EN | ISRIMR_DASH_INTR_CMAC_RESET); + tp->intr_mask |= (ISRIMR_DASH_INTR_EN | ISRIMR_DASH_INTR_CMAC_RESET); + } + } +#endif + } +} + +static void +rtl8127_setup_mqs_reg(struct rtl8127_private *tp) +{ + u16 hw_clo_ptr0_reg, sw_tail_ptr0_reg; + u16 reg_len; + int i; + + //tx + tp->tx_ring[0].tdsar_reg = TxDescStartAddrLow; + for (i = 1; i < tp->HwSuppNumTxQueues; i++) + tp->tx_ring[i].tdsar_reg = (u16)(TNPDS_Q1_LOW_8125 + (i - 1) * 8); + + switch (tp->HwSuppTxNoCloseVer) { + case 4: + case 5: + hw_clo_ptr0_reg = HW_CLO_PTR0_8126; + sw_tail_ptr0_reg = SW_TAIL_PTR0_8126; + reg_len = 4; + break; + case 6: + hw_clo_ptr0_reg = HW_CLO_PTR0_8125BP; + sw_tail_ptr0_reg = SW_TAIL_PTR0_8125BP; + reg_len = 8; + break; + default: + hw_clo_ptr0_reg = HW_CLO_PTR0_8125; + sw_tail_ptr0_reg = SW_TAIL_PTR0_8125; + reg_len = 4; + break; + } + + for (i = 0; i < tp->HwSuppNumTxQueues; i++) { + tp->tx_ring[i].hw_clo_ptr_reg = (u16)(hw_clo_ptr0_reg + i * reg_len); + tp->tx_ring[i].sw_tail_ptr_reg = (u16)(sw_tail_ptr0_reg + i * reg_len); + } + + //rx + tp->rx_ring[0].rdsar_reg = RxDescAddrLow; + for (i = 1; i < tp->HwSuppNumRxQueues; i++) + tp->rx_ring[i].rdsar_reg = (u16)(RDSAR_Q1_LOW_8125 + (i - 1) * 8); + + tp->isr_reg[0] = ISR0_8125; + for (i = 1; i < tp->hw_supp_irq_nvecs; i++) + tp->isr_reg[i] = (u16)(ISR1_8125 + (i - 1) * 4); + + tp->imr_reg[0] = IMR0_8125; + for (i = 1; i < tp->hw_supp_irq_nvecs; i++) + tp->imr_reg[i] = (u16)(IMR1_8125 + (i - 1) * 4); +} + +static void +rtl8127_init_software_variable(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + struct pci_dev *pdev = tp->pci_dev; + +#ifdef ENABLE_LIB_SUPPORT + tp->ring_lib_enabled = 1; +#endif + + switch (tp->mcfg) { + default: + tp->HwSuppDashVer = 0; + break; + } + tp->AllowAccessDashOcp = rtl8127_is_allow_access_dash_ocp(tp); + + tp->HwPkgDet = rtl8127_mac_ocp_read(tp, 0xDC00); + tp->HwPkgDet = (tp->HwPkgDet >> 3) & 0x07; + + if (HW_DASH_SUPPORT_TYPE_3(tp) && tp->HwPkgDet == 0x06) + eee_enable = 0; + + tp->HwSuppNowIsOobVer = 1; + + tp->HwPcieSNOffset = 0x168; + +#ifdef ENABLE_REALWOW_SUPPORT + rtl8127_get_realwow_hw_version(dev); +#endif //ENABLE_REALWOW_SUPPORT + + if (HW_DASH_SUPPORT_DASH(tp) && rtl8127_check_dash(tp)) + tp->DASH = 1; + else + tp->DASH = 0; + + if (tp->DASH) { + if (HW_DASH_SUPPORT_TYPE_3(tp)) { + u64 CmacMemPhysAddress; + void __iomem *cmac_ioaddr = NULL; + + //map CMAC IO space + CmacMemPhysAddress = rtl8127_csi_other_fun_read(tp, 0, 0x18); + if (!(CmacMemPhysAddress & BIT_0)) { + if (CmacMemPhysAddress & BIT_2) + CmacMemPhysAddress |= (u64)rtl8127_csi_other_fun_read(tp, 0, 0x1C) << 32; + + CmacMemPhysAddress &= 0xFFFFFFF0; + /* ioremap MMIO region */ + cmac_ioaddr = ioremap(CmacMemPhysAddress, R8127_REGS_SIZE); + } + + if (cmac_ioaddr == NULL) { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_probe(tp)) + dev_err(&pdev->dev, "cannot remap CMAC MMIO, aborting\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + } + + if (cmac_ioaddr == NULL) + tp->DASH = 0; + else + tp->mapped_cmac_ioaddr = cmac_ioaddr; + } + + eee_enable = 0; + } + + if (HW_DASH_SUPPORT_TYPE_3(tp)) + tp->cmac_ioaddr = tp->mapped_cmac_ioaddr; + + if (aspm) { + tp->org_pci_offset_99 = rtl8127_csi_fun0_read_byte(tp, 0x99); + tp->org_pci_offset_99 &= ~(BIT_5|BIT_6); + + tp->org_pci_offset_180 = rtl8127_csi_fun0_read_byte(tp, 0x22c); + } + + pci_read_config_byte(pdev, 0x80, &tp->org_pci_offset_80); + pci_read_config_byte(pdev, 0x81, &tp->org_pci_offset_81); + + tp->use_timer_interrupt = TRUE; + + tp->HwSuppMaxPhyLinkSpeed = 10000; + + if (timer_count == 0 || tp->mcfg == CFG_METHOD_DEFAULT) + tp->use_timer_interrupt = FALSE; + + tp->ShortPacketSwChecksum = TRUE; + tp->UseSwPaddingShortPkt = TRUE; + + tp->HwSuppMagicPktVer = WAKEUP_MAGIC_PACKET_V3; + + tp->HwSuppLinkChgWakeUpVer = 3; + + tp->HwSuppD0SpeedUpVer = 2; + + tp->HwSuppCheckPhyDisableModeVer = 3; + + tp->HwSuppTxNoCloseVer = 6; + + switch (tp->HwSuppTxNoCloseVer) { + case 5: + case 6: + tp->MaxTxDescPtrMask = MAX_TX_NO_CLOSE_DESC_PTR_MASK_V4; + break; + case 4: + tp->MaxTxDescPtrMask = MAX_TX_NO_CLOSE_DESC_PTR_MASK_V3; + break; + case 3: + tp->MaxTxDescPtrMask = MAX_TX_NO_CLOSE_DESC_PTR_MASK_V2; + break; + default: + tx_no_close_enable = 0; + break; + } + + if (tp->HwSuppTxNoCloseVer > 0 && tx_no_close_enable == 1) + tp->EnableTxNoClose = TRUE; + + switch (tp->mcfg) { + case CFG_METHOD_1: + tp->sw_ram_code_ver = NIC_RAMCODE_VERSION_CFG_METHOD_1; + break; + case CFG_METHOD_2: + tp->sw_ram_code_ver = NIC_RAMCODE_VERSION_CFG_METHOD_2; + break; + default: + break; + } + + if (tp->HwIcVerUnknown) { + tp->NotWrRamCodeToMicroP = TRUE; + tp->NotWrMcuPatchCode = TRUE; + } + + tp->HwSuppMacMcuVer = 2; + + tp->MacMcuPageSize = RTL8127_MAC_MCU_PAGE_SIZE; + + tp->HwSuppNumTxQueues = 2; + tp->HwSuppNumRxQueues = 4; + + //init interrupt + tp->HwSuppIsrVer = 6; + + tp->HwCurrIsrVer = tp->HwSuppIsrVer; + if (tp->HwCurrIsrVer > 1) { + if (!(tp->features & RTL_FEATURE_MSIX) || + tp->irq_nvecs < tp->min_irq_nvecs) + tp->HwCurrIsrVer = 1; + } + + tp->num_tx_rings = 1; +#ifdef ENABLE_MULTIPLE_TX_QUEUE +#ifndef ENABLE_LIB_SUPPORT + tp->num_tx_rings = tp->HwSuppNumTxQueues; +#endif +#endif + if (tp->HwCurrIsrVer < 2 || + (tp->HwCurrIsrVer == 2 && tp->irq_nvecs < 19)) + tp->num_tx_rings = 1; + + //RSS + tp->HwSuppRssVer = 5; + tp->HwSuppIndirTblEntries = 128; + + tp->num_rx_rings = 1; +#ifdef ENABLE_RSS_SUPPORT +#ifdef ENABLE_LIB_SUPPORT + if (tp->HwSuppRssVer > 0) + tp->EnableRss = 1; +#else + if (tp->HwSuppRssVer > 0 && tp->HwCurrIsrVer > 1) { + u8 rss_queue_num = netif_get_num_default_rss_queues(); + tp->num_rx_rings = (tp->HwSuppNumRxQueues > rss_queue_num)? + rss_queue_num : tp->HwSuppNumRxQueues; + + if (!(tp->num_rx_rings >= 2 && tp->irq_nvecs >= tp->num_rx_rings)) + tp->num_rx_rings = 1; + + if (tp->num_rx_rings >= 2) + tp->EnableRss = 1; + } +#endif +#endif + + //interrupt mask + rtl8127_setup_interrupt_mask(tp); + + rtl8127_setup_mqs_reg(tp); + + rtl8127_set_ring_size(tp, NUM_RX_DESC, NUM_TX_DESC); + + tp->HwSuppPtpVer = 2; +#ifdef ENABLE_PTP_SUPPORT + if (tp->HwSuppPtpVer > 0) + tp->EnablePtp = 1; +#endif + + tp->HwSuppIntMitiVer = 6; + + tp->HwSuppTcamVer = 2; + + tp->TcamNotValidReg = TCAM_NOTVALID_ADDR_V2; + tp->TcamValidReg = TCAM_VALID_ADDR_V2; + tp->TcamMaAddrcOffset = TCAM_MAC_ADDR_V2; + tp->TcamVlanTagOffset = TCAM_VLAN_TAG_V2; + + tp->HwSuppExtendTallyCounterVer = 1; + + timer_count_v2 = (timer_count / 0x200); + + tp->HwSuppRxDescType = RX_DESC_RING_TYPE_4; + + tp->InitRxDescType = RX_DESC_RING_TYPE_1; + tp->RxDescLength = RX_DESC_LEN_TYPE_1; + switch (tp->HwSuppRxDescType) { + case RX_DESC_RING_TYPE_3: + if (tp->EnableRss) { + tp->InitRxDescType = RX_DESC_RING_TYPE_3; + tp->RxDescLength = RX_DESC_LEN_TYPE_3; + } + break; + case RX_DESC_RING_TYPE_4: + if (tp->EnableRss) { + tp->InitRxDescType = RX_DESC_RING_TYPE_4; + tp->RxDescLength = RX_DESC_LEN_TYPE_4; + } + break; + } + + tp->rtl8127_rx_config = rtl_chip_info[tp->chipset].RCR_Cfg; + if (tp->InitRxDescType == RX_DESC_RING_TYPE_3) + tp->rtl8127_rx_config |= EnableRxDescV3; + else if (tp->InitRxDescType == RX_DESC_RING_TYPE_4) + tp->rtl8127_rx_config &= ~EnableRxDescV4_1; + + tp->NicCustLedValue = RTL_R16(tp, CustomLED); + + tp->wol_opts = rtl8127_get_hw_wol(tp); + tp->wol_enabled = (tp->wol_opts) ? WOL_ENABLED : WOL_DISABLED; + + rtl8127_set_link_option(tp, autoneg_mode, speed_mode, duplex_mode, + rtl8127_fc_full); + + tp->max_jumbo_frame_size = rtl_chip_info[tp->chipset].jumbo_frame_sz; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + /* MTU range: 60 - hw-specific max */ + dev->min_mtu = ETH_MIN_MTU; + dev->max_mtu = tp->max_jumbo_frame_size; +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + + if (tp->mcfg != CFG_METHOD_DEFAULT) { + struct ethtool_keee *eee = &tp->eee; + + eee->eee_enabled = eee_enable; +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0) + eee->supported = SUPPORTED_100baseT_Full | + SUPPORTED_1000baseT_Full | + SUPPORTED_2500baseX_Full; + eee->advertised = mmd_eee_adv_to_ethtool_adv_t(MDIO_EEE_1000T | MDIO_EEE_100TX); + eee->advertised |= SUPPORTED_2500baseX_Full; +#else + linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, eee->supported); + linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT, eee->supported); + linkmode_set_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, eee->supported); + linkmode_set_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, eee->supported); + linkmode_set_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, eee->supported); + linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, eee->advertised); + linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT, eee->advertised); + linkmode_set_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, eee->advertised); + linkmode_set_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, eee->advertised); + linkmode_set_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, eee->advertised); +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0) */ + eee->tx_lpi_enabled = eee_enable; + eee->tx_lpi_timer = dev->mtu + ETH_HLEN + 0x20; + } + +#ifdef ENABLE_RSS_SUPPORT + if (tp->EnableRss) + rtl8127_init_rss(tp); +#endif +} + +static void +rtl8127_release_board(struct pci_dev *pdev, + struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + void __iomem *ioaddr = tp->mmio_addr; + + rtl8127_rar_set(tp, tp->org_mac_addr); + tp->wol_enabled = WOL_DISABLED; + + if (!tp->DASH) + rtl8127_phy_power_down(dev); + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) + FreeAllocatedDashShareMemory(dev); +#endif + + if (tp->mapped_cmac_ioaddr != NULL) + iounmap(tp->mapped_cmac_ioaddr); + + iounmap(ioaddr); + pci_release_regions(pdev); + pci_clear_mwi(pdev); + pci_disable_device(pdev); + free_netdev(dev); +} + +static void +rtl8127_hw_address_set(struct net_device *dev, u8 mac_addr[MAC_ADDR_LEN]) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0) + eth_hw_addr_set(dev, mac_addr); +#else + memcpy(dev->dev_addr, mac_addr, MAC_ADDR_LEN); +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0) +} + +static int +rtl8127_get_mac_address(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i; + u8 mac_addr[MAC_ADDR_LEN]; + + for (i = 0; i < MAC_ADDR_LEN; i++) + mac_addr[i] = RTL_R8(tp, MAC0 + i); + + *(u32*)&mac_addr[0] = RTL_R32(tp, BACKUP_ADDR0_8125); + *(u16*)&mac_addr[4] = RTL_R16(tp, BACKUP_ADDR1_8125); + + if (!is_valid_ether_addr(mac_addr)) { + netif_err(tp, probe, dev, "Invalid ether addr %pM\n", + mac_addr); + eth_random_addr(mac_addr); + dev->addr_assign_type = NET_ADDR_RANDOM; + netif_info(tp, probe, dev, "Random ether addr %pM\n", + mac_addr); + tp->random_mac = 1; + } + + rtl8127_hw_address_set(dev, mac_addr); + rtl8127_rar_set(tp, mac_addr); + + /* keep the original MAC address */ + memcpy(tp->org_mac_addr, dev->dev_addr, MAC_ADDR_LEN); +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,13) + memcpy(dev->perm_addr, dev->dev_addr, MAC_ADDR_LEN); +#endif + return 0; +} + +/** + * rtl8127_set_mac_address - Change the Ethernet Address of the NIC + * @dev: network interface device structure + * @p: pointer to an address structure + * + * Return 0 on success, negative on failure + **/ +static int +rtl8127_set_mac_address(struct net_device *dev, + void *p) +{ + struct rtl8127_private *tp = netdev_priv(dev); + struct sockaddr *addr = p; + + if (!is_valid_ether_addr(addr->sa_data)) + return -EADDRNOTAVAIL; + + rtl8127_hw_address_set(dev, addr->sa_data); + + rtl8127_rar_set(tp, dev->dev_addr); + + return 0; +} + +/****************************************************************************** + * rtl8127_rar_set - Puts an ethernet address into a receive address register. + * + * tp - The private data structure for driver + * addr - Address to put into receive address register + *****************************************************************************/ +void +rtl8127_rar_set(struct rtl8127_private *tp, + const u8 *addr) +{ + uint32_t rar_low = 0; + uint32_t rar_high = 0; + + rar_low = ((uint32_t) addr[0] | + ((uint32_t) addr[1] << 8) | + ((uint32_t) addr[2] << 16) | + ((uint32_t) addr[3] << 24)); + + rar_high = ((uint32_t) addr[4] | + ((uint32_t) addr[5] << 8)); + + rtl8127_enable_cfg9346_write(tp); + RTL_W32(tp, MAC0, rar_low); + RTL_W32(tp, MAC4, rar_high); + + rtl8127_disable_cfg9346_write(tp); +} + +#ifdef ETHTOOL_OPS_COMPAT +static int ethtool_get_settings(struct net_device *dev, void *useraddr) +{ + struct ethtool_cmd cmd = { ETHTOOL_GSET }; + int err; + + if (!ethtool_ops->get_settings) + return -EOPNOTSUPP; + + err = ethtool_ops->get_settings(dev, &cmd); + if (err < 0) + return err; + + if (copy_to_user(useraddr, &cmd, sizeof(cmd))) + return -EFAULT; + return 0; +} + +static int ethtool_set_settings(struct net_device *dev, void *useraddr) +{ + struct ethtool_cmd cmd; + + if (!ethtool_ops->set_settings) + return -EOPNOTSUPP; + + if (copy_from_user(&cmd, useraddr, sizeof(cmd))) + return -EFAULT; + + return ethtool_ops->set_settings(dev, &cmd); +} + +static int ethtool_get_drvinfo(struct net_device *dev, void *useraddr) +{ + struct ethtool_drvinfo info; + struct ethtool_ops *ops = ethtool_ops; + + if (!ops->get_drvinfo) + return -EOPNOTSUPP; + + memset(&info, 0, sizeof(info)); + info.cmd = ETHTOOL_GDRVINFO; + ops->get_drvinfo(dev, &info); + + if (ops->self_test_count) + info.testinfo_len = ops->self_test_count(dev); + if (ops->get_stats_count) + info.n_stats = ops->get_stats_count(dev); + if (ops->get_regs_len) + info.regdump_len = ops->get_regs_len(dev); + if (ops->get_eeprom_len) + info.eedump_len = ops->get_eeprom_len(dev); + + if (copy_to_user(useraddr, &info, sizeof(info))) + return -EFAULT; + return 0; +} + +static int ethtool_get_regs(struct net_device *dev, char *useraddr) +{ + struct ethtool_regs regs; + struct ethtool_ops *ops = ethtool_ops; + void *regbuf; + int reglen, ret; + + if (!ops->get_regs || !ops->get_regs_len) + return -EOPNOTSUPP; + + if (copy_from_user(®s, useraddr, sizeof(regs))) + return -EFAULT; + + reglen = ops->get_regs_len(dev); + if (regs.len > reglen) + regs.len = reglen; + + regbuf = kmalloc(reglen, GFP_USER); + if (!regbuf) + return -ENOMEM; + + ops->get_regs(dev, ®s, regbuf); + + ret = -EFAULT; + if (copy_to_user(useraddr, ®s, sizeof(regs))) + goto out; + useraddr += offsetof(struct ethtool_regs, data); + if (copy_to_user(useraddr, regbuf, reglen)) + goto out; + ret = 0; + +out: + kfree(regbuf); + return ret; +} + +static int ethtool_get_wol(struct net_device *dev, char *useraddr) +{ + struct ethtool_wolinfo wol = { ETHTOOL_GWOL }; + + if (!ethtool_ops->get_wol) + return -EOPNOTSUPP; + + ethtool_ops->get_wol(dev, &wol); + + if (copy_to_user(useraddr, &wol, sizeof(wol))) + return -EFAULT; + return 0; +} + +static int ethtool_set_wol(struct net_device *dev, char *useraddr) +{ + struct ethtool_wolinfo wol; + + if (!ethtool_ops->set_wol) + return -EOPNOTSUPP; + + if (copy_from_user(&wol, useraddr, sizeof(wol))) + return -EFAULT; + + return ethtool_ops->set_wol(dev, &wol); +} + +static int ethtool_get_msglevel(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata = { ETHTOOL_GMSGLVL }; + + if (!ethtool_ops->get_msglevel) + return -EOPNOTSUPP; + + edata.data = ethtool_ops->get_msglevel(dev); + + if (copy_to_user(useraddr, &edata, sizeof(edata))) + return -EFAULT; + return 0; +} + +static int ethtool_set_msglevel(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata; + + if (!ethtool_ops->set_msglevel) + return -EOPNOTSUPP; + + if (copy_from_user(&edata, useraddr, sizeof(edata))) + return -EFAULT; + + ethtool_ops->set_msglevel(dev, edata.data); + return 0; +} + +static int ethtool_nway_reset(struct net_device *dev) +{ + if (!ethtool_ops->nway_reset) + return -EOPNOTSUPP; + + return ethtool_ops->nway_reset(dev); +} + +static int ethtool_get_link(struct net_device *dev, void *useraddr) +{ + struct ethtool_value edata = { ETHTOOL_GLINK }; + + if (!ethtool_ops->get_link) + return -EOPNOTSUPP; + + edata.data = ethtool_ops->get_link(dev); + + if (copy_to_user(useraddr, &edata, sizeof(edata))) + return -EFAULT; + return 0; +} + +static int ethtool_get_eeprom(struct net_device *dev, void *useraddr) +{ + struct ethtool_eeprom eeprom; + struct ethtool_ops *ops = ethtool_ops; + u8 *data; + int ret; + + if (!ops->get_eeprom || !ops->get_eeprom_len) + return -EOPNOTSUPP; + + if (copy_from_user(&eeprom, useraddr, sizeof(eeprom))) + return -EFAULT; + + /* Check for wrap and zero */ + if (eeprom.offset + eeprom.len <= eeprom.offset) + return -EINVAL; + + /* Check for exceeding total eeprom len */ + if (eeprom.offset + eeprom.len > ops->get_eeprom_len(dev)) + return -EINVAL; + + data = kmalloc(eeprom.len, GFP_USER); + if (!data) + return -ENOMEM; + + ret = -EFAULT; + if (copy_from_user(data, useraddr + sizeof(eeprom), eeprom.len)) + goto out; + + ret = ops->get_eeprom(dev, &eeprom, data); + if (ret) + goto out; + + ret = -EFAULT; + if (copy_to_user(useraddr, &eeprom, sizeof(eeprom))) + goto out; + if (copy_to_user(useraddr + sizeof(eeprom), data, eeprom.len)) + goto out; + ret = 0; + +out: + kfree(data); + return ret; +} + +static int ethtool_set_eeprom(struct net_device *dev, void *useraddr) +{ + struct ethtool_eeprom eeprom; + struct ethtool_ops *ops = ethtool_ops; + u8 *data; + int ret; + + if (!ops->set_eeprom || !ops->get_eeprom_len) + return -EOPNOTSUPP; + + if (copy_from_user(&eeprom, useraddr, sizeof(eeprom))) + return -EFAULT; + + /* Check for wrap and zero */ + if (eeprom.offset + eeprom.len <= eeprom.offset) + return -EINVAL; + + /* Check for exceeding total eeprom len */ + if (eeprom.offset + eeprom.len > ops->get_eeprom_len(dev)) + return -EINVAL; + + data = kmalloc(eeprom.len, GFP_USER); + if (!data) + return -ENOMEM; + + ret = -EFAULT; + if (copy_from_user(data, useraddr + sizeof(eeprom), eeprom.len)) + goto out; + + ret = ops->set_eeprom(dev, &eeprom, data); + if (ret) + goto out; + + if (copy_to_user(useraddr + sizeof(eeprom), data, eeprom.len)) + ret = -EFAULT; + +out: + kfree(data); + return ret; +} + +static int ethtool_get_coalesce(struct net_device *dev, void *useraddr) +{ + struct ethtool_coalesce coalesce = { ETHTOOL_GCOALESCE }; + + if (!ethtool_ops->get_coalesce) + return -EOPNOTSUPP; + + ethtool_ops->get_coalesce(dev, &coalesce); + + if (copy_to_user(useraddr, &coalesce, sizeof(coalesce))) + return -EFAULT; + return 0; +} + +static int ethtool_set_coalesce(struct net_device *dev, void *useraddr) +{ + struct ethtool_coalesce coalesce; + + if (!ethtool_ops->get_coalesce) + return -EOPNOTSUPP; + + if (copy_from_user(&coalesce, useraddr, sizeof(coalesce))) + return -EFAULT; + + return ethtool_ops->set_coalesce(dev, &coalesce); +} + +static int ethtool_get_ringparam(struct net_device *dev, void *useraddr) +{ + struct ethtool_ringparam ringparam = { ETHTOOL_GRINGPARAM }; + + if (!ethtool_ops->get_ringparam) + return -EOPNOTSUPP; + + ethtool_ops->get_ringparam(dev, &ringparam); + + if (copy_to_user(useraddr, &ringparam, sizeof(ringparam))) + return -EFAULT; + return 0; +} + +static int ethtool_set_ringparam(struct net_device *dev, void *useraddr) +{ + struct ethtool_ringparam ringparam; + + if (!ethtool_ops->get_ringparam) + return -EOPNOTSUPP; + + if (copy_from_user(&ringparam, useraddr, sizeof(ringparam))) + return -EFAULT; + + return ethtool_ops->set_ringparam(dev, &ringparam); +} + +static int ethtool_get_pauseparam(struct net_device *dev, void *useraddr) +{ + struct ethtool_pauseparam pauseparam = { ETHTOOL_GPAUSEPARAM }; + + if (!ethtool_ops->get_pauseparam) + return -EOPNOTSUPP; + + ethtool_ops->get_pauseparam(dev, &pauseparam); + + if (copy_to_user(useraddr, &pauseparam, sizeof(pauseparam))) + return -EFAULT; + return 0; +} + +static int ethtool_set_pauseparam(struct net_device *dev, void *useraddr) +{ + struct ethtool_pauseparam pauseparam; + + if (!ethtool_ops->get_pauseparam) + return -EOPNOTSUPP; + + if (copy_from_user(&pauseparam, useraddr, sizeof(pauseparam))) + return -EFAULT; + + return ethtool_ops->set_pauseparam(dev, &pauseparam); +} + +static int ethtool_get_rx_csum(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata = { ETHTOOL_GRXCSUM }; + + if (!ethtool_ops->get_rx_csum) + return -EOPNOTSUPP; + + edata.data = ethtool_ops->get_rx_csum(dev); + + if (copy_to_user(useraddr, &edata, sizeof(edata))) + return -EFAULT; + return 0; +} + +static int ethtool_set_rx_csum(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata; + + if (!ethtool_ops->set_rx_csum) + return -EOPNOTSUPP; + + if (copy_from_user(&edata, useraddr, sizeof(edata))) + return -EFAULT; + + ethtool_ops->set_rx_csum(dev, edata.data); + return 0; +} + +static int ethtool_get_tx_csum(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata = { ETHTOOL_GTXCSUM }; + + if (!ethtool_ops->get_tx_csum) + return -EOPNOTSUPP; + + edata.data = ethtool_ops->get_tx_csum(dev); + + if (copy_to_user(useraddr, &edata, sizeof(edata))) + return -EFAULT; + return 0; +} + +static int ethtool_set_tx_csum(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata; + + if (!ethtool_ops->set_tx_csum) + return -EOPNOTSUPP; + + if (copy_from_user(&edata, useraddr, sizeof(edata))) + return -EFAULT; + + return ethtool_ops->set_tx_csum(dev, edata.data); +} + +static int ethtool_get_sg(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata = { ETHTOOL_GSG }; + + if (!ethtool_ops->get_sg) + return -EOPNOTSUPP; + + edata.data = ethtool_ops->get_sg(dev); + + if (copy_to_user(useraddr, &edata, sizeof(edata))) + return -EFAULT; + return 0; +} + +static int ethtool_set_sg(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata; + + if (!ethtool_ops->set_sg) + return -EOPNOTSUPP; + + if (copy_from_user(&edata, useraddr, sizeof(edata))) + return -EFAULT; + + return ethtool_ops->set_sg(dev, edata.data); +} + +static int ethtool_get_tso(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata = { ETHTOOL_GTSO }; + + if (!ethtool_ops->get_tso) + return -EOPNOTSUPP; + + edata.data = ethtool_ops->get_tso(dev); + + if (copy_to_user(useraddr, &edata, sizeof(edata))) + return -EFAULT; + return 0; +} + +static int ethtool_set_tso(struct net_device *dev, char *useraddr) +{ + struct ethtool_value edata; + + if (!ethtool_ops->set_tso) + return -EOPNOTSUPP; + + if (copy_from_user(&edata, useraddr, sizeof(edata))) + return -EFAULT; + + return ethtool_ops->set_tso(dev, edata.data); +} + +static int ethtool_self_test(struct net_device *dev, char *useraddr) +{ + struct ethtool_test test; + struct ethtool_ops *ops = ethtool_ops; + u64 *data; + int ret; + + if (!ops->self_test || !ops->self_test_count) + return -EOPNOTSUPP; + + if (copy_from_user(&test, useraddr, sizeof(test))) + return -EFAULT; + + test.len = ops->self_test_count(dev); + data = kmalloc(test.len * sizeof(u64), GFP_USER); + if (!data) + return -ENOMEM; + + ops->self_test(dev, &test, data); + + ret = -EFAULT; + if (copy_to_user(useraddr, &test, sizeof(test))) + goto out; + useraddr += sizeof(test); + if (copy_to_user(useraddr, data, test.len * sizeof(u64))) + goto out; + ret = 0; + +out: + kfree(data); + return ret; +} + +static int ethtool_get_strings(struct net_device *dev, void *useraddr) +{ + struct ethtool_gstrings gstrings; + struct ethtool_ops *ops = ethtool_ops; + u8 *data; + int ret; + + if (!ops->get_strings) + return -EOPNOTSUPP; + + if (copy_from_user(&gstrings, useraddr, sizeof(gstrings))) + return -EFAULT; + + switch (gstrings.string_set) { + case ETH_SS_TEST: + if (!ops->self_test_count) + return -EOPNOTSUPP; + gstrings.len = ops->self_test_count(dev); + break; + case ETH_SS_STATS: + if (!ops->get_stats_count) + return -EOPNOTSUPP; + gstrings.len = ops->get_stats_count(dev); + break; + default: + return -EINVAL; + } + + data = kmalloc(gstrings.len * ETH_GSTRING_LEN, GFP_USER); + if (!data) + return -ENOMEM; + + ops->get_strings(dev, gstrings.string_set, data); + + ret = -EFAULT; + if (copy_to_user(useraddr, &gstrings, sizeof(gstrings))) + goto out; + useraddr += sizeof(gstrings); + if (copy_to_user(useraddr, data, gstrings.len * ETH_GSTRING_LEN)) + goto out; + ret = 0; + +out: + kfree(data); + return ret; +} + +static int ethtool_phys_id(struct net_device *dev, void *useraddr) +{ + struct ethtool_value id; + + if (!ethtool_ops->phys_id) + return -EOPNOTSUPP; + + if (copy_from_user(&id, useraddr, sizeof(id))) + return -EFAULT; + + return ethtool_ops->phys_id(dev, id.data); +} + +static int ethtool_get_stats(struct net_device *dev, void *useraddr) +{ + struct ethtool_stats stats; + struct ethtool_ops *ops = ethtool_ops; + u64 *data; + int ret; + + if (!ops->get_ethtool_stats || !ops->get_stats_count) + return -EOPNOTSUPP; + + if (copy_from_user(&stats, useraddr, sizeof(stats))) + return -EFAULT; + + stats.n_stats = ops->get_stats_count(dev); + data = kmalloc(stats.n_stats * sizeof(u64), GFP_USER); + if (!data) + return -ENOMEM; + + ops->get_ethtool_stats(dev, &stats, data); + + ret = -EFAULT; + if (copy_to_user(useraddr, &stats, sizeof(stats))) + goto out; + useraddr += sizeof(stats); + if (copy_to_user(useraddr, data, stats.n_stats * sizeof(u64))) + goto out; + ret = 0; + +out: + kfree(data); + return ret; +} + +static int ethtool_ioctl(struct ifreq *ifr) +{ + struct net_device *dev = __dev_get_by_name(ifr->ifr_name); + void *useraddr = (void *) ifr->ifr_data; + u32 ethcmd; + + /* + * XXX: This can be pushed down into the ethtool_* handlers that + * need it. Keep existing behaviour for the moment. + */ + if (!capable(CAP_NET_ADMIN)) + return -EPERM; + + if (!dev || !netif_device_present(dev)) + return -ENODEV; + + if (copy_from_user(ðcmd, useraddr, sizeof (ethcmd))) + return -EFAULT; + + switch (ethcmd) { + case ETHTOOL_GSET: + return ethtool_get_settings(dev, useraddr); + case ETHTOOL_SSET: + return ethtool_set_settings(dev, useraddr); + case ETHTOOL_GDRVINFO: + return ethtool_get_drvinfo(dev, useraddr); + case ETHTOOL_GREGS: + return ethtool_get_regs(dev, useraddr); + case ETHTOOL_GWOL: + return ethtool_get_wol(dev, useraddr); + case ETHTOOL_SWOL: + return ethtool_set_wol(dev, useraddr); + case ETHTOOL_GMSGLVL: + return ethtool_get_msglevel(dev, useraddr); + case ETHTOOL_SMSGLVL: + return ethtool_set_msglevel(dev, useraddr); + case ETHTOOL_NWAY_RST: + return ethtool_nway_reset(dev); + case ETHTOOL_GLINK: + return ethtool_get_link(dev, useraddr); + case ETHTOOL_GEEPROM: + return ethtool_get_eeprom(dev, useraddr); + case ETHTOOL_SEEPROM: + return ethtool_set_eeprom(dev, useraddr); + case ETHTOOL_GCOALESCE: + return ethtool_get_coalesce(dev, useraddr); + case ETHTOOL_SCOALESCE: + return ethtool_set_coalesce(dev, useraddr); + case ETHTOOL_GRINGPARAM: + return ethtool_get_ringparam(dev, useraddr); + case ETHTOOL_SRINGPARAM: + return ethtool_set_ringparam(dev, useraddr); + case ETHTOOL_GPAUSEPARAM: + return ethtool_get_pauseparam(dev, useraddr); + case ETHTOOL_SPAUSEPARAM: + return ethtool_set_pauseparam(dev, useraddr); + case ETHTOOL_GRXCSUM: + return ethtool_get_rx_csum(dev, useraddr); + case ETHTOOL_SRXCSUM: + return ethtool_set_rx_csum(dev, useraddr); + case ETHTOOL_GTXCSUM: + return ethtool_get_tx_csum(dev, useraddr); + case ETHTOOL_STXCSUM: + return ethtool_set_tx_csum(dev, useraddr); + case ETHTOOL_GSG: + return ethtool_get_sg(dev, useraddr); + case ETHTOOL_SSG: + return ethtool_set_sg(dev, useraddr); + case ETHTOOL_GTSO: + return ethtool_get_tso(dev, useraddr); + case ETHTOOL_STSO: + return ethtool_set_tso(dev, useraddr); + case ETHTOOL_TEST: + return ethtool_self_test(dev, useraddr); + case ETHTOOL_GSTRINGS: + return ethtool_get_strings(dev, useraddr); + case ETHTOOL_PHYS_ID: + return ethtool_phys_id(dev, useraddr); + case ETHTOOL_GSTATS: + return ethtool_get_stats(dev, useraddr); + default: + return -EOPNOTSUPP; + } + + return -EOPNOTSUPP; +} +#endif //ETHTOOL_OPS_COMPAT + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,15,0) +static int rtl8127_siocdevprivate(struct net_device *dev, struct ifreq *ifr, + void __user *data, int cmd) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int ret = 0; + + switch (cmd) { +#ifdef ENABLE_DASH_SUPPORT + case SIOCDEVPRIVATE_RTLDASH: + if (!netif_running(dev)) { + ret = -ENODEV; + break; + } + if (!capable(CAP_NET_ADMIN)) { + ret = -EPERM; + break; + } + + ret = rtl8127_dash_ioctl(dev, ifr); + break; +#endif + +#ifdef ENABLE_REALWOW_SUPPORT + case SIOCDEVPRIVATE_RTLREALWOW: + if (!netif_running(dev)) { + ret = -ENODEV; + break; + } + + ret = rtl8127_realwow_ioctl(dev, ifr); + break; +#endif + + case SIOCRTLTOOL: + if (!capable(CAP_NET_ADMIN)) { + ret = -EPERM; + break; + } + + ret = rtl8127_tool_ioctl(tp, ifr); + break; + + default: + ret = -EOPNOTSUPP; + } + + return ret; +} +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,15,0) + +static int +rtl8127_do_ioctl(struct net_device *dev, + struct ifreq *ifr, + int cmd) +{ + struct rtl8127_private *tp = netdev_priv(dev); + struct mii_ioctl_data *data = if_mii(ifr); + int ret = 0; + + switch (cmd) { + case SIOCGMIIPHY: + data->phy_id = 32; /* Internal PHY */ + break; + + case SIOCGMIIREG: + rtl8127_mdio_write(tp, 0x1F, 0x0000); + data->val_out = rtl8127_mdio_read(tp, data->reg_num); + break; + + case SIOCSMIIREG: + if (!capable(CAP_NET_ADMIN)) + return -EPERM; + rtl8127_mdio_write(tp, 0x1F, 0x0000); + rtl8127_mdio_write(tp, data->reg_num, data->val_in); + break; + +#ifdef ETHTOOL_OPS_COMPAT + case SIOCETHTOOL: + ret = ethtool_ioctl(ifr); + break; +#endif + +#ifdef ENABLE_PTP_SUPPORT + case SIOCSHWTSTAMP: + case SIOCGHWTSTAMP: + if (tp->EnablePtp) + ret = rtl8127_ptp_ioctl(dev, ifr, cmd); + else + ret = -EOPNOTSUPP; + break; +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,15,0) +#ifdef ENABLE_DASH_SUPPORT + case SIOCDEVPRIVATE_RTLDASH: + if (!netif_running(dev)) { + ret = -ENODEV; + break; + } + if (!capable(CAP_NET_ADMIN)) { + ret = -EPERM; + break; + } + + ret = rtl8127_dash_ioctl(dev, ifr); + break; +#endif + +#ifdef ENABLE_REALWOW_SUPPORT + case SIOCDEVPRIVATE_RTLREALWOW: + if (!netif_running(dev)) { + ret = -ENODEV; + break; + } + + if (!capable(CAP_NET_ADMIN)) { + ret = -EPERM; + break; + } + + ret = rtl8127_realwow_ioctl(dev, ifr); + break; +#endif + + case SIOCRTLTOOL: + if (!capable(CAP_NET_ADMIN)) { + ret = -EPERM; + break; + } + + ret = rtl8127_tool_ioctl(tp, ifr); + break; +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(5,15,0) + + default: + ret = -EOPNOTSUPP; + break; + } + + return ret; +} + +static void +rtl8127_phy_power_up(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned long flags; + + if (rtl8127_is_in_phy_disable_mode(dev)) + return; + + spin_lock_irqsave(&tp->phy_lock, flags); + + rtl8127_mdio_write(tp, 0x1F, 0x0000); + rtl8127_mdio_write(tp, MII_BMCR, BMCR_ANENABLE); + + //wait ups resume (phy state 3) + rtl8127_wait_phy_ups_resume(dev, 3); + + spin_unlock_irqrestore(&tp->phy_lock, flags); +} + +static void +rtl8127_phy_power_down(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned long flags; + + spin_lock_irqsave(&tp->phy_lock, flags); + rtl8127_mdio_write(tp, 0x1F, 0x0000); + rtl8127_mdio_write(tp, MII_BMCR, BMCR_ANENABLE | BMCR_PDOWN); + spin_unlock_irqrestore(&tp->phy_lock, flags); +} + +static int __devinit +rtl8127_init_board(struct pci_dev *pdev, + struct net_device **dev_out, + void __iomem **ioaddr_out) +{ + void __iomem *ioaddr; + struct net_device *dev; + struct rtl8127_private *tp; + int rc = -ENOMEM, i, pm_cap; + + assert(ioaddr_out != NULL); + + /* dev zeroed in alloc_etherdev */ + dev = alloc_etherdev_mq(sizeof (*tp), R8127_MAX_QUEUES); + if (dev == NULL) { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_drv(&debug)) + dev_err(&pdev->dev, "unable to alloc new ethernet\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + goto err_out; + } + + SET_MODULE_OWNER(dev); + SET_NETDEV_DEV(dev, &pdev->dev); + tp = netdev_priv(dev); + tp->dev = dev; + tp->pci_dev = pdev; + tp->msg_enable = netif_msg_init(debug.msg_enable, R8127_MSG_DEFAULT); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) + if (!aspm) + pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1 | + PCIE_LINK_STATE_CLKPM); +#endif + + /* enable device (incl. PCI PM wakeup and hotplug setup) */ + rc = pci_enable_device(pdev); + if (rc < 0) { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_probe(tp)) + dev_err(&pdev->dev, "enable failure\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + goto err_out_free_dev; + } + + if (pci_set_mwi(pdev) < 0) { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_drv(&debug)) + dev_info(&pdev->dev, "Mem-Wr-Inval unavailable.\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + } + + /* save power state before pci_enable_device overwrites it */ + pm_cap = pci_find_capability(pdev, PCI_CAP_ID_PM); + if (pm_cap) { + u16 pwr_command; + + pci_read_config_word(pdev, pm_cap + PCI_PM_CTRL, &pwr_command); + } else { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_probe(tp)) + dev_err(&pdev->dev, "PowerManagement capability not found.\n"); +#else + printk("PowerManagement capability not found.\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + + } + + /* make sure PCI base addr 1 is MMIO */ + if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_probe(tp)) + dev_err(&pdev->dev, "region #1 not an MMIO resource, aborting\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + rc = -ENODEV; + goto err_out_mwi; + } + /* check for weird/broken PCI region reporting */ + if (pci_resource_len(pdev, 2) < R8127_REGS_SIZE) { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_probe(tp)) + dev_err(&pdev->dev, "Invalid PCI region size(s), aborting\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + rc = -ENODEV; + goto err_out_mwi; + } + + rc = pci_request_regions(pdev, MODULENAME); + if (rc < 0) { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_probe(tp)) + dev_err(&pdev->dev, "could not request regions.\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + goto err_out_mwi; + } + + if ((sizeof(dma_addr_t) > 4) && + use_dac && + !dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) && + !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) { + dev->features |= NETIF_F_HIGHDMA; + } else { + rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); + if (rc < 0) { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_probe(tp)) + dev_err(&pdev->dev, "DMA configuration failed.\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + goto err_out_free_res; + } + } + + /* ioremap MMIO region */ + ioaddr = ioremap(pci_resource_start(pdev, 2), pci_resource_len(pdev, 2)); + if (ioaddr == NULL) { +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_probe(tp)) + dev_err(&pdev->dev, "cannot remap MMIO, aborting\n"); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + rc = -EIO; + goto err_out_free_res; + } + + tp->mmio_addr = ioaddr; + + /* Identify chip attached to board */ + rtl8127_get_mac_version(tp); + + rtl8127_print_mac_version(tp); + + for (i = ARRAY_SIZE(rtl_chip_info) - 1; i >= 0; i--) { + if (tp->mcfg == rtl_chip_info[i].mcfg) + break; + } + + if (i < 0) { + /* Unknown chip: assume array element #0, original RTL-8125 */ +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (netif_msg_probe(tp)) + dev_printk(KERN_DEBUG, &pdev->dev, "unknown chip version, assuming %s\n", rtl_chip_info[0].name); +#else + printk("Realtek unknown chip version, assuming %s\n", rtl_chip_info[0].name); +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) + i++; + } + + tp->chipset = i; + + *ioaddr_out = ioaddr; + *dev_out = dev; +out: + return rc; + +err_out_free_res: + pci_release_regions(pdev); +err_out_mwi: + pci_clear_mwi(pdev); + pci_disable_device(pdev); +err_out_free_dev: + free_netdev(dev); +err_out: + *ioaddr_out = NULL; + *dev_out = NULL; + goto out; +} + +static void +rtl8127_esd_checker(struct rtl8127_private *tp) +{ + struct net_device *dev = tp->dev; + struct pci_dev *pdev = tp->pci_dev; + u8 cmd; + u16 io_base_l; + u16 mem_base_l; + u16 mem_base_h; + u8 ilr; + u16 resv_0x1c_h; + u16 resv_0x1c_l; + u16 resv_0x20_l; + u16 resv_0x20_h; + u16 resv_0x24_l; + u16 resv_0x24_h; + u16 resv_0x2c_h; + u16 resv_0x2c_l; + u32 pci_sn_l; + u32 pci_sn_h; + + if (unlikely(tp->rtk_enable_diag)) + goto exit; + + tp->esd_flag = 0; + + pci_read_config_byte(pdev, PCI_COMMAND, &cmd); + if (cmd != tp->pci_cfg_space.cmd) { + printk(KERN_ERR "%s: cmd = 0x%02x, should be 0x%02x \n.", dev->name, cmd, tp->pci_cfg_space.cmd); + pci_write_config_byte(pdev, PCI_COMMAND, tp->pci_cfg_space.cmd); + tp->esd_flag |= BIT_0; + + pci_read_config_byte(pdev, PCI_COMMAND, &cmd); + if (cmd == 0xff) { + printk(KERN_ERR "%s: pci link is down \n.", dev->name); + goto exit; + } + } + + pci_read_config_word(pdev, PCI_BASE_ADDRESS_0, &io_base_l); + if (io_base_l != tp->pci_cfg_space.io_base_l) { + printk(KERN_ERR "%s: io_base_l = 0x%04x, should be 0x%04x \n.", dev->name, io_base_l, tp->pci_cfg_space.io_base_l); + pci_write_config_word(pdev, PCI_BASE_ADDRESS_0, tp->pci_cfg_space.io_base_l); + tp->esd_flag |= BIT_1; + } + + pci_read_config_word(pdev, PCI_BASE_ADDRESS_2, &mem_base_l); + if (mem_base_l != tp->pci_cfg_space.mem_base_l) { + printk(KERN_ERR "%s: mem_base_l = 0x%04x, should be 0x%04x \n.", dev->name, mem_base_l, tp->pci_cfg_space.mem_base_l); + pci_write_config_word(pdev, PCI_BASE_ADDRESS_2, tp->pci_cfg_space.mem_base_l); + tp->esd_flag |= BIT_2; + } + + pci_read_config_word(pdev, PCI_BASE_ADDRESS_2 + 2, &mem_base_h); + if (mem_base_h!= tp->pci_cfg_space.mem_base_h) { + printk(KERN_ERR "%s: mem_base_h = 0x%04x, should be 0x%04x \n.", dev->name, mem_base_h, tp->pci_cfg_space.mem_base_h); + pci_write_config_word(pdev, PCI_BASE_ADDRESS_2 + 2, tp->pci_cfg_space.mem_base_h); + tp->esd_flag |= BIT_3; + } + + pci_read_config_word(pdev, PCI_BASE_ADDRESS_3, &resv_0x1c_l); + if (resv_0x1c_l != tp->pci_cfg_space.resv_0x1c_l) { + printk(KERN_ERR "%s: resv_0x1c_l = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x1c_l, tp->pci_cfg_space.resv_0x1c_l); + pci_write_config_word(pdev, PCI_BASE_ADDRESS_3, tp->pci_cfg_space.resv_0x1c_l); + tp->esd_flag |= BIT_4; + } + + pci_read_config_word(pdev, PCI_BASE_ADDRESS_3 + 2, &resv_0x1c_h); + if (resv_0x1c_h != tp->pci_cfg_space.resv_0x1c_h) { + printk(KERN_ERR "%s: resv_0x1c_h = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x1c_h, tp->pci_cfg_space.resv_0x1c_h); + pci_write_config_word(pdev, PCI_BASE_ADDRESS_3 + 2, tp->pci_cfg_space.resv_0x1c_h); + tp->esd_flag |= BIT_5; + } + + pci_read_config_word(pdev, PCI_BASE_ADDRESS_4, &resv_0x20_l); + if (resv_0x20_l != tp->pci_cfg_space.resv_0x20_l) { + printk(KERN_ERR "%s: resv_0x20_l = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x20_l, tp->pci_cfg_space.resv_0x20_l); + pci_write_config_word(pdev, PCI_BASE_ADDRESS_4, tp->pci_cfg_space.resv_0x20_l); + tp->esd_flag |= BIT_6; + } + + pci_read_config_word(pdev, PCI_BASE_ADDRESS_4 + 2, &resv_0x20_h); + if (resv_0x20_h != tp->pci_cfg_space.resv_0x20_h) { + printk(KERN_ERR "%s: resv_0x20_h = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x20_h, tp->pci_cfg_space.resv_0x20_h); + pci_write_config_word(pdev, PCI_BASE_ADDRESS_4 + 2, tp->pci_cfg_space.resv_0x20_h); + tp->esd_flag |= BIT_7; + } + + pci_read_config_word(pdev, PCI_BASE_ADDRESS_5, &resv_0x24_l); + if (resv_0x24_l != tp->pci_cfg_space.resv_0x24_l) { + printk(KERN_ERR "%s: resv_0x24_l = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x24_l, tp->pci_cfg_space.resv_0x24_l); + pci_write_config_word(pdev, PCI_BASE_ADDRESS_5, tp->pci_cfg_space.resv_0x24_l); + tp->esd_flag |= BIT_8; + } + + pci_read_config_word(pdev, PCI_BASE_ADDRESS_5 + 2, &resv_0x24_h); + if (resv_0x24_h != tp->pci_cfg_space.resv_0x24_h) { + printk(KERN_ERR "%s: resv_0x24_h = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x24_h, tp->pci_cfg_space.resv_0x24_h); + pci_write_config_word(pdev, PCI_BASE_ADDRESS_5 + 2, tp->pci_cfg_space.resv_0x24_h); + tp->esd_flag |= BIT_9; + } + + pci_read_config_byte(pdev, PCI_INTERRUPT_LINE, &ilr); + if (ilr != tp->pci_cfg_space.ilr) { + printk(KERN_ERR "%s: ilr = 0x%02x, should be 0x%02x \n.", dev->name, ilr, tp->pci_cfg_space.ilr); + pci_write_config_byte(pdev, PCI_INTERRUPT_LINE, tp->pci_cfg_space.ilr); + tp->esd_flag |= BIT_10; + } + + pci_read_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID, &resv_0x2c_l); + if (resv_0x2c_l != tp->pci_cfg_space.resv_0x2c_l) { + printk(KERN_ERR "%s: resv_0x2c_l = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x2c_l, tp->pci_cfg_space.resv_0x2c_l); + pci_write_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID, tp->pci_cfg_space.resv_0x2c_l); + tp->esd_flag |= BIT_11; + } + + pci_read_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID + 2, &resv_0x2c_h); + if (resv_0x2c_h != tp->pci_cfg_space.resv_0x2c_h) { + printk(KERN_ERR "%s: resv_0x2c_h = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x2c_h, tp->pci_cfg_space.resv_0x2c_h); + pci_write_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID + 2, tp->pci_cfg_space.resv_0x2c_h); + tp->esd_flag |= BIT_12; + } + + if (tp->HwPcieSNOffset > 0) { + pci_sn_l = rtl8127_csi_read(tp, tp->HwPcieSNOffset); + if (pci_sn_l != tp->pci_cfg_space.pci_sn_l) { + printk(KERN_ERR "%s: pci_sn_l = 0x%08x, should be 0x%08x \n.", dev->name, pci_sn_l, tp->pci_cfg_space.pci_sn_l); + rtl8127_csi_write(tp, tp->HwPcieSNOffset, tp->pci_cfg_space.pci_sn_l); + tp->esd_flag |= BIT_13; + } + + pci_sn_h = rtl8127_csi_read(tp, tp->HwPcieSNOffset + 4); + if (pci_sn_h != tp->pci_cfg_space.pci_sn_h) { + printk(KERN_ERR "%s: pci_sn_h = 0x%08x, should be 0x%08x \n.", dev->name, pci_sn_h, tp->pci_cfg_space.pci_sn_h); + rtl8127_csi_write(tp, tp->HwPcieSNOffset + 4, tp->pci_cfg_space.pci_sn_h); + tp->esd_flag |= BIT_14; + } + } + + if (tp->esd_flag != 0) { + printk(KERN_ERR "%s: esd_flag = 0x%04x\n.\n", dev->name, tp->esd_flag); + netif_carrier_off(dev); + netif_tx_disable(dev); + rtl8127_hw_reset(dev); + rtl8127_tx_clear(tp); + rtl8127_rx_clear(tp); + rtl8127_init_ring(dev); + rtl8127_up(dev); + rtl8127_enable_hw_linkchg_interrupt(tp); + rtl8127_set_speed(dev, tp->autoneg, tp->speed, tp->duplex, tp->advertising); + tp->esd_flag = 0; + } +exit: + return; +} +/* +static void +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) +rtl8127_esd_timer(unsigned long __opaque) +#else +rtl8127_esd_timer(struct timer_list *t) +#endif +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) + struct net_device *dev = (struct net_device *)__opaque; + struct rtl8127_private *tp = netdev_priv(dev); + struct timer_list *timer = &tp->esd_timer; +#else + struct rtl8127_private *tp = from_timer(tp, t, esd_timer); + //struct net_device *dev = tp->dev; + struct timer_list *timer = t; +#endif + rtl8127_esd_checker(tp); + + mod_timer(timer, jiffies + timeout); +} +*/ + +/* +static void +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) +rtl8127_link_timer(unsigned long __opaque) +#else +rtl8127_link_timer(struct timer_list *t) +#endif +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) + struct net_device *dev = (struct net_device *)__opaque; + struct rtl8127_private *tp = netdev_priv(dev); + struct timer_list *timer = &tp->link_timer; +#else + struct rtl8127_private *tp = from_timer(tp, t, link_timer); + struct net_device *dev = tp->dev; + struct timer_list *timer = t; +#endif + rtl8127_check_link_status(dev); + + mod_timer(timer, jiffies + RTL8127_LINK_TIMEOUT); +} +*/ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,14,0) +static int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries, + int minvec, int maxvec) +{ + int nvec = maxvec; + int rc; + + if (maxvec < minvec) + return -ERANGE; + + do { + rc = pci_enable_msix(dev, entries, nvec); + if (rc < 0) { + return rc; + } else if (rc > 0) { + if (rc < minvec) + return -ENOSPC; + nvec = rc; + } + } while (rc); + + return nvec; +} +#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,14,0) */ + +static int rtl8127_enable_msix(struct rtl8127_private *tp) +{ + int i, nvecs = 0; + struct msix_entry msix_ent[R8127_MAX_MSIX_VEC]; + //struct net_device *dev = tp->dev; + //const int len = sizeof(tp->irq_tbl[0].name); + + for (i = 0; i < R8127_MAX_MSIX_VEC; i++) { + msix_ent[i].entry = i; + msix_ent[i].vector = 0; + } + + nvecs = pci_enable_msix_range(tp->pci_dev, msix_ent, + tp->min_irq_nvecs, tp->max_irq_nvecs); + if (nvecs < 0) + goto out; + + for (i = 0; i < nvecs; i++) { + struct r8127_irq *irq = &tp->irq_tbl[i]; + irq->vector = msix_ent[i].vector; + //snprintf(irq->name, len, "%s-%d", dev->name, i); + //irq->handler = rtl8127_interrupt_msix; + } + +out: + return nvecs; +} + +/* Cfg9346_Unlock assumed. */ +static int rtl8127_try_msi(struct rtl8127_private *tp) +{ + struct pci_dev *pdev = tp->pci_dev; + unsigned int hw_supp_irq_nvecs; + unsigned msi = 0; + int nvecs = 1; + + hw_supp_irq_nvecs = R8127_MAX_MSIX_VEC_8125B; + tp->hw_supp_irq_nvecs = clamp_val(hw_supp_irq_nvecs, 1, + R8127_MAX_MSIX_VEC); + + tp->max_irq_nvecs = tp->hw_supp_irq_nvecs; + tp->min_irq_nvecs = R8127_MIN_MSIX_VEC_8127; +#ifdef DISABLE_MULTI_MSIX_VECTOR + tp->max_irq_nvecs = 1; +#endif + +#if defined(RTL_USE_NEW_INTR_API) + if ((nvecs = pci_alloc_irq_vectors(pdev, tp->min_irq_nvecs, tp->max_irq_nvecs, PCI_IRQ_MSIX)) > 0) + msi |= RTL_FEATURE_MSIX; + else if ((nvecs = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES)) > 0 && + pci_dev_msi_enabled(pdev)) + msi |= RTL_FEATURE_MSI; +#elif LINUX_VERSION_CODE > KERNEL_VERSION(2,6,13) + if ((nvecs = rtl8127_enable_msix(tp)) > 0) + msi |= RTL_FEATURE_MSIX; + else if (!pci_enable_msi(pdev)) + msi |= RTL_FEATURE_MSI; +#endif + if (!(msi & (RTL_FEATURE_MSI | RTL_FEATURE_MSIX))) + dev_info(&pdev->dev, "no MSI/MSI-X. Back to INTx.\n"); + + if (!(msi & RTL_FEATURE_MSIX) || nvecs < 1) + nvecs = 1; + + tp->irq_nvecs = nvecs; + + tp->features |= msi; + + return nvecs; +} + +static void rtl8127_disable_msi(struct pci_dev *pdev, struct rtl8127_private *tp) +{ +#if defined(RTL_USE_NEW_INTR_API) + if (tp->features & (RTL_FEATURE_MSI | RTL_FEATURE_MSIX)) + pci_free_irq_vectors(pdev); +#elif LINUX_VERSION_CODE > KERNEL_VERSION(2,6,13) + if (tp->features & (RTL_FEATURE_MSIX)) + pci_disable_msix(pdev); + else if (tp->features & (RTL_FEATURE_MSI)) + pci_disable_msi(pdev); +#endif + tp->features &= ~(RTL_FEATURE_MSI | RTL_FEATURE_MSIX); +} + +static int rtl8127_get_irq(struct pci_dev *pdev) +{ +#if defined(RTL_USE_NEW_INTR_API) + return pci_irq_vector(pdev, 0); +#else + return pdev->irq; +#endif +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,11,0) +static void +rtl8127_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) +{ + struct rtl8127_private *tp = netdev_priv(dev); + struct rtl8127_counters *counters = tp->tally_vaddr; + dma_addr_t paddr = tp->tally_paddr; + + if (!counters) + return; + + netdev_stats_to_stats64(stats, &dev->stats); + dev_fetch_sw_netstats(stats, dev->tstats); + + /* + * Fetch additional counter values missing in stats collected by driver + * from tally counters. + */ + rtl8127_dump_tally_counter(tp, paddr); + + stats->tx_errors = le64_to_cpu(counters->tx_errors); + stats->collisions = le32_to_cpu(counters->tx_multi_collision); + stats->tx_aborted_errors = le16_to_cpu(counters->tx_aborted); + stats->rx_missed_errors = le16_to_cpu(counters->rx_missed); +} +#else +/** + * rtl8127_get_stats - Get rtl8127 read/write statistics + * @dev: The Ethernet Device to get statistics for + * + * Get TX/RX statistics for rtl8127 + */ +static struct +net_device_stats *rtl8127_get_stats(struct net_device *dev) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) + struct rtl8127_private *tp = netdev_priv(dev); +#endif + return &RTLDEV->stats; +} +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,36) + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29) +static const struct net_device_ops rtl8127_netdev_ops = { + .ndo_open = rtl8127_open, + .ndo_stop = rtl8127_close, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,11,0) + .ndo_get_stats64 = rtl8127_get_stats64, +#else + .ndo_get_stats = rtl8127_get_stats, +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,11,0) + .ndo_start_xmit = rtl8127_start_xmit, + .ndo_tx_timeout = rtl8127_tx_timeout, + .ndo_change_mtu = rtl8127_change_mtu, + .ndo_set_mac_address = rtl8127_set_mac_address, +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,15,0) + .ndo_do_ioctl = rtl8127_do_ioctl, +#else + .ndo_siocdevprivate = rtl8127_siocdevprivate, + .ndo_eth_ioctl = rtl8127_do_ioctl, +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(5,15,0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,1,0) + .ndo_set_multicast_list = rtl8127_set_rx_mode, +#else + .ndo_set_rx_mode = rtl8127_set_rx_mode, +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) +#ifdef CONFIG_R8127_VLAN + .ndo_vlan_rx_register = rtl8127_vlan_rx_register, +#endif +#else + .ndo_fix_features = rtl8127_fix_features, + .ndo_set_features = rtl8127_set_features, +#endif +#ifdef CONFIG_NET_POLL_CONTROLLER + .ndo_poll_controller = rtl8127_netpoll, +#endif +}; +#endif + + +#ifdef CONFIG_R8127_NAPI + +static int rtl8127_poll(napi_ptr napi, napi_budget budget) +{ + struct r8127_napi *r8127napi = RTL_GET_PRIV(napi, struct r8127_napi); + struct rtl8127_private *tp = r8127napi->priv; + RTL_GET_NETDEV(tp) + unsigned int work_to_do = RTL_NAPI_QUOTA(budget, dev); + unsigned int work_done = 0; + int i; + + for (i = 0; i < tp->num_tx_rings; i++) + rtl8127_tx_interrupt(&tp->tx_ring[i], budget); + + for (i = 0; i < tp->num_rx_rings; i++) + work_done += rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[i], budget); + + work_done = min(work_done, work_to_do); + + RTL_NAPI_QUOTA_UPDATE(dev, work_done, budget); + + if (work_done < work_to_do) { +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) + HandleDashInterrupt(tp->dev); +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + if (RTL_NETIF_RX_COMPLETE(dev, napi, work_done) == FALSE) + return RTL_NAPI_RETURN_VALUE; +#else + RTL_NETIF_RX_COMPLETE(dev, napi, work_done); +#endif + /* + * 20040426: the barrier is not strictly required but the + * behavior of the irq handler could be less predictable + * without it. Btw, the lack of flush for the posted pci + * write is safe - FR + */ + smp_wmb(); + + rtl8127_switch_to_timer_interrupt(tp); + } + + return RTL_NAPI_RETURN_VALUE; +} + +static int rtl8127_poll_msix_ring(napi_ptr napi, napi_budget budget) +{ + struct r8127_napi *r8127napi = RTL_GET_PRIV(napi, struct r8127_napi); + struct rtl8127_private *tp = r8127napi->priv; + RTL_GET_NETDEV(tp) + unsigned int work_to_do = RTL_NAPI_QUOTA(budget, dev); + unsigned int work_done = 0; + const int message_id = r8127napi->index; + + if (message_id < tp->num_tx_rings) + rtl8127_tx_interrupt_with_vector(tp, message_id, budget); + + if (message_id < tp->num_rx_rings) + work_done += rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[message_id], budget); + + RTL_NAPI_QUOTA_UPDATE(dev, work_done, budget); + + if (work_done < work_to_do) { +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH && message_id == 0) + HandleDashInterrupt(tp->dev); +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + if (RTL_NETIF_RX_COMPLETE(dev, napi, work_done) == FALSE) + return RTL_NAPI_RETURN_VALUE; +#else + RTL_NETIF_RX_COMPLETE(dev, napi, work_done); +#endif + /* + * 20040426: the barrier is not strictly required but the + * behavior of the irq handler could be less predictable + * without it. Btw, the lack of flush for the posted pci + * write is safe - FR + */ + smp_wmb(); + + rtl8127_enable_hw_interrupt_v2(tp, message_id); + } + + return RTL_NAPI_RETURN_VALUE; +} + +static int rtl8127_poll_msix_tx(napi_ptr napi, napi_budget budget) +{ + struct r8127_napi *r8127napi = RTL_GET_PRIV(napi, struct r8127_napi); + struct rtl8127_private *tp = r8127napi->priv; + RTL_GET_NETDEV(tp) + unsigned int work_to_do = RTL_NAPI_QUOTA(budget, dev); + unsigned int work_done = 0; + const int message_id = r8127napi->index; + + //suppress unused variable + (void)(dev); + + rtl8127_tx_interrupt_with_vector(tp, message_id, budget); + + RTL_NAPI_QUOTA_UPDATE(dev, work_done, budget); + + if (work_done < work_to_do) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + if (RTL_NETIF_RX_COMPLETE(dev, napi, work_done) == FALSE) + return RTL_NAPI_RETURN_VALUE; +#else + RTL_NETIF_RX_COMPLETE(dev, napi, work_done); +#endif + /* + * 20040426: the barrier is not strictly required but the + * behavior of the irq handler could be less predictable + * without it. Btw, the lack of flush for the posted pci + * write is safe - FR + */ + smp_wmb(); + + rtl8127_enable_hw_interrupt_v2(tp, message_id); + } + + return RTL_NAPI_RETURN_VALUE; +} + +static int rtl8127_poll_msix_other(napi_ptr napi, napi_budget budget) +{ + struct r8127_napi *r8127napi = RTL_GET_PRIV(napi, struct r8127_napi); + struct rtl8127_private *tp = r8127napi->priv; + RTL_GET_NETDEV(tp) + unsigned int work_to_do = RTL_NAPI_QUOTA(budget, dev); + const int message_id = r8127napi->index; + + //suppress unused variable + (void)(dev); + (void)(work_to_do); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + RTL_NETIF_RX_COMPLETE(dev, napi, work_to_do); +#else + RTL_NETIF_RX_COMPLETE(dev, napi, work_to_do); +#endif + + rtl8127_enable_hw_interrupt_v2(tp, message_id); + + return 1; +} + +static int rtl8127_poll_msix_rx(napi_ptr napi, napi_budget budget) +{ + struct r8127_napi *r8127napi = RTL_GET_PRIV(napi, struct r8127_napi); + struct rtl8127_private *tp = r8127napi->priv; + RTL_GET_NETDEV(tp) + unsigned int work_to_do = RTL_NAPI_QUOTA(budget, dev); + unsigned int work_done = 0; + const int message_id = r8127napi->index; + + if (message_id < tp->num_rx_rings) + work_done += rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[message_id], budget); + + RTL_NAPI_QUOTA_UPDATE(dev, work_done, budget); + + if (work_done < work_to_do) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + if (RTL_NETIF_RX_COMPLETE(dev, napi, work_done) == FALSE) + return RTL_NAPI_RETURN_VALUE; +#else + RTL_NETIF_RX_COMPLETE(dev, napi, work_done); +#endif + /* + * 20040426: the barrier is not strictly required but the + * behavior of the irq handler could be less predictable + * without it. Btw, the lack of flush for the posted pci + * write is safe - FR + */ + smp_wmb(); + + rtl8127_enable_hw_interrupt_v2(tp, message_id); + } + + return RTL_NAPI_RETURN_VALUE; +} + +void rtl8127_enable_napi(struct rtl8127_private *tp) +{ +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + int i; + + for (i = 0; i < tp->irq_nvecs; i++) + RTL_NAPI_ENABLE(tp->dev, &tp->r8127napi[i].napi); +#endif +} + +static void rtl8127_disable_napi(struct rtl8127_private *tp) +{ +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + int i; + + for (i = 0; i < tp->irq_nvecs; i++) + RTL_NAPI_DISABLE(tp->dev, &tp->r8127napi[i].napi); +#endif +} + +static void rtl8127_del_napi(struct rtl8127_private *tp) +{ +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + int i; + + for (i = 0; i < tp->irq_nvecs; i++) + RTL_NAPI_DEL((&tp->r8127napi[i])); +#endif +} +#endif //CONFIG_R8127_NAPI + +static void rtl8127_init_napi(struct rtl8127_private *tp) +{ + int i; + + for (i=0; iirq_nvecs; i++) { + struct r8127_napi *r8127napi = &tp->r8127napi[i]; +#ifdef CONFIG_R8127_NAPI + int (*poll)(struct napi_struct *, int); + + poll = rtl8127_poll; + if (tp->features & RTL_FEATURE_MSIX) { + switch (tp->HwCurrIsrVer) { + case 6: + if (i < R8127_MAX_RX_QUEUES_VEC_V4) + poll = rtl8127_poll_msix_rx; + else if (i == 8 || i == 9) + poll = rtl8127_poll_msix_tx; + else + poll = rtl8127_poll_msix_other; + break; + case 5: + if (i < R8127_MAX_RX_QUEUES_VEC_V3) + poll = rtl8127_poll_msix_rx; + else if (i == 16 || i == 17) + poll = rtl8127_poll_msix_tx; + else + poll = rtl8127_poll_msix_other; + break; + case 2: + if (i < R8127_MAX_RX_QUEUES_VEC_V3) + poll = rtl8127_poll_msix_rx; + else if (i == 16 || i == 18) + poll = rtl8127_poll_msix_tx; + else + poll = rtl8127_poll_msix_other; + break; + case 3: + case 4: + if (i < R8127_MAX_RX_QUEUES_VEC_V3) + poll = rtl8127_poll_msix_ring; + else + poll = rtl8127_poll_msix_other; + break; + } + } + + RTL_NAPI_CONFIG(tp->dev, r8127napi, poll, R8127_NAPI_WEIGHT); +#endif + + r8127napi->priv = tp; + r8127napi->index = i; + } +} + +static int +rtl8127_set_real_num_queue(struct rtl8127_private *tp) +{ + int retval = 0; + + retval = netif_set_real_num_tx_queues(tp->dev, tp->num_tx_rings); + if (retval < 0) + goto exit; + + retval = netif_set_real_num_rx_queues(tp->dev, tp->num_rx_rings); + if (retval < 0) + goto exit; + +exit: + return retval; +} + +static int __devinit +rtl8127_init_one(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + struct net_device *dev = NULL; + struct rtl8127_private *tp; + void __iomem *ioaddr = NULL; + static int board_idx = -1; + + int rc; + + assert(pdev != NULL); + assert(ent != NULL); + + board_idx++; + + if (netif_msg_drv(&debug)) + printk(KERN_INFO "%s Ethernet controller driver %s loaded\n", + MODULENAME, RTL8127_VERSION); + + rc = rtl8127_init_board(pdev, &dev, &ioaddr); + if (rc) + goto out; + + tp = netdev_priv(dev); + assert(ioaddr != NULL); + + spin_lock_init(&tp->phy_lock); + + tp->set_speed = rtl8127_set_speed_xmii; + tp->get_settings = rtl8127_gset_xmii; + tp->phy_reset_enable = rtl8127_xmii_reset_enable; + tp->phy_reset_pending = rtl8127_xmii_reset_pending; + tp->link_ok = rtl8127_xmii_link_ok; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,11,0) + dev->tstats = devm_netdev_alloc_pcpu_stats(&pdev->dev, + struct pcpu_sw_netstats); + if (!dev->tstats) + goto err_out_1; +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,11,0) + + rc = rtl8127_try_msi(tp); + if (rc < 0) { + dev_err(&pdev->dev, "Can't allocate interrupt\n"); + goto err_out_1; + } + + rtl8127_init_software_variable(dev); + + RTL_NET_DEVICE_OPS(rtl8127_netdev_ops); + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22) + SET_ETHTOOL_OPS(dev, &rtl8127_ethtool_ops); +#endif + + dev->watchdog_timeo = RTL8127_TX_TIMEOUT; + dev->irq = rtl8127_get_irq(pdev); + dev->base_addr = (unsigned long) ioaddr; + + rtl8127_init_napi(tp); + +#ifdef CONFIG_R8127_VLAN + if (tp->mcfg != CFG_METHOD_DEFAULT) { + dev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX; +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) + dev->vlan_rx_kill_vid = rtl8127_vlan_rx_kill_vid; +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) + } +#endif + + /* There has been a number of reports that using SG/TSO results in + * tx timeouts. However for a lot of people SG/TSO works fine. + * Therefore disable both features by default, but allow users to + * enable them. Use at own risk! + */ + tp->cp_cmd |= RTL_R16(tp, CPlusCmd); + if (tp->mcfg != CFG_METHOD_DEFAULT) { + dev->features |= NETIF_F_IP_CSUM; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + tp->cp_cmd |= RxChkSum; +#else + dev->features |= NETIF_F_RXCSUM; + dev->features |= NETIF_F_SG | NETIF_F_TSO; + dev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO | + NETIF_F_RXCSUM | NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX; + dev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO | + NETIF_F_HIGHDMA; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,15,0) + dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(3,15,0) + dev->hw_features |= NETIF_F_RXALL; + dev->hw_features |= NETIF_F_RXFCS; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,22) + dev->hw_features |= NETIF_F_IPV6_CSUM | NETIF_F_TSO6; + dev->features |= NETIF_F_IPV6_CSUM; + dev->features |= NETIF_F_TSO6; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,19,0) + netif_set_tso_max_size(dev, LSO_64K); + netif_set_tso_max_segs(dev, NIC_MAX_PHYS_BUF_COUNT_LSO2); +#else //LINUX_VERSION_CODE >= KERNEL_VERSION(5,19,0) + netif_set_gso_max_size(dev, LSO_64K); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,18,0) + dev->gso_max_segs = NIC_MAX_PHYS_BUF_COUNT_LSO2; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,7,0) + dev->gso_min_segs = NIC_MIN_PHYS_BUF_COUNT; +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,7,0) +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(3,18,0) +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,19,0) + +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,22) +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + +#ifdef ENABLE_RSS_SUPPORT + if (tp->EnableRss) { + dev->hw_features |= NETIF_F_RXHASH; + dev->features |= NETIF_F_RXHASH; + } +#endif + } + + netdev_sw_irq_coalesce_default_on(dev); + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) + AllocateDashShareMemory(dev); +#endif + +#ifdef ENABLE_LIB_SUPPORT + BLOCKING_INIT_NOTIFIER_HEAD(&tp->lib_nh); +#endif + rtl8127_init_all_schedule_work(tp); + + rc = rtl8127_set_real_num_queue(tp); + if (rc < 0) + goto err_out; + + rtl8127_exit_oob(dev); + + rtl8127_powerup_pll(dev); + + rtl8127_hw_init(dev); + + rtl8127_hw_reset(dev); + + /* Get production from EEPROM */ + rtl8127_eeprom_type(tp); + + if (tp->eeprom_type == EEPROM_TYPE_93C46 || tp->eeprom_type == EEPROM_TYPE_93C56) + rtl8127_set_eeprom_sel_low(tp); + + rtl8127_get_mac_address(dev); + + tp->fw_name = rtl_chip_fw_infos[tp->mcfg].fw_name; + + tp->tally_vaddr = dma_alloc_coherent(&pdev->dev, sizeof(*tp->tally_vaddr), + &tp->tally_paddr, GFP_KERNEL); + if (!tp->tally_vaddr) { + rc = -ENOMEM; + goto err_out; + } + + rtl8127_tally_counter_clear(tp); + + pci_set_drvdata(pdev, dev); + + rc = register_netdev(dev); + if (rc) + goto err_out; + + printk(KERN_INFO "%s: This product is covered by one or more of the following patents: US6,570,884, US6,115,776, and US6,327,625.\n", MODULENAME); + + rtl8127_disable_rxdvgate(dev); + + device_set_wakeup_enable(&pdev->dev, tp->wol_enabled); + + netif_carrier_off(dev); + +#ifdef ENABLE_R8127_SYSFS + rtl8127_sysfs_init(dev); +#endif /* ENABLE_R8127_SYSFS */ + + printk(KERN_INFO "%s", GPL_CLAIM); + +out: + return rc; + +err_out: + if (tp->tally_vaddr != NULL) { + dma_free_coherent(&pdev->dev, sizeof(*tp->tally_vaddr), tp->tally_vaddr, + tp->tally_paddr); + + tp->tally_vaddr = NULL; + } +#ifdef CONFIG_R8127_NAPI + rtl8127_del_napi(tp); +#endif + rtl8127_disable_msi(pdev, tp); + +err_out_1: + rtl8127_release_board(pdev, dev); + + goto out; +} + +static void __devexit +rtl8127_remove_one(struct pci_dev *pdev) +{ + struct net_device *dev = pci_get_drvdata(pdev); + struct rtl8127_private *tp = netdev_priv(dev); + + assert(dev != NULL); + assert(tp != NULL); + + set_bit(R8127_FLAG_DOWN, tp->task_flags); + + rtl8127_cancel_all_schedule_work(tp); + + if (HW_DASH_SUPPORT_DASH(tp)) + rtl8127_driver_stop(tp); + + rtl8127_disable_pci_offset_180(tp); + +#ifdef ENABLE_R8127_SYSFS + rtl8127_sysfs_remove(dev); +#endif //ENABLE_R8127_SYSFS + + unregister_netdev(dev); +#ifdef CONFIG_R8127_NAPI + rtl8127_del_napi(tp); +#endif + rtl8127_disable_msi(pdev, tp); +#ifdef ENABLE_R8127_PROCFS + rtl8127_proc_remove(dev); +#endif + if (tp->tally_vaddr != NULL) { + dma_free_coherent(&pdev->dev, sizeof(*tp->tally_vaddr), tp->tally_vaddr, tp->tally_paddr); + tp->tally_vaddr = NULL; + } + + rtl8127_release_board(pdev, dev); + +#ifdef ENABLE_USE_FIRMWARE_FILE + rtl8127_release_firmware(tp); +#endif + + pci_set_drvdata(pdev, NULL); +} + +#ifdef ENABLE_PAGE_REUSE +static inline unsigned int rtl8127_rx_page_order(unsigned rx_buf_sz, unsigned page_size) +{ + unsigned truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + + SKB_DATA_ALIGN(rx_buf_sz + R8127_RX_ALIGN); + + return get_order(truesize * 2); +} +#endif //ENABLE_PAGE_REUSE + +static void +rtl8127_set_rxbufsize(struct rtl8127_private *tp, + struct net_device *dev) +{ + unsigned int mtu = dev->mtu; + + tp->rms = (mtu > ETH_DATA_LEN) ? + mtu + ETH_HLEN + RT_VALN_HLEN + ETH_FCS_LEN: + RX_BUF_SIZE; + tp->rx_buf_sz = tp->rms; +#ifdef ENABLE_RX_PACKET_FRAGMENT + tp->rx_buf_sz = SKB_DATA_ALIGN(RX_BUF_SIZE); +#endif //ENABLE_RX_PACKET_FRAGMENT +#ifdef ENABLE_PAGE_REUSE + tp->rx_buf_page_order = rtl8127_rx_page_order(tp->rx_buf_sz, PAGE_SIZE); + tp->rx_buf_page_size = rtl8127_rx_page_size(tp->rx_buf_page_order); +#endif //ENABLE_PAGE_REUSE +} + +static void +rtl8127_set_rms(struct rtl8127_private *tp, u16 rms) +{ + RTL_W16(tp, RxMaxSize, rms | AcceppVlanPhys); +} + +static void rtl8127_free_irq(struct rtl8127_private *tp) +{ + int i; + + for (i=0; iirq_nvecs; i++) { + struct r8127_irq *irq = &tp->irq_tbl[i]; + struct r8127_napi *r8127napi = &tp->r8127napi[i]; + + if (irq->requested) { + irq->requested = 0; +#if defined(RTL_USE_NEW_INTR_API) + pci_free_irq(tp->pci_dev, i, r8127napi); +#else + free_irq(irq->vector, r8127napi); +#endif + } + } +} + +static int rtl8127_alloc_irq(struct rtl8127_private *tp) +{ + struct net_device *dev = tp->dev; + int rc = 0; + struct r8127_irq *irq; + struct r8127_napi *r8127napi; + int i = 0; + const int len = sizeof(tp->irq_tbl[0].name); + +#if defined(RTL_USE_NEW_INTR_API) + for (i=0; iirq_nvecs; i++) { + irq = &tp->irq_tbl[i]; + if (tp->features & RTL_FEATURE_MSIX && + tp->HwCurrIsrVer > 1) + irq->handler = rtl8127_interrupt_msix; + else + irq->handler = rtl8127_interrupt; + + r8127napi = &tp->r8127napi[i]; + snprintf(irq->name, len, "%s-%d", dev->name, i); + rc = pci_request_irq(tp->pci_dev, i, irq->handler, NULL, r8127napi, + irq->name); + if (rc) + break; + + irq->vector = pci_irq_vector(tp->pci_dev, i); + irq->requested = 1; + } +#else + unsigned long irq_flags = 0; +#ifdef ENABLE_LIB_SUPPORT + irq_flags |= IRQF_NO_SUSPEND; +#endif + if (tp->features & RTL_FEATURE_MSIX && + tp->HwCurrIsrVer > 1) { + for (i=0; iirq_nvecs; i++) { + irq = &tp->irq_tbl[i]; + irq->handler = rtl8127_interrupt_msix; + r8127napi = &tp->r8127napi[i]; + snprintf(irq->name, len, "%s-%d", dev->name, i); + rc = request_irq(irq->vector, irq->handler, irq_flags, irq->name, r8127napi); + + if (rc) + break; + + irq->requested = 1; + } + } else { + irq = &tp->irq_tbl[0]; + irq->handler = rtl8127_interrupt; + r8127napi = &tp->r8127napi[0]; + snprintf(irq->name, len, "%s-0", dev->name); + if (!(tp->features & RTL_FEATURE_MSIX)) + irq->vector = dev->irq; + irq_flags |= (tp->features & (RTL_FEATURE_MSI | RTL_FEATURE_MSIX)) ? 0 : SA_SHIRQ; + rc = request_irq(irq->vector, irq->handler, irq_flags, irq->name, r8127napi); + + if (rc == 0) + irq->requested = 1; + } +#endif + if (rc) + rtl8127_free_irq(tp); + + return rc; +} + +static int rtl8127_alloc_tx_desc(struct rtl8127_private *tp) +{ + struct rtl8127_tx_ring *ring; + struct pci_dev *pdev = tp->pci_dev; + int i; + + for (i = 0; i < tp->num_tx_rings; i++) { + ring = &tp->tx_ring[i]; + ring->TxDescAllocSize = (ring->num_tx_desc + 1) * sizeof(struct TxDesc); + ring->TxDescArray = dma_alloc_coherent(&pdev->dev, + ring->TxDescAllocSize, + &ring->TxPhyAddr, + GFP_KERNEL); + + if (!ring->TxDescArray) + return -1; + } + + return 0; +} + +static int rtl8127_alloc_rx_desc(struct rtl8127_private *tp) +{ + struct rtl8127_rx_ring *ring; + struct pci_dev *pdev = tp->pci_dev; + int i; + + for (i = 0; i < tp->num_rx_rings; i++) { + ring = &tp->rx_ring[i]; + ring->RxDescAllocSize = (ring->num_rx_desc + 1) * tp->RxDescLength; + ring->RxDescArray = dma_alloc_coherent(&pdev->dev, + ring->RxDescAllocSize, + &ring->RxPhyAddr, + GFP_KERNEL); + + if (!ring->RxDescArray) + return -1; + } + + return 0; +} + +static void rtl8127_free_tx_desc(struct rtl8127_private *tp) +{ + struct rtl8127_tx_ring *ring; + struct pci_dev *pdev = tp->pci_dev; + int i; + + for (i = 0; i < tp->num_tx_rings; i++) { + ring = &tp->tx_ring[i]; + if (ring->TxDescArray) { + dma_free_coherent(&pdev->dev, + ring->TxDescAllocSize, + ring->TxDescArray, + ring->TxPhyAddr); + ring->TxDescArray = NULL; + } + } +} + +static void rtl8127_free_rx_desc(struct rtl8127_private *tp) +{ + struct rtl8127_rx_ring *ring; + struct pci_dev *pdev = tp->pci_dev; + int i; + + for (i = 0; i < tp->num_rx_rings; i++) { + ring = &tp->rx_ring[i]; + if (ring->RxDescArray) { + dma_free_coherent(&pdev->dev, + ring->RxDescAllocSize, + ring->RxDescArray, + ring->RxPhyAddr); + ring->RxDescArray = NULL; + } + } +} + +static void rtl8127_free_alloc_resources(struct rtl8127_private *tp) +{ + rtl8127_free_rx_desc(tp); + + rtl8127_free_tx_desc(tp); +} + +#ifdef ENABLE_USE_FIRMWARE_FILE +static void rtl8127_request_firmware(struct rtl8127_private *tp) +{ + struct rtl8127_fw *rtl_fw; + + /* firmware loaded already or no firmware available */ + if (tp->rtl_fw || !tp->fw_name) + return; + + rtl_fw = kzalloc(sizeof(*rtl_fw), GFP_KERNEL); + if (!rtl_fw) + return; + + rtl_fw->phy_write = rtl8127_mdio_write; + rtl_fw->phy_read = rtl8127_mdio_read; + rtl_fw->mac_mcu_write = mac_mcu_write; + rtl_fw->mac_mcu_read = mac_mcu_read; + rtl_fw->fw_name = tp->fw_name; + rtl_fw->dev = tp_to_dev(tp); + + if (rtl8127_fw_request_firmware(rtl_fw)) + kfree(rtl_fw); + else + tp->rtl_fw = rtl_fw; +} +#endif + +int rtl8127_open(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int retval; + + retval = -ENOMEM; + +#ifdef ENABLE_R8127_PROCFS + rtl8127_proc_init(dev); +#endif + rtl8127_set_rxbufsize(tp, dev); + /* + * Rx and Tx descriptors needs 256 bytes alignment. + * pci_alloc_consistent provides more. + */ + if (rtl8127_alloc_tx_desc(tp) < 0 || rtl8127_alloc_rx_desc(tp) < 0) + goto err_free_all_allocated_mem; + + retval = rtl8127_init_ring(dev); + if (retval < 0) + goto err_free_all_allocated_mem; + + retval = rtl8127_alloc_irq(tp); + if (retval < 0) + goto err_free_all_allocated_mem; + + if (netif_msg_probe(tp)) { + printk(KERN_INFO "%s: 0x%lx, " + "%2.2x:%2.2x:%2.2x:%2.2x:%2.2x:%2.2x, " + "IRQ %d\n", + dev->name, + dev->base_addr, + dev->dev_addr[0], dev->dev_addr[1], + dev->dev_addr[2], dev->dev_addr[3], + dev->dev_addr[4], dev->dev_addr[5], dev->irq); + } + +#ifdef ENABLE_USE_FIRMWARE_FILE + rtl8127_request_firmware(tp); +#endif + pci_set_master(tp->pci_dev); + +#ifdef CONFIG_R8127_NAPI + rtl8127_enable_napi(tp); +#endif + + rtl8127_exit_oob(dev); + + rtl8127_up(dev); + +#ifdef ENABLE_PTP_SUPPORT + if (tp->EnablePtp) + rtl8127_ptp_init(tp); +#endif + clear_bit(R8127_FLAG_DOWN, tp->task_flags); + + if (tp->resume_not_chg_speed) + _rtl8127_check_link_status(dev, R8127_LINK_STATE_UNKNOWN); + else + rtl8127_set_speed(dev, tp->autoneg, tp->speed, tp->duplex, tp->advertising); + + if (tp->esd_flag == 0) { + //rtl8127_request_esd_timer(dev); + + rtl8127_schedule_esd_work(tp); + } + + //rtl8127_request_link_timer(dev); + + rtl8127_enable_hw_linkchg_interrupt(tp); + +out: + + return retval; + +err_free_all_allocated_mem: + rtl8127_free_alloc_resources(tp); + + goto out; +} + +static void +_rtl8127_set_l1_l0s_entry_latency(struct rtl8127_private *tp, u8 setting) +{ + u32 csi_tmp; + u32 temp; + + temp = setting & 0x3f; + temp <<= 24; + /*set PCI configuration space offset 0x70F to setting*/ + /*When the register offset of PCI configuration space larger than 0xff, use CSI to access it.*/ + + csi_tmp = rtl8127_csi_read(tp, 0x70c) & 0xc0ffffff; + rtl8127_csi_write(tp, 0x70c, csi_tmp | temp); +} + +static void +rtl8127_set_l1_l0s_entry_latency(struct rtl8127_private *tp) +{ + _rtl8127_set_l1_l0s_entry_latency(tp, 0x27); +} + +static void +_rtl8127_set_mrrs(struct rtl8127_private *tp, u8 setting) +{ + //Set PCI configuration space offset 0x79 to setting + + struct pci_dev *pdev = tp->pci_dev; + u8 device_control; + + pci_read_config_byte(pdev, 0x79, &device_control); + device_control &= ~0x70; + device_control |= setting; + pci_write_config_byte(pdev, 0x79, device_control); +} + +static void +rtl8127_set_mrrs(struct rtl8127_private *tp) +{ + if (hwoptimize & HW_PATCH_SOC_LAN) + return; + + _rtl8127_set_mrrs(tp, 0x40); +} + +static void +rtl8127_disable_l1_timeout(struct rtl8127_private *tp) +{ + rtl8127_csi_write(tp, 0x890, rtl8127_csi_read(tp, 0x890) & ~BIT(0)); +} + +void +rtl8127_hw_set_rx_packet_filter(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + u32 mc_filter[2]; /* Multicast hash filter */ + int rx_mode; + u32 tmp = 0; + + if (dev->flags & IFF_PROMISC) { + /* Unconditionally log net taps. */ + if (netif_msg_link(tp)) + printk(KERN_NOTICE "%s: Promiscuous mode enabled.\n", + dev->name); + + rx_mode = + AcceptBroadcast | AcceptMulticast | AcceptMyPhys | + AcceptAllPhys; + mc_filter[1] = mc_filter[0] = 0xffffffff; + } else if (dev->flags & IFF_ALLMULTI) { + /* accept all multicasts. */ + rx_mode = AcceptBroadcast | AcceptMulticast | AcceptMyPhys; + mc_filter[1] = mc_filter[0] = 0xffffffff; + } else { +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,35) + struct dev_mc_list *mclist; + unsigned int i; + + rx_mode = AcceptBroadcast | AcceptMyPhys; + mc_filter[1] = mc_filter[0] = 0; + for (i = 0, mclist = dev->mc_list; mclist && i < dev->mc_count; + i++, mclist = mclist->next) { + int bit_nr = ether_crc(ETH_ALEN, mclist->dmi_addr) >> 26; + mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 31); + rx_mode |= AcceptMulticast; + } +#else + struct netdev_hw_addr *ha; + + rx_mode = AcceptBroadcast | AcceptMyPhys; + mc_filter[1] = mc_filter[0] = 0; + netdev_for_each_mc_addr(ha, dev) { + int bit_nr = ether_crc(ETH_ALEN, ha->addr) >> 26; + mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 31); + rx_mode |= AcceptMulticast; + } +#endif + } + + if (dev->features & NETIF_F_RXALL) + rx_mode |= (AcceptErr | AcceptRunt); + + tmp = mc_filter[0]; + mc_filter[0] = swab32(mc_filter[1]); + mc_filter[1] = swab32(tmp); + + tmp = tp->rtl8127_rx_config | rx_mode | (RTL_R32(tp, RxConfig) & rtl_chip_info[tp->chipset].RxConfigMask); + + RTL_W32(tp, RxConfig, tmp); + RTL_W32(tp, MAR0 + 0, mc_filter[0]); + RTL_W32(tp, MAR0 + 4, mc_filter[1]); +} + +static void +rtl8127_set_rx_mode(struct net_device *dev) +{ + rtl8127_hw_set_rx_packet_filter(dev); +} + +void +rtl8127_set_rx_q_num(struct rtl8127_private *tp, + unsigned int num_rx_queues) +{ + u16 q_ctrl; + u16 rx_q_num; + + rx_q_num = (u16)ilog2(num_rx_queues); + rx_q_num &= (BIT_0 | BIT_1 | BIT_2); + rx_q_num <<= 2; + q_ctrl = RTL_R16(tp, Q_NUM_CTRL_8125); + q_ctrl &= ~(BIT_2 | BIT_3 | BIT_4); + q_ctrl |= rx_q_num; + RTL_W16(tp, Q_NUM_CTRL_8125, q_ctrl); +} + +void +rtl8127_set_tx_q_num(struct rtl8127_private *tp, + unsigned int num_tx_queues) +{ + u16 mac_ocp_data; + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xE63E); + mac_ocp_data &= ~(BIT_11 | BIT_10); + mac_ocp_data |= ((ilog2(num_tx_queues) & 0x03) << 10); + rtl8127_mac_ocp_write(tp, 0xE63E, mac_ocp_data); +} + +void +rtl8127_enable_mcu(struct rtl8127_private *tp, bool enable) +{ + if (FALSE == HW_SUPPORT_MAC_MCU(tp)) + return; + + if (enable) + rtl8127_set_mac_ocp_bit(tp, 0xC0B4, BIT_0); + else + rtl8127_clear_mac_ocp_bit(tp, 0xC0B4, BIT_0); +} + +static void +rtl8127_clear_tcam_entries(struct rtl8127_private *tp) +{ + if (FALSE == HW_SUPPORT_TCAM(tp)) + return; + + rtl8127_set_mac_ocp_bit(tp, 0xEB54, BIT_0); + fsleep(1); + rtl8127_clear_mac_ocp_bit(tp, 0xEB54, BIT_0); +} + +static u8 +rtl8127_get_l1off_cap_bits(struct rtl8127_private *tp) +{ + u8 l1offCapBits = 0; + + l1offCapBits = (BIT_0 | BIT_1); + l1offCapBits |= (BIT_2 | BIT_3); + + return l1offCapBits; +} + +void +rtl8127_hw_config(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + struct pci_dev *pdev = tp->pci_dev; + u16 mac_ocp_data; + + rtl8127_disable_rx_packet_filter(tp); + + rtl8127_hw_reset(dev); + + rtl8127_enable_cfg9346_write(tp); + rtl8127_enable_force_clkreq(tp, 0); + rtl8127_enable_aspm_clkreq_lock(tp, 0); + + rtl8127_set_eee_lpi_timer(tp); + + //keep magic packet only + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xC0B6); + mac_ocp_data &= BIT_0; + rtl8127_mac_ocp_write(tp, 0xC0B6, mac_ocp_data); + + rtl8127_tally_counter_addr_fill(tp); + + rtl8127_enable_extend_tally_couter(tp); + + rtl8127_desc_addr_fill(tp); + + /* Set DMA burst size and Interframe Gap Time */ + RTL_W32(tp, TxConfig, (TX_DMA_BURST_unlimited << TxDMAShift) | + (InterFrameGap << TxInterFrameGapShift)); + + if (tp->EnableTxNoClose) + RTL_W32(tp, TxConfig, (RTL_R32(tp, TxConfig) | BIT_6)); + + if (enable_double_vlan) + rtl8127_enable_double_vlan(tp); + else + rtl8127_disable_double_vlan(tp); + + rtl8127_set_l1_l0s_entry_latency(tp); + + rtl8127_set_mrrs(tp); + + rtl8127_disable_l1_timeout(tp); + +#ifdef ENABLE_RSS_SUPPORT + rtl8127_config_rss(tp); +#else + RTL_W32(tp, RSS_CTRL_8125, 0x00); +#endif + rtl8127_set_rx_q_num(tp, rtl8127_tot_rx_rings(tp)); + + RTL_W8(tp, Config1, RTL_R8(tp, Config1) & ~0x10); + + rtl8127_mac_ocp_write(tp, 0xC140, 0xFFFF); + rtl8127_mac_ocp_write(tp, 0xC142, 0xFFFF); + + //new tx desc format + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xEB58); + mac_ocp_data &= ~(BIT_0 | BIT_1); + mac_ocp_data |= (BIT_0); + rtl8127_mac_ocp_write(tp, 0xEB58, mac_ocp_data); + + if (tp->EnableTxNoClose) + RTL_W8(tp, 0x20E4, RTL_R8(tp, 0x20E4) | BIT_2); + else + RTL_W8(tp, 0x20E4, RTL_R8(tp, 0x20E4) & ~BIT_2); + + if (tp->HwSuppRxDescType == RX_DESC_RING_TYPE_4) { + if (tp->InitRxDescType == RX_DESC_RING_TYPE_4) + RTL_W8(tp, 0xd8, RTL_R8(tp, 0xd8) | + EnableRxDescV4_0); + else + RTL_W8(tp, 0xd8, RTL_R8(tp, 0xd8) & + ~EnableRxDescV4_0); + } + + if (tp->mcfg == CFG_METHOD_2) { + rtl8127_clear_mac_ocp_bit(tp, 0xE00C, BIT_12); + + rtl8127_clear_mac_ocp_bit(tp, 0xC0C2, BIT_6); + } + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xE614); + mac_ocp_data &= ~(BIT_11 | BIT_10 | BIT_9 | BIT_8); + mac_ocp_data |= (15 << 8); + rtl8127_mac_ocp_write(tp, 0xE614, mac_ocp_data); + + rtl8127_set_tx_q_num(tp, rtl8127_tot_tx_rings(tp)); + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xE63E); + mac_ocp_data &= ~(BIT_5 | BIT_4); + mac_ocp_data |= ((0x02 & 0x03) << 4); + rtl8127_mac_ocp_write(tp, 0xE63E, mac_ocp_data); + + rtl8127_enable_mcu(tp, 0); + rtl8127_enable_mcu(tp, 1); + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xC0B4); + mac_ocp_data |= (BIT_3 | BIT_2); + rtl8127_mac_ocp_write(tp, 0xC0B4, mac_ocp_data); + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xEB6A); + mac_ocp_data &= ~(BIT_7 | BIT_6 | BIT_5 | BIT_4 | BIT_3 | BIT_2 | BIT_1 | BIT_0); + mac_ocp_data |= (BIT_5 | BIT_4 | BIT_1 | BIT_0); + rtl8127_mac_ocp_write(tp, 0xEB6A, mac_ocp_data); + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xEB50); + mac_ocp_data &= ~(BIT_9 | BIT_8 | BIT_7 | BIT_6 | BIT_5); + mac_ocp_data |= (BIT_6); + rtl8127_mac_ocp_write(tp, 0xEB50, mac_ocp_data); + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xE056); + mac_ocp_data &= ~(BIT_7 | BIT_6 | BIT_5 | BIT_4); + //mac_ocp_data |= (BIT_4 | BIT_5); + rtl8127_mac_ocp_write(tp, 0xE056, mac_ocp_data); + + RTL_W8(tp, TDFNR, 0x10); + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xE040); + mac_ocp_data &= ~(BIT_12); + rtl8127_mac_ocp_write(tp, 0xE040, mac_ocp_data); + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xEA1C); + mac_ocp_data &= ~(BIT_1 | BIT_0); + mac_ocp_data |= (BIT_0); + rtl8127_mac_ocp_write(tp, 0xEA1C, mac_ocp_data); + + rtl8127_mac_ocp_write(tp, 0xE0C0, 0x4000); + + rtl8127_set_mac_ocp_bit(tp, 0xE052, (BIT_6 | BIT_5)); + rtl8127_clear_mac_ocp_bit(tp, 0xE052, BIT_3 | BIT_7); + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xD430); + mac_ocp_data &= ~(BIT_11 | BIT_10 | BIT_9 | BIT_8 | BIT_7 | BIT_6 | BIT_5 | BIT_4 | BIT_3 | BIT_2 | BIT_1 | BIT_0); + mac_ocp_data |= 0x45F; + rtl8127_mac_ocp_write(tp, 0xD430, mac_ocp_data); + + //rtl8127_mac_ocp_write(tp, 0xE0C0, 0x4F87); + if (!tp->DASH) + RTL_W8(tp, 0xD0, RTL_R8(tp, 0xD0) | BIT_6 | BIT_7); + else + RTL_W8(tp, 0xD0, RTL_R8(tp, 0xD0) & ~(BIT_6 | BIT_7)); + + rtl8127_disable_eee_plus(tp); + + mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xEA1C); + mac_ocp_data &= ~(BIT_2); + mac_ocp_data &= ~(BIT_9 | BIT_8); + rtl8127_mac_ocp_write(tp, 0xEA1C, mac_ocp_data); + + rtl8127_clear_tcam_entries(tp); + + RTL_W16(tp, 0x1880, RTL_R16(tp, 0x1880) & ~(BIT_4 | BIT_5)); + + rtl8127_clear_set_mac_ocp_bit(tp, 0xD40C, 0xE038, 0x8020); + + /* csum offload command for RTL8125 */ + tp->tx_tcp_csum_cmd = TxTCPCS_C; + tp->tx_udp_csum_cmd = TxUDPCS_C; + tp->tx_ip_csum_cmd = TxIPCS_C; + tp->tx_ipv6_csum_cmd = TxIPV6F_C; + + /* config interrupt type for RTL8125B */ + if (tp->HwSuppIsrVer > 1) + rtl8127_hw_set_interrupt_type(tp, tp->HwCurrIsrVer); + + //other hw parameters + rtl8127_hw_clear_timer_int(dev); + + rtl8127_hw_clear_int_miti(dev); + + if (tp->use_timer_interrupt && + (tp->HwCurrIsrVer > 1) && + (tp->HwSuppIntMitiVer > 3) && + (tp->features & RTL_FEATURE_MSIX)) { + int i; + for (i = 0; i < tp->irq_nvecs; i++) + rtl8127_hw_set_timer_int(tp, i, timer_count_v2); + } + + rtl8127_enable_exit_l1_mask(tp); + + rtl8127_mac_ocp_write(tp, 0xE098, 0xC302); + + if (aspm && (tp->org_pci_offset_99 & (BIT_2 | BIT_5 | BIT_6))) + rtl8127_init_pci_offset_99(tp); + else + rtl8127_disable_pci_offset_99(tp); + + if (aspm && (tp->org_pci_offset_180 & rtl8127_get_l1off_cap_bits(tp))) + rtl8127_init_pci_offset_180(tp); + else + rtl8127_disable_pci_offset_180(tp); + + tp->cp_cmd &= ~(EnableBist | Macdbgo_oe | Force_halfdup | + Force_rxflow_en | Force_txflow_en | Cxpl_dbg_sel | + ASF | Macdbgo_sel); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0) + RTL_W16(tp, CPlusCmd, tp->cp_cmd); +#else + rtl8127_hw_set_features(dev, dev->features); +#endif + rtl8127_set_rms(tp, tp->rms); + + rtl8127_disable_rxdvgate(dev); + + if (!tp->pci_cfg_is_read) { + pci_read_config_byte(pdev, PCI_COMMAND, &tp->pci_cfg_space.cmd); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_0, &tp->pci_cfg_space.io_base_l); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_0 + 2, &tp->pci_cfg_space.io_base_h); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_2, &tp->pci_cfg_space.mem_base_l); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_2 + 2, &tp->pci_cfg_space.mem_base_h); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_3, &tp->pci_cfg_space.resv_0x1c_l); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_3 + 2, &tp->pci_cfg_space.resv_0x1c_h); + pci_read_config_byte(pdev, PCI_INTERRUPT_LINE, &tp->pci_cfg_space.ilr); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_4, &tp->pci_cfg_space.resv_0x20_l); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_4 + 2, &tp->pci_cfg_space.resv_0x20_h); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_5, &tp->pci_cfg_space.resv_0x24_l); + pci_read_config_word(pdev, PCI_BASE_ADDRESS_5 + 2, &tp->pci_cfg_space.resv_0x24_h); + pci_read_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID, &tp->pci_cfg_space.resv_0x2c_l); + pci_read_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID + 2, &tp->pci_cfg_space.resv_0x2c_h); + if (tp->HwPcieSNOffset > 0) { + tp->pci_cfg_space.pci_sn_l = rtl8127_csi_read(tp, tp->HwPcieSNOffset); + tp->pci_cfg_space.pci_sn_h = rtl8127_csi_read(tp, tp->HwPcieSNOffset + 4); + } + + tp->pci_cfg_is_read = 1; + } + + /* Set Rx packet filter */ + rtl8127_hw_set_rx_packet_filter(dev); + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH && !tp->dash_printer_enabled) + NICChkTypeEnableDashInterrupt(tp); +#endif + + rtl8127_enable_aspm_clkreq_lock(tp, aspm ? 1 : 0); + + rtl8127_disable_cfg9346_write(tp); + + fsleep(10); +} + +void +rtl8127_hw_start(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + +#ifdef ENABLE_LIB_SUPPORT + rtl8127_init_lib_ring(tp); +#endif + + RTL_W8(tp, ChipCmd, CmdTxEnb | CmdRxEnb); + + rtl8127_enable_hw_interrupt(tp); + + rtl8127_lib_reset_complete(tp); +} + +static int +rtl8127_change_mtu(struct net_device *dev, + int new_mtu) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int ret = 0; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,10,0) + if (new_mtu < ETH_MIN_MTU) + return -EINVAL; + else if (new_mtu > tp->max_jumbo_frame_size) + new_mtu = tp->max_jumbo_frame_size; +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,10,0) + + dev->mtu = new_mtu; + + tp->eee.tx_lpi_timer = dev->mtu + ETH_HLEN + 0x20; + + if (!netif_running(dev)) + goto out; + + rtl8127_down(dev); + + rtl8127_set_rxbufsize(tp, dev); + + ret = rtl8127_init_ring(dev); + + if (ret < 0) + goto err_out; + +#ifdef CONFIG_R8127_NAPI + rtl8127_enable_napi(tp); +#endif//CONFIG_R8127_NAPI + + if (tp->link_ok(dev)) + rtl8127_link_on_patch(dev); + else + rtl8127_link_down_patch(dev); + + //mod_timer(&tp->esd_timer, jiffies + RTL8127_ESD_TIMEOUT); + //mod_timer(&tp->link_timer, jiffies + RTL8127_LINK_TIMEOUT); +out: +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0) + netdev_update_features(dev); +#endif + +err_out: + return ret; +} + +static inline void +rtl8127_set_desc_dma_addr(struct rtl8127_private *tp, + struct RxDesc *desc, + dma_addr_t mapping) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + ((struct RxDescV3 *)desc)->addr = cpu_to_le64(mapping); + break; + case RX_DESC_RING_TYPE_4: + ((struct RxDescV4 *)desc)->addr = cpu_to_le64(mapping); + break; + default: + desc->addr = cpu_to_le64(mapping); + break; + } +} + +static inline void +rtl8127_mark_to_asic_v1(struct RxDesc *desc, + u32 rx_buf_sz) +{ + u32 eor = le32_to_cpu(desc->opts1) & RingEnd; + + WRITE_ONCE(desc->opts1, cpu_to_le32(DescOwn | eor | rx_buf_sz)); +} + +static inline void +rtl8127_mark_to_asic_v3(struct RxDescV3 *descv3, + u32 rx_buf_sz) +{ + u32 eor = le32_to_cpu(descv3->RxDescNormalDDWord4.opts1) & RingEnd; + + WRITE_ONCE(descv3->RxDescNormalDDWord4.opts1, cpu_to_le32(DescOwn | eor | rx_buf_sz)); +} + +static inline void +rtl8127_mark_to_asic_v4(struct RxDescV4 *descv4, + u32 rx_buf_sz) +{ + u32 eor = le32_to_cpu(descv4->RxDescNormalDDWord2.opts1) & RingEnd; + + WRITE_ONCE(descv4->RxDescNormalDDWord2.opts1, cpu_to_le32(DescOwn | eor | rx_buf_sz)); +} + +void +rtl8127_mark_to_asic(struct rtl8127_private *tp, + struct RxDesc *desc, + u32 rx_buf_sz) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + rtl8127_mark_to_asic_v3((struct RxDescV3 *)desc, rx_buf_sz); + break; + case RX_DESC_RING_TYPE_4: + rtl8127_mark_to_asic_v4((struct RxDescV4 *)desc, rx_buf_sz); + break; + default: + rtl8127_mark_to_asic_v1(desc, rx_buf_sz); + break; + } +} + +static inline void +rtl8127_map_to_asic(struct rtl8127_private *tp, + struct rtl8127_rx_ring *ring, + struct RxDesc *desc, + dma_addr_t mapping, + u32 rx_buf_sz, + const u32 cur_rx) +{ + ring->RxDescPhyAddr[cur_rx] = mapping; + rtl8127_set_desc_dma_addr(tp, desc, mapping); + wmb(); + rtl8127_mark_to_asic(tp, desc, rx_buf_sz); +} + +#ifdef ENABLE_PAGE_REUSE + +static int +rtl8127_alloc_rx_page(struct rtl8127_private *tp, struct rtl8127_rx_ring *ring, + struct rtl8127_rx_buffer *rxb) +{ + struct page *page; + dma_addr_t dma; + unsigned int order = tp->rx_buf_page_order; + + //get free page + page = dev_alloc_pages(order); + + if (unlikely(!page)) + return -ENOMEM; + + dma = dma_map_page_attrs(&tp->pci_dev->dev, page, 0, + tp->rx_buf_page_size, + DMA_FROM_DEVICE, + (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)); + + if (unlikely(dma_mapping_error(&tp->pci_dev->dev, dma))) { + __free_pages(page, order); + return -ENOMEM; + } + + rxb->page = page; + rxb->data = page_address(page); + rxb->page_offset = ring->rx_offset; + rxb->dma = dma; + + //after page alloc, page refcount already = 1 + + return 0; +} + +static void +rtl8127_free_rx_page(struct rtl8127_private *tp, struct rtl8127_rx_buffer *rxb) +{ + if (!rxb->page) + return; + + dma_unmap_page_attrs(&tp->pci_dev->dev, rxb->dma, + tp->rx_buf_page_size, + DMA_FROM_DEVICE, + (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)); + __free_pages(rxb->page, tp->rx_buf_page_order); + rxb->page = NULL; +} + +static void +_rtl8127_rx_clear(struct rtl8127_private *tp, struct rtl8127_rx_ring *ring) +{ + int i; + struct rtl8127_rx_buffer *rxb; + + for (i = 0; i < ring->num_rx_desc; i++) { + rxb = &ring->rx_buffer[i]; + if (rxb->skb) { + dev_kfree_skb(rxb->skb); + rxb->skb = NULL; + } + rtl8127_free_rx_page(tp, rxb); + } +} + +static u32 +rtl8127_rx_fill(struct rtl8127_private *tp, + struct rtl8127_rx_ring *ring, + struct net_device *dev, + u32 start, + u32 end, + u8 in_intr) +{ + u32 cur; + struct rtl8127_rx_buffer *rxb; + + for (cur = start; end - cur > 0; cur++) { + int ret, i = cur % ring->num_rx_desc; + + rxb = &ring->rx_buffer[i]; + if (rxb->page) + continue; + + ret = rtl8127_alloc_rx_page(tp, ring, rxb); + if (ret) + break; + + dma_sync_single_range_for_device(tp_to_dev(tp), + rxb->dma, + rxb->page_offset, + tp->rx_buf_sz, + DMA_FROM_DEVICE); + + rtl8127_map_to_asic(tp, ring, + rtl8127_get_rxdesc(tp, ring->RxDescArray, i), + rxb->dma + rxb->page_offset, + tp->rx_buf_sz, i); + } + return cur - start; +} + +#else //ENABLE_PAGE_REUSE + +static void +rtl8127_free_rx_skb(struct rtl8127_private *tp, + struct rtl8127_rx_ring *ring, + struct sk_buff **sk_buff, + struct RxDesc *desc, + const u32 cur_rx) +{ + struct pci_dev *pdev = tp->pci_dev; + + dma_unmap_single(&pdev->dev, ring->RxDescPhyAddr[cur_rx], tp->rx_buf_sz, + DMA_FROM_DEVICE); + dev_kfree_skb(*sk_buff); + *sk_buff = NULL; + rtl8127_make_unusable_by_asic(tp, desc); +} + +static int +rtl8127_alloc_rx_skb(struct rtl8127_private *tp, + struct rtl8127_rx_ring *ring, + struct sk_buff **sk_buff, + struct RxDesc *desc, + int rx_buf_sz, + const u32 cur_rx, + u8 in_intr) +{ + struct sk_buff *skb; + dma_addr_t mapping; + int ret = 0; + + if (in_intr) + skb = RTL_ALLOC_SKB_INTR(&tp->r8127napi[ring->index].napi, rx_buf_sz + R8127_RX_ALIGN); + else + skb = dev_alloc_skb(rx_buf_sz + R8127_RX_ALIGN); + + if (unlikely(!skb)) + goto err_out; + + if (!in_intr || !R8127_USE_NAPI_ALLOC_SKB) + skb_reserve(skb, R8127_RX_ALIGN); + + mapping = dma_map_single(tp_to_dev(tp), skb->data, rx_buf_sz, + DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(tp_to_dev(tp), mapping))) { + if (unlikely(net_ratelimit())) + netif_err(tp, drv, tp->dev, "Failed to map RX DMA!\n"); + goto err_out; + } + + *sk_buff = skb; + rtl8127_map_to_asic(tp, ring, desc, mapping, rx_buf_sz, cur_rx); +out: + return ret; + +err_out: + if (skb) + dev_kfree_skb(skb); + ret = -ENOMEM; + rtl8127_make_unusable_by_asic(tp, desc); + goto out; +} + +static void +_rtl8127_rx_clear(struct rtl8127_private *tp, struct rtl8127_rx_ring *ring) +{ + int i; + + for (i = 0; i < ring->num_rx_desc; i++) { + if (ring->Rx_skbuff[i]) { + rtl8127_free_rx_skb(tp, + ring, + ring->Rx_skbuff + i, + rtl8127_get_rxdesc(tp, ring->RxDescArray, i), + i); + ring->Rx_skbuff[i] = NULL; + } + } +} + +static u32 +rtl8127_rx_fill(struct rtl8127_private *tp, + struct rtl8127_rx_ring *ring, + struct net_device *dev, + u32 start, + u32 end, + u8 in_intr) +{ + u32 cur; + + for (cur = start; end - cur > 0; cur++) { + int ret, i = cur % ring->num_rx_desc; + + if (ring->Rx_skbuff[i]) + continue; + + ret = rtl8127_alloc_rx_skb(tp, + ring, + ring->Rx_skbuff + i, + rtl8127_get_rxdesc(tp, ring->RxDescArray, i), + tp->rx_buf_sz, + i, + in_intr); + if (ret < 0) + break; + } + return cur - start; +} + +#endif //ENABLE_PAGE_REUSE + +void +rtl8127_rx_clear(struct rtl8127_private *tp) +{ + int i; + + for (i = 0; i < tp->num_rx_rings; i++) { + struct rtl8127_rx_ring *ring = &tp->rx_ring[i]; + + _rtl8127_rx_clear(tp, ring); + } +} + +static void +rtl8127_mark_as_last_descriptor_v1(struct RxDesc *desc) +{ + desc->opts1 |= cpu_to_le32(RingEnd); +} + +static void +rtl8127_mark_as_last_descriptor_v3(struct RxDescV3 *descv3) +{ + descv3->RxDescNormalDDWord4.opts1 |= cpu_to_le32(RingEnd); +} + +static void +rtl8127_mark_as_last_descriptor_v4(struct RxDescV4 *descv4) +{ + descv4->RxDescNormalDDWord2.opts1 |= cpu_to_le32(RingEnd); +} + +void +rtl8127_mark_as_last_descriptor(struct rtl8127_private *tp, + struct RxDesc *desc) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + rtl8127_mark_as_last_descriptor_v3((struct RxDescV3 *)desc); + break; + case RX_DESC_RING_TYPE_4: + rtl8127_mark_as_last_descriptor_v4((struct RxDescV4 *)desc); + break; + default: + rtl8127_mark_as_last_descriptor_v1(desc); + break; + } +} + +static void +rtl8127_desc_addr_fill(struct rtl8127_private *tp) +{ + int i; + + for (i = 0; i < tp->num_tx_rings; i++) { + struct rtl8127_tx_ring *ring = &tp->tx_ring[i]; + RTL_W32(tp, ring->tdsar_reg, ((u64)ring->TxPhyAddr & DMA_BIT_MASK(32))); + RTL_W32(tp, ring->tdsar_reg + 4, ((u64)ring->TxPhyAddr >> 32)); + } + + for (i = 0; i < tp->num_rx_rings; i++) { + struct rtl8127_rx_ring *ring = &tp->rx_ring[i]; + RTL_W32(tp, ring->rdsar_reg, ((u64)ring->RxPhyAddr & DMA_BIT_MASK(32))); + RTL_W32(tp, ring->rdsar_reg + 4, ((u64)ring->RxPhyAddr >> 32)); + } +} + +static void +rtl8127_tx_desc_init(struct rtl8127_private *tp) +{ + int i = 0; + + for (i = 0; i < tp->num_tx_rings; i++) { + struct rtl8127_tx_ring *ring = &tp->tx_ring[i]; + memset(ring->TxDescArray, 0x0, ring->TxDescAllocSize); + + ring->TxDescArray[ring->num_tx_desc - 1].opts1 = cpu_to_le32(RingEnd); + } +} + +static void +rtl8127_rx_desc_init(struct rtl8127_private *tp) +{ + int i; + + for (i = 0; i < tp->num_rx_rings; i++) { + struct rtl8127_rx_ring *ring = &tp->rx_ring[i]; + memset(ring->RxDescArray, 0x0, ring->RxDescAllocSize); + } +} + +int +rtl8127_init_ring(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + int i; + + rtl8127_init_ring_indexes(tp); + + rtl8127_tx_desc_init(tp); + rtl8127_rx_desc_init(tp); + + for (i = 0; i < tp->num_tx_rings; i++) { + struct rtl8127_tx_ring *ring = &tp->tx_ring[i]; + memset(ring->tx_skb, 0x0, sizeof(ring->tx_skb)); + } + + for (i = 0; i < tp->num_rx_rings; i++) { + struct rtl8127_rx_ring *ring = &tp->rx_ring[i]; +#ifdef ENABLE_PAGE_REUSE + ring->rx_offset = R8127_RX_ALIGN; +#else + memset(ring->Rx_skbuff, 0x0, sizeof(ring->Rx_skbuff)); +#endif //ENABLE_PAGE_REUSE + if (rtl8127_rx_fill(tp, ring, dev, 0, ring->num_rx_desc, 0) != ring->num_rx_desc) + goto err_out; + + rtl8127_mark_as_last_descriptor(tp, rtl8127_get_rxdesc(tp, ring->RxDescArray, ring->num_rx_desc - 1)); + } + + return 0; + +err_out: + rtl8127_rx_clear(tp); + return -ENOMEM; +} + +static void +rtl8127_unmap_tx_skb(struct pci_dev *pdev, + struct ring_info *tx_skb, + struct TxDesc *desc) +{ + unsigned int len = tx_skb->len; + + dma_unmap_single(&pdev->dev, le64_to_cpu(desc->addr), len, DMA_TO_DEVICE); + + desc->opts1 = cpu_to_le32(RTK_MAGIC_DEBUG_VALUE); + desc->opts2 = 0x00; + desc->addr = RTL8127_MAGIC_NUMBER; + tx_skb->len = 0; +} + +static void +rtl8127_tx_clear_range(struct rtl8127_private *tp, + struct rtl8127_tx_ring *ring, + u32 start, + unsigned int n) +{ + unsigned int i; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,22) + struct net_device *dev = tp->dev; +#endif + + for (i = 0; i < n; i++) { + unsigned int entry = (start + i) % ring->num_tx_desc; + struct ring_info *tx_skb = ring->tx_skb + entry; + unsigned int len = tx_skb->len; + + if (len) { + struct sk_buff *skb = tx_skb->skb; + + rtl8127_unmap_tx_skb(tp->pci_dev, tx_skb, + ring->TxDescArray + entry); + if (skb) { + RTLDEV->stats.tx_dropped++; + dev_kfree_skb_any(skb); + tx_skb->skb = NULL; + } + } + } +} + +void +rtl8127_tx_clear(struct rtl8127_private *tp) +{ + int i; + + for (i = 0; i < tp->num_tx_rings; i++) { + struct rtl8127_tx_ring *ring = &tp->tx_ring[i]; + rtl8127_tx_clear_range(tp, ring, ring->dirty_tx, ring->num_tx_desc); + ring->cur_tx = ring->dirty_tx = 0; + } +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) +static void rtl8127_schedule_reset_work(struct rtl8127_private *tp) +{ +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + set_bit(R8127_FLAG_TASK_RESET_PENDING, tp->task_flags); + schedule_delayed_work(&tp->reset_task, 4); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) +} + +static void rtl8127_schedule_esd_work(struct rtl8127_private *tp) +{ +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + set_bit(R8127_FLAG_TASK_ESD_CHECK_PENDING, tp->task_flags); + schedule_delayed_work(&tp->esd_task, RTL8127_ESD_TIMEOUT); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) +} + +static void rtl8127_schedule_linkchg_work(struct rtl8127_private *tp) +{ +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + set_bit(R8127_FLAG_TASK_LINKCHG_CHECK_PENDING, tp->task_flags); + schedule_delayed_work(&tp->linkchg_task, 4); +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) +} + +#define rtl8127_cancel_schedule_reset_work(a) +#define rtl8127_cancel_schedule_esd_work(a) +#define rtl8127_cancel_schedule_linkchg_work(a) + +#else +static void rtl8127_schedule_reset_work(struct rtl8127_private *tp) +{ + set_bit(R8127_FLAG_TASK_RESET_PENDING, tp->task_flags); + schedule_delayed_work(&tp->reset_task, 4); +} + +static void rtl8127_cancel_schedule_reset_work(struct rtl8127_private *tp) +{ + struct work_struct *work = &tp->reset_task.work; + + if (!work->func) + return; + + cancel_delayed_work_sync(&tp->reset_task); +} + +static void rtl8127_schedule_esd_work(struct rtl8127_private *tp) +{ + set_bit(R8127_FLAG_TASK_ESD_CHECK_PENDING, tp->task_flags); + schedule_delayed_work(&tp->esd_task, RTL8127_ESD_TIMEOUT); +} + +static void rtl8127_cancel_schedule_esd_work(struct rtl8127_private *tp) +{ + struct work_struct *work = &tp->esd_task.work; + + if (!work->func) + return; + + cancel_delayed_work_sync(&tp->esd_task); +} + +static void rtl8127_schedule_linkchg_work(struct rtl8127_private *tp) +{ + set_bit(R8127_FLAG_TASK_LINKCHG_CHECK_PENDING, tp->task_flags); + schedule_delayed_work(&tp->linkchg_task, 4); +} + +static void rtl8127_cancel_schedule_linkchg_work(struct rtl8127_private *tp) +{ + struct work_struct *work = &tp->linkchg_task.work; + + if (!work->func) + return; + + cancel_delayed_work_sync(&tp->linkchg_task); +} +#endif + +static void rtl8127_init_all_schedule_work(struct rtl8127_private *tp) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) + INIT_WORK(&tp->reset_task, rtl8127_reset_task, dev); + INIT_WORK(&tp->esd_task, rtl8127_esd_task, dev); + INIT_WORK(&tp->linkchg_task, rtl8127_linkchg_task, dev); +#else + INIT_DELAYED_WORK(&tp->reset_task, rtl8127_reset_task); + INIT_DELAYED_WORK(&tp->esd_task, rtl8127_esd_task); + INIT_DELAYED_WORK(&tp->linkchg_task, rtl8127_linkchg_task); +#endif +} + +static void rtl8127_cancel_all_schedule_work(struct rtl8127_private *tp) +{ + rtl8127_cancel_schedule_reset_work(tp); + rtl8127_cancel_schedule_esd_work(tp); + rtl8127_cancel_schedule_linkchg_work(tp); +} + +static void +rtl8127_wait_for_irq_complete(struct rtl8127_private *tp) +{ + if (tp->features & RTL_FEATURE_MSIX) { + int i; + for (i = 0; i < tp->irq_nvecs; i++) + synchronize_irq(tp->irq_tbl[i].vector); + } else { + synchronize_irq(tp->dev->irq); + } +} + +void +_rtl8127_wait_for_quiescence(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + /* Wait for any pending NAPI task to complete */ +#ifdef CONFIG_R8127_NAPI + rtl8127_disable_napi(tp); +#endif//CONFIG_R8127_NAPI + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,5,67) + /* Give a racing hard_start_xmit a few cycles to complete. */ + synchronize_net(); +#endif + + rtl8127_irq_mask_and_ack(tp); + + rtl8127_wait_for_irq_complete(tp); +} + +static void +rtl8127_wait_for_quiescence(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + //suppress unused variable + (void)(tp); + + _rtl8127_wait_for_quiescence(dev); + +#ifdef CONFIG_R8127_NAPI + rtl8127_enable_napi(tp); +#endif//CONFIG_R8127_NAPI +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) +static void rtl8127_reset_task(void *_data) +{ + struct net_device *dev = _data; + struct rtl8127_private *tp = netdev_priv(dev); +#else +static void rtl8127_reset_task(struct work_struct *work) +{ + struct rtl8127_private *tp = + container_of(work, struct rtl8127_private, reset_task.work); + struct net_device *dev = tp->dev; +#endif + int i; + + rtnl_lock(); + + if (!netif_running(dev) || + test_bit(R8127_FLAG_DOWN, tp->task_flags) || + !test_and_clear_bit(R8127_FLAG_TASK_RESET_PENDING, tp->task_flags)) + goto out_unlock; + + netdev_err(dev, "Device reseting!\n"); + + netif_carrier_off(dev); + netif_tx_disable(dev); + _rtl8127_wait_for_quiescence(dev); + rtl8127_hw_reset(dev); + + rtl8127_tx_clear(tp); + + rtl8127_init_ring_indexes(tp); + + rtl8127_tx_desc_init(tp); + for (i = 0; i < tp->num_rx_rings; i++) { + struct rtl8127_rx_ring *ring; + u32 entry; + + ring = &tp->rx_ring[i]; + for (entry = 0; entry < ring->num_rx_desc; entry++) { + struct RxDesc *desc; + + desc = rtl8127_get_rxdesc(tp, ring->RxDescArray, entry); + rtl8127_mark_to_asic(tp, desc, tp->rx_buf_sz); + } + } + +#ifdef ENABLE_PTP_SUPPORT + rtl8127_ptp_reset(tp); +#endif + +#ifdef CONFIG_R8127_NAPI + rtl8127_enable_napi(tp); +#endif //CONFIG_R8127_NAPI + + if (tp->resume_not_chg_speed) { + _rtl8127_check_link_status(dev, R8127_LINK_STATE_UNKNOWN); + + tp->resume_not_chg_speed = 0; + } else { + rtl8127_enable_hw_linkchg_interrupt(tp); + + rtl8127_set_speed(dev, tp->autoneg, tp->speed, tp->duplex, tp->advertising); + } + +out_unlock: + rtnl_unlock(); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) +static void rtl8127_esd_task(void *_data) +{ + struct net_device *dev = _data; + struct rtl8127_private *tp = netdev_priv(dev); +#else +static void rtl8127_esd_task(struct work_struct *work) +{ + struct rtl8127_private *tp = + container_of(work, struct rtl8127_private, esd_task.work); + struct net_device *dev = tp->dev; +#endif + rtnl_lock(); + + if (!netif_running(dev) || + test_bit(R8127_FLAG_DOWN, tp->task_flags) || + !test_and_clear_bit(R8127_FLAG_TASK_ESD_CHECK_PENDING, tp->task_flags)) + goto out_unlock; + + rtl8127_esd_checker(tp); + + rtl8127_schedule_esd_work(tp); + +out_unlock: + rtnl_unlock(); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) +static void rtl8127_linkchg_task(void *_data) +{ + struct net_device *dev = _data; + //struct rtl8127_private *tp = netdev_priv(dev); +#else +static void rtl8127_linkchg_task(struct work_struct *work) +{ + struct rtl8127_private *tp = + container_of(work, struct rtl8127_private, linkchg_task.work); + struct net_device *dev = tp->dev; +#endif + rtnl_lock(); + + if (!netif_running(dev) || + test_bit(R8127_FLAG_DOWN, tp->task_flags) || + !test_and_clear_bit(R8127_FLAG_TASK_LINKCHG_CHECK_PENDING, tp->task_flags)) + goto out_unlock; + + rtl8127_check_link_status(dev); + +out_unlock: + rtnl_unlock(); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0) +static void +rtl8127_tx_timeout(struct net_device *dev, unsigned int txqueue) +#else +static void +rtl8127_tx_timeout(struct net_device *dev) +#endif +{ + struct rtl8127_private *tp = netdev_priv(dev); + + netdev_err(dev, "Transmit timeout reset Device!\n"); + + /* Let's wait a bit while any (async) irq lands on */ + rtl8127_schedule_reset_work(tp); +} + +static u32 +rtl8127_get_txd_opts1(struct rtl8127_tx_ring *ring, + u32 opts1, + u32 len, + unsigned int entry) +{ + u32 status = opts1 | len; + + if (entry == ring->num_tx_desc - 1) + status |= RingEnd; + + return status; +} + +static int +rtl8127_xmit_frags(struct rtl8127_private *tp, + struct rtl8127_tx_ring *ring, + struct sk_buff *skb, + const u32 *opts) +{ + struct skb_shared_info *info = skb_shinfo(skb); + unsigned int cur_frag, entry; + struct TxDesc *txd = NULL; + const unsigned char nr_frags = info->nr_frags; + unsigned long PktLenCnt = 0; + bool LsoPatchEnabled = FALSE; + + entry = ring->cur_tx; + for (cur_frag = 0; cur_frag < nr_frags; cur_frag++) { + skb_frag_t *frag = info->frags + cur_frag; + dma_addr_t mapping; + u32 status, len; + void *addr; + + entry = (entry + 1) % ring->num_tx_desc; + + txd = ring->TxDescArray + entry; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,2,0) + len = frag->size; + addr = ((void *) page_address(frag->page)) + frag->page_offset; +#else + len = skb_frag_size(frag); + addr = skb_frag_address(frag); +#endif + mapping = dma_map_single(tp_to_dev(tp), addr, len, DMA_TO_DEVICE); + + if (unlikely(dma_mapping_error(tp_to_dev(tp), mapping))) { + if (unlikely(net_ratelimit())) + netif_err(tp, drv, tp->dev, + "Failed to map TX fragments DMA!\n"); + goto err_out; + } + + /* anti gcc 2.95.3 bugware (sic) */ + status = rtl8127_get_txd_opts1(ring, opts[0], len, entry); + if (cur_frag == (nr_frags - 1) || LsoPatchEnabled == TRUE) + status |= LastFrag; + + txd->addr = cpu_to_le64(mapping); + + ring->tx_skb[entry].len = len; + + txd->opts2 = cpu_to_le32(opts[1]); + wmb(); + txd->opts1 = cpu_to_le32(status); + + PktLenCnt += len; + } + + return cur_frag; + +err_out: + rtl8127_tx_clear_range(tp, ring, ring->cur_tx + 1, cur_frag); + return -EIO; +} + +static inline +__be16 get_protocol(struct sk_buff *skb) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,37) + return vlan_get_protocol(skb); +#else + __be16 protocol; + + if (skb->protocol == htons(ETH_P_8021Q)) + protocol = vlan_eth_hdr(skb)->h_vlan_encapsulated_proto; + else + protocol = skb->protocol; + + return protocol; +#endif +} + +static inline +u8 rtl8127_get_l4_protocol(struct sk_buff *skb) +{ + int no = skb_network_offset(skb); + struct ipv6hdr *i6h, _i6h; + struct iphdr *ih, _ih; + u8 ip_protocol = IPPROTO_RAW; + + switch (get_protocol(skb)) { + case __constant_htons(ETH_P_IP): + ih = skb_header_pointer(skb, no, sizeof(_ih), &_ih); + if (ih) + ip_protocol = ih->protocol; + break; + case __constant_htons(ETH_P_IPV6): + i6h = skb_header_pointer(skb, no, sizeof(_i6h), &_i6h); + if (i6h) + ip_protocol = i6h->nexthdr; + break; + } + + return ip_protocol; +} + +static bool rtl8127_skb_pad_with_len(struct sk_buff *skb, unsigned int len) +{ + if (skb_padto(skb, len)) + return false; + skb_put(skb, len - skb->len); + return true; +} + +static bool rtl8127_skb_pad(struct sk_buff *skb) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,19,0) + return rtl8127_skb_pad_with_len(skb, ETH_ZLEN); +#else + return !eth_skb_pad(skb); +#endif +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0) +/* msdn_giant_send_check() + * According to the document of microsoft, the TCP Pseudo Header excludes the + * packet length for IPv6 TCP large packets. + */ +static int msdn_giant_send_check(struct sk_buff *skb) +{ + const struct ipv6hdr *ipv6h; + struct tcphdr *th; + int ret; + + ret = skb_cow_head(skb, 0); + if (ret) + return ret; + + ipv6h = ipv6_hdr(skb); + th = tcp_hdr(skb); + + th->check = 0; + th->check = ~tcp_v6_check(0, &ipv6h->saddr, &ipv6h->daddr, 0); + + return ret; +} +#endif + +static bool rtl8127_require_pad_ptp_pkt(struct rtl8127_private *tp) +{ + return false; +} + +#define MIN_PATCH_LEN (47) +static u32 +rtl8127_get_patch_pad_len(struct rtl8127_private *tp, + struct sk_buff *skb) +{ + u32 pad_len = 0; + int trans_data_len; + u32 hdr_len; + u32 pkt_len = skb->len; + u8 ip_protocol; + bool has_trans = skb_transport_header_was_set(skb); + + if (!rtl8127_require_pad_ptp_pkt(tp)) + goto no_padding; + + if (!(has_trans && (pkt_len < 175))) //128 + MIN_PATCH_LEN + goto no_padding; + + ip_protocol = rtl8127_get_l4_protocol(skb); + if (!(ip_protocol == IPPROTO_TCP || ip_protocol == IPPROTO_UDP)) + goto no_padding; + + trans_data_len = pkt_len - + (skb->transport_header - + skb_headroom(skb)); + if (ip_protocol == IPPROTO_UDP) { + if (trans_data_len > 3 && trans_data_len < MIN_PATCH_LEN) { + u16 dest_port = 0; + + skb_copy_bits(skb, skb->transport_header - skb_headroom(skb) + 2, &dest_port, 2); + dest_port = ntohs(dest_port); + + if (dest_port == 0x13f || + dest_port == 0x140) { + pad_len = MIN_PATCH_LEN - trans_data_len; + goto out; + } + } + } + + hdr_len = 0; + if (ip_protocol == IPPROTO_TCP) + hdr_len = 20; + else if (ip_protocol == IPPROTO_UDP) + hdr_len = 8; + if (trans_data_len < hdr_len) + pad_len = hdr_len - trans_data_len; + +out: + if ((pkt_len + pad_len) < ETH_ZLEN) + pad_len = ETH_ZLEN - pkt_len; + + return pad_len; + +no_padding: + + return 0; +} + +static bool +rtl8127_tso_csum(struct sk_buff *skb, + struct net_device *dev, + u32 *opts, + unsigned int *bytecount, + unsigned short *gso_segs) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned long large_send = 0; + u32 csum_cmd = 0; + u8 sw_calc_csum = false; + u8 check_patch_required = true; + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + if (dev->features & (NETIF_F_TSO | NETIF_F_TSO6)) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) + u32 mss = skb_shinfo(skb)->tso_size; +#else + u32 mss = skb_shinfo(skb)->gso_size; +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) + + /* TCP Segmentation Offload (or TCP Large Send) */ + if (mss) { + union { + struct iphdr *v4; + struct ipv6hdr *v6; + unsigned char *hdr; + } ip; + union { + struct tcphdr *tcp; + struct udphdr *udp; + unsigned char *hdr; + } l4; + u32 l4_offset, hdr_len; + + ip.hdr = skb_network_header(skb); + l4.hdr = skb_checksum_start(skb); + + l4_offset = skb_transport_offset(skb); + assert((l4_offset%2) == 0); + switch (get_protocol(skb)) { + case __constant_htons(ETH_P_IP): + if (l4_offset <= GTTCPHO_MAX) { + opts[0] |= GiantSendv4; + opts[0] |= l4_offset << GTTCPHO_SHIFT; + opts[1] |= min(mss, MSS_MAX) << 18; + large_send = 1; + } + break; + case __constant_htons(ETH_P_IPV6): +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0) + if (msdn_giant_send_check(skb)) + return false; +#endif + if (l4_offset <= GTTCPHO_MAX) { + opts[0] |= GiantSendv6; + opts[0] |= l4_offset << GTTCPHO_SHIFT; + opts[1] |= min(mss, MSS_MAX) << 18; + large_send = 1; + } + break; + default: + if (unlikely(net_ratelimit())) + dprintk("tso proto=%x!\n", skb->protocol); + break; + } + + if (large_send == 0) + return false; + + + /* compute length of segmentation header */ + hdr_len = (l4.tcp->doff * 4) + l4_offset; + /* update gso size and bytecount with header size */ + *gso_segs = skb_shinfo(skb)->gso_segs; + *bytecount += (*gso_segs - 1) * hdr_len; + + return true; + } + } +#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + + if (skb->ip_summed == CHECKSUM_PARTIAL) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) + const struct iphdr *ip = skb->nh.iph; + + if (dev->features & NETIF_F_IP_CSUM) { + if (ip->protocol == IPPROTO_TCP) + csum_cmd = tp->tx_ip_csum_cmd | tp->tx_tcp_csum_cmd; + else if (ip->protocol == IPPROTO_UDP) + csum_cmd = tp->tx_ip_csum_cmd | tp->tx_udp_csum_cmd; + else if (ip->protocol == IPPROTO_IP) + csum_cmd = tp->tx_ip_csum_cmd; + } +#else + u8 ip_protocol = IPPROTO_RAW; + + switch (get_protocol(skb)) { + case __constant_htons(ETH_P_IP): + if (dev->features & NETIF_F_IP_CSUM) { + ip_protocol = ip_hdr(skb)->protocol; + csum_cmd = tp->tx_ip_csum_cmd; + } + break; + case __constant_htons(ETH_P_IPV6): + if (dev->features & NETIF_F_IPV6_CSUM) { + if (skb_transport_offset(skb) > 0 && skb_transport_offset(skb) <= TCPHO_MAX) { + ip_protocol = ipv6_hdr(skb)->nexthdr; + csum_cmd = tp->tx_ipv6_csum_cmd; + csum_cmd |= skb_transport_offset(skb) << TCPHO_SHIFT; + } + } + break; + default: + if (unlikely(net_ratelimit())) + dprintk("checksum_partial proto=%x!\n", skb->protocol); + break; + } + + if (ip_protocol == IPPROTO_TCP) + csum_cmd |= tp->tx_tcp_csum_cmd; + else if (ip_protocol == IPPROTO_UDP) + csum_cmd |= tp->tx_udp_csum_cmd; +#endif + if (csum_cmd == 0) { + sw_calc_csum = true; +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + WARN_ON(1); /* we need a WARN() */ +#endif + } + + if (ip_protocol == IPPROTO_TCP) + check_patch_required = false; + } + + if (check_patch_required) { + u32 pad_len = rtl8127_get_patch_pad_len(tp, skb); + + if (pad_len > 0) { + if (!rtl8127_skb_pad_with_len(skb, skb->len + pad_len)) + return false; + + if (csum_cmd != 0) + sw_calc_csum = true; + } + } + + if (skb->len < ETH_ZLEN) { + if (tp->UseSwPaddingShortPkt || + (tp->ShortPacketSwChecksum && csum_cmd != 0)) { + if (!rtl8127_skb_pad(skb)) + return false; + + if (csum_cmd != 0) + sw_calc_csum = true; + } + } + + if (sw_calc_csum) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,10) && LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,7) + skb_checksum_help(&skb, 0); +#elif LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) && LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,10) + skb_checksum_help(skb, 0); +#else + skb_checksum_help(skb); +#endif + } else + opts[1] |= csum_cmd; + + return true; +} + +static bool rtl8127_tx_slots_avail(struct rtl8127_private *tp, + struct rtl8127_tx_ring *ring) +{ + unsigned int slots_avail = READ_ONCE(ring->dirty_tx) + ring->num_tx_desc + - READ_ONCE(ring->cur_tx); + + /* A skbuff with nr_frags needs nr_frags+1 entries in the tx queue */ + return slots_avail > MAX_SKB_FRAGS; +} + +static inline u32 +rtl8127_fast_mod_mask(const u32 input, const u32 mask) +{ + return input > mask ? input & mask : input; +} + +static void rtl8127_doorbell(struct rtl8127_private *tp, + struct rtl8127_tx_ring *ring) +{ + if (tp->EnableTxNoClose) { + if (tp->HwSuppTxNoCloseVer > 3) + RTL_W32(tp, ring->sw_tail_ptr_reg, ring->cur_tx); + else + RTL_W16(tp, ring->sw_tail_ptr_reg, ring->cur_tx); + } else { + /* set polling bit */ + RTL_W32(tp, TPPOLL_8125, BIT(ring->index)); + } +} + +static netdev_tx_t +rtl8127_start_xmit(struct sk_buff *skb, + struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + unsigned int bytecount; + unsigned short gso_segs; + struct ring_info *last; + unsigned int last_entry; + unsigned int entry; + struct TxDesc *txd; + dma_addr_t mapping; + u32 len; + u32 opts[2]; + netdev_tx_t ret = NETDEV_TX_OK; + int frags; + u8 EnableTxNoClose = tp->EnableTxNoClose; + const u16 queue_mapping = skb_get_queue_mapping(skb); + struct rtl8127_tx_ring *ring; + bool stop_queue; + + assert(queue_mapping < tp->num_tx_rings); + + ring = &tp->tx_ring[queue_mapping]; + + if (unlikely(!rtl8127_tx_slots_avail(tp, ring))) { + if (netif_msg_drv(tp)) { + printk(KERN_ERR + "%s: BUG! Tx Ring[%d] full when queue awake!\n", + dev->name, + queue_mapping); + } + goto err_stop; + } + + entry = ring->cur_tx % ring->num_tx_desc; + txd = ring->TxDescArray + entry; + + if (!EnableTxNoClose) { + if (unlikely(le32_to_cpu(txd->opts1) & DescOwn)) { + if (netif_msg_drv(tp)) { + printk(KERN_ERR + "%s: BUG! Tx Desc is own by hardware!\n", + dev->name); + } + goto err_stop; + } + } + + bytecount = skb->len; + gso_segs = 1; + + opts[0] = DescOwn; + opts[1] = rtl8127_tx_vlan_tag(tp, skb); + + if (unlikely(!rtl8127_tso_csum(skb, dev, opts, &bytecount, &gso_segs))) + goto err_dma_0; + + frags = rtl8127_xmit_frags(tp, ring, skb, opts); + if (unlikely(frags < 0)) + goto err_dma_0; + if (frags) { + len = skb_headlen(skb); + opts[0] |= FirstFrag; + } else { + len = skb->len; + opts[0] |= FirstFrag | LastFrag; + } + + opts[0] = rtl8127_get_txd_opts1(ring, opts[0], len, entry); + mapping = dma_map_single(tp_to_dev(tp), skb->data, len, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(tp_to_dev(tp), mapping))) { + if (unlikely(net_ratelimit())) + netif_err(tp, drv, dev, "Failed to map TX DMA!\n"); + goto err_dma_1; + } + +#ifdef ENABLE_PTP_SUPPORT + if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { + if (!test_and_set_bit_lock(__RTL8127_PTP_TX_IN_PROGRESS, &tp->state)) { + if (tp->hwtstamp_config.tx_type == HWTSTAMP_TX_ON && + !tp->ptp_tx_skb) { + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + + tp->ptp_tx_skb = skb_get(skb); + tp->ptp_tx_start = jiffies; + schedule_work(&tp->ptp_tx_work); + } else + tp->tx_hwtstamp_skipped++; + } + } +#endif + /* set first fragment's length */ + ring->tx_skb[entry].len = len; + + /* set skb to last fragment */ + last_entry = (entry + frags) % ring->num_tx_desc; + last = &ring->tx_skb[last_entry]; + last->skb = skb; + last->gso_segs = gso_segs; + last->bytecount = bytecount; + + txd->addr = cpu_to_le64(mapping); + txd->opts2 = cpu_to_le32(opts[1]); + wmb(); + txd->opts1 = cpu_to_le32(opts[0]); + + netdev_tx_sent_queue(txring_txq(ring), bytecount); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) + dev->trans_start = jiffies; +#else + skb_tx_timestamp(skb); +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) + + /* rtl_tx needs to see descriptor changes before updated tp->cur_tx */ + smp_wmb(); + + WRITE_ONCE(ring->cur_tx, ring->cur_tx + frags + 1); + + stop_queue = !rtl8127_tx_slots_avail(tp, ring); + if (unlikely(stop_queue)) { + /* Avoid wrongly optimistic queue wake-up: rtl_tx thread must + * not miss a ring update when it notices a stopped queue. + */ + smp_wmb(); + netif_stop_subqueue(dev, queue_mapping); + } + + if (netif_xmit_stopped(txring_txq(ring)) || !netdev_xmit_more()) + rtl8127_doorbell(tp, ring); + + if (unlikely(stop_queue)) { + /* Sync with rtl_tx: + * - publish queue status and cur_tx ring index (write barrier) + * - refresh dirty_tx ring index (read barrier). + * May the current thread have a pessimistic view of the ring + * status and forget to wake up queue, a racing rtl_tx thread + * can't. + */ + smp_mb(); + if (rtl8127_tx_slots_avail(tp, ring)) + netif_start_subqueue(dev, queue_mapping); + } +out: + return ret; +err_dma_1: + rtl8127_tx_clear_range(tp, ring, ring->cur_tx + 1, frags); +err_dma_0: + RTLDEV->stats.tx_dropped++; + dev_kfree_skb_any(skb); + ret = NETDEV_TX_OK; + goto out; +err_stop: + netif_stop_subqueue(dev, queue_mapping); + ret = NETDEV_TX_BUSY; + RTLDEV->stats.tx_dropped++; + goto out; +} + +/* recycle tx no close desc*/ +static int +rtl8127_tx_interrupt_noclose(struct rtl8127_tx_ring *ring, int budget) +{ + unsigned int total_bytes = 0, total_packets = 0; + struct rtl8127_private *tp = ring->priv; + struct net_device *dev = tp->dev; + unsigned int dirty_tx, tx_left; + unsigned int tx_desc_closed; + unsigned int count = 0; + + dirty_tx = ring->dirty_tx; + ring->NextHwDesCloPtr = rtl8127_get_hw_clo_ptr(ring); + tx_desc_closed = rtl8127_fast_mod_mask(ring->NextHwDesCloPtr - + ring->BeginHwDesCloPtr, + tp->MaxTxDescPtrMask); + tx_left = min((READ_ONCE(ring->cur_tx) - dirty_tx), tx_desc_closed); + ring->BeginHwDesCloPtr += tx_left; + + while (tx_left > 0) { + unsigned int entry = dirty_tx % ring->num_tx_desc; + struct ring_info *tx_skb = ring->tx_skb + entry; + + rtl8127_unmap_tx_skb(tp->pci_dev, + tx_skb, + ring->TxDescArray + entry); + + if (tx_skb->skb != NULL) { + /* update the statistics for this packet */ + total_bytes += tx_skb->bytecount; + total_packets += tx_skb->gso_segs; + + RTL_NAPI_CONSUME_SKB_ANY(tx_skb->skb, budget); + tx_skb->skb = NULL; + } + dirty_tx++; + tx_left--; + } + + if (total_packets) { + netdev_tx_completed_queue(txring_txq(ring), + total_packets, total_bytes); + + RTLDEV->stats.tx_bytes += total_bytes; + RTLDEV->stats.tx_packets+= total_packets; + } + + if (ring->dirty_tx != dirty_tx) { + count = dirty_tx - ring->dirty_tx; + WRITE_ONCE(ring->dirty_tx, dirty_tx); + smp_wmb(); + if (__netif_subqueue_stopped(dev, ring->index) && + (rtl8127_tx_slots_avail(tp, ring))) { + netif_start_subqueue(dev, ring->index); + } + } + + return count; +} + +/* recycle tx close desc*/ +static int +rtl8127_tx_interrupt_close(struct rtl8127_tx_ring *ring, int budget) +{ + unsigned int total_bytes = 0, total_packets = 0; + struct rtl8127_private *tp = ring->priv; + struct net_device *dev = tp->dev; + unsigned int dirty_tx, tx_left; + unsigned int count = 0; + + dirty_tx = ring->dirty_tx; + tx_left = READ_ONCE(ring->cur_tx) - dirty_tx; + + while (tx_left > 0) { + unsigned int entry = dirty_tx % ring->num_tx_desc; + struct ring_info *tx_skb = ring->tx_skb + entry; + + if (le32_to_cpu(READ_ONCE(ring->TxDescArray[entry].opts1)) & DescOwn) + break; + + rtl8127_unmap_tx_skb(tp->pci_dev, + tx_skb, + ring->TxDescArray + entry); + + if (tx_skb->skb != NULL) { + /* update the statistics for this packet */ + total_bytes += tx_skb->bytecount; + total_packets += tx_skb->gso_segs; + + RTL_NAPI_CONSUME_SKB_ANY(tx_skb->skb, budget); + tx_skb->skb = NULL; + } + dirty_tx++; + tx_left--; + } + + if (total_packets) { + netdev_tx_completed_queue(txring_txq(ring), + total_packets, total_bytes); + + RTLDEV->stats.tx_bytes += total_bytes; + RTLDEV->stats.tx_packets+= total_packets; + } + + if (ring->dirty_tx != dirty_tx) { + count = dirty_tx - ring->dirty_tx; + WRITE_ONCE(ring->dirty_tx, dirty_tx); + smp_wmb(); + if (__netif_subqueue_stopped(dev, ring->index) && + (rtl8127_tx_slots_avail(tp, ring))) { + netif_start_subqueue(dev, ring->index); + } + + if (READ_ONCE(ring->cur_tx) != dirty_tx) + rtl8127_doorbell(tp, ring); + } + + return count; +} + +static int +rtl8127_tx_interrupt(struct rtl8127_tx_ring *ring, int budget) +{ + struct rtl8127_private *tp = ring->priv; + + if (tp->EnableTxNoClose) + return rtl8127_tx_interrupt_noclose(ring, budget); + else + return rtl8127_tx_interrupt_close(ring, budget); +} + +static int +rtl8127_tx_interrupt_with_vector(struct rtl8127_private *tp, + const int message_id, + int budget) +{ + int count = 0; + + switch (tp->HwCurrIsrVer) { + case 3: + case 4: + if (message_id < tp->num_tx_rings) + count += rtl8127_tx_interrupt(&tp->tx_ring[message_id], budget); + break; + case 5: + if (message_id == 16) + count += rtl8127_tx_interrupt(&tp->tx_ring[0], budget); +#ifdef ENABLE_MULTIPLE_TX_QUEUE + else if (message_id == 17 && tp->num_tx_rings > 1) + count += rtl8127_tx_interrupt(&tp->tx_ring[1], budget); +#endif + break; + case 6: + if (message_id == 8) + count += rtl8127_tx_interrupt(&tp->tx_ring[0], budget); +#ifdef ENABLE_MULTIPLE_TX_QUEUE + else if (message_id == 9 && tp->num_tx_rings > 1) + count += rtl8127_tx_interrupt(&tp->tx_ring[1], budget); +#endif + break; + default: + if (message_id == 16) + count += rtl8127_tx_interrupt(&tp->tx_ring[0], budget); +#ifdef ENABLE_MULTIPLE_TX_QUEUE + else if (message_id == 18 && tp->num_tx_rings > 1) + count += rtl8127_tx_interrupt(&tp->tx_ring[1], budget); +#endif + break; + } + + return count; +} + +static inline int +rtl8127_fragmented_frame(struct rtl8127_private *tp, u32 status) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + return (status & (FirstFrag_V3 | LastFrag_V3)) != (FirstFrag_V3 | LastFrag_V3); + case RX_DESC_RING_TYPE_4: + return (status & (FirstFrag_V4 | LastFrag_V4)) != (FirstFrag_V4 | LastFrag_V4); + default: + return (status & (FirstFrag | LastFrag)) != (FirstFrag | LastFrag); + } +} + +static inline int +rtl8127_is_non_eop(struct rtl8127_private *tp, u32 status) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + return !(status & LastFrag_V3); + case RX_DESC_RING_TYPE_4: + return !(status & LastFrag_V4); + default: + return !(status & LastFrag); + } +} + +static inline int +rtl8127_rx_desc_type(u32 status) +{ + return ((status >> 26) & 0x0F); +} + +static inline void +rtl8127_rx_v1_csum(struct rtl8127_private *tp, + struct sk_buff *skb, + struct RxDesc *desc) +{ + u32 opts1 = le32_to_cpu(desc->opts1); + + if (((opts1 & RxTCPT) && !(opts1 & RxTCPF)) || + ((opts1 & RxUDPT) && !(opts1 & RxUDPF))) + skb->ip_summed = CHECKSUM_UNNECESSARY; + else + skb_checksum_none_assert(skb); +} + +static inline void +rtl8127_rx_v3_csum(struct rtl8127_private *tp, + struct sk_buff *skb, + struct RxDescV3 *descv3) +{ + u32 opts2 = le32_to_cpu(descv3->RxDescNormalDDWord4.opts2); + + /* rx csum offload for RTL8125 */ + if (((opts2 & RxTCPT_v3) && !(opts2 & RxTCPF_v3)) || + ((opts2 & RxUDPT_v3) && !(opts2 & RxUDPF_v3))) + skb->ip_summed = CHECKSUM_UNNECESSARY; + else + skb_checksum_none_assert(skb); +} + +static inline void +rtl8127_rx_v4_csum(struct rtl8127_private *tp, + struct sk_buff *skb, + struct RxDescV4 *descv4) +{ + u32 opts1 = le32_to_cpu(descv4->RxDescNormalDDWord2.opts1); + + /* rx csum offload for RTL8125 */ + if (((opts1 & RxTCPT_v4) && !(opts1 & RxTCPF_v4)) || + ((opts1 & RxUDPT_v4) && !(opts1 & RxUDPF_v4))) + skb->ip_summed = CHECKSUM_UNNECESSARY; + else + skb_checksum_none_assert(skb); +} + +static inline void +rtl8127_rx_csum(struct rtl8127_private *tp, + struct sk_buff *skb, + struct RxDesc *desc) +{ + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + rtl8127_rx_v3_csum(tp, skb, (struct RxDescV3 *)desc); + break; + case RX_DESC_RING_TYPE_4: + rtl8127_rx_v4_csum(tp, skb, (struct RxDescV4 *)desc); + break; + default: + rtl8127_rx_v1_csum(tp, skb, desc); + break; + } +} + +/* +static inline int +rtl8127_try_rx_copy(struct rtl8127_private *tp, + struct rtl8127_rx_ring *ring, + struct sk_buff **sk_buff, + int pkt_size, + struct RxDesc *desc, + int rx_buf_sz) +{ + int ret = -1; + + struct sk_buff *skb; + + skb = RTL_ALLOC_SKB_INTR(&tp->r8127napi[ring->index].napi, pkt_size + R8127_RX_ALIGN); + if (skb) { + u8 *data; + + data = sk_buff[0]->data; + if (!R8127_USE_NAPI_ALLOC_SKB) + skb_reserve(skb, R8127_RX_ALIGN); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,37) + prefetch(data - R8127_RX_ALIGN); +#endif + eth_copy_and_sum(skb, data, pkt_size, 0); + *sk_buff = skb; + rtl8127_mark_to_asic(tp, desc, rx_buf_sz); + ret = 0; + } + + return ret; +} +*/ + +static inline void +rtl8127_rx_skb(struct rtl8127_private *tp, + struct sk_buff *skb, + u32 ring_index) +{ +#ifdef CONFIG_R8127_NAPI +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29) + netif_receive_skb(skb); +#else + napi_gro_receive(&tp->r8127napi[ring_index].napi, skb); +#endif +#else + netif_rx(skb); +#endif +} + +static int +rtl8127_check_rx_desc_error(struct net_device *dev, + struct rtl8127_private *tp, + u32 status) +{ + int ret = 0; + + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + if (unlikely(status & RxRES_V3)) { + if (status & (RxRWT_V3 | RxRUNT_V3)) + RTLDEV->stats.rx_length_errors++; + if (status & RxCRC_V3) + RTLDEV->stats.rx_crc_errors++; + + ret = -1; + } + break; + case RX_DESC_RING_TYPE_4: + if (unlikely(status & RxRES_V4)) { + if (status & RxRUNT_V4) + RTLDEV->stats.rx_length_errors++; + if (status & RxCRC_V4) + RTLDEV->stats.rx_crc_errors++; + + ret = -1; + } + break; + default: + if (unlikely(status & RxRES)) { + if (status & (RxRWT | RxRUNT)) + RTLDEV->stats.rx_length_errors++; + if (status & RxCRC) + RTLDEV->stats.rx_crc_errors++; + + ret = -1; + } + break; + } + + return ret; +} + +#ifdef ENABLE_PAGE_REUSE + +static inline bool +rtl8127_reuse_rx_ok(struct page *page) +{ + /* avoid re-using remote pages */ + if (!dev_page_is_reusable(page)) { + //printk(KERN_INFO "r8127 page pfmemalloc, can't reuse!\n"); + return false; + } + /* if we are only owner of page we can reuse it */ + if (unlikely(page_ref_count(page) != 1)) { + //printk(KERN_INFO "r8127 page refcnt %d, can't reuse!\n", page_ref_count(page)); + return false; + } + + return true; +} + +static void +rtl8127_reuse_rx_buffer(struct rtl8127_private *tp, struct rtl8127_rx_ring *ring, u32 cur_rx, struct rtl8127_rx_buffer *rxb) +{ + struct page *page = rxb->page; + + u32 dirty_rx = ring->dirty_rx; + u32 entry = dirty_rx % ring->num_rx_desc; + struct rtl8127_rx_buffer *nrxb = &ring->rx_buffer[entry]; + + u32 noffset; + + //the page gonna be shared by us and kernel, keep page ref = 2 + page_ref_inc(page); + + //flip the buffer in page to use next + noffset = rxb->page_offset ^ (tp->rx_buf_page_size / 2); //one page, two buffer, ping-pong + + nrxb->dma = rxb->dma; + nrxb->page_offset = noffset; + nrxb->data = rxb->data; + + if (cur_rx != dirty_rx) { + //move the buffer to other slot + nrxb->page = page; + rxb->page = NULL; + } +} + +static void rtl8127_put_rx_buffer(struct rtl8127_private *tp, + struct rtl8127_rx_ring *ring, + u32 cur_rx, + struct rtl8127_rx_buffer *rxb) +{ + struct rtl8127_rx_buffer *nrxb; + struct page *page = rxb->page; + u32 entry; + + entry = ring->dirty_rx % ring->num_rx_desc; + nrxb = &ring->rx_buffer[entry]; + if (likely(rtl8127_reuse_rx_ok(page))) { + /* hand second half of page back to the ring */ + rtl8127_reuse_rx_buffer(tp, ring, cur_rx, rxb); + } else { + tp->page_reuse_fail_cnt++; + + dma_unmap_page_attrs(&tp->pci_dev->dev, rxb->dma, + tp->rx_buf_page_size, + DMA_FROM_DEVICE, + (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)); + //the page ref is kept 1, uniquely owned by kernel now + rxb->page = NULL; + + return; + } + + dma_sync_single_range_for_device(tp_to_dev(tp), + nrxb->dma, + nrxb->page_offset, + tp->rx_buf_sz, + DMA_FROM_DEVICE); + + rtl8127_map_to_asic(tp, ring, + rtl8127_get_rxdesc(tp, ring->RxDescArray, entry), + nrxb->dma + nrxb->page_offset, + tp->rx_buf_sz, entry); + + ring->dirty_rx++; +} + +#endif //ENABLE_PAGE_REUSE + +static int +rtl8127_rx_interrupt(struct net_device *dev, + struct rtl8127_private *tp, + struct rtl8127_rx_ring *ring, + napi_budget budget) +{ + unsigned int cur_rx, rx_left; + unsigned int delta, count = 0; + unsigned int entry; + struct RxDesc *desc; + struct sk_buff *skb; + u32 status; + u32 rx_quota; + u32 ring_index = ring->index; +#ifdef ENABLE_PAGE_REUSE + struct rtl8127_rx_buffer *rxb; +#else //ENABLE_PAGE_REUSE + u64 rx_buf_phy_addr; +#endif //ENABLE_PAGE_REUSE + unsigned int total_rx_multicast_packets = 0; + unsigned int total_rx_bytes = 0, total_rx_packets = 0; + + assert(dev != NULL); + assert(tp != NULL); + + if (ring->RxDescArray == NULL) + goto rx_out; + + rx_quota = RTL_RX_QUOTA(budget); + cur_rx = ring->cur_rx; + rx_left = ring->num_rx_desc + ring->dirty_rx - cur_rx; + rx_left = rtl8127_rx_quota(rx_left, (u32)rx_quota); + + for (; rx_left > 0; rx_left--, cur_rx++) { +#ifndef ENABLE_PAGE_REUSE + const void *rx_buf; +#endif //!ENABLE_PAGE_REUSE + u32 pkt_size; + + entry = cur_rx % ring->num_rx_desc; + desc = rtl8127_get_rxdesc(tp, ring->RxDescArray, entry); + status = le32_to_cpu(rtl8127_rx_desc_opts1(tp, desc)); + if (status & DescOwn) { + RTL_R8(tp, tp->imr_reg[0]); + status = le32_to_cpu(rtl8127_rx_desc_opts1(tp, desc)); + if (status & DescOwn) + break; + } + + rmb(); + + if (unlikely(rtl8127_check_rx_desc_error(dev, tp, status) < 0)) { + if (netif_msg_rx_err(tp)) { + printk(KERN_INFO + "%s: Rx ERROR. status = %08x\n", + dev->name, status); + } + + RTLDEV->stats.rx_errors++; + + if (!(dev->features & NETIF_F_RXALL)) + goto release_descriptor; + } + pkt_size = status & 0x00003fff; + if (likely(!(dev->features & NETIF_F_RXFCS))) { +#ifdef ENABLE_RX_PACKET_FRAGMENT + if (rtl8127_is_non_eop(tp, status) && + pkt_size == tp->rx_buf_sz) { + struct RxDesc *desc_next; + unsigned int entry_next; + int pkt_size_next; + u32 status_next; + + entry_next = (cur_rx + 1) % ring->num_rx_desc; + desc_next = rtl8127_get_rxdesc(tp, ring->RxDescArray, entry_next); + status_next = le32_to_cpu(rtl8127_rx_desc_opts1(tp, desc_next)); + if (!(status_next & DescOwn)) { + pkt_size_next = status_next & 0x00003fff; + if (pkt_size_next < ETH_FCS_LEN) + pkt_size -= (ETH_FCS_LEN - pkt_size_next); + } + } +#endif //ENABLE_RX_PACKET_FRAGMENT + if (!rtl8127_is_non_eop(tp, status)) { + if (pkt_size < ETH_FCS_LEN) { +#ifdef ENABLE_RX_PACKET_FRAGMENT + pkt_size = 0; +#else + goto drop_packet; +#endif //ENABLE_RX_PACKET_FRAGMENT + } else + pkt_size -= ETH_FCS_LEN; + } + } + + if (unlikely(pkt_size > tp->rx_buf_sz)) + goto drop_packet; + +#if !defined(ENABLE_RX_PACKET_FRAGMENT) || !defined(ENABLE_PAGE_REUSE) + /* + * The driver does not support incoming fragmented + * frames. They are seen as a symptom of over-mtu + * sized frames. + */ + if (unlikely(rtl8127_fragmented_frame(tp, status))) + goto drop_packet; +#endif //!ENABLE_RX_PACKET_FRAGMENT || !ENABLE_PAGE_REUSE + +#ifdef ENABLE_PAGE_REUSE + rxb = &ring->rx_buffer[entry]; + skb = rxb->skb; + rxb->skb = NULL; + if (!skb) { + skb = RTL_BUILD_SKB_INTR(rxb->data + rxb->page_offset - ring->rx_offset, tp->rx_buf_page_size / 2); + if (!skb) { + //netdev_err(tp->dev, "Failed to allocate RX skb!\n"); + goto drop_packet; + } + + skb->dev = dev; + if (!R8127_USE_NAPI_ALLOC_SKB) + skb_reserve(skb, R8127_RX_ALIGN); + skb_put(skb, pkt_size); +#ifdef ENABLE_RSS_SUPPORT + rtl8127_rx_hash(tp, desc, skb); +#endif + rtl8127_rx_csum(tp, skb, desc); + } else + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rxb->page, + rxb->page_offset, pkt_size, tp->rx_buf_page_size / 2); + //recycle desc + rtl8127_put_rx_buffer(tp, ring, cur_rx, rxb); + + dma_sync_single_range_for_cpu(tp_to_dev(tp), + rxb->dma, + rxb->page_offset, + tp->rx_buf_sz, + DMA_FROM_DEVICE); +#else //ENABLE_PAGE_REUSE + skb = RTL_ALLOC_SKB_INTR(&tp->r8127napi[ring->index].napi, pkt_size + R8127_RX_ALIGN); + if (!skb) { + //netdev_err(tp->dev, "Failed to allocate RX skb!\n"); + goto drop_packet; + } + + skb->dev = dev; + if (!R8127_USE_NAPI_ALLOC_SKB) + skb_reserve(skb, R8127_RX_ALIGN); + skb_put(skb, pkt_size); + + rx_buf_phy_addr = ring->RxDescPhyAddr[entry]; + dma_sync_single_for_cpu(tp_to_dev(tp), + rx_buf_phy_addr, tp->rx_buf_sz, + DMA_FROM_DEVICE); + rx_buf = ring->Rx_skbuff[entry]->data; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,37) + prefetch(rx_buf - R8127_RX_ALIGN); +#endif + eth_copy_and_sum(skb, rx_buf, pkt_size, 0); + + dma_sync_single_for_device(tp_to_dev(tp), rx_buf_phy_addr, + tp->rx_buf_sz, DMA_FROM_DEVICE); +#endif //ENABLE_PAGE_REUSE + +#ifdef ENABLE_PTP_SUPPORT + if (tp->flags & RTL_FLAG_RX_HWTSTAMP_ENABLED) + rtl8127_rx_ptp_timestamp(tp, skb); +#endif // ENABLE_PTP_SUPPORT + +#ifdef ENABLE_RX_PACKET_FRAGMENT + if (rtl8127_is_non_eop(tp, status)) { + unsigned int entry_next; + entry_next = (entry + 1) % ring->num_rx_desc; + rxb = &ring->rx_buffer[entry_next]; + rxb->skb = skb; + continue; + } +#endif //ENABLE_RX_PACKET_FRAGMENT + +#ifndef ENABLE_PAGE_REUSE +#ifdef ENABLE_RSS_SUPPORT + rtl8127_rx_hash(tp, desc, skb); +#endif + rtl8127_rx_csum(tp, skb, desc); +#endif /* !ENABLE_PAGE_REUSE */ + + skb->protocol = eth_type_trans(skb, dev); + + total_rx_bytes += skb->len; + + if (skb->pkt_type == PACKET_MULTICAST) + total_rx_multicast_packets++; + + if (rtl8127_rx_vlan_skb(tp, desc, skb) < 0) + rtl8127_rx_skb(tp, skb, ring_index); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,11,0) + dev->last_rx = jiffies; +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,11,0) + total_rx_packets++; + +#ifdef ENABLE_PAGE_REUSE + rxb->skb = NULL; + continue; +#endif + +release_descriptor: + switch (tp->InitRxDescType) { + case RX_DESC_RING_TYPE_3: + case RX_DESC_RING_TYPE_4: + rtl8127_set_desc_dma_addr(tp, desc, + ring->RxDescPhyAddr[entry]); + wmb(); + break; + } + rtl8127_mark_to_asic(tp, desc, tp->rx_buf_sz); + continue; +drop_packet: + RTLDEV->stats.rx_dropped++; + RTLDEV->stats.rx_length_errors++; + goto release_descriptor; + } + + count = cur_rx - ring->cur_rx; + ring->cur_rx = cur_rx; + + delta = rtl8127_rx_fill(tp, ring, dev, ring->dirty_rx, ring->cur_rx, 1); + if (!delta && count && netif_msg_intr(tp)) + printk(KERN_INFO "%s: no Rx buffer allocated\n", dev->name); + ring->dirty_rx += delta; + + RTLDEV->stats.rx_bytes += total_rx_bytes; + RTLDEV->stats.rx_packets += total_rx_packets; + RTLDEV->stats.multicast += total_rx_multicast_packets; + + /* + * FIXME: until there is periodic timer to try and refill the ring, + * a temporary shortage may definitely kill the Rx process. + * - disable the asic to try and avoid an overflow and kick it again + * after refill ? + * - how do others driver handle this condition (Uh oh...). + */ + if ((ring->dirty_rx + ring->num_rx_desc == ring->cur_rx) && netif_msg_intr(tp)) + printk(KERN_EMERG "%s: Rx buffers exhausted\n", dev->name); + +rx_out: + return total_rx_packets; +} + +static bool +rtl8127_linkchg_interrupt(struct rtl8127_private *tp, u32 status) +{ + switch (tp->HwCurrIsrVer) { + case 2: + case 3: + return status & ISRIMR_V2_LINKCHG; + case 4: + return status & ISRIMR_V4_LINKCHG; + case 5: + return status & ISRIMR_V5_LINKCHG; + case 6: + return status & ISRIMR_V6_LINKCHG; + default: + return status & LinkChg; + } +} + +static u32 +rtl8127_get_linkchg_message_id(struct rtl8127_private *tp) +{ + switch (tp->HwCurrIsrVer) { + case 4: + case 6: + return 29; + case 5: + return 18; + default: + return 21; + } +} + +/* + *The interrupt handler does all of the Rx thread work and cleans up after + *the Tx thread. + */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) +static irqreturn_t rtl8127_interrupt(int irq, void *dev_instance, struct pt_regs *regs) +#else +static irqreturn_t rtl8127_interrupt(int irq, void *dev_instance) +#endif +{ + struct r8127_napi *r8127napi = dev_instance; + struct rtl8127_private *tp = r8127napi->priv; + struct net_device *dev = tp->dev; + u32 status; + int handled = 0; + + do { + status = RTL_R32(tp, tp->isr_reg[0]); + + if (!(tp->features & (RTL_FEATURE_MSI | RTL_FEATURE_MSIX))) { + /* hotplug/major error/no more work/shared irq */ + if (!status) + break; + + if (status == 0xFFFFFFFF) + break; + + if (!(status & (tp->intr_mask | tp->timer_intr_mask))) + break; + } + + handled = 1; + +#if defined(RTL_USE_NEW_INTR_API) + if (!tp->irq_tbl[0].requested) + break; +#endif + rtl8127_disable_hw_interrupt(tp); + + RTL_W32(tp, tp->isr_reg[0], status&~RxFIFOOver); + + if (rtl8127_linkchg_interrupt(tp, status)) + rtl8127_schedule_linkchg_work(tp); + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) { + if (HW_DASH_SUPPORT_TYPE_3(tp)) { + u8 DashIntType2Status; + + if (status & ISRIMR_DASH_INTR_CMAC_RESET) + tp->CmacResetIntr = TRUE; + + DashIntType2Status = RTL_CMAC_R8(tp, CMAC_IBISR0); + if (DashIntType2Status & ISRIMR_DASH_TYPE2_ROK) + tp->RcvFwDashOkEvt = TRUE; + if (DashIntType2Status & ISRIMR_DASH_TYPE2_TOK) + tp->SendFwHostOkEvt = TRUE; + if (DashIntType2Status & ISRIMR_DASH_TYPE2_RX_DISABLE_IDLE) + tp->DashFwDisableRx = TRUE; + + RTL_CMAC_W8(tp, CMAC_IBISR0, DashIntType2Status); + } + } +#endif + +#ifdef CONFIG_R8127_NAPI + if (status & tp->intr_mask || tp->keep_intr_cnt-- > 0) { + if (status & tp->intr_mask) + tp->keep_intr_cnt = RTK_KEEP_INTERRUPT_COUNT; + + if (likely(RTL_NETIF_RX_SCHEDULE_PREP(dev, &tp->r8127napi[0].napi))) + __RTL_NETIF_RX_SCHEDULE(dev, &tp->r8127napi[0].napi); + else if (netif_msg_intr(tp)) + printk(KERN_INFO "%s: interrupt %04x in poll\n", + dev->name, status); + } else { + tp->keep_intr_cnt = RTK_KEEP_INTERRUPT_COUNT; + rtl8127_switch_to_hw_interrupt(tp); + } +#else + if (status & tp->intr_mask || tp->keep_intr_cnt-- > 0) { + u32 budget = ~(u32)0; + int i; + + if (status & tp->intr_mask) + tp->keep_intr_cnt = RTK_KEEP_INTERRUPT_COUNT; + + for (i = 0; i < tp->num_tx_rings; i++) + rtl8127_tx_interrupt(&tp->tx_ring[i], ~(u32)0); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) + rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[0], &budget); +#else + rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[0], budget); +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) + +#ifdef ENABLE_DASH_SUPPORT + if (tp->DASH) { + struct net_device *dev = tp->dev; + + HandleDashInterrupt(dev); + } +#endif + + rtl8127_switch_to_timer_interrupt(tp); + } else { + tp->keep_intr_cnt = RTK_KEEP_INTERRUPT_COUNT; + rtl8127_switch_to_hw_interrupt(tp); + } +#endif + } while (false); + + return IRQ_RETVAL(handled); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) +static irqreturn_t rtl8127_interrupt_msix(int irq, void *dev_instance, struct pt_regs *regs) +#else +static irqreturn_t rtl8127_interrupt_msix(int irq, void *dev_instance) +#endif +{ + struct r8127_napi *r8127napi = dev_instance; + struct rtl8127_private *tp = r8127napi->priv; + struct net_device *dev = tp->dev; + int message_id = r8127napi->index; +#ifndef CONFIG_R8127_NAPI + u32 budget = ~(u32)0; +#endif + + do { +#if defined(RTL_USE_NEW_INTR_API) + if (!tp->irq_tbl[message_id].requested) + break; +#endif + //link change + if (message_id == rtl8127_get_linkchg_message_id(tp)) { + rtl8127_disable_hw_interrupt_v2(tp, message_id); + rtl8127_clear_hw_isr_v2(tp, message_id); + rtl8127_schedule_linkchg_work(tp); + break; + } + +#ifdef CONFIG_R8127_NAPI + if (likely(RTL_NETIF_RX_SCHEDULE_PREP(dev, &r8127napi->napi))) { + rtl8127_disable_hw_interrupt_v2(tp, message_id); + __RTL_NETIF_RX_SCHEDULE(dev, &r8127napi->napi); + } else if (netif_msg_intr(tp)) + printk(KERN_INFO "%s: interrupt message id %d in poll_msix\n", + dev->name, message_id); + rtl8127_clear_hw_isr_v2(tp, message_id); +#else + rtl8127_disable_hw_interrupt_v2(tp, message_id); + + rtl8127_clear_hw_isr_v2(tp, message_id); + + rtl8127_tx_interrupt_with_vector(tp, message_id, ~(u32)0); + + if (message_id < tp->num_rx_rings) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) + rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[message_id], &budget); +#else + rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[message_id], budget); +#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) + } + + rtl8127_enable_hw_interrupt_v2(tp, message_id); +#endif + + } while (false); + + return IRQ_HANDLED; +} + +static void rtl8127_down(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + //rtl8127_delete_esd_timer(dev, &tp->esd_timer); + + //rtl8127_delete_link_timer(dev, &tp->link_timer); + + netif_carrier_off(dev); + + netif_tx_disable(dev); + + _rtl8127_wait_for_quiescence(dev); + + rtl8127_hw_reset(dev); + + rtl8127_tx_clear(tp); + + rtl8127_rx_clear(tp); +} + +static int rtl8127_resource_freed(struct rtl8127_private *tp) +{ + int i; + + for (i = 0; i < tp->num_tx_rings; i++) + if (tp->tx_ring[i].TxDescArray) + return 0; + + for (i = 0; i < tp->num_rx_rings; i++) + if (tp->rx_ring[i].RxDescArray) + return 0; + + return 1; +} + +int rtl8127_close(struct net_device *dev) +{ + struct rtl8127_private *tp = netdev_priv(dev); + + if (!rtl8127_resource_freed(tp)) { + set_bit(R8127_FLAG_DOWN, tp->task_flags); + + rtl8127_down(dev); + + pci_clear_master(tp->pci_dev); + +#ifdef ENABLE_PTP_SUPPORT + rtl8127_ptp_stop(tp); +#endif + rtl8127_hw_d3_para(dev); + + rtl8127_powerdown_pll(dev, 0); + + rtl8127_free_irq(tp); + + rtl8127_free_alloc_resources(tp); + } else { + rtl8127_hw_d3_para(dev); + + rtl8127_powerdown_pll(dev, 0); + } + + return 0; +} + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,11) +static void rtl8127_shutdown(struct pci_dev *pdev) +{ + struct net_device *dev = pci_get_drvdata(pdev); + struct rtl8127_private *tp = netdev_priv(dev); + + rtnl_lock(); + + if (HW_DASH_SUPPORT_DASH(tp)) + rtl8127_driver_stop(tp); + + rtl8127_disable_pci_offset_180(tp); + + if (s5_keep_curr_mac == 0 && tp->random_mac == 0) + rtl8127_rar_set(tp, tp->org_mac_addr); + + if (s5wol == 0) + tp->wol_enabled = WOL_DISABLED; + + rtl8127_close(dev); + rtl8127_disable_msi(pdev, tp); + + rtnl_unlock(); + + if (system_state == SYSTEM_POWER_OFF) { + pci_clear_master(tp->pci_dev); + pci_wake_from_d3(pdev, tp->wol_enabled); + pci_set_power_state(pdev, PCI_D3hot); + } +} +#endif + +#ifdef CONFIG_PM + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,11) +static int +rtl8127_suspend(struct pci_dev *pdev, u32 state) +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29) +static int +rtl8127_suspend(struct device *device) +#else +static int +rtl8127_suspend(struct pci_dev *pdev, pm_message_t state) +#endif +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29) + struct pci_dev *pdev = to_pci_dev(device); + struct net_device *dev = pci_get_drvdata(pdev); +#else + struct net_device *dev = pci_get_drvdata(pdev); +#endif + struct rtl8127_private *tp = netdev_priv(dev); +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,10) + u32 pci_pm_state = pci_choose_state(pdev, state); +#endif + rtnl_lock(); + + if (!netif_running(dev)) + goto out; + + set_bit(R8127_FLAG_DOWN, tp->task_flags); + + netif_carrier_off(dev); + + netif_tx_disable(dev); + + netif_device_detach(dev); + +#ifdef ENABLE_PTP_SUPPORT + rtl8127_ptp_suspend(tp); +#endif + rtl8127_hw_reset(dev); + + pci_clear_master(pdev); + + rtl8127_hw_d3_para(dev); + + rtl8127_powerdown_pll(dev, 1); +out: + if (HW_DASH_SUPPORT_DASH(tp)) + rtl8127_driver_stop(tp); + + rtnl_unlock(); + + pci_disable_device(pdev); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,10) + pci_save_state(pdev, &pci_pm_state); +#else + pci_save_state(pdev); +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29) + pci_enable_wake(pdev, pci_choose_state(pdev, state), tp->wol_enabled); +#endif + + pci_prepare_to_sleep(pdev); + + return 0; +} + +static int +rtl8127_hw_d3_not_power_off(struct net_device *dev) +{ + return rtl8127_check_hw_phy_mcu_code_ver(dev); +} + +static int rtl8127_wait_phy_nway_complete_sleep(struct rtl8127_private *tp) +{ + int i, val; + + for (i = 0; i < 30; i++) { + val = rtl8127_mdio_read(tp, MII_BMSR) & BMSR_ANEGCOMPLETE; + if (val) + return 0; + + fsleep(100000); + } + + return -1; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29) +static int +rtl8127_resume(struct pci_dev *pdev) +#else +static int +rtl8127_resume(struct device *device) +#endif +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29) + struct pci_dev *pdev = to_pci_dev(device); + struct net_device *dev = pci_get_drvdata(pdev); +#else + struct net_device *dev = pci_get_drvdata(pdev); +#endif + struct rtl8127_private *tp = netdev_priv(dev); +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,10) + u32 pci_pm_state = PCI_D0; +#endif + u32 err; + + rtnl_lock(); + + err = pci_enable_device(pdev); + if (err) { + dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); + goto out_unlock; + } +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,10) + pci_restore_state(pdev, &pci_pm_state); +#else + pci_restore_state(pdev); +#endif + pci_enable_wake(pdev, PCI_D0, 0); + + /* restore last modified mac address */ + rtl8127_rar_set(tp, dev->dev_addr); + + tp->resume_not_chg_speed = 0; + if (tp->check_keep_link_speed && + //tp->link_ok(dev) && + rtl8127_hw_d3_not_power_off(dev) && + rtl8127_wait_phy_nway_complete_sleep(tp) == 0) + tp->resume_not_chg_speed = 1; + + if (!netif_running(dev)) + goto out_unlock; + + pci_set_master(pdev); + + rtl8127_exit_oob(dev); + + rtl8127_up(dev); + + clear_bit(R8127_FLAG_DOWN, tp->task_flags); + + rtl8127_schedule_reset_work(tp); + + rtl8127_schedule_esd_work(tp); + + //mod_timer(&tp->esd_timer, jiffies + RTL8127_ESD_TIMEOUT); + //mod_timer(&tp->link_timer, jiffies + RTL8127_LINK_TIMEOUT); +out_unlock: + netif_device_attach(dev); + + rtnl_unlock(); + + return err; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29) + +static struct dev_pm_ops rtl8127_pm_ops = { + .suspend = rtl8127_suspend, + .resume = rtl8127_resume, + .freeze = rtl8127_suspend, + .thaw = rtl8127_resume, + .poweroff = rtl8127_suspend, + .restore = rtl8127_resume, +}; + +#define RTL8127_PM_OPS (&rtl8127_pm_ops) + +#endif + +#else /* !CONFIG_PM */ + +#define RTL8127_PM_OPS NULL + +#endif /* CONFIG_PM */ + +static struct pci_driver rtl8127_pci_driver = { + .name = MODULENAME, + .id_table = rtl8127_pci_tbl, + .probe = rtl8127_init_one, + .remove = __devexit_p(rtl8127_remove_one), +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,11) + .shutdown = rtl8127_shutdown, +#endif +#ifdef CONFIG_PM +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29) + .suspend = rtl8127_suspend, + .resume = rtl8127_resume, +#else + .driver.pm = RTL8127_PM_OPS, +#endif +#endif +}; + +static int __init +rtl8127_init_module(void) +{ + int ret = 0; +#ifdef ENABLE_R8127_PROCFS + rtl8127_proc_module_init(); +#endif + +#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) + + ret = pci_register_driver(&rtl8127_pci_driver); +#else + ret = pci_module_init(&rtl8127_pci_driver); +#endif + + return ret; +} + +static void __exit +rtl8127_cleanup_module(void) +{ + pci_unregister_driver(&rtl8127_pci_driver); + +#ifdef ENABLE_R8127_PROCFS + if (rtl8127_proc) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) + remove_proc_subtree(MODULENAME, init_net.proc_net); +#else +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,32) + remove_proc_entry(MODULENAME, init_net.proc_net); +#else + remove_proc_entry(MODULENAME, proc_net); +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,32) +#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) + rtl8127_proc = NULL; + } +#endif +} + +module_init(rtl8127_init_module); +module_exit(rtl8127_cleanup_module); diff --git a/drivers/net/ethernet/realtek/r8127/r8127_realwow.h b/drivers/net/ethernet/realtek/r8127/r8127_realwow.h new file mode 100755 index 0000000000000..a869b6c532b33 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/r8127_realwow.h @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#ifndef _LINUX_R8127_REALWOW_H +#define _LINUX_R8127_REALWOW_H + +#define SIOCDEVPRIVATE_RTLREALWOW SIOCDEVPRIVATE+3 + +#define MAX_RealWoW_KCP_SIZE (100) +#define MAX_RealWoW_Payload (64) + +#define KA_TX_PACKET_SIZE (100) +#define KA_WAKEUP_PATTERN_SIZE (120) + +//HwSuppKeepAliveOffloadVer +#define HW_SUPPORT_KCP_OFFLOAD(_M) ((_M)->HwSuppKCPOffloadVer > 0) + +enum rtl_realwow_cmd { + + RTL_REALWOW_SET_KCP_DISABLE=0, + RTL_REALWOW_SET_KCP_INFO, + RTL_REALWOW_SET_KCP_CONTENT, + + RTL_REALWOW_SET_KCP_ACKPKTINFO, + RTL_REALWOW_SET_KCP_WPINFO, + RTL_REALWOW_SET_KCPDHCP_TIMEOUT, + + RTLT_REALWOW_COMMAND_INVALID +}; + +struct rtl_realwow_ioctl_struct { + __u32 cmd; + __u32 offset; + __u32 len; + union { + __u32 data; + void *data_buffer; + }; +}; + +typedef struct _MP_KCPInfo { + u8 DIPv4[4]; + u8 MacID[6]; + u16 UdpPort[2]; + u8 PKTLEN[2]; + + u16 ackLostCnt; + u8 KCP_WakePattern[MAX_RealWoW_Payload]; + u8 KCP_AckPacket[MAX_RealWoW_Payload]; + u32 KCP_interval; + u8 KCP_WakePattern_Len; + u8 KCP_AckPacket_Len; + u8 KCP_TxPacket[2][KA_TX_PACKET_SIZE]; +} MP_KCP_INFO, *PMP_KCP_INFO; + +typedef struct _KCPInfo { + u32 nId; // = id + u8 DIPv4[4]; + u8 MacID[6]; + u16 UdpPort; + u16 PKTLEN; +} KCPInfo, *PKCPInfo; + +typedef struct _KCPContent { + u32 id; // = id + u32 mSec; // = msec + u32 size; // =size + u8 bPacket[MAX_RealWoW_KCP_SIZE]; // put packet here +} KCPContent, *PKCPContent; + +typedef struct _RealWoWAckPktInfo { + u16 ackLostCnt; + u16 patterntSize; + u8 pattern[MAX_RealWoW_Payload]; +} RealWoWAckPktInfo,*PRealWoWAckPktInfo; + +typedef struct _RealWoWWPInfo { + u16 patterntSize; + u8 pattern[MAX_RealWoW_Payload]; +} RealWoWWPInfo,*PRealWoWWPInfo; + +int rtl8127_realwow_ioctl(struct net_device *dev, struct ifreq *ifr); +void rtl8127_realwow_hw_init(struct net_device *dev); +void rtl8127_get_realwow_hw_version(struct net_device *dev); +void rtl8127_set_realwow_d3_para(struct net_device *dev); + +#endif /* _LINUX_R8127_REALWOW_H */ diff --git a/drivers/net/ethernet/realtek/r8127/r8127_rss.h b/drivers/net/ethernet/realtek/r8127/r8127_rss.h new file mode 100755 index 0000000000000..8e92bb830c969 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/r8127_rss.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#ifndef _LINUX_R8127_RSS_H +#define _LINUX_R8127_RSS_H + +#include +#include + +#define RTL8127_RSS_KEY_SIZE 40 /* size of RSS Hash Key in bytes */ +#define RTL8127_MAX_INDIRECTION_TABLE_ENTRIES 128 + +enum rtl8127_rss_flag { + RTL_8125_RSS_FLAG_HASH_UDP_IPV4 = (1 << 0), + RTL_8125_RSS_FLAG_HASH_UDP_IPV6 = (1 << 1), +}; + +struct rtl8127_private; +struct RxDesc; + +int rtl8127_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd, + u32 *rule_locs); +int rtl8127_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd); +u32 rtl8127_get_rxfh_key_size(struct net_device *netdev); +u32 rtl8127_rss_indir_size(struct net_device *netdev); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,8,0) +int rtl8127_get_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh); +int rtl8127_set_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack); +#else +int rtl8127_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, + u8 *hfunc); +int rtl8127_set_rxfh(struct net_device *netdev, const u32 *indir, + const u8 *key, const u8 hfunc); +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6,8,0) */ +void rtl8127_rx_hash(struct rtl8127_private *tp, + struct RxDesc *desc, + struct sk_buff *skb); +void _rtl8127_config_rss(struct rtl8127_private *tp); +void rtl8127_config_rss(struct rtl8127_private *tp); +void rtl8127_init_rss(struct rtl8127_private *tp); +u32 rtl8127_rss_indir_tbl_entries(struct rtl8127_private *tp); +void rtl8127_disable_rss(struct rtl8127_private *tp); + +#endif /* _LINUX_R8127_RSS_H */ diff --git a/drivers/net/ethernet/realtek/r8127/rtl_eeprom.c b/drivers/net/ethernet/realtek/r8127/rtl_eeprom.c new file mode 100755 index 0000000000000..c95bbf0d19537 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/rtl_eeprom.c @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#include +#include +#include +#include +#include +#include + +#include + +#include "r8127.h" +#include "rtl_eeprom.h" + +//------------------------------------------------------------------- +//rtl8127_eeprom_type(): +// tell the eeprom type +//return value: +// 0: the eeprom type is 93C46 +// 1: the eeprom type is 93C56 or 93C66 +//------------------------------------------------------------------- +void rtl8127_eeprom_type(struct rtl8127_private *tp) +{ + u16 magic = 0; + + if (tp->mcfg == CFG_METHOD_DEFAULT) + goto out_no_eeprom; + + if(RTL_R8(tp, 0xD2)&0x04) { + //not support + //tp->eeprom_type = EEPROM_TWSI; + //tp->eeprom_len = 256; + goto out_no_eeprom; + } else if(RTL_R32(tp, RxConfig) & RxCfg_9356SEL) { + tp->eeprom_type = EEPROM_TYPE_93C56; + tp->eeprom_len = 256; + } else { + tp->eeprom_type = EEPROM_TYPE_93C46; + tp->eeprom_len = 128; + } + + magic = rtl8127_eeprom_read_sc(tp, 0); + +out_no_eeprom: + if ((magic != 0x8129) && (magic != 0x8128)) { + tp->eeprom_type = EEPROM_TYPE_NONE; + tp->eeprom_len = 0; + } +} + +void rtl8127_eeprom_cleanup(struct rtl8127_private *tp) +{ + u8 x; + + x = RTL_R8(tp, Cfg9346); + x &= ~(Cfg9346_EEDI | Cfg9346_EECS); + + RTL_W8(tp, Cfg9346, x); + + rtl8127_raise_clock(tp, &x); + rtl8127_lower_clock(tp, &x); +} + +static int rtl8127_eeprom_cmd_done(struct rtl8127_private *tp) +{ + u8 x; + int i; + + rtl8127_stand_by(tp); + + for (i = 0; i < 50000; i++) { + x = RTL_R8(tp, Cfg9346); + + if (x & Cfg9346_EEDO) { + fsleep(RTL_CLOCK_RATE * 2 * 3); + return 0; + } + fsleep(1); + } + + return -1; +} + +//------------------------------------------------------------------- +//rtl8127_eeprom_read_sc(): +// read one word from eeprom +//------------------------------------------------------------------- +u16 rtl8127_eeprom_read_sc(struct rtl8127_private *tp, u16 reg) +{ + int addr_sz = 6; + u8 x; + u16 data; + + if(tp->eeprom_type == EEPROM_TYPE_NONE) + return -1; + + if (tp->eeprom_type==EEPROM_TYPE_93C46) + addr_sz = 6; + else if (tp->eeprom_type==EEPROM_TYPE_93C56) + addr_sz = 8; + + x = Cfg9346_EEM1 | Cfg9346_EECS; + RTL_W8(tp, Cfg9346, x); + + rtl8127_shift_out_bits(tp, RTL_EEPROM_READ_OPCODE, 3); + rtl8127_shift_out_bits(tp, reg, addr_sz); + + data = rtl8127_shift_in_bits(tp); + + rtl8127_eeprom_cleanup(tp); + + RTL_W8(tp, Cfg9346, 0); + + return data; +} + +//------------------------------------------------------------------- +//rtl8127_eeprom_write_sc(): +// write one word to a specific address in the eeprom +//------------------------------------------------------------------- +void rtl8127_eeprom_write_sc(struct rtl8127_private *tp, u16 reg, u16 data) +{ + u8 x; + int addr_sz = 6; + int w_dummy_addr = 4; + + if(tp->eeprom_type == EEPROM_TYPE_NONE) + return; + + if (tp->eeprom_type==EEPROM_TYPE_93C46) { + addr_sz = 6; + w_dummy_addr = 4; + } else if (tp->eeprom_type==EEPROM_TYPE_93C56) { + addr_sz = 8; + w_dummy_addr = 6; + } + + x = Cfg9346_EEM1 | Cfg9346_EECS; + RTL_W8(tp, Cfg9346, x); + + rtl8127_shift_out_bits(tp, RTL_EEPROM_EWEN_OPCODE, 5); + rtl8127_shift_out_bits(tp, reg, w_dummy_addr); + rtl8127_stand_by(tp); + + rtl8127_shift_out_bits(tp, RTL_EEPROM_ERASE_OPCODE, 3); + rtl8127_shift_out_bits(tp, reg, addr_sz); + if (rtl8127_eeprom_cmd_done(tp) < 0) + return; + rtl8127_stand_by(tp); + + rtl8127_shift_out_bits(tp, RTL_EEPROM_WRITE_OPCODE, 3); + rtl8127_shift_out_bits(tp, reg, addr_sz); + rtl8127_shift_out_bits(tp, data, 16); + if (rtl8127_eeprom_cmd_done(tp) < 0) + return; + rtl8127_stand_by(tp); + + rtl8127_shift_out_bits(tp, RTL_EEPROM_EWDS_OPCODE, 5); + rtl8127_shift_out_bits(tp, reg, w_dummy_addr); + + rtl8127_eeprom_cleanup(tp); + RTL_W8(tp, Cfg9346, 0); +} + +void rtl8127_raise_clock(struct rtl8127_private *tp, u8 *x) +{ + *x = *x | Cfg9346_EESK; + RTL_W8(tp, Cfg9346, *x); + fsleep(RTL_CLOCK_RATE); +} + +void rtl8127_lower_clock(struct rtl8127_private *tp, u8 *x) +{ + + *x = *x & ~Cfg9346_EESK; + RTL_W8(tp, Cfg9346, *x); + fsleep(RTL_CLOCK_RATE); +} + +void rtl8127_shift_out_bits(struct rtl8127_private *tp, int data, int count) +{ + u8 x; + int mask; + + mask = 0x01 << (count - 1); + x = RTL_R8(tp, Cfg9346); + x &= ~(Cfg9346_EEDI | Cfg9346_EEDO); + + do { + if (data & mask) + x |= Cfg9346_EEDI; + else + x &= ~Cfg9346_EEDI; + + RTL_W8(tp, Cfg9346, x); + fsleep(RTL_CLOCK_RATE); + rtl8127_raise_clock(tp, &x); + rtl8127_lower_clock(tp, &x); + mask = mask >> 1; + } while(mask); + + x &= ~Cfg9346_EEDI; + RTL_W8(tp, Cfg9346, x); +} + +u16 rtl8127_shift_in_bits(struct rtl8127_private *tp) +{ + u8 x; + u16 d, i; + + x = RTL_R8(tp, Cfg9346); + x &= ~(Cfg9346_EEDI | Cfg9346_EEDO); + + d = 0; + + for (i = 0; i < 16; i++) { + d = d << 1; + rtl8127_raise_clock(tp, &x); + + x = RTL_R8(tp, Cfg9346); + x &= ~Cfg9346_EEDI; + + if (x & Cfg9346_EEDO) + d |= 1; + + rtl8127_lower_clock(tp, &x); + } + + return d; +} + +void rtl8127_stand_by(struct rtl8127_private *tp) +{ + u8 x; + + x = RTL_R8(tp, Cfg9346); + x &= ~(Cfg9346_EECS | Cfg9346_EESK); + RTL_W8(tp, Cfg9346, x); + fsleep(RTL_CLOCK_RATE); + + x |= Cfg9346_EECS; + RTL_W8(tp, Cfg9346, x); +} + +void rtl8127_set_eeprom_sel_low(struct rtl8127_private *tp) +{ + RTL_W8(tp, Cfg9346, Cfg9346_EEM1); + RTL_W8(tp, Cfg9346, Cfg9346_EEM1 | Cfg9346_EESK); + + fsleep(20); + + RTL_W8(tp, Cfg9346, Cfg9346_EEM1); +} diff --git a/drivers/net/ethernet/realtek/r8127/rtl_eeprom.h b/drivers/net/ethernet/realtek/r8127/rtl_eeprom.h new file mode 100755 index 0000000000000..e4d8c6c3f6765 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/rtl_eeprom.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#ifndef _LINUX_RTLEEPROM_H +#define _LINUX_RTLEEPROM_H + +//EEPROM opcodes +#define RTL_EEPROM_READ_OPCODE 06 +#define RTL_EEPROM_WRITE_OPCODE 05 +#define RTL_EEPROM_ERASE_OPCODE 07 +#define RTL_EEPROM_EWEN_OPCODE 19 +#define RTL_EEPROM_EWDS_OPCODE 16 + +#define RTL_CLOCK_RATE 3 + +void rtl8127_eeprom_type(struct rtl8127_private *tp); +void rtl8127_eeprom_cleanup(struct rtl8127_private *tp); +u16 rtl8127_eeprom_read_sc(struct rtl8127_private *tp, u16 reg); +void rtl8127_eeprom_write_sc(struct rtl8127_private *tp, u16 reg, u16 data); +void rtl8127_shift_out_bits(struct rtl8127_private *tp, int data, int count); +u16 rtl8127_shift_in_bits(struct rtl8127_private *tp); +void rtl8127_raise_clock(struct rtl8127_private *tp, u8 *x); +void rtl8127_lower_clock(struct rtl8127_private *tp, u8 *x); +void rtl8127_stand_by(struct rtl8127_private *tp); +void rtl8127_set_eeprom_sel_low(struct rtl8127_private *tp); + +#endif /* _LINUX_RTLEEPROM_H */ diff --git a/drivers/net/ethernet/realtek/r8127/rtltool.c b/drivers/net/ethernet/realtek/r8127/rtltool.c new file mode 100755 index 0000000000000..ba3a0cfd32420 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/rtltool.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "r8127.h" +#include "rtl_eeprom.h" +#include "rtltool.h" + +int rtl8127_tool_ioctl(struct rtl8127_private *tp, struct ifreq *ifr) +{ + struct rtltool_cmd my_cmd; + int ret; + + if (copy_from_user(&my_cmd, ifr->ifr_data, sizeof(my_cmd))) + return -EFAULT; + + ret = 0; + switch (my_cmd.cmd) { + case RTLTOOL_READ_MAC: + if ((my_cmd.offset + my_cmd.len) > pci_resource_len(tp->pci_dev, 2)) { + ret = -EINVAL; + break; + } + + if (my_cmd.len==1) + my_cmd.data = readb(tp->mmio_addr+my_cmd.offset); + else if (my_cmd.len==2) + my_cmd.data = readw(tp->mmio_addr+(my_cmd.offset&~1)); + else if (my_cmd.len==4) + my_cmd.data = readl(tp->mmio_addr+(my_cmd.offset&~3)); + else { + ret = -EOPNOTSUPP; + break; + } + + if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) { + ret = -EFAULT; + break; + } + break; + + case RTLTOOL_WRITE_MAC: + if ((my_cmd.offset + my_cmd.len) > pci_resource_len(tp->pci_dev, 2)) { + ret = -EINVAL; + break; + } + + if (my_cmd.len==1) + writeb(my_cmd.data, tp->mmio_addr+my_cmd.offset); + else if (my_cmd.len==2) + writew(my_cmd.data, tp->mmio_addr+(my_cmd.offset&~1)); + else if (my_cmd.len==4) + writel(my_cmd.data, tp->mmio_addr+(my_cmd.offset&~3)); + else { + ret = -EOPNOTSUPP; + break; + } + + break; + + case RTLTOOL_READ_PHY: + my_cmd.data = rtl8127_mdio_prot_read(tp, my_cmd.offset); + if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) { + ret = -EFAULT; + break; + } + + break; + + case RTLTOOL_WRITE_PHY: + rtl8127_mdio_prot_write(tp, my_cmd.offset, my_cmd.data); + break; + + case RTLTOOL_READ_EPHY: + my_cmd.data = rtl8127_ephy_read(tp, my_cmd.offset); + if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) { + ret = -EFAULT; + break; + } + + break; + + case RTLTOOL_WRITE_EPHY: + rtl8127_ephy_write(tp, my_cmd.offset, my_cmd.data); + break; + + case RTLTOOL_READ_ERI: + my_cmd.data = 0; + if (my_cmd.len==1 || my_cmd.len==2 || my_cmd.len==4) { + my_cmd.data = rtl8127_eri_read(tp, my_cmd.offset, my_cmd.len, ERIAR_ExGMAC); + } else { + ret = -EOPNOTSUPP; + break; + } + + if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) { + ret = -EFAULT; + break; + } + + break; + + case RTLTOOL_WRITE_ERI: + if (my_cmd.len==1 || my_cmd.len==2 || my_cmd.len==4) { + rtl8127_eri_write(tp, my_cmd.offset, my_cmd.len, my_cmd.data, ERIAR_ExGMAC); + } else { + ret = -EOPNOTSUPP; + break; + } + break; + + case RTLTOOL_READ_PCI: + my_cmd.data = 0; + if (my_cmd.len==1) + pci_read_config_byte(tp->pci_dev, my_cmd.offset, + (u8 *)&my_cmd.data); + else if (my_cmd.len==2) + pci_read_config_word(tp->pci_dev, my_cmd.offset, + (u16 *)&my_cmd.data); + else if (my_cmd.len==4) + pci_read_config_dword(tp->pci_dev, my_cmd.offset, + &my_cmd.data); + else { + ret = -EOPNOTSUPP; + break; + } + + if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) { + ret = -EFAULT; + break; + } + break; + + case RTLTOOL_WRITE_PCI: + if (my_cmd.len==1) + pci_write_config_byte(tp->pci_dev, my_cmd.offset, + my_cmd.data); + else if (my_cmd.len==2) + pci_write_config_word(tp->pci_dev, my_cmd.offset, + my_cmd.data); + else if (my_cmd.len==4) + pci_write_config_dword(tp->pci_dev, my_cmd.offset, + my_cmd.data); + else { + ret = -EOPNOTSUPP; + break; + } + + break; + + case RTLTOOL_READ_EEPROM: + my_cmd.data = rtl8127_eeprom_read_sc(tp, my_cmd.offset); + if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) { + ret = -EFAULT; + break; + } + + break; + + case RTLTOOL_WRITE_EEPROM: + rtl8127_eeprom_write_sc(tp, my_cmd.offset, my_cmd.data); + break; + + case RTL_READ_OOB_MAC: + rtl8127_oob_mutex_lock(tp); + my_cmd.data = rtl8127_ocp_read(tp, my_cmd.offset, 4); + rtl8127_oob_mutex_unlock(tp); + if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) { + ret = -EFAULT; + break; + } + break; + + case RTL_WRITE_OOB_MAC: + if (my_cmd.len == 0 || my_cmd.len > 4) + return -EOPNOTSUPP; + + rtl8127_oob_mutex_lock(tp); + rtl8127_ocp_write(tp, my_cmd.offset, my_cmd.len, my_cmd.data); + rtl8127_oob_mutex_unlock(tp); + break; + + case RTL_ENABLE_PCI_DIAG: + tp->rtk_enable_diag = 1; + + dprintk("enable rtk diag\n"); + break; + + case RTL_DISABLE_PCI_DIAG: + tp->rtk_enable_diag = 0; + + dprintk("disable rtk diag\n"); + break; + + case RTL_READ_MAC_OCP: + if (my_cmd.offset % 2) + return -EOPNOTSUPP; + + my_cmd.data = rtl8127_mac_ocp_read(tp, my_cmd.offset); + if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) { + ret = -EFAULT; + break; + } + break; + + case RTL_WRITE_MAC_OCP: + if ((my_cmd.offset % 2) || (my_cmd.len != 2)) + return -EOPNOTSUPP; + + rtl8127_mac_ocp_write(tp, my_cmd.offset, (u16)my_cmd.data); + break; + + case RTL_DIRECT_READ_PHY_OCP: + my_cmd.data = rtl8127_mdio_prot_direct_read_phy_ocp(tp, my_cmd.offset); + if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) { + ret = -EFAULT; + break; + } + + break; + + case RTL_DIRECT_WRITE_PHY_OCP: + rtl8127_mdio_prot_direct_write_phy_ocp(tp, my_cmd.offset, my_cmd.data); + break; + + default: + ret = -EOPNOTSUPP; + break; + } + + return ret; +} diff --git a/drivers/net/ethernet/realtek/r8127/rtltool.h b/drivers/net/ethernet/realtek/r8127/rtltool.h new file mode 100755 index 0000000000000..7b0ee22707be0 --- /dev/null +++ b/drivers/net/ethernet/realtek/r8127/rtltool.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* +################################################################################ +# +# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet +# controllers with PCI-Express interface. +# +# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 2 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, see . +# +# Author: +# Realtek NIC software team +# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan +# +################################################################################ +*/ + +/************************************************************************************ + * This product is covered by one or more of the following patents: + * US6,570,884, US6,115,776, and US6,327,625. + ***********************************************************************************/ + +#ifndef _LINUX_RTLTOOL_H +#define _LINUX_RTLTOOL_H + +#define SIOCRTLTOOL SIOCDEVPRIVATE+1 + +enum rtl_cmd { + RTLTOOL_READ_MAC=0, + RTLTOOL_WRITE_MAC, + RTLTOOL_READ_PHY, + RTLTOOL_WRITE_PHY, + RTLTOOL_READ_EPHY, + RTLTOOL_WRITE_EPHY, + RTLTOOL_READ_ERI, + RTLTOOL_WRITE_ERI, + RTLTOOL_READ_PCI, + RTLTOOL_WRITE_PCI, + RTLTOOL_READ_EEPROM, + RTLTOOL_WRITE_EEPROM, + + RTL_READ_OOB_MAC, + RTL_WRITE_OOB_MAC, + + RTL_ENABLE_PCI_DIAG, + RTL_DISABLE_PCI_DIAG, + + RTL_READ_MAC_OCP, + RTL_WRITE_MAC_OCP, + + RTL_DIRECT_READ_PHY_OCP, + RTL_DIRECT_WRITE_PHY_OCP, + + RTLTOOL_INVALID +}; + +struct rtltool_cmd { + __u32 cmd; + __u32 offset; + __u32 len; + __u32 data; +}; + +enum mode_access { + MODE_NONE=0, + MODE_READ, + MODE_WRITE +}; + +#ifdef __KERNEL__ +int rtl8127_tool_ioctl(struct rtl8127_private *tp, struct ifreq *ifr); +#endif + +#endif /* _LINUX_RTLTOOL_H */ diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index 4f27f2bbb391e..fbd0d7d903b50 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -241,10 +241,8 @@ static const struct pci_device_id rtl8169_pci_tbl[] = { { 0x0001, 0x8168, PCI_ANY_ID, 0x2410 }, { PCI_VDEVICE(REALTEK, 0x8125) }, { PCI_VDEVICE(REALTEK, 0x8126) }, - { PCI_VDEVICE(REALTEK, 0x8127) }, { PCI_VDEVICE(REALTEK, 0x3000) }, { PCI_VDEVICE(REALTEK, 0x5000) }, - { PCI_VDEVICE(REALTEK, 0x0e10) }, {} }; diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/main.c b/drivers/net/wireless/mediatek/mt76/mt7925/main.c index 2d358a96640c9..45810ff17c019 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/main.c @@ -245,6 +245,7 @@ int mt7925_init_mlo_caps(struct mt792x_phy *phy) { struct wiphy *wiphy = phy->mt76->hw->wiphy; static const u8 ext_capa_sta[] = { + [0] = WLAN_EXT_CAPA1_EXT_CHANNEL_SWITCHING, [2] = WLAN_EXT_CAPA3_MULTI_BSSID_SUPPORT, [7] = WLAN_EXT_CAPA8_OPMODE_NOTIF, }; @@ -438,6 +439,8 @@ mt7925_add_interface(struct ieee80211_hw *hw, struct ieee80211_vif *vif) if (phy->chip_cap & MT792x_CHIP_CAP_RSSI_NOTIFY_EVT_EN) vif->driver_flags |= IEEE80211_VIF_SUPPORTS_CQM_RSSI; + INIT_WORK(&mvif->csa_work, mt7925_csa_work); + timer_setup(&mvif->csa_timer, mt792x_csa_timer, 0); out: mt792x_mutex_release(dev); @@ -1749,6 +1752,10 @@ static int mt7925_add_chanctx(struct ieee80211_hw *hw, struct ieee80211_chanctx_conf *ctx) { + struct mt792x_dev *dev = mt792x_hw_dev(hw); + + dev->new_ctx = ctx; + return 0; } @@ -1756,6 +1763,11 @@ static void mt7925_remove_chanctx(struct ieee80211_hw *hw, struct ieee80211_chanctx_conf *ctx) { + struct mt792x_dev *dev = mt792x_hw_dev(hw); + + if (dev->new_ctx == ctx) + dev->new_ctx = NULL; + } static void @@ -2144,6 +2156,11 @@ static void mt7925_unassign_vif_chanctx(struct ieee80211_hw *hw, mctx->bss_conf = NULL; mconf->mt76.ctx = NULL; mutex_unlock(&dev->mt76.mutex); + + if (link_conf->csa_active) { + timer_delete_sync(&mvif->csa_timer); + cancel_work_sync(&mvif->csa_work); + } } static void mt7925_rfkill_poll(struct ieee80211_hw *hw) @@ -2158,6 +2175,121 @@ static void mt7925_rfkill_poll(struct ieee80211_hw *hw) wiphy_rfkill_set_hw_state(hw->wiphy, ret == 0); } +static int mt7925_switch_vif_chanctx(struct ieee80211_hw *hw, + struct ieee80211_vif_chanctx_switch *vifs, + int n_vifs, + enum ieee80211_chanctx_switch_mode mode) +{ + return mt7925_assign_vif_chanctx(hw, vifs->vif, vifs->link_conf, + vifs->new_ctx); +} + +void mt7925_csa_work(struct work_struct *work) +{ + struct mt792x_vif *mvif; + struct mt792x_dev *dev; + struct ieee80211_vif *vif; + struct ieee80211_bss_conf *link_conf; + struct mt792x_bss_conf *mconf; + u8 link_id, roc_rtype; + int ret = 0; + + mvif = (struct mt792x_vif *)container_of(work, struct mt792x_vif, + csa_work); + dev = mvif->phy->dev; + vif = container_of((void *)mvif, struct ieee80211_vif, drv_priv); + + if (ieee80211_vif_is_mld(vif)) + return; + + if (!dev->new_ctx) + return; + + link_id = 0; + mconf = &mvif->bss_conf; + link_conf = &vif->bss_conf; + roc_rtype = MT7925_ROC_REQ_JOIN; + + mt792x_mutex_acquire(dev); + ret = mt7925_set_roc(mvif->phy, mconf, dev->new_ctx->def.chan, + 4000, roc_rtype); + mt792x_mutex_release(dev); + if (!ret) { + mt792x_mutex_acquire(dev); + ret = mt7925_mcu_set_chctx(mvif->phy->mt76, &mconf->mt76, link_conf, + dev->new_ctx); + mt792x_mutex_release(dev); + + mt7925_abort_roc(mvif->phy, mconf); + } + + ieee80211_chswitch_done(vif, !ret, link_id); +} + +static int mt7925_pre_channel_switch(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_channel_switch *chsw) +{ + if (ieee80211_vif_is_mld(vif)) + return -EOPNOTSUPP; + + if (vif->type != NL80211_IFTYPE_STATION || !vif->cfg.assoc) + return -EOPNOTSUPP; + + if (!cfg80211_chandef_usable(hw->wiphy, &chsw->chandef, + IEEE80211_CHAN_DISABLED)) + return -EOPNOTSUPP; + + return 0; +} + +static void mt7925_channel_switch(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_channel_switch *chsw) +{ + struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + u16 beacon_interval; + + if (ieee80211_vif_is_mld(vif)) + return; + + beacon_interval = vif->bss_conf.beacon_int; + + mvif->csa_timer.expires = TU_TO_EXP_TIME(beacon_interval * chsw->count); + add_timer(&mvif->csa_timer); +} + +static void mt7925_abort_channel_switch(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_bss_conf *link_conf) +{ + struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + + timer_delete_sync(&mvif->csa_timer); + cancel_work_sync(&mvif->csa_work); +} + +static void mt7925_channel_switch_rx_beacon(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_channel_switch *chsw) +{ + struct mt792x_dev *dev = mt792x_hw_dev(hw); + struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + u16 beacon_interval; + + if (ieee80211_vif_is_mld(vif)) + return; + + beacon_interval = vif->bss_conf.beacon_int; + + if (cfg80211_chandef_identical(&chsw->chandef, + &dev->new_ctx->def) && + chsw->count) { + mod_timer(&mvif->csa_timer, + TU_TO_EXP_TIME(beacon_interval * chsw->count)); + } +} + const struct ieee80211_ops mt7925_ops = { .tx = mt792x_tx, .start = mt7925_start, @@ -2221,6 +2353,12 @@ const struct ieee80211_ops mt7925_ops = { .change_vif_links = mt7925_change_vif_links, .change_sta_links = mt7925_change_sta_links, .rfkill_poll = mt7925_rfkill_poll, + + .switch_vif_chanctx = mt7925_switch_vif_chanctx, + .pre_channel_switch = mt7925_pre_channel_switch, + .channel_switch = mt7925_channel_switch, + .abort_channel_switch = mt7925_abort_channel_switch, + .channel_switch_rx_beacon = mt7925_channel_switch_rx_beacon, }; EXPORT_SYMBOL_GPL(mt7925_ops); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c index cf0fdea45cf73..279ace8ab883d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c @@ -1330,6 +1330,8 @@ int mt7925_mcu_set_mlo_roc(struct mt792x_bss_conf *mconf, u16 sel_links, .roc[1].len = cpu_to_le16(sizeof(struct roc_acquire_tlv)) }; + struct wiphy *wiphy = mvif->phy->mt76->hw->wiphy; + if (!mconf || hweight16(vif->valid_links) < 2 || hweight16(sel_links) != 2) return -EPERM; @@ -1352,7 +1354,8 @@ int mt7925_mcu_set_mlo_roc(struct mt792x_bss_conf *mconf, u16 sel_links, is_AG_band |= links[i].chan->band == NL80211_BAND_2GHZ; } - if (vif->cfg.eml_cap & IEEE80211_EML_CAP_EMLSR_SUPP) + if (!(wiphy->iftype_ext_capab[0].mld_capa_and_ops & + IEEE80211_MLD_CAP_OP_MAX_SIMUL_LINKS)) type = is_AG_band ? MT7925_ROC_REQ_MLSR_AG : MT7925_ROC_REQ_MLSR_AA; else diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mt7925.h b/drivers/net/wireless/mediatek/mt76/mt7925/mt7925.h index 6b9bf1b890320..5030d7714bcf2 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mt7925.h +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mt7925.h @@ -298,6 +298,7 @@ int mt7925_mcu_uni_rx_ba(struct mt792x_dev *dev, void mt7925_mlo_pm_work(struct work_struct *work); void mt7925_scan_work(struct work_struct *work); void mt7925_roc_work(struct work_struct *work); +void mt7925_csa_work(struct work_struct *work); int mt7925_mcu_uni_bss_ps(struct mt792x_dev *dev, struct ieee80211_bss_conf *link_conf); void mt7925_coredump_work(struct work_struct *work); diff --git a/drivers/net/wireless/mediatek/mt76/mt792x_core.c b/drivers/net/wireless/mediatek/mt76/mt792x_core.c index f2ed16feb6c1b..1d8b8c00b7eef 100644 --- a/drivers/net/wireless/mediatek/mt76/mt792x_core.c +++ b/drivers/net/wireless/mediatek/mt76/mt792x_core.c @@ -691,9 +691,7 @@ int mt792x_init_wiphy(struct ieee80211_hw *hw) ieee80211_hw_set(hw, SUPPORTS_MULTI_BSSID); ieee80211_hw_set(hw, SUPPORTS_ONLY_HE_MULTI_BSSID); - if (is_mt7921(&dev->mt76)) { - ieee80211_hw_set(hw, CHANCTX_STA_CSA); - } + ieee80211_hw_set(hw, CHANCTX_STA_CSA); if (dev->pm.enable) ieee80211_hw_set(hw, CONNECTION_MONITOR); diff --git a/drivers/nvme/host/Makefile b/drivers/nvme/host/Makefile index 6414ec968f99a..2fdd327bf6a88 100644 --- a/drivers/nvme/host/Makefile +++ b/drivers/nvme/host/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 ccflags-y += -I$(src) - +ccflags-y += -DCONFIG_NVFS obj-$(CONFIG_NVME_CORE) += nvme-core.o obj-$(CONFIG_BLK_DEV_NVME) += nvme.o obj-$(CONFIG_NVME_FABRICS) += nvme-fabrics.o @@ -20,10 +20,11 @@ nvme-core-$(CONFIG_NVME_HWMON) += hwmon.o nvme-core-$(CONFIG_NVME_HOST_AUTH) += auth.o nvme-y += pci.o - +nvme-y += nvfs-dma.o nvme-fabrics-y += fabrics.o nvme-rdma-y += rdma.o +nvme-rdma-y += nvfs-rdma.o nvme-fc-y += fc.o diff --git a/drivers/nvme/host/nvfs-dma.c b/drivers/nvme/host/nvfs-dma.c new file mode 100644 index 0000000000000..8d821e7b08846 --- /dev/null +++ b/drivers/nvme/host/nvfs-dma.c @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ +#ifdef CONFIG_NVFS +#define NVFS_USE_DMA_ITER_API +#define MODULE_PREFIX nvme_v2 +#include "nvfs.h" + +struct nvfs_dma_rw_blk_iter_ops *nvfs_ops = NULL; + +atomic_t nvfs_shutdown = ATOMIC_INIT(1); + +DEFINE_PER_CPU(long, nvfs_n_ops); + +#define NVIDIA_FS_COMPAT_FT(ops) \ + (NVIDIA_FS_CHECK_FT_BLK_DMA_MAP_ITER_START(ops) && NVIDIA_FS_CHECK_FT_BLK_DMA_MAP_ITER_NEXT(ops)) + +// protected via nvfs_module_mutex +int REGISTER_FUNC(struct nvfs_dma_rw_blk_iter_ops *ops) +{ + if (NVIDIA_FS_COMPAT_FT(ops)) { + nvfs_ops = ops; + atomic_set(&nvfs_shutdown, 0); + return 0; + } else + return -EOPNOTSUPP; + +} +EXPORT_SYMBOL_GPL(REGISTER_FUNC); + +// protected via nvfs_module_mutex +void UNREGISTER_FUNC(void) +{ + (void) atomic_cmpxchg(&nvfs_shutdown, 0, 1); + do { + msleep(NVFS_HOLD_TIME_MS); + } while(nvfs_count_ops()); + nvfs_ops = NULL; +} +EXPORT_SYMBOL_GPL(UNREGISTER_FUNC); +#endif diff --git a/drivers/nvme/host/nvfs-dma.h b/drivers/nvme/host/nvfs-dma.h new file mode 100644 index 0000000000000..6dc6654adb2a8 --- /dev/null +++ b/drivers/nvme/host/nvfs-dma.h @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#ifndef NVFS_DMA_H +#define NVFS_DMA_H + +/* Forward declarations for functions from pci.c that we need */ +static blk_status_t nvme_pci_setup_data_prp(struct request *req, + struct blk_dma_iter *iter); +static blk_status_t nvme_pci_setup_data_sgl(struct request *req, + struct blk_dma_iter *iter); +static inline struct dma_pool *nvme_dma_pool(struct nvme_queue *nvmeq, + struct nvme_iod *iod); +static inline dma_addr_t nvme_pci_first_desc_dma_addr(struct nvme_command *cmd); + +static inline bool nvme_nvfs_unmap_sgls(struct request *req) +{ + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; + struct device *dma_dev = nvmeq->dev->dev; + unsigned int sqe_dma_len = le32_to_cpu(iod->cmd.common.dptr.sgl.length); + struct nvme_sgl_desc *sg_list = iod->descriptors[0]; + enum dma_data_direction dir = rq_dma_dir(req); + + /* + * nr_descriptors == 0 means dma_pool_alloc failed before any SGL + * entries were recorded; the first iter mapping is handled by + * nvme_nvfs_map_data() directly, so nothing to unmap here. + */ + if (iod->nr_descriptors) { + unsigned int nr_entries = sqe_dma_len / sizeof(*sg_list), i; + + for (i = 0; i < nr_entries; i++) { + nvfs_ops->nvfs_dma_unmap_page(dma_dev, + iod->nvfs_cookie, + le64_to_cpu(sg_list[i].addr), + le32_to_cpu(sg_list[i].length), + dir); + } + } + + return true; +} + +static inline bool nvme_nvfs_unmap_prps(struct request *req) +{ + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; + struct device *dma_dev = nvmeq->dev->dev; + enum dma_data_direction dma_dir = rq_dma_dir(req); + unsigned int i; + + /* Check if dma_vecs was allocated - if setup failed early, it might be NULL */ + if (!iod->dma_vecs) + return true; + + /* Unmap all DMA vectors - pass page pointer from dma_vecs */ + for (i = 0; i < iod->nr_dma_vecs; i++) { + nvfs_ops->nvfs_dma_unmap_page(dma_dev, + iod->nvfs_cookie, + iod->dma_vecs[i].addr, + iod->dma_vecs[i].len, + dma_dir); + } + + /* Free the dma_vecs mempool allocation */ + mempool_free(iod->dma_vecs, nvmeq->dev->dmavec_mempool); + iod->dma_vecs = NULL; + iod->nr_dma_vecs = 0; + + return true; +} + +static inline void nvme_nvfs_free_descriptors(struct request *req) +{ + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; + const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1; + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + dma_addr_t dma_addr = nvme_pci_first_desc_dma_addr(&iod->cmd); + int i; + + if (iod->nr_descriptors == 1) { + dma_pool_free(nvme_dma_pool(nvmeq, iod), iod->descriptors[0], + dma_addr); + return; + } + + for (i = 0; i < iod->nr_descriptors; i++) { + __le64 *prp_list = iod->descriptors[i]; + dma_addr_t next_dma_addr = le64_to_cpu(prp_list[last_prp]); + + dma_pool_free(nvmeq->descriptor_pools.large, prp_list, + dma_addr); + dma_addr = next_dma_addr; + } +} + +static inline bool nvme_nvfs_unmap_data(struct request *req) +{ + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + bool ret; + + /* Check if this was an NVFS I/O by checking the IOD_NVFS_IO flag */ + if (!(iod->flags & IOD_NVFS_IO)) + return false; + + /* Clear the NVFS flag */ + iod->flags &= ~IOD_NVFS_IO; + + /* Call appropriate unmap function based on command type */ + if (nvme_pci_cmd_use_sgl(&iod->cmd)) + ret = nvme_nvfs_unmap_sgls(req); + else + ret = nvme_nvfs_unmap_prps(req); + + if (iod->nr_descriptors) + nvme_nvfs_free_descriptors(req); + + nvfs_put_ops(); + return ret; +} + +static inline blk_status_t nvme_nvfs_map_data(struct request *req, + bool *is_nvfs_io) +{ + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; + struct nvme_dev *dev = nvmeq->dev; + struct device *dma_dev = nvmeq->dev->dev; + enum nvme_use_sgl use_sgl = nvme_pci_use_sgls(dev, req); + struct blk_dma_iter iter; + blk_status_t ret = BLK_STS_RESOURCE; + + *is_nvfs_io = false; + + /* Check integrity and try to get nvfs_ops */ + if (blk_integrity_rq(req) || !nvfs_get_ops()) { + return ret; + } + + /* Initialize total_len for this request */ + iod->total_len = 0; + + if (!nvfs_ops->nvfs_blk_rq_dma_map_iter_start(req, dma_dev, + &iod->dma_state, &iter, &iod->nvfs_cookie)) { + nvfs_put_ops(); + if (iter.status == BLK_STS_IOERR) { + /* GPU DMA error — do not fall through to CPU path */ + *is_nvfs_io = true; + ret = iter.status; + } + /* else: CPU page, let caller fall through to CPU path */ + return ret; + } + + /* NVFS can handle this request, set the flag */ + *is_nvfs_io = true; + iod->flags |= IOD_NVFS_IO; + + if (use_sgl == SGL_FORCED || + (use_sgl == SGL_SUPPORTED && + (sgl_threshold && nvme_pci_avg_seg_size(req) >= sgl_threshold))) + ret = nvme_pci_setup_data_sgl(req, &iter); + else + ret = nvme_pci_setup_data_prp(req, &iter); + + /* If setup failed, cleanup: unmap DMA, clear flag, release ops */ + if (ret != BLK_STS_OK) { + /* + * If setup failed before any mappings were tracked (dma_vecs is + * NULL for PRP, or nr_descriptors is 0 for SGL), the first page + * mapped by nvfs_blk_rq_dma_map_iter_start() won't be covered by + * nvme_nvfs_unmap_data(). Unmap it directly using iter. + */ + bool early_fail = nvme_pci_cmd_use_sgl(&iod->cmd) ? + !iod->nr_descriptors : !iod->dma_vecs; + if (early_fail) + nvfs_ops->nvfs_dma_unmap_page(dma_dev, iod->nvfs_cookie, + iter.addr, iter.len, rq_dma_dir(req)); + nvme_nvfs_unmap_data(req); + } + + return ret; +} + +#endif /* NVFS_DMA_H */ diff --git a/drivers/nvme/host/nvfs-rdma.c b/drivers/nvme/host/nvfs-rdma.c new file mode 100644 index 0000000000000..75a269143f3a8 --- /dev/null +++ b/drivers/nvme/host/nvfs-rdma.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#ifdef CONFIG_NVFS +#define MODULE_PREFIX nvme_rdma_v1 +#include "nvfs.h" + +struct nvfs_dma_rw_ops *nvfs_ops; + +atomic_t nvfs_shutdown = ATOMIC_INIT(1); + +DEFINE_PER_CPU(long, nvfs_n_ops); + +// must have for compatability +#define NVIDIA_FS_COMPAT_FT(ops) \ + (NVIDIA_FS_CHECK_FT_SGLIST_PREP(ops) && NVIDIA_FS_CHECK_FT_SGLIST_DMA(ops)) + +// protected via nvfs_module_mutex +int REGISTER_FUNC(struct nvfs_dma_rw_ops *ops) +{ + if (NVIDIA_FS_COMPAT_FT(ops)) { + nvfs_ops = ops; + atomic_set(&nvfs_shutdown, 0); + return 0; + } else + return -EOPNOTSUPP; + +} +EXPORT_SYMBOL_GPL(REGISTER_FUNC); + +// protected via nvfs_module_mutex +void UNREGISTER_FUNC(void) +{ + (void) atomic_cmpxchg(&nvfs_shutdown, 0, 1); + do { + msleep(NVFS_HOLD_TIME_MS); + } while(nvfs_count_ops()); + nvfs_ops = NULL; +} +EXPORT_SYMBOL_GPL(UNREGISTER_FUNC); +#endif diff --git a/drivers/nvme/host/nvfs-rdma.h b/drivers/nvme/host/nvfs-rdma.h new file mode 100644 index 0000000000000..f9051e2ab22b3 --- /dev/null +++ b/drivers/nvme/host/nvfs-rdma.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#ifndef NVFS_RDMA_H +#define NVFS_RDMA_H + +static bool nvme_rdma_nvfs_unmap_data(struct ib_device *ibdev, + struct request *rq) + +{ + struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq); + enum dma_data_direction dma_dir = rq_dma_dir(rq); + int count; + + if (!blk_integrity_rq(rq) && nvfs_ops != NULL) { + count = nvfs_ops->nvfs_dma_unmap_sg(ibdev->dma_device, req->data_sgl.sg_table.sgl, req->data_sgl.nents, + dma_dir); + if (count) { + nvfs_put_ops(); + sg_free_table_chained(&req->data_sgl.sg_table, NVME_INLINE_SG_CNT); + return true; + } + } + return false; +} + +static int nvme_rdma_nvfs_map_data(struct ib_device *ibdev, struct request *rq, bool *is_nvfs_io, int* count) +{ + struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq); + enum dma_data_direction dma_dir = rq_dma_dir(rq); + int ret = 0; + + *is_nvfs_io = false; + *count = 0; + if (!blk_integrity_rq(rq) && nvfs_get_ops()) { + + // associates bio pages to scatterlist + *count = nvfs_ops->nvfs_blk_rq_map_sg(rq->q, rq , req->data_sgl.sg_table.sgl); + if (!*count) { + nvfs_put_ops(); + return 0; // fall to cpu path + } + + *is_nvfs_io = true; + if (unlikely((*count == NVFS_IO_ERR))) { + nvfs_put_ops(); + pr_err("%s: failed to map sg_nents=:%d\n", __func__, req->data_sgl.nents); + return -EIO; + } + req->data_sgl.nents = *count; + + *count = nvfs_ops->nvfs_dma_map_sg_attrs(ibdev->dma_device, + req->data_sgl.sg_table.sgl, + req->data_sgl.nents, + dma_dir, + DMA_ATTR_NO_WARN); + + if (unlikely((*count == NVFS_IO_ERR))) { + nvfs_put_ops(); + return -EIO; + } + + if (unlikely(*count == NVFS_CPU_REQ)) { + nvfs_put_ops(); + return -EIO; + } + + return ret; + } + + // Fall to CPU path + return 0; +} + +#endif diff --git a/drivers/nvme/host/nvfs.h b/drivers/nvme/host/nvfs.h new file mode 100644 index 0000000000000..0101a88dcc6c7 --- /dev/null +++ b/drivers/nvme/host/nvfs.h @@ -0,0 +1,156 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ +#ifndef NVFS_H +#define NVFS_H + +#include +#include +#include +#include +#include +#include +#include + +/* Forward declarations */ +struct blk_dma_iter; +struct dma_iova_state; + +#define REGSTR2(x) x##_register_nvfs_dma_ops +#define REGSTR(x) REGSTR2(x) + +#define UNREGSTR2(x) x##_unregister_nvfs_dma_ops +#define UNREGSTR(x) UNREGSTR2(x) + +#define REGISTER_FUNC REGSTR(MODULE_PREFIX) +#define UNREGISTER_FUNC UNREGSTR(MODULE_PREFIX) + +#define NVFS_IO_ERR -1 +#define NVFS_CPU_REQ -2 + +#define NVFS_HOLD_TIME_MS 1000 + +#ifdef NVFS_USE_DMA_ITER_API +extern struct nvfs_dma_rw_blk_iter_ops *nvfs_ops; +#else +extern struct nvfs_dma_rw_ops *nvfs_ops; +#endif + +extern atomic_t nvfs_shutdown; + +DECLARE_PER_CPU(long, nvfs_n_ops); + +static inline long nvfs_count_ops(void) +{ + int i; + long sum = 0; + for_each_possible_cpu(i) + sum += per_cpu(nvfs_n_ops, i); + return sum; +} + +static inline bool nvfs_get_ops(void) +{ + if (nvfs_ops && !atomic_read(&nvfs_shutdown)) { + this_cpu_inc(nvfs_n_ops); + return true; + } + return false; +} + +static inline void nvfs_put_ops(void) +{ + this_cpu_dec(nvfs_n_ops); +} + + +struct nvfs_dma_rw_blk_iter_ops { + unsigned long long ft_bmap; // feature bitmap + + int (*nvfs_blk_rq_dma_map_iter_start) (struct request *req, + struct device *dma_dev, + struct dma_iova_state *state, + struct blk_dma_iter *iter, + void **cookie); + + int (*nvfs_blk_rq_dma_map_iter_next) (struct request *req, + struct device *dma_dev, + struct dma_iova_state *state, + struct blk_dma_iter *iter); + + int (*nvfs_dma_unmap_page) (struct device *device, + void* cookie, + dma_addr_t addr, + size_t size, + enum dma_data_direction dir); + + bool (*nvfs_is_gpu_page) (struct page *page); + + unsigned int (*nvfs_gpu_index) (struct page *page); + + unsigned int (*nvfs_device_priority) (struct device *dev, unsigned int gpu_index); + +}; + +struct nvfs_dma_rw_ops { + unsigned long long ft_bmap; // feature bitmap + + int (*nvfs_blk_rq_map_sg) (struct request_queue *q, + struct request *req, + struct scatterlist *sglist); + + int (*nvfs_dma_map_sg_attrs) (struct device *device, + struct scatterlist *sglist, + int nents, + enum dma_data_direction dma_dir, + unsigned long attrs); + + int (*nvfs_dma_unmap_sg) (struct device *device, + struct scatterlist *sglist, + int nents, + enum dma_data_direction dma_dir); + + bool (*nvfs_is_gpu_page) (struct page *page); + + unsigned int (*nvfs_gpu_index) (struct page *page); + + unsigned int (*nvfs_device_priority) (struct device *dev, unsigned int gpu_index); +}; + +// feature list for dma_ops, values indicate bit pos +enum ft_bits { + nvfs_ft_prep_sglist = 1ULL << 0, + nvfs_ft_map_sglist = 1ULL << 1, + nvfs_ft_is_gpu_page = 1ULL << 2, + nvfs_ft_device_priority = 1ULL << 3, + nvfs_ft_blk_dma_map_iter_start = 1ULL << 5, + nvfs_ft_blk_dma_map_iter_next = 1ULL << 6, +}; + +// check features for use in registration with vendor drivers +#define NVIDIA_FS_CHECK_FT_SGLIST_PREP(ops) ((ops)->ft_bmap & nvfs_ft_prep_sglist) +#define NVIDIA_FS_CHECK_FT_SGLIST_DMA(ops) ((ops)->ft_bmap & nvfs_ft_map_sglist) +#define NVIDIA_FS_CHECK_FT_GPU_PAGE(ops) ((ops)->ft_bmap & nvfs_ft_is_gpu_page) +#define NVIDIA_FS_CHECK_FT_DEVICE_PRIORITY(ops) ((ops)->ft_bmap & nvfs_ft_device_priority) +#define NVIDIA_FS_CHECK_FT_BLK_DMA_MAP_ITER_START(ops) ((ops)->ft_bmap & nvfs_ft_blk_dma_map_iter_start) +#define NVIDIA_FS_CHECK_FT_BLK_DMA_MAP_ITER_NEXT(ops) ((ops)->ft_bmap & nvfs_ft_blk_dma_map_iter_next) + +#ifdef NVFS_USE_DMA_ITER_API +int REGISTER_FUNC(struct nvfs_dma_rw_blk_iter_ops *ops); +#else +int REGISTER_FUNC(struct nvfs_dma_rw_ops *ops); +#endif + +void UNREGISTER_FUNC(void); + +#endif /* NVFS_H */ diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index db5fc9bf66272..2a56a97937c3c 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -30,6 +30,10 @@ #include "trace.h" #include "nvme.h" +#ifdef CONFIG_NVFS +#define NVFS_USE_DMA_ITER_API +#include "nvfs.h" +#endif #define SQ_SIZE(q) ((q)->q_depth << (q)->sqes) #define CQ_SIZE(q) ((q)->q_depth * sizeof(struct nvme_completion)) @@ -37,6 +41,15 @@ /* Optimisation for I/Os between 4k and 128k */ #define NVME_SMALL_POOL_SIZE 256 +#ifdef CONFIG_NVFS +/* GPU physical pages are minimum 64K. Worst-case SGL entries with misalignment: + * ceil(payload/64K) + 1. Safe payload for small pool = (entries - 1) * 64K, + * where entries = NVME_SMALL_POOL_SIZE / sizeof(struct nvme_sgl_desc). */ +#define NVFS_GPU_PAGE_SIZE (64UL * 1024) +#define NVFS_SMALL_POOL_PAYLOAD \ + ((NVME_SMALL_POOL_SIZE / sizeof(struct nvme_sgl_desc) - 1) * NVFS_GPU_PAGE_SIZE) +#endif + /* * Arbitrary upper bound. */ @@ -418,6 +431,11 @@ enum nvme_iod_flags { /* Metadata using non-coalesced MPTR */ IOD_SINGLE_META_SEGMENT = 1U << 7, + +#ifdef CONFIG_NVFS + /* NVFS GPU Direct Storage I/O */ + IOD_NVFS_IO = 1U << 31, +#endif }; struct nvme_dma_vec { @@ -431,7 +449,11 @@ struct nvme_dma_vec { struct nvme_iod { struct nvme_request req; struct nvme_command cmd; +#ifdef CONFIG_NVFS + u32 flags; +#else u8 flags; +#endif u8 nr_descriptors; size_t total_len; @@ -444,6 +466,9 @@ struct nvme_iod { size_t meta_total_len; struct dma_iova_state meta_dma_state; struct nvme_sgl_desc *meta_descriptor; +#ifdef CONFIG_NVFS + void *nvfs_cookie; +#endif }; static inline unsigned int nvme_dbbuf_size(struct nvme_dev *dev) @@ -924,6 +949,10 @@ static void nvme_unmap_metadata(struct request *req) iod->meta_descriptor, iod->meta_dma); } +#ifdef CONFIG_NVFS +#include "nvfs-dma.h" +#endif + static void nvme_unmap_data(struct request *req) { enum pci_p2pdma_map_type map = PCI_P2PDMA_MAP_NONE; @@ -932,6 +961,12 @@ static void nvme_unmap_data(struct request *req) struct device *dma_dev = nvmeq->dev->dev; unsigned int attrs = 0; +#ifdef CONFIG_NVFS + /* Check if this was an NVFS I/O and handle unmapping */ + if (nvme_nvfs_unmap_data(req)) + return; +#endif + if (iod->flags & IOD_SINGLE_SEGMENT) { static_assert(offsetof(union nvme_data_ptr, prp1) == offsetof(union nvme_data_ptr, sgl.addr)); @@ -991,6 +1026,20 @@ static bool nvme_pci_prp_iter_next(struct request *req, struct device *dma_dev, { if (iter->len) return true; +#ifdef CONFIG_NVFS + { + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + if (iod->flags & IOD_NVFS_IO) { + if (!nvfs_ops->nvfs_blk_rq_dma_map_iter_next(req, dma_dev, + &iod->dma_state, iter)) + return false; + iod->dma_vecs[iod->nr_dma_vecs].addr = iter->addr; + iod->dma_vecs[iod->nr_dma_vecs].len = iter->len; + iod->nr_dma_vecs++; + return true; + } + } +#endif if (!blk_rq_dma_map_iter_next(req, dma_dev, iter)) return false; return nvme_pci_prp_save_mapping(req, dma_dev, iter); @@ -1006,6 +1055,16 @@ static blk_status_t nvme_pci_setup_data_prp(struct request *req, unsigned int prp_len, i; __le64 *prp_list; +#ifdef CONFIG_NVFS + if (iod->flags & IOD_NVFS_IO) { + iod->dma_vecs = mempool_alloc(nvmeq->dev->dmavec_mempool, GFP_ATOMIC); + if (!iod->dma_vecs) + return BLK_STS_RESOURCE; + iod->dma_vecs[0].addr = iter->addr; + iod->dma_vecs[0].len = iter->len; + iod->nr_dma_vecs = 1; + } else +#endif if (!nvme_pci_prp_save_mapping(req, nvmeq->dev->dev, iter)) return iter->status; @@ -1104,6 +1163,11 @@ static blk_status_t nvme_pci_setup_data_prp(struct request *req, */ iod->cmd.common.dptr.prp1 = cpu_to_le64(prp1_dma); iod->cmd.common.dptr.prp2 = cpu_to_le64(prp2_dma); +#ifdef CONFIG_NVFS + /* For NVFS, don't call nvme_unmap_data - cleanup happens in nvme_nvfs_unmap_data */ + if (iod->flags & IOD_NVFS_IO) + return iter->status; +#endif if (unlikely(iter->status)) nvme_unmap_data(req); return iter->status; @@ -1144,12 +1208,34 @@ static blk_status_t nvme_pci_setup_data_sgl(struct request *req, /* set the transfer type as SGL */ iod->cmd.common.flags = NVME_CMD_SGL_METABUF; - if (entries == 1 || blk_rq_dma_map_coalesce(&iod->dma_state)) { - nvme_pci_sgl_set_data(&iod->cmd.common.dptr.sgl, iter); - iod->total_len += iter->len; - return BLK_STS_OK; +#ifdef CONFIG_NVFS + if (!(iod->flags & IOD_NVFS_IO)) +#endif + { + if (entries == 1 || blk_rq_dma_map_coalesce(&iod->dma_state)) { + nvme_pci_sgl_set_data(&iod->cmd.common.dptr.sgl, iter); + iod->total_len += iter->len; + return BLK_STS_OK; + } } +#ifdef CONFIG_NVFS + if (iod->flags & IOD_NVFS_IO) { + /* + * blk_rq_nr_phys_segments() reflects shadow buffer contiguity, + * not GPU physical segments. GPU pages are 64K minimum; worst-case + * entries with misalignment = ceil(payload/64K) + 1. + * Small pool (16 entries) is safe for payload < NVFS_SMALL_POOL_PAYLOAD. + * Large pool capacity = NVME_CTRL_PAGE_SIZE / sizeof(*sg_list) = 256. + */ + if (blk_rq_payload_bytes(req) < NVFS_SMALL_POOL_PAYLOAD) { + entries = NVME_SMALL_POOL_SIZE / sizeof(*sg_list); + iod->flags |= IOD_SMALL_DESCRIPTOR; + } else { + entries = NVME_CTRL_PAGE_SIZE / sizeof(*sg_list); + } + } else +#endif if (entries <= NVME_SMALL_POOL_SIZE / sizeof(*sg_list)) iod->flags |= IOD_SMALL_DESCRIPTOR; @@ -1166,9 +1252,21 @@ static blk_status_t nvme_pci_setup_data_sgl(struct request *req, } nvme_pci_sgl_set_data(&sg_list[mapped++], iter); iod->total_len += iter->len; - } while (blk_rq_dma_map_iter_next(req, nvmeq->dev->dev, iter)); + } while ( +#ifdef CONFIG_NVFS + (iod->flags & IOD_NVFS_IO) ? + (mapped < entries && + nvfs_ops->nvfs_blk_rq_dma_map_iter_next(req, nvmeq->dev->dev, + &iod->dma_state, iter)) : +#endif + blk_rq_dma_map_iter_next(req, nvmeq->dev->dev, iter)); nvme_pci_sgl_set_seg(&iod->cmd.common.dptr.sgl, sgl_dma, mapped); +#ifdef CONFIG_NVFS + /* For NVFS, don't call nvme_unmap_data - cleanup happens in nvme_nvfs_unmap_data */ + if (iod->flags & IOD_NVFS_IO) + return iter->status; +#endif if (unlikely(iter->status)) nvme_unmap_data(req); return iter->status; @@ -1222,6 +1320,12 @@ static blk_status_t nvme_map_data(struct request *req) struct blk_dma_iter iter; blk_status_t ret; +#ifdef CONFIG_NVFS + bool is_nvfs_io = false; + ret = nvme_nvfs_map_data(req, &is_nvfs_io); + if (is_nvfs_io) + return ret; +#endif /* * Try to skip the DMA iterator for single segment requests, as that * significantly improves performances for small I/O sizes. diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 57111139e84fa..53b4823d57c5f 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -27,6 +27,9 @@ #include "nvme.h" #include "fabrics.h" +#ifdef CONFIG_NVFS +#include "nvfs.h" +#endif #define NVME_RDMA_CM_TIMEOUT_MS 3000 /* 3 second */ @@ -1212,6 +1215,9 @@ static int nvme_rdma_inv_rkey(struct nvme_rdma_queue *queue, return ib_post_send(queue->qp, &wr, NULL); } +#ifdef CONFIG_NVFS +#include "nvfs-rdma.h" +#endif static void nvme_rdma_dma_unmap_req(struct ib_device *ibdev, struct request *rq) { struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq); @@ -1223,6 +1229,11 @@ static void nvme_rdma_dma_unmap_req(struct ib_device *ibdev, struct request *rq) NVME_INLINE_METADATA_SG_CNT); } +#ifdef CONFIG_NVFS + if (nvme_rdma_nvfs_unmap_data(ibdev, rq)) + return; +#endif + ib_dma_unmap_sg(ibdev, req->data_sgl.sg_table.sgl, req->data_sgl.nents, rq_dma_dir(rq)); sg_free_table_chained(&req->data_sgl.sg_table, NVME_INLINE_SG_CNT); @@ -1476,6 +1487,17 @@ static int nvme_rdma_dma_map_req(struct ib_device *ibdev, struct request *rq, if (ret) return -ENOMEM; +#ifdef CONFIG_NVFS + { + bool is_nvfs_io = false; + ret = nvme_rdma_nvfs_map_data(ibdev, rq, &is_nvfs_io, count); + if (is_nvfs_io) { + if (ret) + goto out_free_table; + return 0; + } + } +#endif req->data_sgl.nents = blk_rq_map_sg(rq, req->data_sgl.sg_table.sgl); *count = ib_dma_map_sg(ibdev, req->data_sgl.sg_table.sgl, diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index acc71a26733f9..255ebd948dfe1 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -398,6 +398,19 @@ static void nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd *cmd) static void nvmet_tcp_fatal_error(struct nvmet_tcp_queue *queue) { + /* + * Keep rcv_state at RECV_ERR even for the internal -ESHUTDOWN path. + * nvmet_tcp_handle_icreq() can return -ESHUTDOWN after the ICReq has + * already been consumed and queue teardown has started. + * + * If nvmet_tcp_data_ready() or nvmet_tcp_write_space() queues + * nvmet_tcp_io_work() again before nvmet_tcp_release_queue_work() + * cancels it, the queue must not keep that old receive state. + * Otherwise the next nvmet_tcp_io_work() run can reach + * nvmet_tcp_done_recv_pdu() and try to handle the same ICReq again. + * + * That is why queue->rcv_state needs to be updated before we return. + */ queue->rcv_state = NVMET_TCP_RECV_ERR; if (queue->nvme_sq.ctrl) nvmet_ctrl_fatal_error(queue->nvme_sq.ctrl); @@ -922,11 +935,24 @@ static int nvmet_tcp_handle_icreq(struct nvmet_tcp_queue *queue) iov.iov_len = sizeof(*icresp); ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); if (ret < 0) { + spin_lock_bh(&queue->state_lock); + if (queue->state == NVMET_TCP_Q_DISCONNECTING) { + spin_unlock_bh(&queue->state_lock); + return -ESHUTDOWN; + } queue->state = NVMET_TCP_Q_FAILED; + spin_unlock_bh(&queue->state_lock); return ret; /* queue removal will cleanup */ } + spin_lock_bh(&queue->state_lock); + if (queue->state == NVMET_TCP_Q_DISCONNECTING) { + spin_unlock_bh(&queue->state_lock); + /* Tell nvmet_tcp_socket_error() teardown is in progress. */ + return -ESHUTDOWN; + } queue->state = NVMET_TCP_Q_LIVE; + spin_unlock_bh(&queue->state_lock); nvmet_prepare_receive_pdu(queue); return 0; } diff --git a/drivers/pinctrl/mediatek/Kconfig b/drivers/pinctrl/mediatek/Kconfig index 4819617d93683..92f4f394b71e7 100644 --- a/drivers/pinctrl/mediatek/Kconfig +++ b/drivers/pinctrl/mediatek/Kconfig @@ -281,6 +281,18 @@ config PINCTRL_MT8189 In MTK platform, we support virtual gpio and use it to map specific eint which doesn't have real gpio pin. +config PINCTRL_MT8901 + bool "MediaTek MT8901 pin control" + depends on ACPI + depends on ARM64 || COMPILE_TEST + default ARM64 && ARCH_MEDIATEK + select PINCTRL_MTK_PARIS + help + Say yes here to support pin controller and gpio driver + on MediaTek MT8901 SoC. + In MTK platform, we support virtual gpio and use it to + map specific eint which doesn't have real gpio pin. + config PINCTRL_MT8192 bool "MediaTek MT8192 pin control" depends on OF diff --git a/drivers/pinctrl/mediatek/Makefile b/drivers/pinctrl/mediatek/Makefile index ae765bd999657..57c69b1e5c2d4 100644 --- a/drivers/pinctrl/mediatek/Makefile +++ b/drivers/pinctrl/mediatek/Makefile @@ -43,3 +43,4 @@ obj-$(CONFIG_PINCTRL_MT8196) += pinctrl-mt8196.o obj-$(CONFIG_PINCTRL_MT8365) += pinctrl-mt8365.o obj-$(CONFIG_PINCTRL_MT8516) += pinctrl-mt8516.o obj-$(CONFIG_PINCTRL_MT6397) += pinctrl-mt6397.o +obj-$(CONFIG_PINCTRL_MT8901) += pinctrl-mt8901.o diff --git a/drivers/pinctrl/mediatek/mtk-eint.c b/drivers/pinctrl/mediatek/mtk-eint.c index 2a3c04eedc5f3..3e6b121cf593a 100644 --- a/drivers/pinctrl/mediatek/mtk-eint.c +++ b/drivers/pinctrl/mediatek/mtk-eint.c @@ -71,6 +71,10 @@ const unsigned int debounce_time_mt6878[] = { }; EXPORT_SYMBOL_GPL(debounce_time_mt6878); +const unsigned int debounce_time_mt8901[] = { + 156, 313, 625, 1250, 20000, 40000, 80000, 160000, 320000, 640000, 0}; +EXPORT_SYMBOL_GPL(debounce_time_mt8901); + static void __iomem *mtk_eint_get_offset(struct mtk_eint *eint, unsigned int eint_num, unsigned int offset) diff --git a/drivers/pinctrl/mediatek/mtk-eint.h b/drivers/pinctrl/mediatek/mtk-eint.h index 3cdd6f6310cd0..1b185f660affa 100644 --- a/drivers/pinctrl/mediatek/mtk-eint.h +++ b/drivers/pinctrl/mediatek/mtk-eint.h @@ -53,6 +53,7 @@ extern const unsigned int debounce_time_mt2701[]; extern const unsigned int debounce_time_mt6765[]; extern const unsigned int debounce_time_mt6795[]; extern const unsigned int debounce_time_mt6878[]; +extern const unsigned int debounce_time_mt8901[]; struct mtk_eint; diff --git a/drivers/pinctrl/mediatek/pinctrl-mt8901.c b/drivers/pinctrl/mediatek/pinctrl-mt8901.c new file mode 100644 index 0000000000000..623cd0cd58f05 --- /dev/null +++ b/drivers/pinctrl/mediatek/pinctrl-mt8901.c @@ -0,0 +1,1460 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2025 MediaTek Inc. + * + */ + +#include +#include +#include "pinctrl-mtk-mt8901.h" +#include "pinctrl-paris.h" + +#define PIN_FIELD_BASE(s_pin, e_pin, i_base, s_addr, x_addrs, s_bit, x_bits) \ + PIN_FIELD_CALC(s_pin, e_pin, i_base, s_addr, x_addrs, s_bit, x_bits, 32, 0) +#define PINS_FIELD_BASE(s_pin, e_pin, i_base, s_addr, x_addrs, s_bit, x_bits) \ + PIN_FIELD_CALC(s_pin, e_pin, i_base, s_addr, x_addrs, s_bit, x_bits, 32, 1) + +static const struct mtk_pin_field_calc mt8901_pin_mode_range[] = { + PIN_FIELD(0, 181, 0x0300, 0x10, 0, 4), +}; + +static const struct mtk_pin_field_calc mt8901_pin_dir_range[] = { + PIN_FIELD(0, 181, 0x0000, 0x10, 0, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_di_range[] = { + PIN_FIELD(0, 181, 0x0200, 0x10, 0, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_do_range[] = { + PIN_FIELD(0, 181, 0x0100, 0x10, 0, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_smt_range[] = { + PIN_FIELD_BASE(0, 0, 8, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(1, 1, 8, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(2, 2, 8, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(3, 3, 8, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(4, 4, 8, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(5, 5, 8, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(6, 6, 8, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(7, 7, 8, 0x00c0, 0x10, 7, 1), + PIN_FIELD_BASE(8, 8, 2, 0x0120, 0x10, 13, 1), + PIN_FIELD_BASE(9, 9, 2, 0x0120, 0x10, 14, 1), + PIN_FIELD_BASE(10, 10, 2, 0x0120, 0x10, 15, 1), + PIN_FIELD_BASE(11, 11, 2, 0x0120, 0x10, 16, 1), + PIN_FIELD_BASE(12, 12, 1, 0x0140, 0x10, 17, 1), + PIN_FIELD_BASE(13, 13, 1, 0x0140, 0x10, 18, 1), + PIN_FIELD_BASE(14, 14, 1, 0x0140, 0x10, 14, 1), + PIN_FIELD_BASE(15, 15, 1, 0x0140, 0x10, 16, 1), + PIN_FIELD_BASE(16, 16, 1, 0x0140, 0x10, 19, 1), + PIN_FIELD_BASE(17, 17, 1, 0x0140, 0x10, 20, 1), + PIN_FIELD_BASE(18, 18, 1, 0x0140, 0x10, 21, 1), + PIN_FIELD_BASE(19, 19, 1, 0x0140, 0x10, 27, 1), + PIN_FIELD_BASE(20, 20, 1, 0x0140, 0x10, 28, 1), + PIN_FIELD_BASE(21, 21, 1, 0x0140, 0x10, 26, 1), + PIN_FIELD_BASE(22, 22, 1, 0x0140, 0x10, 25, 1), + PIN_FIELD_BASE(23, 23, 1, 0x0140, 0x10, 0, 1), + PIN_FIELD_BASE(24, 24, 1, 0x0140, 0x10, 1, 1), + PIN_FIELD_BASE(25, 25, 1, 0x0140, 0x10, 2, 1), + PIN_FIELD_BASE(26, 26, 1, 0x0140, 0x10, 3, 1), + PIN_FIELD_BASE(27, 27, 1, 0x0140, 0x10, 4, 1), + PIN_FIELD_BASE(28, 28, 1, 0x0140, 0x10, 5, 1), + PIN_FIELD_BASE(29, 29, 1, 0x0140, 0x10, 6, 1), + PIN_FIELD_BASE(30, 30, 1, 0x0140, 0x10, 7, 1), + PIN_FIELD_BASE(31, 31, 1, 0x0140, 0x10, 8, 1), + PIN_FIELD_BASE(32, 32, 9, 0x0100, 0x10, 0, 1), + PIN_FIELD_BASE(33, 33, 1, 0x0140, 0x10, 9, 1), + PIN_FIELD_BASE(34, 34, 1, 0x0140, 0x10, 10, 1), + PIN_FIELD_BASE(35, 35, 1, 0x0140, 0x10, 11, 1), + PIN_FIELD_BASE(36, 36, 9, 0x0100, 0x10, 8, 1), + PIN_FIELD_BASE(37, 37, 2, 0x0120, 0x10, 17, 1), + PIN_FIELD_BASE(38, 38, 2, 0x0120, 0x10, 18, 1), + PIN_FIELD_BASE(39, 39, 1, 0x0140, 0x10, 12, 1), + PIN_FIELD_BASE(40, 40, 2, 0x0120, 0x10, 1, 1), + PIN_FIELD_BASE(41, 41, 2, 0x0120, 0x10, 2, 1), + PIN_FIELD_BASE(42, 42, 2, 0x0120, 0x10, 3, 1), + PIN_FIELD_BASE(43, 43, 2, 0x0120, 0x10, 4, 1), + PIN_FIELD_BASE(44, 44, 2, 0x0120, 0x10, 5, 1), + PIN_FIELD_BASE(45, 45, 2, 0x0120, 0x10, 6, 1), + PIN_FIELD_BASE(46, 46, 1, 0x0140, 0x10, 13, 1), + PIN_FIELD_BASE(47, 47, 1, 0x0140, 0x10, 15, 1), + PIN_FIELD_BASE(48, 48, 2, 0x0120, 0x10, 7, 1), + PIN_FIELD_BASE(49, 49, 2, 0x0120, 0x10, 8, 1), + PIN_FIELD_BASE(50, 50, 2, 0x0120, 0x10, 9, 1), + PIN_FIELD_BASE(51, 51, 2, 0x0120, 0x10, 10, 1), + PIN_FIELD_BASE(52, 52, 2, 0x0120, 0x10, 11, 1), + PIN_FIELD_BASE(53, 53, 2, 0x0120, 0x10, 12, 1), + PIN_FIELD_BASE(54, 54, 5, 0x0120, 0x10, 10, 1), + PIN_FIELD_BASE(55, 55, 5, 0x0120, 0x10, 11, 1), + PIN_FIELD_BASE(56, 56, 1, 0x0140, 0x10, 22, 1), + PIN_FIELD_BASE(57, 57, 1, 0x0140, 0x10, 23, 1), + PIN_FIELD_BASE(58, 58, 1, 0x0140, 0x10, 24, 1), + PIN_FIELD_BASE(59, 59, 2, 0x0120, 0x10, 0, 1), + PIN_FIELD_BASE(60, 60, 9, 0x0100, 0x10, 1, 1), + PIN_FIELD_BASE(61, 61, 9, 0x0100, 0x10, 2, 1), + PIN_FIELD_BASE(62, 62, 9, 0x0100, 0x10, 3, 1), + PIN_FIELD_BASE(63, 63, 9, 0x0100, 0x10, 4, 1), + PIN_FIELD_BASE(64, 64, 9, 0x0100, 0x10, 5, 1), + PIN_FIELD_BASE(65, 65, 9, 0x0100, 0x10, 6, 1), + PIN_FIELD_BASE(66, 66, 5, 0x0120, 0x10, 0, 1), + PIN_FIELD_BASE(67, 67, 5, 0x0120, 0x10, 1, 1), + PIN_FIELD_BASE(68, 68, 9, 0x0100, 0x10, 7, 1), + PIN_FIELD_BASE(69, 69, 7, 0x0110, 0x10, 0, 1), + PIN_FIELD_BASE(70, 70, 7, 0x0110, 0x10, 1, 1), + PIN_FIELD_BASE(71, 71, 7, 0x0110, 0x10, 2, 1), + PIN_FIELD_BASE(72, 72, 7, 0x0110, 0x10, 3, 1), + PIN_FIELD_BASE(73, 73, 7, 0x0110, 0x10, 4, 1), + PIN_FIELD_BASE(74, 74, 7, 0x0110, 0x10, 5, 1), + PIN_FIELD_BASE(75, 75, 7, 0x0110, 0x10, 6, 1), + PIN_FIELD_BASE(76, 76, 7, 0x0110, 0x10, 7, 1), + PIN_FIELD_BASE(77, 77, 7, 0x0110, 0x10, 8, 1), + PIN_FIELD_BASE(78, 78, 7, 0x0110, 0x10, 9, 1), + PIN_FIELD_BASE(79, 79, 7, 0x0110, 0x10, 10, 1), + PIN_FIELD_BASE(80, 80, 7, 0x0110, 0x10, 11, 1), + PIN_FIELD_BASE(81, 81, 7, 0x0110, 0x10, 12, 1), + PIN_FIELD_BASE(82, 82, 7, 0x0110, 0x10, 13, 1), + PIN_FIELD_BASE(83, 83, 7, 0x0110, 0x10, 14, 1), + PIN_FIELD_BASE(84, 84, 7, 0x0110, 0x10, 15, 1), + PIN_FIELD_BASE(85, 85, 7, 0x0110, 0x10, 16, 1), + PIN_FIELD_BASE(86, 86, 7, 0x0110, 0x10, 17, 1), + PIN_FIELD_BASE(87, 87, 7, 0x0110, 0x10, 18, 1), + PIN_FIELD_BASE(88, 88, 7, 0x0110, 0x10, 19, 1), + PIN_FIELD_BASE(89, 89, 7, 0x0110, 0x10, 20, 1), + PIN_FIELD_BASE(90, 90, 7, 0x0110, 0x10, 21, 1), + PIN_FIELD_BASE(91, 91, 7, 0x0110, 0x10, 22, 1), + PIN_FIELD_BASE(92, 92, 3, 0x0130, 0x10, 16, 1), + PIN_FIELD_BASE(93, 93, 3, 0x0130, 0x10, 17, 1), + PIN_FIELD_BASE(94, 94, 3, 0x0130, 0x10, 18, 1), + PIN_FIELD_BASE(95, 95, 3, 0x0130, 0x10, 19, 1), + PIN_FIELD_BASE(96, 96, 3, 0x0130, 0x10, 20, 1), + PIN_FIELD_BASE(97, 97, 3, 0x0130, 0x10, 21, 1), + PIN_FIELD_BASE(98, 98, 3, 0x0130, 0x10, 22, 1), + PIN_FIELD_BASE(99, 99, 3, 0x0130, 0x10, 23, 1), + PIN_FIELD_BASE(100, 100, 3, 0x0130, 0x10, 24, 1), + PIN_FIELD_BASE(101, 101, 3, 0x0130, 0x10, 25, 1), + PIN_FIELD_BASE(102, 102, 3, 0x0130, 0x10, 26, 1), + PIN_FIELD_BASE(103, 103, 3, 0x0130, 0x10, 27, 1), + PIN_FIELD_BASE(104, 104, 3, 0x0130, 0x10, 28, 1), + PIN_FIELD_BASE(105, 105, 3, 0x0130, 0x10, 29, 1), + PIN_FIELD_BASE(106, 106, 8, 0x00c0, 0x10, 8, 1), + PIN_FIELD_BASE(107, 107, 8, 0x00c0, 0x10, 9, 1), + PIN_FIELD_BASE(108, 108, 8, 0x00c0, 0x10, 10, 1), + PIN_FIELD_BASE(109, 109, 8, 0x00c0, 0x10, 11, 1), + PIN_FIELD_BASE(110, 110, 8, 0x00c0, 0x10, 12, 1), + PIN_FIELD_BASE(111, 111, 8, 0x00c0, 0x10, 13, 1), + PIN_FIELD_BASE(112, 112, 5, 0x0120, 0x10, 15, 1), + PIN_FIELD_BASE(113, 113, 5, 0x0120, 0x10, 16, 1), + PIN_FIELD_BASE(114, 114, 5, 0x0120, 0x10, 17, 1), + PIN_FIELD_BASE(115, 115, 5, 0x0120, 0x10, 18, 1), + PIN_FIELD_BASE(116, 116, 5, 0x0120, 0x10, 19, 1), + PIN_FIELD_BASE(117, 117, 4, 0x0110, 0x10, 8, 1), + PIN_FIELD_BASE(118, 118, 4, 0x0110, 0x10, 9, 1), + PIN_FIELD_BASE(119, 119, 4, 0x0110, 0x10, 10, 1), + PIN_FIELD_BASE(120, 120, 4, 0x0110, 0x10, 11, 1), + PIN_FIELD_BASE(121, 121, 4, 0x0110, 0x10, 12, 1), + PIN_FIELD_BASE(122, 122, 4, 0x0110, 0x10, 13, 1), + PIN_FIELD_BASE(123, 123, 4, 0x0110, 0x10, 14, 1), + PIN_FIELD_BASE(124, 124, 4, 0x0110, 0x10, 15, 1), + PIN_FIELD_BASE(125, 125, 4, 0x0110, 0x10, 16, 1), + PIN_FIELD_BASE(126, 126, 5, 0x0120, 0x10, 6, 1), + PIN_FIELD_BASE(127, 127, 5, 0x0120, 0x10, 7, 1), + PIN_FIELD_BASE(128, 128, 5, 0x0120, 0x10, 8, 1), + PIN_FIELD_BASE(129, 129, 5, 0x0120, 0x10, 9, 1), + PIN_FIELD_BASE(130, 130, 4, 0x0110, 0x10, 17, 1), + PIN_FIELD_BASE(131, 131, 4, 0x0110, 0x10, 18, 1), + PIN_FIELD_BASE(132, 132, 4, 0x0110, 0x10, 19, 1), + PIN_FIELD_BASE(133, 133, 4, 0x0110, 0x10, 20, 1), + PIN_FIELD_BASE(134, 134, 3, 0x0130, 0x10, 0, 1), + PIN_FIELD_BASE(135, 135, 3, 0x0130, 0x10, 1, 1), + PIN_FIELD_BASE(136, 136, 3, 0x0130, 0x10, 2, 1), + PIN_FIELD_BASE(137, 137, 3, 0x0130, 0x10, 3, 1), + PIN_FIELD_BASE(138, 138, 3, 0x0130, 0x10, 4, 1), + PIN_FIELD_BASE(139, 139, 3, 0x0130, 0x10, 5, 1), + PIN_FIELD_BASE(140, 140, 3, 0x0130, 0x10, 6, 1), + PIN_FIELD_BASE(141, 141, 3, 0x0130, 0x10, 7, 1), + PIN_FIELD_BASE(142, 142, 3, 0x0130, 0x10, 8, 1), + PIN_FIELD_BASE(143, 143, 3, 0x0130, 0x10, 9, 1), + PIN_FIELD_BASE(144, 144, 3, 0x0130, 0x10, 10, 1), + PIN_FIELD_BASE(145, 145, 3, 0x0130, 0x10, 11, 1), + PIN_FIELD_BASE(146, 146, 3, 0x0130, 0x10, 12, 1), + PIN_FIELD_BASE(147, 147, 3, 0x0130, 0x10, 13, 1), + PIN_FIELD_BASE(148, 148, 3, 0x0130, 0x10, 14, 1), + PIN_FIELD_BASE(149, 149, 3, 0x0130, 0x10, 15, 1), + PIN_FIELD_BASE(150, 150, 4, 0x0110, 0x10, 21, 1), + PIN_FIELD_BASE(151, 151, 4, 0x0110, 0x10, 26, 1), + PIN_FIELD_BASE(152, 152, 4, 0x0110, 0x10, 25, 1), + PIN_FIELD_BASE(153, 153, 4, 0x0110, 0x10, 24, 1), + PIN_FIELD_BASE(154, 154, 4, 0x0110, 0x10, 22, 1), + PIN_FIELD_BASE(155, 155, 4, 0x0110, 0x10, 23, 1), + PIN_FIELD_BASE(156, 156, 4, 0x0110, 0x10, 0, 1), + PIN_FIELD_BASE(157, 157, 4, 0x0110, 0x10, 1, 1), + PIN_FIELD_BASE(158, 158, 4, 0x0110, 0x10, 2, 1), + PIN_FIELD_BASE(159, 159, 4, 0x0110, 0x10, 3, 1), + PIN_FIELD_BASE(160, 160, 4, 0x0110, 0x10, 4, 1), + PIN_FIELD_BASE(161, 161, 4, 0x0110, 0x10, 5, 1), + PIN_FIELD_BASE(162, 162, 4, 0x0110, 0x10, 6, 1), + PIN_FIELD_BASE(163, 163, 4, 0x0110, 0x10, 7, 1), + PIN_FIELD_BASE(164, 164, 5, 0x0120, 0x10, 12, 1), + PIN_FIELD_BASE(165, 165, 5, 0x0120, 0x10, 3, 1), + PIN_FIELD_BASE(166, 166, 5, 0x0120, 0x10, 4, 1), + PIN_FIELD_BASE(167, 167, 5, 0x0120, 0x10, 2, 1), + PIN_FIELD_BASE(168, 168, 5, 0x0120, 0x10, 13, 1), + PIN_FIELD_BASE(169, 169, 5, 0x0120, 0x10, 14, 1), + PIN_FIELD_BASE(170, 170, 5, 0x0120, 0x10, 5, 1), + PIN_FIELD_BASE(171, 171, 6, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(172, 172, 6, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(173, 173, 6, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(174, 174, 6, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(175, 175, 6, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(176, 176, 6, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(177, 177, 6, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(178, 178, 6, 0x00c0, 0x10, 7, 1), + PIN_FIELD_BASE(179, 179, 6, 0x00c0, 0x10, 8, 1), + PIN_FIELD_BASE(180, 180, 6, 0x00c0, 0x10, 9, 1), + PIN_FIELD_BASE(181, 181, 10, 0x0080, 0x10, 0, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_ies_range[] = { + PIN_FIELD_BASE(0, 0, 8, 0x0050, 0x10, 0, 1), + PIN_FIELD_BASE(1, 1, 8, 0x0050, 0x10, 1, 1), + PIN_FIELD_BASE(2, 2, 8, 0x0050, 0x10, 2, 1), + PIN_FIELD_BASE(3, 3, 8, 0x0050, 0x10, 3, 1), + PIN_FIELD_BASE(4, 4, 8, 0x0050, 0x10, 4, 1), + PIN_FIELD_BASE(5, 5, 8, 0x0050, 0x10, 5, 1), + PIN_FIELD_BASE(6, 6, 8, 0x0050, 0x10, 6, 1), + PIN_FIELD_BASE(7, 7, 8, 0x0050, 0x10, 7, 1), + PIN_FIELD_BASE(8, 8, 2, 0x0070, 0x10, 13, 1), + PIN_FIELD_BASE(9, 9, 2, 0x0070, 0x10, 14, 1), + PIN_FIELD_BASE(10, 10, 2, 0x0070, 0x10, 15, 1), + PIN_FIELD_BASE(11, 11, 2, 0x0070, 0x10, 16, 1), + PIN_FIELD_BASE(12, 12, 1, 0x0080, 0x10, 17, 1), + PIN_FIELD_BASE(13, 13, 1, 0x0080, 0x10, 18, 1), + PIN_FIELD_BASE(14, 14, 1, 0x0080, 0x10, 14, 1), + PIN_FIELD_BASE(15, 15, 1, 0x0080, 0x10, 16, 1), + PIN_FIELD_BASE(16, 16, 1, 0x0080, 0x10, 19, 1), + PIN_FIELD_BASE(17, 17, 1, 0x0080, 0x10, 20, 1), + PIN_FIELD_BASE(18, 18, 1, 0x0080, 0x10, 21, 1), + PIN_FIELD_BASE(19, 19, 1, 0x0080, 0x10, 27, 1), + PIN_FIELD_BASE(20, 20, 1, 0x0080, 0x10, 28, 1), + PIN_FIELD_BASE(21, 21, 1, 0x0080, 0x10, 26, 1), + PIN_FIELD_BASE(22, 22, 1, 0x0080, 0x10, 25, 1), + PIN_FIELD_BASE(23, 23, 1, 0x0080, 0x10, 0, 1), + PIN_FIELD_BASE(24, 24, 1, 0x0080, 0x10, 1, 1), + PIN_FIELD_BASE(25, 25, 1, 0x0080, 0x10, 2, 1), + PIN_FIELD_BASE(26, 26, 1, 0x0080, 0x10, 3, 1), + PIN_FIELD_BASE(27, 27, 1, 0x0080, 0x10, 4, 1), + PIN_FIELD_BASE(28, 28, 1, 0x0080, 0x10, 5, 1), + PIN_FIELD_BASE(29, 29, 1, 0x0080, 0x10, 6, 1), + PIN_FIELD_BASE(30, 30, 1, 0x0080, 0x10, 7, 1), + PIN_FIELD_BASE(31, 31, 1, 0x0080, 0x10, 8, 1), + PIN_FIELD_BASE(32, 32, 9, 0x0060, 0x10, 0, 1), + PIN_FIELD_BASE(33, 33, 1, 0x0080, 0x10, 9, 1), + PIN_FIELD_BASE(34, 34, 1, 0x0080, 0x10, 10, 1), + PIN_FIELD_BASE(35, 35, 1, 0x0080, 0x10, 11, 1), + PIN_FIELD_BASE(36, 36, 9, 0x0060, 0x10, 8, 1), + PIN_FIELD_BASE(37, 37, 2, 0x0070, 0x10, 17, 1), + PIN_FIELD_BASE(38, 38, 2, 0x0070, 0x10, 18, 1), + PIN_FIELD_BASE(39, 39, 1, 0x0080, 0x10, 12, 1), + PIN_FIELD_BASE(40, 40, 2, 0x0070, 0x10, 1, 1), + PIN_FIELD_BASE(41, 41, 2, 0x0070, 0x10, 2, 1), + PIN_FIELD_BASE(42, 42, 2, 0x0070, 0x10, 3, 1), + PIN_FIELD_BASE(43, 43, 2, 0x0070, 0x10, 4, 1), + PIN_FIELD_BASE(44, 44, 2, 0x0070, 0x10, 5, 1), + PIN_FIELD_BASE(45, 45, 2, 0x0070, 0x10, 6, 1), + PIN_FIELD_BASE(46, 46, 1, 0x0080, 0x10, 13, 1), + PIN_FIELD_BASE(47, 47, 1, 0x0080, 0x10, 15, 1), + PIN_FIELD_BASE(48, 48, 2, 0x0070, 0x10, 7, 1), + PIN_FIELD_BASE(49, 49, 2, 0x0070, 0x10, 8, 1), + PIN_FIELD_BASE(50, 50, 2, 0x0070, 0x10, 9, 1), + PIN_FIELD_BASE(51, 51, 2, 0x0070, 0x10, 10, 1), + PIN_FIELD_BASE(52, 52, 2, 0x0070, 0x10, 11, 1), + PIN_FIELD_BASE(53, 53, 2, 0x0070, 0x10, 12, 1), + PIN_FIELD_BASE(54, 54, 5, 0x0060, 0x10, 10, 1), + PIN_FIELD_BASE(55, 55, 5, 0x0060, 0x10, 11, 1), + PIN_FIELD_BASE(56, 56, 1, 0x0080, 0x10, 22, 1), + PIN_FIELD_BASE(57, 57, 1, 0x0080, 0x10, 23, 1), + PIN_FIELD_BASE(58, 58, 1, 0x0080, 0x10, 24, 1), + PIN_FIELD_BASE(59, 59, 2, 0x0070, 0x10, 0, 1), + PIN_FIELD_BASE(60, 60, 9, 0x0060, 0x10, 1, 1), + PIN_FIELD_BASE(61, 61, 9, 0x0060, 0x10, 2, 1), + PIN_FIELD_BASE(62, 62, 9, 0x0060, 0x10, 3, 1), + PIN_FIELD_BASE(63, 63, 9, 0x0060, 0x10, 4, 1), + PIN_FIELD_BASE(64, 64, 9, 0x0060, 0x10, 5, 1), + PIN_FIELD_BASE(65, 65, 9, 0x0060, 0x10, 6, 1), + PIN_FIELD_BASE(66, 66, 5, 0x0060, 0x10, 0, 1), + PIN_FIELD_BASE(67, 67, 5, 0x0060, 0x10, 1, 1), + PIN_FIELD_BASE(68, 68, 9, 0x0060, 0x10, 7, 1), + PIN_FIELD_BASE(69, 69, 7, 0x0070, 0x10, 0, 1), + PIN_FIELD_BASE(70, 70, 7, 0x0070, 0x10, 1, 1), + PIN_FIELD_BASE(71, 71, 7, 0x0070, 0x10, 2, 1), + PIN_FIELD_BASE(72, 72, 7, 0x0070, 0x10, 3, 1), + PIN_FIELD_BASE(73, 73, 7, 0x0070, 0x10, 4, 1), + PIN_FIELD_BASE(74, 74, 7, 0x0070, 0x10, 5, 1), + PIN_FIELD_BASE(75, 75, 7, 0x0070, 0x10, 6, 1), + PIN_FIELD_BASE(76, 76, 7, 0x0070, 0x10, 7, 1), + PIN_FIELD_BASE(77, 77, 7, 0x0070, 0x10, 8, 1), + PIN_FIELD_BASE(78, 78, 7, 0x0070, 0x10, 9, 1), + PIN_FIELD_BASE(79, 79, 7, 0x0070, 0x10, 10, 1), + PIN_FIELD_BASE(80, 80, 7, 0x0070, 0x10, 11, 1), + PIN_FIELD_BASE(81, 81, 7, 0x0070, 0x10, 12, 1), + PIN_FIELD_BASE(82, 82, 7, 0x0070, 0x10, 13, 1), + PIN_FIELD_BASE(83, 83, 7, 0x0070, 0x10, 14, 1), + PIN_FIELD_BASE(84, 84, 7, 0x0070, 0x10, 15, 1), + PIN_FIELD_BASE(85, 85, 7, 0x0070, 0x10, 16, 1), + PIN_FIELD_BASE(86, 86, 7, 0x0070, 0x10, 17, 1), + PIN_FIELD_BASE(87, 87, 7, 0x0070, 0x10, 18, 1), + PIN_FIELD_BASE(88, 88, 7, 0x0070, 0x10, 19, 1), + PIN_FIELD_BASE(89, 89, 7, 0x0070, 0x10, 20, 1), + PIN_FIELD_BASE(90, 90, 7, 0x0070, 0x10, 21, 1), + PIN_FIELD_BASE(91, 91, 7, 0x0070, 0x10, 22, 1), + PIN_FIELD_BASE(92, 92, 3, 0x0080, 0x10, 16, 1), + PIN_FIELD_BASE(93, 93, 3, 0x0080, 0x10, 17, 1), + PIN_FIELD_BASE(94, 94, 3, 0x0080, 0x10, 18, 1), + PIN_FIELD_BASE(95, 95, 3, 0x0080, 0x10, 19, 1), + PIN_FIELD_BASE(96, 96, 3, 0x0080, 0x10, 20, 1), + PIN_FIELD_BASE(97, 97, 3, 0x0080, 0x10, 21, 1), + PIN_FIELD_BASE(98, 98, 3, 0x0080, 0x10, 22, 1), + PIN_FIELD_BASE(99, 99, 3, 0x0080, 0x10, 23, 1), + PIN_FIELD_BASE(100, 100, 3, 0x0080, 0x10, 24, 1), + PIN_FIELD_BASE(101, 101, 3, 0x0080, 0x10, 25, 1), + PIN_FIELD_BASE(102, 102, 3, 0x0080, 0x10, 26, 1), + PIN_FIELD_BASE(103, 103, 3, 0x0080, 0x10, 27, 1), + PIN_FIELD_BASE(104, 104, 3, 0x0080, 0x10, 28, 1), + PIN_FIELD_BASE(105, 105, 3, 0x0080, 0x10, 29, 1), + PIN_FIELD_BASE(106, 106, 8, 0x0050, 0x10, 8, 1), + PIN_FIELD_BASE(107, 107, 8, 0x0050, 0x10, 9, 1), + PIN_FIELD_BASE(108, 108, 8, 0x0050, 0x10, 10, 1), + PIN_FIELD_BASE(109, 109, 8, 0x0050, 0x10, 11, 1), + PIN_FIELD_BASE(110, 110, 8, 0x0050, 0x10, 12, 1), + PIN_FIELD_BASE(111, 111, 8, 0x0050, 0x10, 13, 1), + PIN_FIELD_BASE(112, 112, 5, 0x0060, 0x10, 15, 1), + PIN_FIELD_BASE(113, 113, 5, 0x0060, 0x10, 16, 1), + PIN_FIELD_BASE(114, 114, 5, 0x0060, 0x10, 17, 1), + PIN_FIELD_BASE(115, 115, 5, 0x0060, 0x10, 18, 1), + PIN_FIELD_BASE(116, 116, 5, 0x0060, 0x10, 19, 1), + PIN_FIELD_BASE(117, 117, 4, 0x0060, 0x10, 8, 1), + PIN_FIELD_BASE(118, 118, 4, 0x0060, 0x10, 9, 1), + PIN_FIELD_BASE(119, 119, 4, 0x0060, 0x10, 10, 1), + PIN_FIELD_BASE(120, 120, 4, 0x0060, 0x10, 11, 1), + PIN_FIELD_BASE(121, 121, 4, 0x0060, 0x10, 12, 1), + PIN_FIELD_BASE(122, 122, 4, 0x0060, 0x10, 13, 1), + PIN_FIELD_BASE(123, 123, 4, 0x0060, 0x10, 14, 1), + PIN_FIELD_BASE(124, 124, 4, 0x0060, 0x10, 15, 1), + PIN_FIELD_BASE(125, 125, 4, 0x0060, 0x10, 16, 1), + PIN_FIELD_BASE(126, 126, 5, 0x0060, 0x10, 6, 1), + PIN_FIELD_BASE(127, 127, 5, 0x0060, 0x10, 7, 1), + PIN_FIELD_BASE(128, 128, 5, 0x0060, 0x10, 8, 1), + PIN_FIELD_BASE(129, 129, 5, 0x0060, 0x10, 9, 1), + PIN_FIELD_BASE(130, 130, 4, 0x0060, 0x10, 17, 1), + PIN_FIELD_BASE(131, 131, 4, 0x0060, 0x10, 18, 1), + PIN_FIELD_BASE(132, 132, 4, 0x0060, 0x10, 19, 1), + PIN_FIELD_BASE(133, 133, 4, 0x0060, 0x10, 20, 1), + PIN_FIELD_BASE(134, 134, 3, 0x0080, 0x10, 0, 1), + PIN_FIELD_BASE(135, 135, 3, 0x0080, 0x10, 1, 1), + PIN_FIELD_BASE(136, 136, 3, 0x0080, 0x10, 2, 1), + PIN_FIELD_BASE(137, 137, 3, 0x0080, 0x10, 3, 1), + PIN_FIELD_BASE(138, 138, 3, 0x0080, 0x10, 4, 1), + PIN_FIELD_BASE(139, 139, 3, 0x0080, 0x10, 5, 1), + PIN_FIELD_BASE(140, 140, 3, 0x0080, 0x10, 6, 1), + PIN_FIELD_BASE(141, 141, 3, 0x0080, 0x10, 7, 1), + PIN_FIELD_BASE(142, 142, 3, 0x0080, 0x10, 8, 1), + PIN_FIELD_BASE(143, 143, 3, 0x0080, 0x10, 9, 1), + PIN_FIELD_BASE(144, 144, 3, 0x0080, 0x10, 10, 1), + PIN_FIELD_BASE(145, 145, 3, 0x0080, 0x10, 11, 1), + PIN_FIELD_BASE(146, 146, 3, 0x0080, 0x10, 12, 1), + PIN_FIELD_BASE(147, 147, 3, 0x0080, 0x10, 13, 1), + PIN_FIELD_BASE(148, 148, 3, 0x0080, 0x10, 14, 1), + PIN_FIELD_BASE(149, 149, 3, 0x0080, 0x10, 15, 1), + PIN_FIELD_BASE(150, 150, 4, 0x0060, 0x10, 21, 1), + PIN_FIELD_BASE(151, 151, 4, 0x0060, 0x10, 26, 1), + PIN_FIELD_BASE(152, 152, 4, 0x0060, 0x10, 25, 1), + PIN_FIELD_BASE(153, 153, 4, 0x0060, 0x10, 24, 1), + PIN_FIELD_BASE(154, 154, 4, 0x0060, 0x10, 22, 1), + PIN_FIELD_BASE(155, 155, 4, 0x0060, 0x10, 23, 1), + PIN_FIELD_BASE(156, 156, 4, 0x0060, 0x10, 0, 1), + PIN_FIELD_BASE(157, 157, 4, 0x0060, 0x10, 1, 1), + PIN_FIELD_BASE(158, 158, 4, 0x0060, 0x10, 2, 1), + PIN_FIELD_BASE(159, 159, 4, 0x0060, 0x10, 3, 1), + PIN_FIELD_BASE(160, 160, 4, 0x0060, 0x10, 4, 1), + PIN_FIELD_BASE(161, 161, 4, 0x0060, 0x10, 5, 1), + PIN_FIELD_BASE(162, 162, 4, 0x0060, 0x10, 6, 1), + PIN_FIELD_BASE(163, 163, 4, 0x0060, 0x10, 7, 1), + PIN_FIELD_BASE(164, 164, 5, 0x0060, 0x10, 12, 1), + PIN_FIELD_BASE(165, 165, 5, 0x0060, 0x10, 3, 1), + PIN_FIELD_BASE(166, 166, 5, 0x0060, 0x10, 4, 1), + PIN_FIELD_BASE(167, 167, 5, 0x0060, 0x10, 2, 1), + PIN_FIELD_BASE(168, 168, 5, 0x0060, 0x10, 13, 1), + PIN_FIELD_BASE(169, 169, 5, 0x0060, 0x10, 14, 1), + PIN_FIELD_BASE(170, 170, 5, 0x0060, 0x10, 5, 1), + PIN_FIELD_BASE(171, 171, 6, 0x0050, 0x10, 0, 1), + PIN_FIELD_BASE(172, 172, 6, 0x0050, 0x10, 1, 1), + PIN_FIELD_BASE(173, 173, 6, 0x0050, 0x10, 2, 1), + PIN_FIELD_BASE(174, 174, 6, 0x0050, 0x10, 3, 1), + PIN_FIELD_BASE(175, 175, 6, 0x0050, 0x10, 4, 1), + PIN_FIELD_BASE(176, 176, 6, 0x0050, 0x10, 5, 1), + PIN_FIELD_BASE(177, 177, 6, 0x0050, 0x10, 6, 1), + PIN_FIELD_BASE(178, 178, 6, 0x0050, 0x10, 7, 1), + PIN_FIELD_BASE(179, 179, 6, 0x0050, 0x10, 8, 1), + PIN_FIELD_BASE(180, 180, 6, 0x0050, 0x10, 9, 1), + PIN_FIELD_BASE(181, 181, 10, 0x0020, 0x10, 0, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_pupd_range[] = { + PIN_FIELD_BASE(0, 0, 8, 0x0080, 0x10, 0, 1), + PIN_FIELD_BASE(1, 1, 8, 0x0080, 0x10, 1, 1), + PIN_FIELD_BASE(2, 2, 8, 0x0080, 0x10, 2, 1), + PIN_FIELD_BASE(3, 3, 8, 0x0080, 0x10, 3, 1), + PIN_FIELD_BASE(4, 4, 8, 0x0080, 0x10, 4, 1), + PIN_FIELD_BASE(5, 5, 8, 0x0080, 0x10, 5, 1), + PIN_FIELD_BASE(6, 6, 8, 0x0080, 0x10, 6, 1), + PIN_FIELD_BASE(7, 7, 8, 0x0080, 0x10, 7, 1), + PIN_FIELD_BASE(14, 14, 1, 0x00c0, 0x10, 14, 1), + PIN_FIELD_BASE(15, 15, 1, 0x00c0, 0x10, 16, 1), + PIN_FIELD_BASE(16, 16, 1, 0x00c0, 0x10, 17, 1), + PIN_FIELD_BASE(19, 19, 1, 0x00c0, 0x10, 21, 1), + PIN_FIELD_BASE(20, 20, 1, 0x00c0, 0x10, 22, 1), + PIN_FIELD_BASE(21, 21, 1, 0x00c0, 0x10, 20, 1), + PIN_FIELD_BASE(22, 22, 1, 0x00c0, 0x10, 19, 1), + PIN_FIELD_BASE(23, 23, 1, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(24, 24, 1, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(25, 25, 1, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(26, 26, 1, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(27, 27, 1, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(28, 28, 1, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(29, 29, 1, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(30, 30, 1, 0x00c0, 0x10, 7, 1), + PIN_FIELD_BASE(31, 31, 1, 0x00c0, 0x10, 8, 1), + PIN_FIELD_BASE(32, 32, 9, 0x00a0, 0x10, 0, 1), + PIN_FIELD_BASE(33, 33, 1, 0x00c0, 0x10, 9, 1), + PIN_FIELD_BASE(34, 34, 1, 0x00c0, 0x10, 10, 1), + PIN_FIELD_BASE(35, 35, 1, 0x00c0, 0x10, 11, 1), + PIN_FIELD_BASE(36, 36, 9, 0x00a0, 0x10, 6, 1), + PIN_FIELD_BASE(37, 37, 2, 0x00b0, 0x10, 10, 1), + PIN_FIELD_BASE(38, 38, 2, 0x00b0, 0x10, 11, 1), + PIN_FIELD_BASE(39, 39, 1, 0x00c0, 0x10, 12, 1), + PIN_FIELD_BASE(40, 40, 2, 0x00b0, 0x10, 0, 1), + PIN_FIELD_BASE(41, 41, 2, 0x00b0, 0x10, 1, 1), + PIN_FIELD_BASE(42, 42, 2, 0x00b0, 0x10, 2, 1), + PIN_FIELD_BASE(43, 43, 2, 0x00b0, 0x10, 3, 1), + PIN_FIELD_BASE(44, 44, 2, 0x00b0, 0x10, 4, 1), + PIN_FIELD_BASE(45, 45, 2, 0x00b0, 0x10, 5, 1), + PIN_FIELD_BASE(46, 46, 1, 0x00c0, 0x10, 13, 1), + PIN_FIELD_BASE(47, 47, 1, 0x00c0, 0x10, 15, 1), + PIN_FIELD_BASE(48, 48, 2, 0x00b0, 0x10, 6, 1), + PIN_FIELD_BASE(49, 49, 2, 0x00b0, 0x10, 7, 1), + PIN_FIELD_BASE(50, 50, 2, 0x00b0, 0x10, 8, 1), + PIN_FIELD_BASE(51, 51, 2, 0x00b0, 0x10, 9, 1), + PIN_FIELD_BASE(58, 58, 1, 0x00c0, 0x10, 18, 1), + PIN_FIELD_BASE(62, 62, 9, 0x00a0, 0x10, 1, 1), + PIN_FIELD_BASE(63, 63, 9, 0x00a0, 0x10, 2, 1), + PIN_FIELD_BASE(64, 64, 9, 0x00a0, 0x10, 3, 1), + PIN_FIELD_BASE(65, 65, 9, 0x00a0, 0x10, 4, 1), + PIN_FIELD_BASE(68, 68, 9, 0x00a0, 0x10, 5, 1), + PIN_FIELD_BASE(74, 74, 7, 0x00b0, 0x10, 0, 1), + PIN_FIELD_BASE(75, 75, 7, 0x00b0, 0x10, 1, 1), + PIN_FIELD_BASE(76, 76, 7, 0x00b0, 0x10, 2, 1), + PIN_FIELD_BASE(77, 77, 7, 0x00b0, 0x10, 3, 1), + PIN_FIELD_BASE(78, 78, 7, 0x00b0, 0x10, 4, 1), + PIN_FIELD_BASE(79, 79, 7, 0x00b0, 0x10, 5, 1), + PIN_FIELD_BASE(80, 80, 7, 0x00b0, 0x10, 6, 1), + PIN_FIELD_BASE(81, 81, 7, 0x00b0, 0x10, 7, 1), + PIN_FIELD_BASE(82, 82, 7, 0x00b0, 0x10, 8, 1), + PIN_FIELD_BASE(83, 83, 7, 0x00b0, 0x10, 9, 1), + PIN_FIELD_BASE(84, 84, 7, 0x00b0, 0x10, 10, 1), + PIN_FIELD_BASE(85, 85, 7, 0x00b0, 0x10, 11, 1), + PIN_FIELD_BASE(86, 86, 7, 0x00b0, 0x10, 12, 1), + PIN_FIELD_BASE(87, 87, 7, 0x00b0, 0x10, 13, 1), + PIN_FIELD_BASE(90, 90, 7, 0x00b0, 0x10, 14, 1), + PIN_FIELD_BASE(91, 91, 7, 0x00b0, 0x10, 15, 1), + PIN_FIELD_BASE(94, 94, 3, 0x00c0, 0x10, 12, 1), + PIN_FIELD_BASE(95, 95, 3, 0x00c0, 0x10, 13, 1), + PIN_FIELD_BASE(96, 96, 3, 0x00c0, 0x10, 14, 1), + PIN_FIELD_BASE(97, 97, 3, 0x00c0, 0x10, 15, 1), + PIN_FIELD_BASE(98, 98, 3, 0x00c0, 0x10, 16, 1), + PIN_FIELD_BASE(99, 99, 3, 0x00c0, 0x10, 17, 1), + PIN_FIELD_BASE(100, 100, 3, 0x00c0, 0x10, 18, 1), + PIN_FIELD_BASE(101, 101, 3, 0x00c0, 0x10, 19, 1), + PIN_FIELD_BASE(102, 102, 3, 0x00c0, 0x10, 20, 1), + PIN_FIELD_BASE(103, 103, 3, 0x00c0, 0x10, 21, 1), + PIN_FIELD_BASE(104, 104, 3, 0x00c0, 0x10, 22, 1), + PIN_FIELD_BASE(105, 105, 3, 0x00c0, 0x10, 23, 1), + PIN_FIELD_BASE(106, 106, 8, 0x0080, 0x10, 8, 1), + PIN_FIELD_BASE(107, 107, 8, 0x0080, 0x10, 9, 1), + PIN_FIELD_BASE(108, 108, 8, 0x0080, 0x10, 10, 1), + PIN_FIELD_BASE(109, 109, 8, 0x0080, 0x10, 11, 1), + PIN_FIELD_BASE(110, 110, 8, 0x0080, 0x10, 12, 1), + PIN_FIELD_BASE(111, 111, 8, 0x0080, 0x10, 13, 1), + PIN_FIELD_BASE(112, 112, 5, 0x00a0, 0x10, 5, 1), + PIN_FIELD_BASE(113, 113, 5, 0x00a0, 0x10, 6, 1), + PIN_FIELD_BASE(114, 114, 5, 0x00a0, 0x10, 7, 1), + PIN_FIELD_BASE(115, 115, 5, 0x00a0, 0x10, 8, 1), + PIN_FIELD_BASE(116, 116, 5, 0x00a0, 0x10, 9, 1), + PIN_FIELD_BASE(125, 125, 4, 0x00a0, 0x10, 8, 1), + PIN_FIELD_BASE(130, 130, 4, 0x00a0, 0x10, 9, 1), + PIN_FIELD_BASE(131, 131, 4, 0x00a0, 0x10, 10, 1), + PIN_FIELD_BASE(132, 132, 4, 0x00a0, 0x10, 11, 1), + PIN_FIELD_BASE(133, 133, 4, 0x00a0, 0x10, 12, 1), + PIN_FIELD_BASE(138, 138, 3, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(139, 139, 3, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(140, 140, 3, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(141, 141, 3, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(142, 142, 3, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(143, 143, 3, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(144, 144, 3, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(145, 145, 3, 0x00c0, 0x10, 7, 1), + PIN_FIELD_BASE(146, 146, 3, 0x00c0, 0x10, 8, 1), + PIN_FIELD_BASE(147, 147, 3, 0x00c0, 0x10, 9, 1), + PIN_FIELD_BASE(148, 148, 3, 0x00c0, 0x10, 10, 1), + PIN_FIELD_BASE(149, 149, 3, 0x00c0, 0x10, 11, 1), + PIN_FIELD_BASE(150, 150, 4, 0x00a0, 0x10, 13, 1), + PIN_FIELD_BASE(151, 151, 4, 0x00a0, 0x10, 18, 1), + PIN_FIELD_BASE(152, 152, 4, 0x00a0, 0x10, 17, 1), + PIN_FIELD_BASE(153, 153, 4, 0x00a0, 0x10, 16, 1), + PIN_FIELD_BASE(154, 154, 4, 0x00a0, 0x10, 14, 1), + PIN_FIELD_BASE(155, 155, 4, 0x00a0, 0x10, 15, 1), + PIN_FIELD_BASE(156, 156, 4, 0x00a0, 0x10, 0, 1), + PIN_FIELD_BASE(157, 157, 4, 0x00a0, 0x10, 1, 1), + PIN_FIELD_BASE(158, 158, 4, 0x00a0, 0x10, 2, 1), + PIN_FIELD_BASE(159, 159, 4, 0x00a0, 0x10, 3, 1), + PIN_FIELD_BASE(160, 160, 4, 0x00a0, 0x10, 4, 1), + PIN_FIELD_BASE(161, 161, 4, 0x00a0, 0x10, 5, 1), + PIN_FIELD_BASE(162, 162, 4, 0x00a0, 0x10, 6, 1), + PIN_FIELD_BASE(163, 163, 4, 0x00a0, 0x10, 7, 1), + PIN_FIELD_BASE(164, 164, 5, 0x00a0, 0x10, 2, 1), + PIN_FIELD_BASE(167, 167, 5, 0x00a0, 0x10, 0, 1), + PIN_FIELD_BASE(168, 168, 5, 0x00a0, 0x10, 3, 1), + PIN_FIELD_BASE(169, 169, 5, 0x00a0, 0x10, 4, 1), + PIN_FIELD_BASE(170, 170, 5, 0x00a0, 0x10, 1, 1), + PIN_FIELD_BASE(171, 171, 6, 0x0080, 0x10, 0, 1), + PIN_FIELD_BASE(172, 172, 6, 0x0080, 0x10, 1, 1), + PIN_FIELD_BASE(173, 173, 6, 0x0080, 0x10, 2, 1), + PIN_FIELD_BASE(174, 174, 6, 0x0080, 0x10, 3, 1), + PIN_FIELD_BASE(175, 175, 6, 0x0080, 0x10, 4, 1), + PIN_FIELD_BASE(176, 176, 6, 0x0080, 0x10, 5, 1), + PIN_FIELD_BASE(177, 177, 6, 0x0080, 0x10, 6, 1), + PIN_FIELD_BASE(178, 178, 6, 0x0080, 0x10, 7, 1), + PIN_FIELD_BASE(179, 179, 6, 0x0080, 0x10, 8, 1), + PIN_FIELD_BASE(180, 180, 6, 0x0080, 0x10, 9, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_r0_range[] = { + PIN_FIELD_BASE(0, 0, 8, 0x0090, 0x10, 0, 1), + PIN_FIELD_BASE(1, 1, 8, 0x0090, 0x10, 1, 1), + PIN_FIELD_BASE(2, 2, 8, 0x0090, 0x10, 2, 1), + PIN_FIELD_BASE(3, 3, 8, 0x0090, 0x10, 3, 1), + PIN_FIELD_BASE(4, 4, 8, 0x0090, 0x10, 4, 1), + PIN_FIELD_BASE(5, 5, 8, 0x0090, 0x10, 5, 1), + PIN_FIELD_BASE(6, 6, 8, 0x0090, 0x10, 6, 1), + PIN_FIELD_BASE(7, 7, 8, 0x0090, 0x10, 7, 1), + PIN_FIELD_BASE(14, 14, 1, 0x00e0, 0x10, 14, 1), + PIN_FIELD_BASE(15, 15, 1, 0x00e0, 0x10, 16, 1), + PIN_FIELD_BASE(16, 16, 1, 0x00e0, 0x10, 17, 1), + PIN_FIELD_BASE(19, 19, 1, 0x00e0, 0x10, 21, 1), + PIN_FIELD_BASE(20, 20, 1, 0x00e0, 0x10, 22, 1), + PIN_FIELD_BASE(21, 21, 1, 0x00e0, 0x10, 20, 1), + PIN_FIELD_BASE(22, 22, 1, 0x00e0, 0x10, 19, 1), + PIN_FIELD_BASE(23, 23, 1, 0x00e0, 0x10, 0, 1), + PIN_FIELD_BASE(24, 24, 1, 0x00e0, 0x10, 1, 1), + PIN_FIELD_BASE(25, 25, 1, 0x00e0, 0x10, 2, 1), + PIN_FIELD_BASE(26, 26, 1, 0x00e0, 0x10, 3, 1), + PIN_FIELD_BASE(27, 27, 1, 0x00e0, 0x10, 4, 1), + PIN_FIELD_BASE(28, 28, 1, 0x00e0, 0x10, 5, 1), + PIN_FIELD_BASE(29, 29, 1, 0x00e0, 0x10, 6, 1), + PIN_FIELD_BASE(30, 30, 1, 0x00e0, 0x10, 7, 1), + PIN_FIELD_BASE(31, 31, 1, 0x00e0, 0x10, 8, 1), + PIN_FIELD_BASE(32, 32, 9, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(33, 33, 1, 0x00e0, 0x10, 9, 1), + PIN_FIELD_BASE(34, 34, 1, 0x00e0, 0x10, 10, 1), + PIN_FIELD_BASE(35, 35, 1, 0x00e0, 0x10, 11, 1), + PIN_FIELD_BASE(36, 36, 9, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(37, 37, 2, 0x00d0, 0x10, 10, 1), + PIN_FIELD_BASE(38, 38, 2, 0x00d0, 0x10, 11, 1), + PIN_FIELD_BASE(39, 39, 1, 0x00e0, 0x10, 12, 1), + PIN_FIELD_BASE(40, 40, 2, 0x00d0, 0x10, 0, 1), + PIN_FIELD_BASE(41, 41, 2, 0x00d0, 0x10, 1, 1), + PIN_FIELD_BASE(42, 42, 2, 0x00d0, 0x10, 2, 1), + PIN_FIELD_BASE(43, 43, 2, 0x00d0, 0x10, 3, 1), + PIN_FIELD_BASE(44, 44, 2, 0x00d0, 0x10, 4, 1), + PIN_FIELD_BASE(45, 45, 2, 0x00d0, 0x10, 5, 1), + PIN_FIELD_BASE(46, 46, 1, 0x00e0, 0x10, 13, 1), + PIN_FIELD_BASE(47, 47, 1, 0x00e0, 0x10, 15, 1), + PIN_FIELD_BASE(48, 48, 2, 0x00d0, 0x10, 6, 1), + PIN_FIELD_BASE(49, 49, 2, 0x00d0, 0x10, 7, 1), + PIN_FIELD_BASE(50, 50, 2, 0x00d0, 0x10, 8, 1), + PIN_FIELD_BASE(51, 51, 2, 0x00d0, 0x10, 9, 1), + PIN_FIELD_BASE(58, 58, 1, 0x00e0, 0x10, 18, 1), + PIN_FIELD_BASE(62, 62, 9, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(63, 63, 9, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(64, 64, 9, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(65, 65, 9, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(68, 68, 9, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(74, 74, 7, 0x00d0, 0x10, 0, 1), + PIN_FIELD_BASE(75, 75, 7, 0x00d0, 0x10, 1, 1), + PIN_FIELD_BASE(76, 76, 7, 0x00d0, 0x10, 2, 1), + PIN_FIELD_BASE(77, 77, 7, 0x00d0, 0x10, 3, 1), + PIN_FIELD_BASE(78, 78, 7, 0x00d0, 0x10, 4, 1), + PIN_FIELD_BASE(79, 79, 7, 0x00d0, 0x10, 5, 1), + PIN_FIELD_BASE(80, 80, 7, 0x00d0, 0x10, 6, 1), + PIN_FIELD_BASE(81, 81, 7, 0x00d0, 0x10, 7, 1), + PIN_FIELD_BASE(82, 82, 7, 0x00d0, 0x10, 8, 1), + PIN_FIELD_BASE(83, 83, 7, 0x00d0, 0x10, 9, 1), + PIN_FIELD_BASE(84, 84, 7, 0x00d0, 0x10, 10, 1), + PIN_FIELD_BASE(85, 85, 7, 0x00d0, 0x10, 11, 1), + PIN_FIELD_BASE(86, 86, 7, 0x00d0, 0x10, 12, 1), + PIN_FIELD_BASE(87, 87, 7, 0x00d0, 0x10, 13, 1), + PIN_FIELD_BASE(90, 90, 7, 0x00d0, 0x10, 14, 1), + PIN_FIELD_BASE(91, 91, 7, 0x00d0, 0x10, 15, 1), + PIN_FIELD_BASE(94, 94, 3, 0x00e0, 0x10, 12, 1), + PIN_FIELD_BASE(95, 95, 3, 0x00e0, 0x10, 13, 1), + PIN_FIELD_BASE(96, 96, 3, 0x00e0, 0x10, 14, 1), + PIN_FIELD_BASE(97, 97, 3, 0x00e0, 0x10, 15, 1), + PIN_FIELD_BASE(98, 98, 3, 0x00e0, 0x10, 16, 1), + PIN_FIELD_BASE(99, 99, 3, 0x00e0, 0x10, 17, 1), + PIN_FIELD_BASE(100, 100, 3, 0x00e0, 0x10, 18, 1), + PIN_FIELD_BASE(101, 101, 3, 0x00e0, 0x10, 19, 1), + PIN_FIELD_BASE(102, 102, 3, 0x00e0, 0x10, 20, 1), + PIN_FIELD_BASE(103, 103, 3, 0x00e0, 0x10, 21, 1), + PIN_FIELD_BASE(104, 104, 3, 0x00e0, 0x10, 22, 1), + PIN_FIELD_BASE(105, 105, 3, 0x00e0, 0x10, 23, 1), + PIN_FIELD_BASE(106, 106, 8, 0x0090, 0x10, 8, 1), + PIN_FIELD_BASE(107, 107, 8, 0x0090, 0x10, 9, 1), + PIN_FIELD_BASE(108, 108, 8, 0x0090, 0x10, 10, 1), + PIN_FIELD_BASE(109, 109, 8, 0x0090, 0x10, 11, 1), + PIN_FIELD_BASE(110, 110, 8, 0x0090, 0x10, 12, 1), + PIN_FIELD_BASE(111, 111, 8, 0x0090, 0x10, 13, 1), + PIN_FIELD_BASE(112, 112, 5, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(113, 113, 5, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(114, 114, 5, 0x00c0, 0x10, 7, 1), + PIN_FIELD_BASE(115, 115, 5, 0x00c0, 0x10, 8, 1), + PIN_FIELD_BASE(116, 116, 5, 0x00c0, 0x10, 9, 1), + PIN_FIELD_BASE(125, 125, 4, 0x00c0, 0x10, 8, 1), + PIN_FIELD_BASE(130, 130, 4, 0x00c0, 0x10, 9, 1), + PIN_FIELD_BASE(131, 131, 4, 0x00c0, 0x10, 10, 1), + PIN_FIELD_BASE(132, 132, 4, 0x00c0, 0x10, 11, 1), + PIN_FIELD_BASE(133, 133, 4, 0x00c0, 0x10, 12, 1), + PIN_FIELD_BASE(138, 138, 3, 0x00e0, 0x10, 0, 1), + PIN_FIELD_BASE(139, 139, 3, 0x00e0, 0x10, 1, 1), + PIN_FIELD_BASE(140, 140, 3, 0x00e0, 0x10, 2, 1), + PIN_FIELD_BASE(141, 141, 3, 0x00e0, 0x10, 3, 1), + PIN_FIELD_BASE(142, 142, 3, 0x00e0, 0x10, 4, 1), + PIN_FIELD_BASE(143, 143, 3, 0x00e0, 0x10, 5, 1), + PIN_FIELD_BASE(144, 144, 3, 0x00e0, 0x10, 6, 1), + PIN_FIELD_BASE(145, 145, 3, 0x00e0, 0x10, 7, 1), + PIN_FIELD_BASE(146, 146, 3, 0x00e0, 0x10, 8, 1), + PIN_FIELD_BASE(147, 147, 3, 0x00e0, 0x10, 9, 1), + PIN_FIELD_BASE(148, 148, 3, 0x00e0, 0x10, 10, 1), + PIN_FIELD_BASE(149, 149, 3, 0x00e0, 0x10, 11, 1), + PIN_FIELD_BASE(150, 150, 4, 0x00c0, 0x10, 13, 1), + PIN_FIELD_BASE(151, 151, 4, 0x00c0, 0x10, 18, 1), + PIN_FIELD_BASE(152, 152, 4, 0x00c0, 0x10, 17, 1), + PIN_FIELD_BASE(153, 153, 4, 0x00c0, 0x10, 16, 1), + PIN_FIELD_BASE(154, 154, 4, 0x00c0, 0x10, 14, 1), + PIN_FIELD_BASE(155, 155, 4, 0x00c0, 0x10, 15, 1), + PIN_FIELD_BASE(156, 156, 4, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(157, 157, 4, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(158, 158, 4, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(159, 159, 4, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(160, 160, 4, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(161, 161, 4, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(162, 162, 4, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(163, 163, 4, 0x00c0, 0x10, 7, 1), + PIN_FIELD_BASE(164, 164, 5, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(167, 167, 5, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(168, 168, 5, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(169, 169, 5, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(170, 170, 5, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(171, 171, 6, 0x0090, 0x10, 0, 1), + PIN_FIELD_BASE(172, 172, 6, 0x0090, 0x10, 1, 1), + PIN_FIELD_BASE(173, 173, 6, 0x0090, 0x10, 2, 1), + PIN_FIELD_BASE(174, 174, 6, 0x0090, 0x10, 3, 1), + PIN_FIELD_BASE(175, 175, 6, 0x0090, 0x10, 4, 1), + PIN_FIELD_BASE(176, 176, 6, 0x0090, 0x10, 5, 1), + PIN_FIELD_BASE(177, 177, 6, 0x0090, 0x10, 6, 1), + PIN_FIELD_BASE(178, 178, 6, 0x0090, 0x10, 7, 1), + PIN_FIELD_BASE(179, 179, 6, 0x0090, 0x10, 8, 1), + PIN_FIELD_BASE(180, 180, 6, 0x0090, 0x10, 9, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_r1_range[] = { + PIN_FIELD_BASE(0, 0, 8, 0x00a0, 0x10, 0, 1), + PIN_FIELD_BASE(1, 1, 8, 0x00a0, 0x10, 1, 1), + PIN_FIELD_BASE(2, 2, 8, 0x00a0, 0x10, 2, 1), + PIN_FIELD_BASE(3, 3, 8, 0x00a0, 0x10, 3, 1), + PIN_FIELD_BASE(4, 4, 8, 0x00a0, 0x10, 4, 1), + PIN_FIELD_BASE(5, 5, 8, 0x00a0, 0x10, 5, 1), + PIN_FIELD_BASE(6, 6, 8, 0x00a0, 0x10, 6, 1), + PIN_FIELD_BASE(7, 7, 8, 0x00a0, 0x10, 7, 1), + PIN_FIELD_BASE(14, 14, 1, 0x00f0, 0x10, 14, 1), + PIN_FIELD_BASE(15, 15, 1, 0x00f0, 0x10, 16, 1), + PIN_FIELD_BASE(16, 16, 1, 0x00f0, 0x10, 17, 1), + PIN_FIELD_BASE(19, 19, 1, 0x00f0, 0x10, 21, 1), + PIN_FIELD_BASE(20, 20, 1, 0x00f0, 0x10, 22, 1), + PIN_FIELD_BASE(21, 21, 1, 0x00f0, 0x10, 20, 1), + PIN_FIELD_BASE(22, 22, 1, 0x00f0, 0x10, 19, 1), + PIN_FIELD_BASE(23, 23, 1, 0x00f0, 0x10, 0, 1), + PIN_FIELD_BASE(24, 24, 1, 0x00f0, 0x10, 1, 1), + PIN_FIELD_BASE(25, 25, 1, 0x00f0, 0x10, 2, 1), + PIN_FIELD_BASE(26, 26, 1, 0x00f0, 0x10, 3, 1), + PIN_FIELD_BASE(27, 27, 1, 0x00f0, 0x10, 4, 1), + PIN_FIELD_BASE(28, 28, 1, 0x00f0, 0x10, 5, 1), + PIN_FIELD_BASE(29, 29, 1, 0x00f0, 0x10, 6, 1), + PIN_FIELD_BASE(30, 30, 1, 0x00f0, 0x10, 7, 1), + PIN_FIELD_BASE(31, 31, 1, 0x00f0, 0x10, 8, 1), + PIN_FIELD_BASE(32, 32, 9, 0x00d0, 0x10, 0, 1), + PIN_FIELD_BASE(33, 33, 1, 0x00f0, 0x10, 9, 1), + PIN_FIELD_BASE(34, 34, 1, 0x00f0, 0x10, 10, 1), + PIN_FIELD_BASE(35, 35, 1, 0x00f0, 0x10, 11, 1), + PIN_FIELD_BASE(36, 36, 9, 0x00d0, 0x10, 6, 1), + PIN_FIELD_BASE(37, 37, 2, 0x00e0, 0x10, 10, 1), + PIN_FIELD_BASE(38, 38, 2, 0x00e0, 0x10, 11, 1), + PIN_FIELD_BASE(39, 39, 1, 0x00f0, 0x10, 12, 1), + PIN_FIELD_BASE(40, 40, 2, 0x00e0, 0x10, 0, 1), + PIN_FIELD_BASE(41, 41, 2, 0x00e0, 0x10, 1, 1), + PIN_FIELD_BASE(42, 42, 2, 0x00e0, 0x10, 2, 1), + PIN_FIELD_BASE(43, 43, 2, 0x00e0, 0x10, 3, 1), + PIN_FIELD_BASE(44, 44, 2, 0x00e0, 0x10, 4, 1), + PIN_FIELD_BASE(45, 45, 2, 0x00e0, 0x10, 5, 1), + PIN_FIELD_BASE(46, 46, 1, 0x00f0, 0x10, 13, 1), + PIN_FIELD_BASE(47, 47, 1, 0x00f0, 0x10, 15, 1), + PIN_FIELD_BASE(48, 48, 2, 0x00e0, 0x10, 6, 1), + PIN_FIELD_BASE(49, 49, 2, 0x00e0, 0x10, 7, 1), + PIN_FIELD_BASE(50, 50, 2, 0x00e0, 0x10, 8, 1), + PIN_FIELD_BASE(51, 51, 2, 0x00e0, 0x10, 9, 1), + PIN_FIELD_BASE(58, 58, 1, 0x00f0, 0x10, 18, 1), + PIN_FIELD_BASE(62, 62, 9, 0x00d0, 0x10, 1, 1), + PIN_FIELD_BASE(63, 63, 9, 0x00d0, 0x10, 2, 1), + PIN_FIELD_BASE(64, 64, 9, 0x00d0, 0x10, 3, 1), + PIN_FIELD_BASE(65, 65, 9, 0x00d0, 0x10, 4, 1), + PIN_FIELD_BASE(68, 68, 9, 0x00d0, 0x10, 5, 1), + PIN_FIELD_BASE(74, 74, 7, 0x00e0, 0x10, 0, 1), + PIN_FIELD_BASE(75, 75, 7, 0x00e0, 0x10, 1, 1), + PIN_FIELD_BASE(76, 76, 7, 0x00e0, 0x10, 2, 1), + PIN_FIELD_BASE(77, 77, 7, 0x00e0, 0x10, 3, 1), + PIN_FIELD_BASE(78, 78, 7, 0x00e0, 0x10, 4, 1), + PIN_FIELD_BASE(79, 79, 7, 0x00e0, 0x10, 5, 1), + PIN_FIELD_BASE(80, 80, 7, 0x00e0, 0x10, 6, 1), + PIN_FIELD_BASE(81, 81, 7, 0x00e0, 0x10, 7, 1), + PIN_FIELD_BASE(82, 82, 7, 0x00e0, 0x10, 8, 1), + PIN_FIELD_BASE(83, 83, 7, 0x00e0, 0x10, 9, 1), + PIN_FIELD_BASE(84, 84, 7, 0x00e0, 0x10, 10, 1), + PIN_FIELD_BASE(85, 85, 7, 0x00e0, 0x10, 11, 1), + PIN_FIELD_BASE(86, 86, 7, 0x00e0, 0x10, 12, 1), + PIN_FIELD_BASE(87, 87, 7, 0x00e0, 0x10, 13, 1), + PIN_FIELD_BASE(90, 90, 7, 0x00e0, 0x10, 14, 1), + PIN_FIELD_BASE(91, 91, 7, 0x00e0, 0x10, 15, 1), + PIN_FIELD_BASE(94, 94, 3, 0x00f0, 0x10, 12, 1), + PIN_FIELD_BASE(95, 95, 3, 0x00f0, 0x10, 13, 1), + PIN_FIELD_BASE(96, 96, 3, 0x00f0, 0x10, 14, 1), + PIN_FIELD_BASE(97, 97, 3, 0x00f0, 0x10, 15, 1), + PIN_FIELD_BASE(98, 98, 3, 0x00f0, 0x10, 16, 1), + PIN_FIELD_BASE(99, 99, 3, 0x00f0, 0x10, 17, 1), + PIN_FIELD_BASE(100, 100, 3, 0x00f0, 0x10, 18, 1), + PIN_FIELD_BASE(101, 101, 3, 0x00f0, 0x10, 19, 1), + PIN_FIELD_BASE(102, 102, 3, 0x00f0, 0x10, 20, 1), + PIN_FIELD_BASE(103, 103, 3, 0x00f0, 0x10, 21, 1), + PIN_FIELD_BASE(104, 104, 3, 0x00f0, 0x10, 22, 1), + PIN_FIELD_BASE(105, 105, 3, 0x00f0, 0x10, 23, 1), + PIN_FIELD_BASE(106, 106, 8, 0x00a0, 0x10, 8, 1), + PIN_FIELD_BASE(107, 107, 8, 0x00a0, 0x10, 9, 1), + PIN_FIELD_BASE(108, 108, 8, 0x00a0, 0x10, 10, 1), + PIN_FIELD_BASE(109, 109, 8, 0x00a0, 0x10, 11, 1), + PIN_FIELD_BASE(110, 110, 8, 0x00a0, 0x10, 12, 1), + PIN_FIELD_BASE(111, 111, 8, 0x00a0, 0x10, 13, 1), + PIN_FIELD_BASE(112, 112, 5, 0x00d0, 0x10, 5, 1), + PIN_FIELD_BASE(113, 113, 5, 0x00d0, 0x10, 6, 1), + PIN_FIELD_BASE(114, 114, 5, 0x00d0, 0x10, 7, 1), + PIN_FIELD_BASE(115, 115, 5, 0x00d0, 0x10, 8, 1), + PIN_FIELD_BASE(116, 116, 5, 0x00d0, 0x10, 9, 1), + PIN_FIELD_BASE(125, 125, 4, 0x00d0, 0x10, 8, 1), + PIN_FIELD_BASE(130, 130, 4, 0x00d0, 0x10, 9, 1), + PIN_FIELD_BASE(131, 131, 4, 0x00d0, 0x10, 10, 1), + PIN_FIELD_BASE(132, 132, 4, 0x00d0, 0x10, 11, 1), + PIN_FIELD_BASE(133, 133, 4, 0x00d0, 0x10, 12, 1), + PIN_FIELD_BASE(138, 138, 3, 0x00f0, 0x10, 0, 1), + PIN_FIELD_BASE(139, 139, 3, 0x00f0, 0x10, 1, 1), + PIN_FIELD_BASE(140, 140, 3, 0x00f0, 0x10, 2, 1), + PIN_FIELD_BASE(141, 141, 3, 0x00f0, 0x10, 3, 1), + PIN_FIELD_BASE(142, 142, 3, 0x00f0, 0x10, 4, 1), + PIN_FIELD_BASE(143, 143, 3, 0x00f0, 0x10, 5, 1), + PIN_FIELD_BASE(144, 144, 3, 0x00f0, 0x10, 6, 1), + PIN_FIELD_BASE(145, 145, 3, 0x00f0, 0x10, 7, 1), + PIN_FIELD_BASE(146, 146, 3, 0x00f0, 0x10, 8, 1), + PIN_FIELD_BASE(147, 147, 3, 0x00f0, 0x10, 9, 1), + PIN_FIELD_BASE(148, 148, 3, 0x00f0, 0x10, 10, 1), + PIN_FIELD_BASE(149, 149, 3, 0x00f0, 0x10, 11, 1), + PIN_FIELD_BASE(150, 150, 4, 0x00d0, 0x10, 13, 1), + PIN_FIELD_BASE(151, 151, 4, 0x00d0, 0x10, 18, 1), + PIN_FIELD_BASE(152, 152, 4, 0x00d0, 0x10, 17, 1), + PIN_FIELD_BASE(153, 153, 4, 0x00d0, 0x10, 16, 1), + PIN_FIELD_BASE(154, 154, 4, 0x00d0, 0x10, 14, 1), + PIN_FIELD_BASE(155, 155, 4, 0x00d0, 0x10, 15, 1), + PIN_FIELD_BASE(156, 156, 4, 0x00d0, 0x10, 0, 1), + PIN_FIELD_BASE(157, 157, 4, 0x00d0, 0x10, 1, 1), + PIN_FIELD_BASE(158, 158, 4, 0x00d0, 0x10, 2, 1), + PIN_FIELD_BASE(159, 159, 4, 0x00d0, 0x10, 3, 1), + PIN_FIELD_BASE(160, 160, 4, 0x00d0, 0x10, 4, 1), + PIN_FIELD_BASE(161, 161, 4, 0x00d0, 0x10, 5, 1), + PIN_FIELD_BASE(162, 162, 4, 0x00d0, 0x10, 6, 1), + PIN_FIELD_BASE(163, 163, 4, 0x00d0, 0x10, 7, 1), + PIN_FIELD_BASE(164, 164, 5, 0x00d0, 0x10, 2, 1), + PIN_FIELD_BASE(167, 167, 5, 0x00d0, 0x10, 0, 1), + PIN_FIELD_BASE(168, 168, 5, 0x00d0, 0x10, 3, 1), + PIN_FIELD_BASE(169, 169, 5, 0x00d0, 0x10, 4, 1), + PIN_FIELD_BASE(170, 170, 5, 0x00d0, 0x10, 1, 1), + PIN_FIELD_BASE(171, 171, 6, 0x00a0, 0x10, 0, 1), + PIN_FIELD_BASE(172, 172, 6, 0x00a0, 0x10, 1, 1), + PIN_FIELD_BASE(173, 173, 6, 0x00a0, 0x10, 2, 1), + PIN_FIELD_BASE(174, 174, 6, 0x00a0, 0x10, 3, 1), + PIN_FIELD_BASE(175, 175, 6, 0x00a0, 0x10, 4, 1), + PIN_FIELD_BASE(176, 176, 6, 0x00a0, 0x10, 5, 1), + PIN_FIELD_BASE(177, 177, 6, 0x00a0, 0x10, 6, 1), + PIN_FIELD_BASE(178, 178, 6, 0x00a0, 0x10, 7, 1), + PIN_FIELD_BASE(179, 179, 6, 0x00a0, 0x10, 8, 1), + PIN_FIELD_BASE(180, 180, 6, 0x00a0, 0x10, 9, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_pu_range[] = { + PIN_FIELD_BASE(8, 8, 2, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(9, 9, 2, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(10, 10, 2, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(11, 11, 2, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(12, 12, 1, 0x00d0, 0x10, 0, 1), + PIN_FIELD_BASE(13, 13, 1, 0x00d0, 0x10, 1, 1), + PIN_FIELD_BASE(17, 17, 1, 0x00d0, 0x10, 2, 1), + PIN_FIELD_BASE(18, 18, 1, 0x00d0, 0x10, 3, 1), + PIN_FIELD_BASE(52, 52, 2, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(53, 53, 2, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(54, 54, 5, 0x00b0, 0x10, 8, 1), + PIN_FIELD_BASE(55, 55, 5, 0x00b0, 0x10, 9, 1), + PIN_FIELD_BASE(56, 56, 1, 0x00d0, 0x10, 4, 1), + PIN_FIELD_BASE(57, 57, 1, 0x00d0, 0x10, 5, 1), + PIN_FIELD_BASE(59, 59, 2, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(60, 60, 9, 0x00b0, 0x10, 0, 1), + PIN_FIELD_BASE(61, 61, 9, 0x00b0, 0x10, 1, 1), + PIN_FIELD_BASE(66, 66, 5, 0x00b0, 0x10, 0, 1), + PIN_FIELD_BASE(67, 67, 5, 0x00b0, 0x10, 1, 1), + PIN_FIELD_BASE(69, 69, 7, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(70, 70, 7, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(71, 71, 7, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(72, 72, 7, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(73, 73, 7, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(88, 88, 7, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(89, 89, 7, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(92, 92, 3, 0x00d0, 0x10, 4, 1), + PIN_FIELD_BASE(93, 93, 3, 0x00d0, 0x10, 5, 1), + PIN_FIELD_BASE(117, 117, 4, 0x00b0, 0x10, 0, 1), + PIN_FIELD_BASE(118, 118, 4, 0x00b0, 0x10, 1, 1), + PIN_FIELD_BASE(119, 119, 4, 0x00b0, 0x10, 2, 1), + PIN_FIELD_BASE(120, 120, 4, 0x00b0, 0x10, 3, 1), + PIN_FIELD_BASE(121, 121, 4, 0x00b0, 0x10, 4, 1), + PIN_FIELD_BASE(122, 122, 4, 0x00b0, 0x10, 5, 1), + PIN_FIELD_BASE(123, 123, 4, 0x00b0, 0x10, 6, 1), + PIN_FIELD_BASE(124, 124, 4, 0x00b0, 0x10, 7, 1), + PIN_FIELD_BASE(126, 126, 5, 0x00b0, 0x10, 4, 1), + PIN_FIELD_BASE(127, 127, 5, 0x00b0, 0x10, 5, 1), + PIN_FIELD_BASE(128, 128, 5, 0x00b0, 0x10, 6, 1), + PIN_FIELD_BASE(129, 129, 5, 0x00b0, 0x10, 7, 1), + PIN_FIELD_BASE(134, 134, 3, 0x00d0, 0x10, 0, 1), + PIN_FIELD_BASE(135, 135, 3, 0x00d0, 0x10, 1, 1), + PIN_FIELD_BASE(136, 136, 3, 0x00d0, 0x10, 2, 1), + PIN_FIELD_BASE(137, 137, 3, 0x00d0, 0x10, 3, 1), + PIN_FIELD_BASE(165, 165, 5, 0x00b0, 0x10, 2, 1), + PIN_FIELD_BASE(166, 166, 5, 0x00b0, 0x10, 3, 1), + PIN_FIELD_BASE(181, 181, 10, 0x0060, 0x10, 0, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_pd_range[] = { + PIN_FIELD_BASE(8, 8, 2, 0x00a0, 0x10, 3, 1), + PIN_FIELD_BASE(9, 9, 2, 0x00a0, 0x10, 4, 1), + PIN_FIELD_BASE(10, 10, 2, 0x00a0, 0x10, 5, 1), + PIN_FIELD_BASE(11, 11, 2, 0x00a0, 0x10, 6, 1), + PIN_FIELD_BASE(12, 12, 1, 0x00b0, 0x10, 0, 1), + PIN_FIELD_BASE(13, 13, 1, 0x00b0, 0x10, 1, 1), + PIN_FIELD_BASE(17, 17, 1, 0x00b0, 0x10, 2, 1), + PIN_FIELD_BASE(18, 18, 1, 0x00b0, 0x10, 3, 1), + PIN_FIELD_BASE(52, 52, 2, 0x00a0, 0x10, 1, 1), + PIN_FIELD_BASE(53, 53, 2, 0x00a0, 0x10, 2, 1), + PIN_FIELD_BASE(54, 54, 5, 0x0090, 0x10, 8, 1), + PIN_FIELD_BASE(55, 55, 5, 0x0090, 0x10, 9, 1), + PIN_FIELD_BASE(56, 56, 1, 0x00b0, 0x10, 4, 1), + PIN_FIELD_BASE(57, 57, 1, 0x00b0, 0x10, 5, 1), + PIN_FIELD_BASE(59, 59, 2, 0x00a0, 0x10, 0, 1), + PIN_FIELD_BASE(60, 60, 9, 0x0090, 0x10, 0, 1), + PIN_FIELD_BASE(61, 61, 9, 0x0090, 0x10, 1, 1), + PIN_FIELD_BASE(66, 66, 5, 0x0090, 0x10, 0, 1), + PIN_FIELD_BASE(67, 67, 5, 0x0090, 0x10, 1, 1), + PIN_FIELD_BASE(69, 69, 7, 0x00a0, 0x10, 0, 1), + PIN_FIELD_BASE(70, 70, 7, 0x00a0, 0x10, 1, 1), + PIN_FIELD_BASE(71, 71, 7, 0x00a0, 0x10, 2, 1), + PIN_FIELD_BASE(72, 72, 7, 0x00a0, 0x10, 3, 1), + PIN_FIELD_BASE(73, 73, 7, 0x00a0, 0x10, 4, 1), + PIN_FIELD_BASE(88, 88, 7, 0x00a0, 0x10, 5, 1), + PIN_FIELD_BASE(89, 89, 7, 0x00a0, 0x10, 6, 1), + PIN_FIELD_BASE(92, 92, 3, 0x00b0, 0x10, 4, 1), + PIN_FIELD_BASE(93, 93, 3, 0x00b0, 0x10, 5, 1), + PIN_FIELD_BASE(117, 117, 4, 0x0090, 0x10, 0, 1), + PIN_FIELD_BASE(118, 118, 4, 0x0090, 0x10, 1, 1), + PIN_FIELD_BASE(119, 119, 4, 0x0090, 0x10, 2, 1), + PIN_FIELD_BASE(120, 120, 4, 0x0090, 0x10, 3, 1), + PIN_FIELD_BASE(121, 121, 4, 0x0090, 0x10, 4, 1), + PIN_FIELD_BASE(122, 122, 4, 0x0090, 0x10, 5, 1), + PIN_FIELD_BASE(123, 123, 4, 0x0090, 0x10, 6, 1), + PIN_FIELD_BASE(124, 124, 4, 0x0090, 0x10, 7, 1), + PIN_FIELD_BASE(126, 126, 5, 0x0090, 0x10, 4, 1), + PIN_FIELD_BASE(127, 127, 5, 0x0090, 0x10, 5, 1), + PIN_FIELD_BASE(128, 128, 5, 0x0090, 0x10, 6, 1), + PIN_FIELD_BASE(129, 129, 5, 0x0090, 0x10, 7, 1), + PIN_FIELD_BASE(134, 134, 3, 0x00b0, 0x10, 0, 1), + PIN_FIELD_BASE(135, 135, 3, 0x00b0, 0x10, 1, 1), + PIN_FIELD_BASE(136, 136, 3, 0x00b0, 0x10, 2, 1), + PIN_FIELD_BASE(137, 137, 3, 0x00b0, 0x10, 3, 1), + PIN_FIELD_BASE(165, 165, 5, 0x0090, 0x10, 2, 1), + PIN_FIELD_BASE(166, 166, 5, 0x0090, 0x10, 3, 1), + PIN_FIELD_BASE(181, 181, 10, 0x0050, 0x10, 0, 1), +}; + +static const struct mtk_pin_field_calc mt8901_pin_drv_range[] = { + PIN_FIELD_BASE(0, 0, 8, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(1, 1, 8, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(2, 2, 8, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(3, 3, 8, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(4, 4, 8, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(5, 5, 8, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(6, 6, 8, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(7, 7, 8, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(8, 8, 2, 0x0010, 0x10, 9, 3), + PIN_FIELD_BASE(9, 9, 2, 0x0010, 0x10, 12, 3), + PIN_FIELD_BASE(10, 10, 2, 0x0010, 0x10, 15, 3), + PIN_FIELD_BASE(11, 11, 2, 0x0010, 0x10, 18, 3), + PIN_FIELD_BASE(12, 12, 1, 0x0010, 0x10, 21, 3), + PIN_FIELD_BASE(13, 13, 1, 0x0010, 0x10, 24, 3), + PIN_FIELD_BASE(14, 14, 1, 0x0010, 0x10, 9, 3), + PIN_FIELD_BASE(15, 15, 1, 0x0010, 0x10, 15, 3), + PIN_FIELD_BASE(16, 16, 1, 0x0010, 0x10, 27, 3), + PIN_FIELD_BASE(17, 17, 1, 0x0020, 0x10, 0, 3), + PIN_FIELD_BASE(18, 18, 1, 0x0020, 0x10, 3, 3), + PIN_FIELD_BASE(19, 19, 1, 0x0020, 0x10, 21, 3), + PIN_FIELD_BASE(20, 20, 1, 0x0020, 0x10, 24, 3), + PIN_FIELD_BASE(21, 21, 1, 0x0020, 0x10, 18, 3), + PIN_FIELD_BASE(22, 22, 1, 0x0020, 0x10, 15, 3), + PIN_FIELD_BASE(23, 23, 1, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(24, 24, 1, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(25, 25, 1, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(26, 26, 1, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(27, 27, 1, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(28, 28, 1, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(29, 29, 1, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(30, 30, 1, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(31, 31, 1, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(32, 32, 9, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(33, 33, 1, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(34, 34, 1, 0x0010, 0x10, 0, 3), + PIN_FIELD_BASE(35, 35, 1, 0x0010, 0x10, 3, 3), + PIN_FIELD_BASE(36, 36, 9, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(37, 37, 2, 0x0010, 0x10, 21, 3), + PIN_FIELD_BASE(38, 38, 2, 0x0010, 0x10, 24, 3), + PIN_FIELD_BASE(39, 39, 1, 0x0010, 0x10, 6, 3), + PIN_FIELD_BASE(40, 40, 2, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(41, 41, 2, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(42, 42, 2, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(43, 43, 2, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(44, 44, 2, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(45, 45, 2, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(46, 46, 1, 0x0010, 0x10, 12, 3), + PIN_FIELD_BASE(47, 47, 1, 0x0010, 0x10, 18, 3), + PIN_FIELD_BASE(48, 48, 2, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(49, 49, 2, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(50, 50, 2, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(51, 51, 2, 0x0010, 0x10, 0, 3), + PIN_FIELD_BASE(52, 52, 2, 0x0010, 0x10, 3, 3), + PIN_FIELD_BASE(53, 53, 2, 0x0010, 0x10, 6, 3), + PIN_FIELD_BASE(54, 54, 5, 0x0010, 0x10, 0, 3), + PIN_FIELD_BASE(55, 55, 5, 0x0010, 0x10, 3, 3), + PIN_FIELD_BASE(56, 56, 1, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(57, 57, 1, 0x0020, 0x10, 9, 3), + PIN_FIELD_BASE(58, 58, 1, 0x0020, 0x10, 12, 3), + PIN_FIELD_BASE(59, 59, 2, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(60, 60, 9, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(61, 61, 9, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(62, 62, 9, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(63, 63, 9, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(64, 64, 9, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(65, 65, 9, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(66, 66, 5, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(67, 67, 5, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(68, 68, 9, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(69, 69, 7, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(70, 70, 7, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(71, 71, 7, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(72, 72, 7, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(73, 73, 7, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(74, 74, 7, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(75, 75, 7, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(76, 76, 7, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(77, 77, 7, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(78, 78, 7, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(79, 79, 7, 0x0010, 0x10, 0, 3), + PIN_FIELD_BASE(80, 80, 7, 0x0010, 0x10, 3, 3), + PIN_FIELD_BASE(81, 81, 7, 0x0010, 0x10, 6, 3), + PIN_FIELD_BASE(82, 82, 7, 0x0010, 0x10, 9, 3), + PIN_FIELD_BASE(83, 83, 7, 0x0010, 0x10, 12, 3), + PIN_FIELD_BASE(84, 84, 7, 0x0010, 0x10, 15, 3), + PIN_FIELD_BASE(85, 85, 7, 0x0010, 0x10, 18, 3), + PIN_FIELD_BASE(86, 86, 7, 0x0010, 0x10, 21, 3), + PIN_FIELD_BASE(87, 87, 7, 0x0010, 0x10, 24, 3), + PIN_FIELD_BASE(88, 88, 7, 0x0010, 0x10, 27, 3), + PIN_FIELD_BASE(89, 89, 7, 0x0020, 0x10, 0, 3), + PIN_FIELD_BASE(90, 90, 7, 0x0020, 0x10, 3, 3), + PIN_FIELD_BASE(91, 91, 7, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(92, 92, 3, 0x0010, 0x10, 18, 3), + PIN_FIELD_BASE(93, 93, 3, 0x0010, 0x10, 21, 3), + PIN_FIELD_BASE(94, 94, 3, 0x0010, 0x10, 24, 3), + PIN_FIELD_BASE(95, 95, 3, 0x0010, 0x10, 27, 3), + PIN_FIELD_BASE(96, 96, 3, 0x0020, 0x10, 0, 3), + PIN_FIELD_BASE(97, 97, 3, 0x0020, 0x10, 3, 3), + PIN_FIELD_BASE(98, 98, 3, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(99, 99, 3, 0x0020, 0x10, 9, 3), + PIN_FIELD_BASE(100, 100, 3, 0x0020, 0x10, 12, 3), + PIN_FIELD_BASE(101, 101, 3, 0x0020, 0x10, 15, 3), + PIN_FIELD_BASE(102, 102, 3, 0x0020, 0x10, 18, 3), + PIN_FIELD_BASE(103, 103, 3, 0x0020, 0x10, 21, 3), + PIN_FIELD_BASE(104, 104, 3, 0x0020, 0x10, 24, 3), + PIN_FIELD_BASE(105, 105, 3, 0x0020, 0x10, 27, 3), + PIN_FIELD_BASE(106, 106, 8, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(107, 107, 8, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(108, 108, 8, 0x0010, 0x10, 0, 3), + PIN_FIELD_BASE(109, 109, 8, 0x0010, 0x10, 3, 3), + PIN_FIELD_BASE(110, 110, 8, 0x0010, 0x10, 6, 3), + PIN_FIELD_BASE(111, 111, 8, 0x0010, 0x10, 9, 3), + PIN_FIELD_BASE(112, 112, 5, 0x0010, 0x10, 15, 3), + PIN_FIELD_BASE(113, 113, 5, 0x0010, 0x10, 18, 3), + PIN_FIELD_BASE(114, 114, 5, 0x0010, 0x10, 21, 3), + PIN_FIELD_BASE(115, 115, 5, 0x0010, 0x10, 24, 3), + PIN_FIELD_BASE(116, 116, 5, 0x0010, 0x10, 27, 3), + PIN_FIELD_BASE(117, 117, 4, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(118, 118, 4, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(119, 119, 4, 0x0010, 0x10, 0, 3), + PIN_FIELD_BASE(120, 120, 4, 0x0010, 0x10, 3, 3), + PIN_FIELD_BASE(121, 121, 4, 0x0010, 0x10, 6, 3), + PIN_FIELD_BASE(122, 122, 4, 0x0010, 0x10, 9, 3), + PIN_FIELD_BASE(123, 123, 4, 0x0010, 0x10, 12, 3), + PIN_FIELD_BASE(124, 124, 4, 0x0010, 0x10, 15, 3), + PIN_FIELD_BASE(125, 125, 4, 0x0010, 0x10, 18, 3), + PIN_FIELD_BASE(126, 126, 5, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(127, 127, 5, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(128, 128, 5, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(129, 129, 5, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(130, 130, 4, 0x0010, 0x10, 21, 3), + PIN_FIELD_BASE(131, 131, 4, 0x0010, 0x10, 24, 3), + PIN_FIELD_BASE(132, 132, 4, 0x0010, 0x10, 27, 3), + PIN_FIELD_BASE(133, 133, 4, 0x0020, 0x10, 0, 3), + PIN_FIELD_BASE(134, 134, 3, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(135, 135, 3, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(136, 136, 3, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(137, 137, 3, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(138, 138, 3, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(139, 139, 3, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(140, 140, 3, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(141, 141, 3, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(142, 142, 3, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(143, 143, 3, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(144, 144, 3, 0x0010, 0x10, 0, 3), + PIN_FIELD_BASE(145, 145, 3, 0x0010, 0x10, 3, 3), + PIN_FIELD_BASE(146, 146, 3, 0x0010, 0x10, 6, 3), + PIN_FIELD_BASE(147, 147, 3, 0x0010, 0x10, 9, 3), + PIN_FIELD_BASE(148, 148, 3, 0x0010, 0x10, 12, 3), + PIN_FIELD_BASE(149, 149, 3, 0x0010, 0x10, 15, 3), + PIN_FIELD_BASE(150, 150, 4, 0x0020, 0x10, 3, 3), + PIN_FIELD_BASE(151, 151, 4, 0x0020, 0x10, 18, 3), + PIN_FIELD_BASE(152, 152, 4, 0x0020, 0x10, 15, 3), + PIN_FIELD_BASE(153, 153, 4, 0x0020, 0x10, 12, 3), + PIN_FIELD_BASE(154, 154, 4, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(155, 155, 4, 0x0020, 0x10, 9, 3), + PIN_FIELD_BASE(156, 156, 4, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(157, 157, 4, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(158, 158, 4, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(159, 159, 4, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(160, 160, 4, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(161, 161, 4, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(162, 162, 4, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(163, 163, 4, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(164, 164, 5, 0x0010, 0x10, 6, 3), + PIN_FIELD_BASE(165, 165, 5, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(166, 166, 5, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(167, 167, 5, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(168, 168, 5, 0x0010, 0x10, 9, 3), + PIN_FIELD_BASE(169, 169, 5, 0x0010, 0x10, 12, 3), + PIN_FIELD_BASE(170, 170, 5, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(171, 171, 6, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(172, 172, 6, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(173, 173, 6, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(174, 174, 6, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(175, 175, 6, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(176, 176, 6, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(177, 177, 6, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(178, 178, 6, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(179, 179, 6, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(180, 180, 6, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(181, 181, 10, 0x0000, 0x10, 0, 3), +}; + +static const struct mtk_pin_field_calc mt8901_pin_drv_adv_range[] = { + PIN_FIELD_BASE(8, 8, 2, 0x0030, 0x10, 6, 3), + PIN_FIELD_BASE(9, 9, 2, 0x0030, 0x10, 9, 3), + PIN_FIELD_BASE(10, 10, 2, 0x0030, 0x10, 12, 3), + PIN_FIELD_BASE(11, 11, 2, 0x0030, 0x10, 15, 3), + PIN_FIELD_BASE(12, 12, 1, 0x0040, 0x10, 0, 3), + PIN_FIELD_BASE(13, 13, 1, 0x0040, 0x10, 3, 3), + PIN_FIELD_BASE(17, 17, 1, 0x0040, 0x10, 6, 3), + PIN_FIELD_BASE(18, 18, 1, 0x0040, 0x10, 9, 3), + PIN_FIELD_BASE(52, 52, 2, 0x0030, 0x10, 0, 3), + PIN_FIELD_BASE(53, 53, 2, 0x0030, 0x10, 3, 3), + PIN_FIELD_BASE(54, 54, 5, 0x0030, 0x10, 24, 3), + PIN_FIELD_BASE(55, 55, 5, 0x0030, 0x10, 27, 3), + PIN_FIELD_BASE(56, 56, 1, 0x0040, 0x10, 12, 3), + PIN_FIELD_BASE(57, 57, 1, 0x0040, 0x10, 15, 3), + PIN_FIELD_BASE(60, 60, 9, 0x0020, 0x10, 0, 3), + PIN_FIELD_BASE(61, 61, 9, 0x0020, 0x10, 3, 3), + PIN_FIELD_BASE(66, 66, 5, 0x0030, 0x10, 0, 3), + PIN_FIELD_BASE(67, 67, 5, 0x0030, 0x10, 3, 3), + PIN_FIELD_BASE(70, 70, 7, 0x0030, 0x10, 0, 3), + PIN_FIELD_BASE(71, 71, 7, 0x0030, 0x10, 3, 3), + PIN_FIELD_BASE(72, 72, 7, 0x0030, 0x10, 6, 3), + PIN_FIELD_BASE(73, 73, 7, 0x0030, 0x10, 9, 3), + PIN_FIELD_BASE(88, 88, 7, 0x0030, 0x10, 12, 3), + PIN_FIELD_BASE(89, 89, 7, 0x0030, 0x10, 15, 3), + PIN_FIELD_BASE(92, 92, 3, 0x0040, 0x10, 12, 3), + PIN_FIELD_BASE(93, 93, 3, 0x0040, 0x10, 15, 3), + PIN_FIELD_BASE(117, 117, 4, 0x0030, 0x10, 0, 3), + PIN_FIELD_BASE(118, 118, 4, 0x0030, 0x10, 3, 3), + PIN_FIELD_BASE(119, 119, 4, 0x0030, 0x10, 6, 3), + PIN_FIELD_BASE(120, 120, 4, 0x0030, 0x10, 9, 3), + PIN_FIELD_BASE(121, 121, 4, 0x0030, 0x10, 12, 3), + PIN_FIELD_BASE(122, 122, 4, 0x0030, 0x10, 15, 3), + PIN_FIELD_BASE(123, 123, 4, 0x0030, 0x10, 18, 3), + PIN_FIELD_BASE(124, 124, 4, 0x0030, 0x10, 21, 3), + PIN_FIELD_BASE(126, 126, 5, 0x0030, 0x10, 12, 3), + PIN_FIELD_BASE(127, 127, 5, 0x0030, 0x10, 15, 3), + PIN_FIELD_BASE(128, 128, 5, 0x0030, 0x10, 18, 3), + PIN_FIELD_BASE(129, 129, 5, 0x0030, 0x10, 21, 3), + PIN_FIELD_BASE(134, 134, 3, 0x0040, 0x10, 0, 3), + PIN_FIELD_BASE(135, 135, 3, 0x0040, 0x10, 3, 3), + PIN_FIELD_BASE(136, 136, 3, 0x0040, 0x10, 6, 3), + PIN_FIELD_BASE(137, 137, 3, 0x0040, 0x10, 9, 3), + PIN_FIELD_BASE(165, 165, 5, 0x0030, 0x10, 6, 3), + PIN_FIELD_BASE(166, 166, 5, 0x0030, 0x10, 9, 3), +}; + +static const struct mtk_pin_field_calc mt8901_pin_rsel_range[] = { + PIN_FIELD_BASE(8, 8, 2, 0x0110, 0x10, 6, 3), + PIN_FIELD_BASE(9, 9, 2, 0x0110, 0x10, 9, 3), + PIN_FIELD_BASE(10, 10, 2, 0x0110, 0x10, 12, 3), + PIN_FIELD_BASE(11, 11, 2, 0x0110, 0x10, 15, 3), + PIN_FIELD_BASE(12, 12, 1, 0x0130, 0x10, 0, 1), + PIN_FIELD_BASE(13, 13, 1, 0x0130, 0x10, 1, 1), + PIN_FIELD_BASE(17, 17, 1, 0x0130, 0x10, 2, 1), + PIN_FIELD_BASE(18, 18, 1, 0x0130, 0x10, 3, 1), + PIN_FIELD_BASE(52, 52, 2, 0x0110, 0x10, 0, 3), + PIN_FIELD_BASE(53, 53, 2, 0x0110, 0x10, 3, 3), + PIN_FIELD_BASE(54, 54, 5, 0x0110, 0x10, 12, 3), + PIN_FIELD_BASE(55, 55, 5, 0x0110, 0x10, 15, 3), + PIN_FIELD_BASE(56, 56, 1, 0x0130, 0x10, 4, 1), + PIN_FIELD_BASE(57, 57, 1, 0x0130, 0x10, 5, 1), + PIN_FIELD_BASE(60, 60, 9, 0x00f0, 0x10, 0, 3), + PIN_FIELD_BASE(61, 61, 9, 0x00f0, 0x10, 3, 3), + PIN_FIELD_BASE(66, 66, 5, 0x0110, 0x10, 0, 1), + PIN_FIELD_BASE(67, 67, 5, 0x0110, 0x10, 1, 1), + PIN_FIELD_BASE(70, 70, 7, 0x0100, 0x10, 0, 3), + PIN_FIELD_BASE(71, 71, 7, 0x0100, 0x10, 3, 3), + PIN_FIELD_BASE(72, 72, 7, 0x0100, 0x10, 6, 3), + PIN_FIELD_BASE(73, 73, 7, 0x0100, 0x10, 9, 3), + PIN_FIELD_BASE(88, 88, 7, 0x0100, 0x10, 12, 3), + PIN_FIELD_BASE(89, 89, 7, 0x0100, 0x10, 15, 3), + PIN_FIELD_BASE(92, 92, 3, 0x0120, 0x10, 12, 3), + PIN_FIELD_BASE(93, 93, 3, 0x0120, 0x10, 15, 3), + PIN_FIELD_BASE(117, 117, 4, 0x0100, 0x10, 0, 3), + PIN_FIELD_BASE(118, 118, 4, 0x0100, 0x10, 3, 3), + PIN_FIELD_BASE(119, 119, 4, 0x0100, 0x10, 6, 3), + PIN_FIELD_BASE(120, 120, 4, 0x0100, 0x10, 9, 3), + PIN_FIELD_BASE(121, 121, 4, 0x0100, 0x10, 12, 3), + PIN_FIELD_BASE(122, 122, 4, 0x0100, 0x10, 15, 3), + PIN_FIELD_BASE(123, 123, 4, 0x0100, 0x10, 18, 3), + PIN_FIELD_BASE(124, 124, 4, 0x0100, 0x10, 21, 3), + PIN_FIELD_BASE(126, 126, 5, 0x0110, 0x10, 8, 1), + PIN_FIELD_BASE(127, 127, 5, 0x0110, 0x10, 9, 1), + PIN_FIELD_BASE(128, 128, 5, 0x0110, 0x10, 10, 1), + PIN_FIELD_BASE(129, 129, 5, 0x0110, 0x10, 11, 1), + PIN_FIELD_BASE(134, 134, 3, 0x0120, 0x10, 0, 3), + PIN_FIELD_BASE(135, 135, 3, 0x0120, 0x10, 3, 3), + PIN_FIELD_BASE(136, 136, 3, 0x0120, 0x10, 6, 3), + PIN_FIELD_BASE(137, 137, 3, 0x0120, 0x10, 9, 3), + PIN_FIELD_BASE(165, 165, 5, 0x0110, 0x10, 2, 3), + PIN_FIELD_BASE(166, 166, 5, 0x0110, 0x10, 5, 3), +}; + +static const struct mtk_pin_rsel mt8901_pin_rsel_val_range[] = { + 0 +}; + +static const unsigned int mt8901_pull_type[] = { + MTK_PULL_PUPD_R1R0_TYPE, /*0*/ + MTK_PULL_PUPD_R1R0_TYPE, /*1*/ + MTK_PULL_PUPD_R1R0_TYPE, /*2*/ + MTK_PULL_PUPD_R1R0_TYPE, /*3*/ + MTK_PULL_PUPD_R1R0_TYPE, /*4*/ + MTK_PULL_PUPD_R1R0_TYPE, /*5*/ + MTK_PULL_PUPD_R1R0_TYPE, /*6*/ + MTK_PULL_PUPD_R1R0_TYPE, /*7*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*8*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*9*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*10*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*11*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*12*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*13*/ + MTK_PULL_PUPD_R1R0_TYPE, /*14*/ + MTK_PULL_PUPD_R1R0_TYPE, /*15*/ + MTK_PULL_PUPD_R1R0_TYPE, /*16*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*17*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*18*/ + MTK_PULL_PUPD_R1R0_TYPE, /*19*/ + MTK_PULL_PUPD_R1R0_TYPE, /*20*/ + MTK_PULL_PUPD_R1R0_TYPE, /*21*/ + MTK_PULL_PUPD_R1R0_TYPE, /*22*/ + MTK_PULL_PUPD_R1R0_TYPE, /*23*/ + MTK_PULL_PUPD_R1R0_TYPE, /*24*/ + MTK_PULL_PUPD_R1R0_TYPE, /*25*/ + MTK_PULL_PUPD_R1R0_TYPE, /*26*/ + MTK_PULL_PUPD_R1R0_TYPE, /*27*/ + MTK_PULL_PUPD_R1R0_TYPE, /*28*/ + MTK_PULL_PUPD_R1R0_TYPE, /*29*/ + MTK_PULL_PUPD_R1R0_TYPE, /*30*/ + MTK_PULL_PUPD_R1R0_TYPE, /*31*/ + MTK_PULL_PUPD_R1R0_TYPE, /*32*/ + MTK_PULL_PUPD_R1R0_TYPE, /*33*/ + MTK_PULL_PUPD_R1R0_TYPE, /*34*/ + MTK_PULL_PUPD_R1R0_TYPE, /*35*/ + MTK_PULL_PUPD_R1R0_TYPE, /*36*/ + MTK_PULL_PUPD_R1R0_TYPE, /*37*/ + MTK_PULL_PUPD_R1R0_TYPE, /*38*/ + MTK_PULL_PUPD_R1R0_TYPE, /*39*/ + MTK_PULL_PUPD_R1R0_TYPE, /*40*/ + MTK_PULL_PUPD_R1R0_TYPE, /*41*/ + MTK_PULL_PUPD_R1R0_TYPE, /*42*/ + MTK_PULL_PUPD_R1R0_TYPE, /*43*/ + MTK_PULL_PUPD_R1R0_TYPE, /*44*/ + MTK_PULL_PUPD_R1R0_TYPE, /*45*/ + MTK_PULL_PUPD_R1R0_TYPE, /*46*/ + MTK_PULL_PUPD_R1R0_TYPE, /*47*/ + MTK_PULL_PUPD_R1R0_TYPE, /*48*/ + MTK_PULL_PUPD_R1R0_TYPE, /*49*/ + MTK_PULL_PUPD_R1R0_TYPE, /*50*/ + MTK_PULL_PUPD_R1R0_TYPE, /*51*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*52*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*53*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*54*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*55*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*56*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*57*/ + MTK_PULL_PUPD_R1R0_TYPE, /*58*/ + MTK_PULL_PU_PD_TYPE, /*59*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*60*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*61*/ + MTK_PULL_PUPD_R1R0_TYPE, /*62*/ + MTK_PULL_PUPD_R1R0_TYPE, /*63*/ + MTK_PULL_PUPD_R1R0_TYPE, /*64*/ + MTK_PULL_PUPD_R1R0_TYPE, /*65*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*66*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*67*/ + MTK_PULL_PUPD_R1R0_TYPE, /*68*/ + MTK_PULL_PU_PD_TYPE, /*69*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*70*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*71*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*72*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*73*/ + MTK_PULL_PUPD_R1R0_TYPE, /*74*/ + MTK_PULL_PUPD_R1R0_TYPE, /*75*/ + MTK_PULL_PUPD_R1R0_TYPE, /*76*/ + MTK_PULL_PUPD_R1R0_TYPE, /*77*/ + MTK_PULL_PUPD_R1R0_TYPE, /*78*/ + MTK_PULL_PUPD_R1R0_TYPE, /*79*/ + MTK_PULL_PUPD_R1R0_TYPE, /*80*/ + MTK_PULL_PUPD_R1R0_TYPE, /*81*/ + MTK_PULL_PUPD_R1R0_TYPE, /*82*/ + MTK_PULL_PUPD_R1R0_TYPE, /*83*/ + MTK_PULL_PUPD_R1R0_TYPE, /*84*/ + MTK_PULL_PUPD_R1R0_TYPE, /*85*/ + MTK_PULL_PUPD_R1R0_TYPE, /*86*/ + MTK_PULL_PUPD_R1R0_TYPE, /*87*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*88*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*89*/ + MTK_PULL_PUPD_R1R0_TYPE, /*90*/ + MTK_PULL_PUPD_R1R0_TYPE, /*91*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*92*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*93*/ + MTK_PULL_PUPD_R1R0_TYPE, /*94*/ + MTK_PULL_PUPD_R1R0_TYPE, /*95*/ + MTK_PULL_PUPD_R1R0_TYPE, /*96*/ + MTK_PULL_PUPD_R1R0_TYPE, /*97*/ + MTK_PULL_PUPD_R1R0_TYPE, /*98*/ + MTK_PULL_PUPD_R1R0_TYPE, /*99*/ + MTK_PULL_PUPD_R1R0_TYPE, /*100*/ + MTK_PULL_PUPD_R1R0_TYPE, /*101*/ + MTK_PULL_PUPD_R1R0_TYPE, /*102*/ + MTK_PULL_PUPD_R1R0_TYPE, /*103*/ + MTK_PULL_PUPD_R1R0_TYPE, /*104*/ + MTK_PULL_PUPD_R1R0_TYPE, /*105*/ + MTK_PULL_PUPD_R1R0_TYPE, /*106*/ + MTK_PULL_PUPD_R1R0_TYPE, /*107*/ + MTK_PULL_PUPD_R1R0_TYPE, /*108*/ + MTK_PULL_PUPD_R1R0_TYPE, /*109*/ + MTK_PULL_PUPD_R1R0_TYPE, /*110*/ + MTK_PULL_PUPD_R1R0_TYPE, /*111*/ + MTK_PULL_PUPD_R1R0_TYPE, /*112*/ + MTK_PULL_PUPD_R1R0_TYPE, /*113*/ + MTK_PULL_PUPD_R1R0_TYPE, /*114*/ + MTK_PULL_PUPD_R1R0_TYPE, /*115*/ + MTK_PULL_PUPD_R1R0_TYPE, /*116*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*117*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*118*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*119*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*120*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*121*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*122*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*123*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*124*/ + MTK_PULL_PUPD_R1R0_TYPE, /*125*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*126*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*127*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*128*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*129*/ + MTK_PULL_PUPD_R1R0_TYPE, /*130*/ + MTK_PULL_PUPD_R1R0_TYPE, /*131*/ + MTK_PULL_PUPD_R1R0_TYPE, /*132*/ + MTK_PULL_PUPD_R1R0_TYPE, /*133*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*134*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*135*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*136*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*137*/ + MTK_PULL_PUPD_R1R0_TYPE, /*138*/ + MTK_PULL_PUPD_R1R0_TYPE, /*139*/ + MTK_PULL_PUPD_R1R0_TYPE, /*140*/ + MTK_PULL_PUPD_R1R0_TYPE, /*141*/ + MTK_PULL_PUPD_R1R0_TYPE, /*142*/ + MTK_PULL_PUPD_R1R0_TYPE, /*143*/ + MTK_PULL_PUPD_R1R0_TYPE, /*144*/ + MTK_PULL_PUPD_R1R0_TYPE, /*145*/ + MTK_PULL_PUPD_R1R0_TYPE, /*146*/ + MTK_PULL_PUPD_R1R0_TYPE, /*147*/ + MTK_PULL_PUPD_R1R0_TYPE, /*148*/ + MTK_PULL_PUPD_R1R0_TYPE, /*149*/ + MTK_PULL_PUPD_R1R0_TYPE, /*150*/ + MTK_PULL_PUPD_R1R0_TYPE, /*151*/ + MTK_PULL_PUPD_R1R0_TYPE, /*152*/ + MTK_PULL_PUPD_R1R0_TYPE, /*153*/ + MTK_PULL_PUPD_R1R0_TYPE, /*154*/ + MTK_PULL_PUPD_R1R0_TYPE, /*155*/ + MTK_PULL_PUPD_R1R0_TYPE, /*156*/ + MTK_PULL_PUPD_R1R0_TYPE, /*157*/ + MTK_PULL_PUPD_R1R0_TYPE, /*158*/ + MTK_PULL_PUPD_R1R0_TYPE, /*159*/ + MTK_PULL_PUPD_R1R0_TYPE, /*160*/ + MTK_PULL_PUPD_R1R0_TYPE, /*161*/ + MTK_PULL_PUPD_R1R0_TYPE, /*162*/ + MTK_PULL_PUPD_R1R0_TYPE, /*163*/ + MTK_PULL_PUPD_R1R0_TYPE, /*164*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*165*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*166*/ + MTK_PULL_PUPD_R1R0_TYPE, /*167*/ + MTK_PULL_PUPD_R1R0_TYPE, /*168*/ + MTK_PULL_PUPD_R1R0_TYPE, /*169*/ + MTK_PULL_PUPD_R1R0_TYPE, /*170*/ + MTK_PULL_PUPD_R1R0_TYPE, /*171*/ + MTK_PULL_PUPD_R1R0_TYPE, /*172*/ + MTK_PULL_PUPD_R1R0_TYPE, /*173*/ + MTK_PULL_PUPD_R1R0_TYPE, /*174*/ + MTK_PULL_PUPD_R1R0_TYPE, /*175*/ + MTK_PULL_PUPD_R1R0_TYPE, /*176*/ + MTK_PULL_PUPD_R1R0_TYPE, /*177*/ + MTK_PULL_PUPD_R1R0_TYPE, /*178*/ + MTK_PULL_PUPD_R1R0_TYPE, /*179*/ + MTK_PULL_PUPD_R1R0_TYPE, /*180*/ + MTK_PULL_PU_PD_TYPE, /*181*/ +}; + +static const struct mtk_pin_reg_calc mt8901_reg_cals[PINCTRL_PIN_REG_MAX] = { + [PINCTRL_PIN_REG_MODE] = MTK_RANGE(mt8901_pin_mode_range), + [PINCTRL_PIN_REG_DIR] = MTK_RANGE(mt8901_pin_dir_range), + [PINCTRL_PIN_REG_DI] = MTK_RANGE(mt8901_pin_di_range), + [PINCTRL_PIN_REG_DO] = MTK_RANGE(mt8901_pin_do_range), + [PINCTRL_PIN_REG_SMT] = MTK_RANGE(mt8901_pin_smt_range), + [PINCTRL_PIN_REG_IES] = MTK_RANGE(mt8901_pin_ies_range), + [PINCTRL_PIN_REG_PUPD] = MTK_RANGE(mt8901_pin_pupd_range), + [PINCTRL_PIN_REG_R0] = MTK_RANGE(mt8901_pin_r0_range), + [PINCTRL_PIN_REG_R1] = MTK_RANGE(mt8901_pin_r1_range), + [PINCTRL_PIN_REG_PU] = MTK_RANGE(mt8901_pin_pu_range), + [PINCTRL_PIN_REG_PD] = MTK_RANGE(mt8901_pin_pd_range), + [PINCTRL_PIN_REG_DRV] = MTK_RANGE(mt8901_pin_drv_range), + [PINCTRL_PIN_REG_DRV_ADV] = MTK_RANGE(mt8901_pin_drv_adv_range), + [PINCTRL_PIN_REG_RSEL] = MTK_RANGE(mt8901_pin_rsel_range), +}; + +static const char * const mt8901_pinctrl_register_base_name[] = { + "iocfg0", "iocfg_lt2", "iocfg_lt3", "iocfg_rt1", "iocfg_rt2", "iocfg_rt3", + "iocfg_tr", "iocfg_rt0", "iocfg_lt1", "iocfg_lb", "iocfg_rb", +}; + +static const struct mtk_eint_hw mt8901_eint_hw = { + .port_mask = 0xf, + .ports = 7, + .ap_num = 209, + .db_cnt = 32, + .db_time = debounce_time_mt8901, +}; + +static const struct mtk_pin_soc mt8901_data = { + .reg_cal = mt8901_reg_cals, + .pins = mtk_pins_mt8901, + .npins = ARRAY_SIZE(mtk_pins_mt8901), + .ngrps = ARRAY_SIZE(mtk_pins_mt8901), + .eint_hw = &mt8901_eint_hw, + .eint_pin = eint_pins_mt8901, + .nfuncs = 8, + .gpio_m = 0, + .base_names = mt8901_pinctrl_register_base_name, + .nbase_names = ARRAY_SIZE(mt8901_pinctrl_register_base_name), + .pull_type = mt8901_pull_type, + .pin_rsel = mt8901_pin_rsel_val_range, + .npin_rsel = ARRAY_SIZE(mt8901_pin_rsel_val_range), /*numsel*/ + .bias_set_combo = mtk_pinconf_bias_set_combo, + .bias_get_combo = mtk_pinconf_bias_get_combo, + .drive_set = mtk_pinconf_drive_set_rev1, + .drive_get = mtk_pinconf_drive_get_rev1, + .adv_drive_set = mtk_pinconf_adv_drive_set_raw, + .adv_drive_get = mtk_pinconf_adv_drive_get_raw, +}; + +static const struct acpi_device_id mt8901_pinctrl_acpi_match[] = { + {"NVDA9221", (kernel_ulong_t)&mt8901_data }, + { } +}; +MODULE_DEVICE_TABLE(acpi, mt8901_pinctrl_acpi_match); + +static struct platform_driver mt8901_pinctrl_driver = { + .driver = { + .name = "mt8901-pinctrl", + .acpi_match_table = ACPI_PTR(mt8901_pinctrl_acpi_match), + .pm = pm_sleep_ptr(&mtk_paris_pinctrl_pm_ops) + }, + .probe = mtk_paris_pinctrl_probe, +}; + +static int __init mt8901_pinctrl_init(void) +{ + return platform_driver_register(&mt8901_pinctrl_driver); +} + +arch_initcall(mt8901_pinctrl_init); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("MediaTek MT8901 Pinctrl Driver"); diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c index 4918d38abfc29..fc71f9b267c52 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c +++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (C) 2018 MediaTek Inc. + * Copyright (C) 2018-2025 MediaTek Inc. * * Author: Sean Wang * @@ -369,18 +369,30 @@ int mtk_build_eint(struct mtk_pinctrl *hw, struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; int ret, i, j, count_reg_names; + struct fwnode_handle *fwnode = dev_fwnode(&pdev->dev); + struct resource *res; if (!IS_ENABLED(CONFIG_EINT_MTK)) return 0; - if (!of_property_read_bool(np, "interrupt-controller")) + if (is_of_node(fwnode) && !of_property_read_bool(np, "interrupt-controller")) return -ENODEV; hw->eint = devm_kzalloc(hw->dev, sizeof(*hw->eint), GFP_KERNEL); if (!hw->eint) return -ENOMEM; - count_reg_names = of_property_count_strings(np, "reg-names"); + if (is_of_node(fwnode)) { + count_reg_names = of_property_count_strings(np, "reg-names"); + } else { + count_reg_names = 0; + for (i = 0; i < pdev->num_resources; i++) { + struct resource *r = &pdev->resource[i]; + + if (resource_type(r) == IORESOURCE_MEM) + count_reg_names++; + } + } if (count_reg_names < 0) return -EINVAL; @@ -396,14 +408,18 @@ int mtk_build_eint(struct mtk_pinctrl *hw, struct platform_device *pdev) } for (i = hw->soc->nbase_names, j = 0; i < count_reg_names; i++, j++) { - hw->eint->base[j] = of_iomap(np, i); + res = platform_get_resource(pdev, IORESOURCE_MEM, i); + hw->eint->base[j] = is_of_node(fwnode) ? of_iomap(np, i) : + ioremap(res->start, resource_size(res)); if (IS_ERR(hw->eint->base[j])) { ret = PTR_ERR(hw->eint->base[j]); goto err_free_eint; } } - hw->eint->irq = irq_of_parse_and_map(np, 0); + hw->eint->irq = is_of_node(fwnode) + ? irq_of_parse_and_map(np, 0) + : platform_get_irq(pdev, 0); if (!hw->eint->irq) { ret = -EINVAL; goto err_free_eint; diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.h b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.h index fa7c0ed493464..df8dce14744f9 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.h +++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.h @@ -302,6 +302,7 @@ struct mtk_pinctrl { spinlock_t lock; /* identify rsel setting by si unit or rsel define in dts node */ bool rsel_si_unit; + struct pinctrl_gpio_range range; }; void mtk_rmw(struct mtk_pinctrl *pctl, u8 i, u32 reg, u32 mask, u32 set); diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-mt8901.h b/drivers/pinctrl/mediatek/pinctrl-mtk-mt8901.h new file mode 100644 index 0000000000000..fc64fc6ff5f9b --- /dev/null +++ b/drivers/pinctrl/mediatek/pinctrl-mtk-mt8901.h @@ -0,0 +1,2130 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2025 MediaTek Inc. + * + */ + +#ifndef __PINCTRL_MTK_MT8901_H +#define __PINCTRL_MTK_MT8901_H + +#include "pinctrl-paris.h" + +#define INVALID_BASE 0xFF + +static const struct mtk_pin_desc mtk_pins_mt8901[] = { + MTK_PIN( + 0, "GPIO0", + MTK_EINT_FUNCTION(0, 0), + DRV_GRP4, + MTK_FUNCTION(0, "B:GPIO0"), + MTK_FUNCTION(1, "O:ESPI_SCK") + ), + MTK_PIN( + 1, "GPIO1", + MTK_EINT_FUNCTION(0, 1), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO1"), + MTK_FUNCTION(1, "B1_ESPI_IO0") + ), + MTK_PIN( + 2, "GPIO2", + MTK_EINT_FUNCTION(0, 2), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO2"), + MTK_FUNCTION(1, "B1_ESPI_IO1") + ), + MTK_PIN( + 3, "GPIO3", + MTK_EINT_FUNCTION(0, 3), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO3"), + MTK_FUNCTION(1, "B1_ESPI_IO2") + ), + MTK_PIN( + 4, "GPIO4", + MTK_EINT_FUNCTION(0, 4), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO4"), + MTK_FUNCTION(1, "B1_ESPI_IO3") + ), + MTK_PIN( + 5, "GPIO5", + MTK_EINT_FUNCTION(0, 5), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO5"), + MTK_FUNCTION(1, "O_ESPI_CSN") + ), + MTK_PIN( + 6, "GPIO6", + MTK_EINT_FUNCTION(0, 6), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO6"), + MTK_FUNCTION(1, "O_ESPI_RESET_O"), + MTK_FUNCTION(2, "I1_ESPI_RESET_I") + ), + MTK_PIN( + 7, "GPIO7", + MTK_EINT_FUNCTION(0, 7), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO7"), + MTK_FUNCTION(1, "I1_ESPI_ALERT") + ), + MTK_PIN( + 8, "GPIO8", + MTK_EINT_FUNCTION(0, 8), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO8"), + MTK_FUNCTION(1, "B1_I2C_SCL3"), + MTK_FUNCTION(2, "B1_DISP_SCL2"), + MTK_FUNCTION(4, "O_PMSR_SMAP"), + MTK_FUNCTION(6, "O_MD32_0_TXD"), + MTK_FUNCTION(7, "O_MD32_1_TXD") + ), + MTK_PIN( + 9, "GPIO9", + MTK_EINT_FUNCTION(0, 9), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO9"), + MTK_FUNCTION(1, "B1_I2C_SDA3"), + MTK_FUNCTION(2, "B1_DISP_SDA2"), + MTK_FUNCTION(4, "O_PMSR_SMAP_MAX"), + MTK_FUNCTION(6, "I1_MD32_0_RXD"), + MTK_FUNCTION(7, "I1_MD32_1_RXD") + ), + MTK_PIN( + 10, "GPIO10", + MTK_EINT_FUNCTION(0, 10), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO10"), + MTK_FUNCTION(1, "B1_I2C_SCL4"), + MTK_FUNCTION(2, "B1_DISP_SCL2"), + MTK_FUNCTION(4, "O_PMSR_SMAP_MAX_W"), + MTK_FUNCTION(6, "O_MD32_0_GPIO0"), + MTK_FUNCTION(7, "O_MD32_1_GPIO0") + ), + MTK_PIN( + 11, "GPIO11", + MTK_EINT_FUNCTION(0, 11), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO11"), + MTK_FUNCTION(1, "B1_I2C_SDA4"), + MTK_FUNCTION(2, "B1_DISP_SDA2") + ), + MTK_PIN( + 12, "GPIO12", + MTK_EINT_FUNCTION(0, 12), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO12"), + MTK_FUNCTION(1, "B0_SPMI_M_SCL"), + MTK_FUNCTION(2, "B0_TP_GPIO31_AO") + ), + MTK_PIN( + 13, "GPIO13", + MTK_EINT_FUNCTION(0, 13), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO13"), + MTK_FUNCTION(1, "B0_SPMI_M_SDA"), + MTK_FUNCTION(2, "B0_TP_GPIO6_AO") + ), + MTK_PIN( + 14, "GPIO14", + MTK_EINT_FUNCTION(0, 14), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO14"), + MTK_FUNCTION(1, "I0_DPAUX_HPD_IN_2"), + MTK_FUNCTION(7, "O_DBG_MON_A0") + ), + MTK_PIN( + 15, "GPIO15", + MTK_EINT_FUNCTION(0, 15), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO15"), + MTK_FUNCTION(1, "I0_DPAUX_HPD_IN_3"), + MTK_FUNCTION(2, "B0_TP_GPIO25_AO"), + MTK_FUNCTION(7, "O_DBG_MON_A1") + ), + MTK_PIN( + 16, "GPIO16", + MTK_EINT_FUNCTION(0, 16), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO16"), + MTK_FUNCTION(1, "O_USB4_L_TCPC_RESET"), + MTK_FUNCTION(7, "O_DBG_MON_A18") + ), + MTK_PIN( + 17, "GPIO17", + MTK_EINT_FUNCTION(0, 17), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO17"), + MTK_FUNCTION(1, "B0_SPMI_P_SCL") + ), + MTK_PIN( + 18, "GPIO18", + MTK_EINT_FUNCTION(0, 18), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO18"), + MTK_FUNCTION(1, "B0_SPMI_P_SDA") + ), + MTK_PIN( + 19, "GPIO19", + MTK_EINT_FUNCTION(0, 19), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO19"), + MTK_FUNCTION(1, "B0_TP_GPIO29_AO") + ), + MTK_PIN( + 20, "GPIO20", + MTK_EINT_FUNCTION(0, 20), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO20"), + MTK_FUNCTION(1, "B0_TP_GPIO30_AO") + ), + MTK_PIN( + 21, "GPIO21", + MTK_EINT_FUNCTION(0, 21), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO21"), + MTK_FUNCTION(1, "B1_PROCHOT") + ), + MTK_PIN( + 22, "GPIO22", + MTK_EINT_FUNCTION(0, 22), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO22"), + MTK_FUNCTION(1, "I0_RTC32K_CK") + ), + MTK_PIN( + 23, "GPIO23", + MTK_EINT_FUNCTION(0, 23), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO23"), + MTK_FUNCTION(1, "B0_TP_GPIO0_AO") + ), + MTK_PIN( + 24, "GPIO24", + MTK_EINT_FUNCTION(0, 24), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO24"), + MTK_FUNCTION(2, "B0_TP_GPIO1_AO"), + MTK_FUNCTION(3, "O_CMMCLK1"), + MTK_FUNCTION(4, "O_SROOT_GPIO_O"), + MTK_FUNCTION(5, "O_MD32_10_TXD"), + MTK_FUNCTION(6, "O_MD32_11_TXD"), + MTK_FUNCTION(7, "O_DBG_MON_A3") + ), + MTK_PIN( + 25, "GPIO25", + MTK_EINT_FUNCTION(0, 25), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO25"), + MTK_FUNCTION(1, "B0_TP_GPIO2_AO"), + MTK_FUNCTION(2, "I0_VBUSVALID_0P"), + MTK_FUNCTION(4, "I0_SROOT_GPIO_I"), + MTK_FUNCTION(5, "I1_MD32_10_RXD"), + MTK_FUNCTION(6, "I1_MD32_11_RXD"), + MTK_FUNCTION(7, "O_DBG_MON_A4") + ), + MTK_PIN( + 26, "GPIO26", + MTK_EINT_FUNCTION(0, 26), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO26"), + MTK_FUNCTION(1, "B0_TP_GPIO3_AO"), + MTK_FUNCTION(5, "O_MD32_12_TXD"), + MTK_FUNCTION(6, "O_MD32_13_TXD"), + MTK_FUNCTION(7, "O_DBG_MON_A5") + ), + MTK_PIN( + 27, "GPIO27", + MTK_EINT_FUNCTION(0, 27), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO27"), + MTK_FUNCTION(1, "B0_TP_GPIO4_AO"), + MTK_FUNCTION(5, "I1_MD32_12_RXD"), + MTK_FUNCTION(6, "I1_MD32_13_RXD"), + MTK_FUNCTION(7, "O_DBG_MON_A6") + ), + MTK_PIN( + 28, "GPIO28", + MTK_EINT_FUNCTION(0, 28), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO28"), + MTK_FUNCTION(1, "B0_TP_GPIO5_AO"), + MTK_FUNCTION(5, "O_MD32_12_GPIO0"), + MTK_FUNCTION(6, "O_MD32_13_GPIO0"), + MTK_FUNCTION(7, "O_DBG_MON_A7") + ), + MTK_PIN( + 29, "GPIO29", + MTK_EINT_FUNCTION(0, 29), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO29"), + MTK_FUNCTION(1, "B1_THERMTRIP") + ), + MTK_PIN( + 30, "GPIO30", + MTK_EINT_FUNCTION(0, 30), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO30"), + MTK_FUNCTION(1, "B0_TP_GPIO7_AO"), + MTK_FUNCTION(2, "O_CMMCLK0"), + MTK_FUNCTION(4, "I0_CLUSTER0_SLV_CPUEB_JTAG_TRSTN"), + MTK_FUNCTION(5, "I0_CLUSTER1_SLV_CPUEB_JTAG_TRSTN"), + MTK_FUNCTION(6, "I0_OSROOT_GPIO_I"), + MTK_FUNCTION(7, "O_DBG_MON_A8") + ), + MTK_PIN( + 31, "GPIO31", + MTK_EINT_FUNCTION(0, 31), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO31"), + MTK_FUNCTION(1, "B0_TP_GPIO8_AO"), + MTK_FUNCTION(2, "O_CMMCLK1"), + MTK_FUNCTION(4, "I1_CLUSTER0_SLV_CPUEB_JTAG_TMS"), + MTK_FUNCTION(5, "I1_CLUSTER1_SLV_CPUEB_JTAG_TMS"), + MTK_FUNCTION(6, "O_OSROOT_GPIO_O"), + MTK_FUNCTION(7, "O_DBG_MON_A9") + ), + MTK_PIN( + 32, "GPIO32", + MTK_EINT_FUNCTION(0, 32), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO32"), + MTK_FUNCTION(1, "B0_TP_GPIO9_AO"), + MTK_FUNCTION(5, "O_SROOT_UTX"), + MTK_FUNCTION(6, "I1_TP_UCTS1_VLP"), + MTK_FUNCTION(7, "O_DBG_MON_A10") + ), + MTK_PIN( + 33, "GPIO33", + MTK_EINT_FUNCTION(0, 33), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO33"), + MTK_FUNCTION(1, "B0_TP_GPIO10_AO"), + MTK_FUNCTION(2, "O_CMMCLK2"), + MTK_FUNCTION(3, "I0_VBUSVALID_1P"), + MTK_FUNCTION(4, "I1_CLUSTER0_SLV_CPUEB_JTAG_TCK"), + MTK_FUNCTION(5, "I1_CLUSTER1_SLV_CPUEB_JTAG_TCK"), + MTK_FUNCTION(6, "I0_SROOT_TCK"), + MTK_FUNCTION(7, "I0_OSROOT_TCK") + ), + MTK_PIN( + 34, "GPIO34", + MTK_EINT_FUNCTION(0, 34), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO34"), + MTK_FUNCTION(1, "B0_TP_GPIO11_AO"), + MTK_FUNCTION(2, "O_CMMCLK4"), + MTK_FUNCTION(3, "I0_VBUSVALID_3P"), + MTK_FUNCTION(4, "I1_CLUSTER0_SLV_CPUEB_JTAG_TDI"), + MTK_FUNCTION(5, "I1_CLUSTER1_SLV_CPUEB_JTAG_TDI"), + MTK_FUNCTION(6, "I0_SROOT_TDI"), + MTK_FUNCTION(7, "I0_OSROOT_TDI") + ), + MTK_PIN( + 35, "GPIO35", + MTK_EINT_FUNCTION(0, 35), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO35"), + MTK_FUNCTION(1, "B0_TP_GPIO12_AO"), + MTK_FUNCTION(2, "O_SCP_PWM_1_VLP"), + MTK_FUNCTION(4, "O_CLUSTER0_SLV_CPUEB_JTAG_TDO"), + MTK_FUNCTION(5, "O_CLUSTER1_SLV_CPUEB_JTAG_TDO"), + MTK_FUNCTION(6, "O_SROOT_TDO"), + MTK_FUNCTION(7, "O_OSROOT_TDO") + ), + MTK_PIN( + 36, "GPIO36", + MTK_EINT_FUNCTION(0, 36), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO36"), + MTK_FUNCTION(1, "B0_TP_GPIO22_AO"), + MTK_FUNCTION(5, "I1_SROOT_URX"), + MTK_FUNCTION(6, "O_TP_URTS1_VLP"), + MTK_FUNCTION(7, "O_DBG_MON_A31") + ), + MTK_PIN( + 37, "GPIO37", + MTK_EINT_FUNCTION(0, 37), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO37"), + MTK_FUNCTION(1, "B0_TP_GPIO23_AO"), + MTK_FUNCTION(2, "O_CMMCLK3"), + MTK_FUNCTION(4, "O_SCP_PWM_2_VLP"), + MTK_FUNCTION(6, "O_MD32_5_GPIO0"), + MTK_FUNCTION(7, "O_DBG_MON_A11") + ), + MTK_PIN( + 38, "GPIO38", + MTK_EINT_FUNCTION(0, 38), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO38"), + MTK_FUNCTION(1, "B0_TP_GPIO24_AO"), + MTK_FUNCTION(2, "O_SCP_VREQ_VAO") + ), + MTK_PIN( + 39, "GPIO39", + MTK_EINT_FUNCTION(0, 39), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO39"), + MTK_FUNCTION(1, "B0_TP_GPIO13_AO"), + MTK_FUNCTION(2, "O_SCP_PWM_2_VLP"), + MTK_FUNCTION(3, "O_CMMCLK0"), + MTK_FUNCTION(4, "I0_VBUSVALID_2P"), + MTK_FUNCTION(5, "O_MD32_10_GPIO0"), + MTK_FUNCTION(6, "I0_SROOT_TMS"), + MTK_FUNCTION(7, "I0_OSROOT_TMS") + ), + MTK_PIN( + 40, "GPIO40", + MTK_EINT_FUNCTION(0, 40), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO40"), + MTK_FUNCTION(1, "B0_TP_GPIO14_AO"), + MTK_FUNCTION(2, "I0_VBUSVALID_1P"), + MTK_FUNCTION(3, "O_URTS2"), + MTK_FUNCTION(4, "O_TP_URTS2_VLP"), + MTK_FUNCTION(5, "O_SPMI_P_TRIG_FLAG"), + MTK_FUNCTION(6, "I1_MD32_5_RXD"), + MTK_FUNCTION(7, "O_DBG_MON_A13") + ), + MTK_PIN( + 41, "GPIO41", + MTK_EINT_FUNCTION(0, 41), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO41"), + MTK_FUNCTION(1, "B0_TP_GPIO15_AO"), + MTK_FUNCTION(2, "I0_VBUSVALID_0P"), + MTK_FUNCTION(3, "I1_UCTS2"), + MTK_FUNCTION(4, "I1_TP_UCTS2_VLP"), + MTK_FUNCTION(5, "O_SPMI_S_TRIG_FLAG"), + MTK_FUNCTION(6, "O_MD32_5_TXD"), + MTK_FUNCTION(7, "O_DBG_MON_A12") + ), + MTK_PIN( + 42, "GPIO42", + MTK_EINT_FUNCTION(0, 42), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO42"), + MTK_FUNCTION(1, "B0_TP_GPIO16_AO"), + MTK_FUNCTION(2, "O_CMMCLK3"), + MTK_FUNCTION(3, "O_UTXD2"), + MTK_FUNCTION(4, "O_TP_UTXD2_VLP"), + MTK_FUNCTION(5, "O_SPMI_M_TRIG_FLAG"), + MTK_FUNCTION(6, "I0_SROOT_NTRST"), + MTK_FUNCTION(7, "I0_OSROOT_NTRST") + ), + MTK_PIN( + 43, "GPIO43", + MTK_EINT_FUNCTION(0, 43), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO43"), + MTK_FUNCTION(1, "B0_TP_GPIO17_AO"), + MTK_FUNCTION(2, "O_CMMCLK4"), + MTK_FUNCTION(3, "I1_URXD2"), + MTK_FUNCTION(4, "I1_TP_URXD2_VLP"), + MTK_FUNCTION(5, "O_MD32_4_TXD"), + MTK_FUNCTION(6, "O_MD32PCM_UTXD_AO_VLP") + ), + MTK_PIN( + 44, "GPIO44", + MTK_EINT_FUNCTION(0, 44), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO44"), + MTK_FUNCTION(1, "B0_TP_GPIO18_AO"), + MTK_FUNCTION(2, "I1_TP_UCTS2_VLP"), + MTK_FUNCTION(3, "I1_UCTS2"), + MTK_FUNCTION(5, "I1_MD32_4_RXD"), + MTK_FUNCTION(6, "I1_MD32PCM_URXD_AO_VLP") + ), + MTK_PIN( + 45, "GPIO45", + MTK_EINT_FUNCTION(0, 45), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO45"), + MTK_FUNCTION(1, "B0_TP_GPIO19_AO"), + MTK_FUNCTION(2, "O_TP_URTS2_VLP"), + MTK_FUNCTION(3, "O_URTS2"), + MTK_FUNCTION(5, "O_MD32_4_GPIO0"), + MTK_FUNCTION(6, "O_MD32_11_GPIO0") + ), + MTK_PIN( + 46, "GPIO46", + MTK_EINT_FUNCTION(0, 46), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO46"), + MTK_FUNCTION(1, "B0_TP_GPIO20_AO"), + MTK_FUNCTION(2, "I0_VBUSVALID_2P"), + MTK_FUNCTION(3, "O_SCP_VREQ_VAO"), + MTK_FUNCTION(4, "O_SCP_PWM_1_VLP"), + MTK_FUNCTION(5, "O_SROOT_GPIO_O"), + MTK_FUNCTION(6, "O_OSROOT_GPIO_O") + ), + MTK_PIN( + 47, "GPIO47", + MTK_EINT_FUNCTION(0, 47), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO47"), + MTK_FUNCTION(1, "B0_TP_GPIO21_AO"), + MTK_FUNCTION(2, "I0_VBUSVALID_3P"), + MTK_FUNCTION(3, "O_CMMCLK2"), + MTK_FUNCTION(5, "I0_SROOT_GPIO_I"), + MTK_FUNCTION(6, "I0_OSROOT_GPIO_I"), + MTK_FUNCTION(7, "O_DBG_MON_A2") + ), + MTK_PIN( + 48, "GPIO48", + MTK_EINT_FUNCTION(0, 48), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO48"), + MTK_FUNCTION(1, "O_UTXD0"), + MTK_FUNCTION(2, "O_TP_UTXD1_VLP"), + MTK_FUNCTION(6, "O_ADSP_UTXD0"), + MTK_FUNCTION(7, "O_DBG_MON_A19") + ), + MTK_PIN( + 49, "GPIO49", + MTK_EINT_FUNCTION(0, 49), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO49"), + MTK_FUNCTION(1, "I1_URXD0"), + MTK_FUNCTION(2, "I1_TP_URXD1_VLP"), + MTK_FUNCTION(6, "I1_ADSP_URXD0"), + MTK_FUNCTION(7, "O_DBG_MON_A20") + ), + MTK_PIN( + 50, "GPIO50", + MTK_EINT_FUNCTION(0, 50), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO50"), + MTK_FUNCTION(1, "O_TP_UTXD2_VLP"), + MTK_FUNCTION(2, "O_UTXD2"), + MTK_FUNCTION(4, "B0_TP_GPIO26_AO"), + MTK_FUNCTION(5, "O_TP_UTXD1_VLP"), + MTK_FUNCTION(7, "O_SROOT_UTX") + ), + MTK_PIN( + 51, "GPIO51", + MTK_EINT_FUNCTION(0, 51), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO51"), + MTK_FUNCTION(1, "I1_TP_URXD2_VLP"), + MTK_FUNCTION(2, "I1_URXD2"), + MTK_FUNCTION(4, "B0_TP_GPIO27_AO"), + MTK_FUNCTION(5, "I1_TP_URXD1_VLP"), + MTK_FUNCTION(7, "I1_SROOT_URX") + ), + MTK_PIN( + 52, "GPIO52", + MTK_EINT_FUNCTION(0, 52), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO52"), + MTK_FUNCTION(1, "B1_USB4_L_PD_SCL"), + MTK_FUNCTION(7, "O_ADSP_UTXD0") + ), + MTK_PIN( + 53, "GPIO53", + MTK_EINT_FUNCTION(0, 53), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO53"), + MTK_FUNCTION(1, "B1_USB4_L_PD_SDA"), + MTK_FUNCTION(5, "O_MD32_7_TXD"), + MTK_FUNCTION(6, "O_MD32_6_TXD"), + MTK_FUNCTION(7, "I1_ADSP_URXD0") + ), + MTK_PIN( + 54, "GPIO54", + MTK_EINT_FUNCTION(0, 54), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO54"), + MTK_FUNCTION(1, "B1_USB4_R_PD_SCL"), + MTK_FUNCTION(4, "I1_CKM_SCL"), + MTK_FUNCTION(5, "I1_MD32_7_RXD"), + MTK_FUNCTION(6, "I1_USB4_L_PAR_SCL"), + MTK_FUNCTION(7, "O_PBUD_CTRL_UTXD_AO_VLP") + ), + MTK_PIN( + 55, "GPIO55", + MTK_EINT_FUNCTION(0, 55), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO55"), + MTK_FUNCTION(1, "B1_USB4_R_PD_SDA"), + MTK_FUNCTION(4, "B1_CKM_SDA"), + MTK_FUNCTION(5, "O_MD32_7_GPIO0"), + MTK_FUNCTION(6, "B1_USB4_L_PAR_SDA"), + MTK_FUNCTION(7, "I1_PBUD_CTRL_URXD_AO_VLP") + ), + MTK_PIN( + 56, "GPIO56", + MTK_EINT_FUNCTION(0, 56), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO56"), + MTK_FUNCTION(1, "B0_SPMI_S_SCL") + ), + MTK_PIN( + 57, "GPIO57", + MTK_EINT_FUNCTION(0, 57), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO57"), + MTK_FUNCTION(1, "B0_SPMI_S_SDA") + ), + MTK_PIN( + 58, "GPIO58", + MTK_EINT_FUNCTION(0, 58), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO58"), + MTK_FUNCTION(1, "O_WATCHDOG") + ), + MTK_PIN( + 59, "GPIO59", + MTK_EINT_FUNCTION(0, 59), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO59"), + MTK_FUNCTION(1, "B0_PAD_RESET_DRAM_0") + ), + MTK_PIN( + 60, "GPIO60", + MTK_EINT_FUNCTION(0, 60), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO60"), + MTK_FUNCTION(1, "B1_I2C_SCL0"), + MTK_FUNCTION(5, "O_MD32_2_TXD"), + MTK_FUNCTION(6, "O_MD32_3_TXD") + ), + MTK_PIN( + 61, "GPIO61", + MTK_EINT_FUNCTION(0, 61), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO61"), + MTK_FUNCTION(1, "B1_I2C_SDA0"), + MTK_FUNCTION(5, "I1_MD32_2_RXD"), + MTK_FUNCTION(6, "I1_MD32_3_RXD") + ), + MTK_PIN( + 62, "GPIO62", + MTK_EINT_FUNCTION(0, 62), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO62"), + MTK_FUNCTION(1, "I0_DMIC0_DAT"), + MTK_FUNCTION(6, "O_TP_UTXD1_VLP"), + MTK_FUNCTION(7, "O_DBG_MON_B0") + ), + MTK_PIN( + 63, "GPIO63", + MTK_EINT_FUNCTION(0, 63), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO63"), + MTK_FUNCTION(1, "O_DMIC0_CLK"), + MTK_FUNCTION(6, "I1_TP_URXD1_VLP"), + MTK_FUNCTION(7, "O_DBG_MON_B1") + ), + MTK_PIN( + 64, "GPIO64", + MTK_EINT_FUNCTION(0, 64), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO64"), + MTK_FUNCTION(1, "I0_DMIC1_DAT"), + MTK_FUNCTION(5, "O_MD32_2_GPIO0"), + MTK_FUNCTION(6, "O_MD32_3_GPIO0"), + MTK_FUNCTION(7, "O_DBG_MON_B2") + ), + MTK_PIN( + 65, "GPIO65", + MTK_EINT_FUNCTION(0, 65), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO65"), + MTK_FUNCTION(1, "O_DMIC1_CLK"), + MTK_FUNCTION(7, "O_DBG_MON_B3") + ), + MTK_PIN( + 66, "GPIO66", + MTK_EINT_FUNCTION(0, 66), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO66"), + MTK_FUNCTION(1, "O_SOUNDWIRE0_CK"), + MTK_FUNCTION(3, "I1_TP_UCTS0_VLP"), + MTK_FUNCTION(4, "O_SPI_HID_IRQ_S_MON0"), + MTK_FUNCTION(7, "O_VADSP_UTXD0") + ), + MTK_PIN( + 67, "GPIO67", + MTK_EINT_FUNCTION(0, 67), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO67"), + MTK_FUNCTION(1, "B0_SOUNDWIRE0_D0"), + MTK_FUNCTION(3, "O_TP_URTS0_VLP"), + MTK_FUNCTION(7, "I1_VADSP_URXD0") + ), + MTK_PIN( + 68, "GPIO68", + MTK_EINT_FUNCTION(0, 68), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO68"), + MTK_FUNCTION(1, "O_SCP_PWM_0_VLP"), + MTK_FUNCTION(2, "O_PWM_VLP"), + MTK_FUNCTION(3, "B0_TP_GPIO28_AO") + ), + MTK_PIN( + 69, "GPIO69", + MTK_EINT_FUNCTION(0, 69), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO69"), + MTK_FUNCTION(1, "B0_PAD_RESET_DRAM_8") + ), + MTK_PIN( + 70, "GPIO70", + MTK_EINT_FUNCTION(0, 70), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO70"), + MTK_FUNCTION(1, "B1_SCP_SCL0"), + MTK_FUNCTION(4, "I0_VADSP_JTAG0_TCK"), + MTK_FUNCTION(5, "I1_PCIE4_USB3_PAR_SCL"), + MTK_FUNCTION(6, "I1_SCP_JTAG0_TCK_VLP"), + MTK_FUNCTION(7, "B1_SROOT_SCL") + ), + MTK_PIN( + 71, "GPIO71", + MTK_EINT_FUNCTION(0, 71), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO71"), + MTK_FUNCTION(1, "B1_SCP_SDA0"), + MTK_FUNCTION(4, "I1_VADSP_JTAG0_TMS"), + MTK_FUNCTION(5, "B1_PCIE4_USB3_PAR_SDA"), + MTK_FUNCTION(6, "B1_SCP_JTAG0_TMS_VLP"), + MTK_FUNCTION(7, "B1_SROOT_SDA") + ), + MTK_PIN( + 72, "GPIO72", + MTK_EINT_FUNCTION(0, 72), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO72"), + MTK_FUNCTION(1, "B1_SCP_SCL2"), + MTK_FUNCTION(2, "B1_I3C_HCI_AO_SCL"), + MTK_FUNCTION(4, "I1_VADSP_JTAG0_TDI"), + MTK_FUNCTION(5, "I1_PCIE5_PAR_SCL"), + MTK_FUNCTION(6, "I1_SCP_JTAG0_TDI_VLP"), + MTK_FUNCTION(7, "O_OSROOT_UTX") + ), + MTK_PIN( + 73, "GPIO73", + MTK_EINT_FUNCTION(0, 73), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO73"), + MTK_FUNCTION(1, "B1_SCP_SDA2"), + MTK_FUNCTION(2, "B1_I3C_HCI_AO_SDA"), + MTK_FUNCTION(4, "O_VADSP_JTAG0_TDO"), + MTK_FUNCTION(5, "B1_PCIE5_PAR_SDA"), + MTK_FUNCTION(6, "O_SCP_JTAG0_TDO_VLP"), + MTK_FUNCTION(7, "I1_OSROOT_URX") + ), + MTK_PIN( + 74, "GPIO74", + MTK_EINT_FUNCTION(0, 74), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO74"), + MTK_FUNCTION(1, "I0_SCP_SPIS0_SCL"), + MTK_FUNCTION(2, "O_SPI2_CLK"), + MTK_FUNCTION(3, "O_SCP_SPIM0_CK"), + MTK_FUNCTION(4, "I1_SPM_JTAG_TCK_VLP"), + MTK_FUNCTION(5, "I1_SSPM_JTAG_TCK_VLP"), + MTK_FUNCTION(6, "I1_PBUD_CTRL_JTAG_TCK_VLP"), + MTK_FUNCTION(7, "I0_OSROOT_TCK") + ), + MTK_PIN( + 75, "GPIO75", + MTK_EINT_FUNCTION(0, 75), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO75"), + MTK_FUNCTION(1, "B0_SCP_SPIS0_SIO0"), + MTK_FUNCTION(2, "B0_SPI2_MI"), + MTK_FUNCTION(3, "B0_SCP_SPIM0_SIO0"), + MTK_FUNCTION(4, "I1_SPM_JTAG_TDI_VLP"), + MTK_FUNCTION(5, "I1_SSPM_JTAG_TDI_VLP"), + MTK_FUNCTION(6, "I1_PBUD_CTRL_JTAG_TDI_VLP"), + MTK_FUNCTION(7, "I0_OSROOT_TDI") + ), + MTK_PIN( + 76, "GPIO76", + MTK_EINT_FUNCTION(0, 76), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO76"), + MTK_FUNCTION(1, "B0_SCP_SPIS0_SIO1"), + MTK_FUNCTION(2, "B0_SPI2_MO"), + MTK_FUNCTION(3, "B0_SCP_SPIM0_SIO1"), + MTK_FUNCTION(4, "B1_SPM_JTAG_TDO_VLP"), + MTK_FUNCTION(5, "O_SSPM_JTAG_TDO_VLP"), + MTK_FUNCTION(6, "O_PBUD_CTRL_JTAG_TDO_VLP"), + MTK_FUNCTION(7, "O_OSROOT_TDO") + ), + MTK_PIN( + 77, "GPIO77", + MTK_EINT_FUNCTION(0, 77), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO77"), + MTK_FUNCTION(1, "B0_SCP_SPIS0_SIO2"), + MTK_FUNCTION(2, "B0_SPI2_WP"), + MTK_FUNCTION(3, "B0_SCP_SPIM0_SIO2"), + MTK_FUNCTION(4, "I1_SPM_JTAG_TMS_VLP"), + MTK_FUNCTION(5, "I1_SSPM_JTAG_TMS_VLP"), + MTK_FUNCTION(6, "I1_PBUD_CTRL_JTAG_TMS_VLP"), + MTK_FUNCTION(7, "I0_OSROOT_TMS") + ), + MTK_PIN( + 78, "GPIO78", + MTK_EINT_FUNCTION(0, 78), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO78"), + MTK_FUNCTION(1, "B0_SCP_SPIS0_SIO3"), + MTK_FUNCTION(2, "B0_SPI2_HOLD"), + MTK_FUNCTION(3, "B0_SCP_SPIM0_SIO3"), + MTK_FUNCTION(4, "I0_SPM_JTAG_TRSTN_VLP"), + MTK_FUNCTION(5, "I0_SSPM_JTAG_TRSTN_VLP"), + MTK_FUNCTION(6, "I0_PBUD_CTRL_JTAG_TRSTN_VLP"), + MTK_FUNCTION(7, "I0_OSROOT_NTRST") + ), + MTK_PIN( + 79, "GPIO79", + MTK_EINT_FUNCTION(0, 79), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO79"), + MTK_FUNCTION(1, "I1_SCP_SPIS0_CS"), + MTK_FUNCTION(2, "O_SPI2_CSB"), + MTK_FUNCTION(3, "O_SCP_SPIM0_CS"), + MTK_FUNCTION(4, "I1_VADSP_JTAG0_TRSTN"), + MTK_FUNCTION(6, "I0_SCP_JTAG0_TRSTN_VLP") + ), + MTK_PIN( + 80, "GPIO80", + MTK_EINT_FUNCTION(0, 80), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO80"), + MTK_FUNCTION(1, "O_SPI0_CLK"), + MTK_FUNCTION(2, "B0_SPI0_OSROOT_CLK"), + MTK_FUNCTION(4, "I1_SSPM_JTAG_TCK_VLP"), + MTK_FUNCTION(5, "I0_VADSP_JTAG0_TCK"), + MTK_FUNCTION(6, "I1_SPM_JTAG_TCK_VLP"), + MTK_FUNCTION(7, "I1_SCP_JTAG0_TCK_VLP") + ), + MTK_PIN( + 81, "GPIO81", + MTK_EINT_FUNCTION(0, 81), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO81"), + MTK_FUNCTION(1, "B0_SPI0_MI"), + MTK_FUNCTION(2, "I0_SPI0_OSROOT_MI"), + MTK_FUNCTION(4, "I1_SSPM_JTAG_TDI_VLP"), + MTK_FUNCTION(5, "I1_VADSP_JTAG0_TDI"), + MTK_FUNCTION(6, "I1_SPM_JTAG_TDI_VLP"), + MTK_FUNCTION(7, "I1_SCP_JTAG0_TDI_VLP") + ), + MTK_PIN( + 82, "GPIO82", + MTK_EINT_FUNCTION(0, 82), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO82"), + MTK_FUNCTION(1, "B0_SPI0_MO"), + MTK_FUNCTION(2, "O_SPI0_OSROOT_MO"), + MTK_FUNCTION(4, "O_SSPM_JTAG_TDO_VLP"), + MTK_FUNCTION(5, "O_VADSP_JTAG0_TDO"), + MTK_FUNCTION(6, "B1_SPM_JTAG_TDO_VLP"), + MTK_FUNCTION(7, "O_SCP_JTAG0_TDO_VLP") + ), + MTK_PIN( + 83, "GPIO83", + MTK_EINT_FUNCTION(0, 83), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO83"), + MTK_FUNCTION(1, "O_SPI0_CSB0"), + MTK_FUNCTION(2, "O_SPI0_OSROOT_CSB"), + MTK_FUNCTION(4, "I1_SSPM_JTAG_TMS_VLP"), + MTK_FUNCTION(5, "I1_VADSP_JTAG0_TMS"), + MTK_FUNCTION(6, "I1_SPM_JTAG_TMS_VLP"), + MTK_FUNCTION(7, "B1_SCP_JTAG0_TMS_VLP") + ), + MTK_PIN( + 84, "GPIO84", + MTK_EINT_FUNCTION(0, 84), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO84"), + MTK_FUNCTION(1, "O_SPI0_CSB1"), + MTK_FUNCTION(4, "I0_SSPM_JTAG_TRSTN_VLP"), + MTK_FUNCTION(5, "I1_VADSP_JTAG0_TRSTN"), + MTK_FUNCTION(6, "I0_SPM_JTAG_TRSTN_VLP"), + MTK_FUNCTION(7, "I0_SCP_JTAG0_TRSTN_VLP") + ), + MTK_PIN( + 85, "GPIO85", + MTK_EINT_FUNCTION(0, 85), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO85"), + MTK_FUNCTION(1, "O_DISP_PWM"), + MTK_FUNCTION(4, "B1_U4CP_JTAG_TMS"), + MTK_FUNCTION(5, "I0_CLUSTER0_UDI_TDI_1"), + MTK_FUNCTION(6, "I0_CLUSTER1_UDI_TDI_1"), + MTK_FUNCTION(7, "O_DBG_MON_A21") + ), + MTK_PIN( + 86, "GPIO86", + MTK_EINT_FUNCTION(0, 86), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO86"), + MTK_FUNCTION(1, "O_DISP_BL_EN"), + MTK_FUNCTION(4, "I1_U4CP_JTAG_TDI"), + MTK_FUNCTION(5, "O_CLUSTER0_UDI_TDO_1"), + MTK_FUNCTION(6, "O_CLUSTER1_UDI_TDO_1"), + MTK_FUNCTION(7, "O_DBG_MON_A22") + ), + MTK_PIN( + 87, "GPIO87", + MTK_EINT_FUNCTION(0, 87), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO87"), + MTK_FUNCTION(1, "B0_DISP_GPIO_N_1"), + MTK_FUNCTION(4, "O_U4CP_JTAG_TDO"), + MTK_FUNCTION(5, "I0_CLUSTER0_UDI_TDI_2"), + MTK_FUNCTION(6, "I0_CLUSTER1_UDI_TDI_2"), + MTK_FUNCTION(7, "O_DBG_MON_A23") + ), + MTK_PIN( + 88, "GPIO88", + MTK_EINT_FUNCTION(0, 88), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO88"), + MTK_FUNCTION(1, "B1_DISP_SCL1"), + MTK_FUNCTION(4, "I1_U4CP_JTAG_TCK"), + MTK_FUNCTION(5, "I1_EDP0_SCL"), + MTK_FUNCTION(6, "I1_HDMITX_DBG_I2C_SCL") + ), + MTK_PIN( + 89, "GPIO89", + MTK_EINT_FUNCTION(0, 89), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO89"), + MTK_FUNCTION(1, "B1_DISP_SDA1"), + MTK_FUNCTION(4, "I0_U4CP_JTAG_TRSTN"), + MTK_FUNCTION(5, "B1_EDP0_SDA"), + MTK_FUNCTION(6, "B1_HDMITX_DBG_I2C_SDA") + ), + MTK_PIN( + 90, "GPIO90", + MTK_EINT_FUNCTION(0, 90), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO90"), + MTK_FUNCTION(1, "B0_DISP_GPIO_N2"), + MTK_FUNCTION(4, "O_CLKM0_C"), + MTK_FUNCTION(5, "O_CLUSTER0_UDI_TDO_2"), + MTK_FUNCTION(6, "O_CLUSTER1_UDI_TDO_2"), + MTK_FUNCTION(7, "O_DBG_MON_A24") + ), + MTK_PIN( + 91, "GPIO91", + MTK_EINT_FUNCTION(0, 91), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO91"), + MTK_FUNCTION(1, "I0_DPAUX_HPD_IN_4"), + MTK_FUNCTION(4, "O_CLKM1_C"), + MTK_FUNCTION(5, "I0_CLUSTER0_UDI_TDI_3"), + MTK_FUNCTION(6, "I0_CLUSTER1_UDI_TDI_3"), + MTK_FUNCTION(7, "O_DBG_MON_A25") + ), + MTK_PIN( + 92, "GPIO92", + MTK_EINT_FUNCTION(0, 92), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO92"), + MTK_FUNCTION(1, "B1_I2C_SCL5"), + MTK_FUNCTION(2, "O_TP_UTXD0_VLP"), + MTK_FUNCTION(3, "O_SSPM_UTXD_AO_VLP"), + MTK_FUNCTION(4, "O_TSFDC_FOUT"), + MTK_FUNCTION(5, "O_CLUSTER0_UDI_TDO_3"), + MTK_FUNCTION(6, "O_CLUSTER1_UDI_TDO_3"), + MTK_FUNCTION(7, "O_DBG_MON_A26") + ), + MTK_PIN( + 93, "GPIO93", + MTK_EINT_FUNCTION(0, 93), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO93"), + MTK_FUNCTION(1, "B1_I2C_SDA5"), + MTK_FUNCTION(2, "I1_TP_URXD0_VLP"), + MTK_FUNCTION(3, "I1_SSPM_URXD_AO_VLP"), + MTK_FUNCTION(4, "O_TSFDC_SDO"), + MTK_FUNCTION(5, "I0_CLUSTER0_UDI_TDI_4"), + MTK_FUNCTION(6, "I0_CLUSTER1_UDI_TDI_4"), + MTK_FUNCTION(7, "O_DBG_MON_A27") + ), + MTK_PIN( + 94, "GPIO94", + MTK_EINT_FUNCTION(0, 94), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO94"), + MTK_FUNCTION(1, "O_CMFLASH0"), + MTK_FUNCTION(4, "I0_TSFDC_26M"), + MTK_FUNCTION(5, "O_CLUSTER0_UDI_TDO_4"), + MTK_FUNCTION(6, "O_CLUSTER1_UDI_TDO_4"), + MTK_FUNCTION(7, "O_DBG_MON_A28") + ), + MTK_PIN( + 95, "GPIO95", + MTK_EINT_FUNCTION(0, 95), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO95"), + MTK_FUNCTION(1, "O_CMFLASH1"), + MTK_FUNCTION(4, "I0_TSFDC_SCF"), + MTK_FUNCTION(5, "I0_CLUSTER0_UDI_TDI_5"), + MTK_FUNCTION(6, "I0_CLUSTER1_UDI_TDI_5"), + MTK_FUNCTION(7, "O_DBG_MON_A29") + ), + MTK_PIN( + 96, "GPIO96", + MTK_EINT_FUNCTION(0, 96), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO96"), + MTK_FUNCTION(1, "O_CMVREF0"), + MTK_FUNCTION(2, "O_CMFLASH1"), + MTK_FUNCTION(4, "I0_TSFDC_SCK"), + MTK_FUNCTION(5, "O_CLUSTER0_UDI_TDO_5"), + MTK_FUNCTION(6, "O_CLUSTER1_UDI_TDO_5"), + MTK_FUNCTION(7, "O_DBG_MON_A30") + ), + MTK_PIN( + 97, "GPIO97", + MTK_EINT_FUNCTION(0, 97), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO97"), + MTK_FUNCTION(1, "O_CMVREF1"), + MTK_FUNCTION(2, "O_CMFLASH0"), + MTK_FUNCTION(4, "I0_TSFDC_SDI"), + MTK_FUNCTION(5, "I0_CLUSTER0_UDI_TDI_6"), + MTK_FUNCTION(6, "I0_CLUSTER1_UDI_TDI_6"), + MTK_FUNCTION(7, "O_U4CP_UTXD") + ), + MTK_PIN( + 98, "GPIO98", + MTK_EINT_FUNCTION(0, 98), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO98"), + MTK_FUNCTION(2, "O_CMFLASH2"), + MTK_FUNCTION(4, "I0_RG_TSFDC_LDO_EN"), + MTK_FUNCTION(5, "O_CLUSTER0_UDI_TDO_6"), + MTK_FUNCTION(6, "O_CLUSTER1_UDI_TDO_6"), + MTK_FUNCTION(7, "I1_U4CP_URXD") + ), + MTK_PIN( + 99, "GPIO99", + MTK_EINT_FUNCTION(0, 99), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO99"), + MTK_FUNCTION(1, "I0_MCU_M_PMIC_POC_I"), + MTK_FUNCTION(4, "I0_DA_TSFDC_LDO_MODE"), + MTK_FUNCTION(5, "I0_CLUSTER0_UDI_TDI_7"), + MTK_FUNCTION(6, "I0_CLUSTER1_UDI_TDI_7"), + MTK_FUNCTION(7, "O_U4CP_URTS") + ), + MTK_PIN( + 100, "GPIO100", + MTK_EINT_FUNCTION(0, 100), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO100"), + MTK_FUNCTION(1, "I0_MCU_B_PMIC_POC_I"), + MTK_FUNCTION(4, "I0_RG_TSFDC_LDO_REFSEL1"), + MTK_FUNCTION(5, "O_CLUSTER0_UDI_TDO_7"), + MTK_FUNCTION(6, "O_CLUSTER1_UDI_TDO_7"), + MTK_FUNCTION(7, "I1_U4CP_UCTS") + ), + MTK_PIN( + 101, "GPIO101", + MTK_EINT_FUNCTION(0, 101), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO101"), + MTK_FUNCTION(1, "O_CMFLASH2"), + MTK_FUNCTION(2, "O_CMVREF1"), + MTK_FUNCTION(3, "I1_UCTS0"), + MTK_FUNCTION(4, "I0_RG_TSFDC_LDO_REFSEL0"), + MTK_FUNCTION(5, "I1_U4CP_JTAG_TCK"), + MTK_FUNCTION(6, "I1_PBUD_CTRL_JTAG_TCK_VCORE"), + MTK_FUNCTION(7, "O_CLKM0_A") + ), + MTK_PIN( + 102, "GPIO102", + MTK_EINT_FUNCTION(0, 102), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO102"), + MTK_FUNCTION(1, "O_CMFLASH3"), + MTK_FUNCTION(2, "O_CMVREF0"), + MTK_FUNCTION(3, "O_URTS0"), + MTK_FUNCTION(4, "O_TSFDC_BG_COMP"), + MTK_FUNCTION(5, "B1_U4CP_JTAG_TMS"), + MTK_FUNCTION(6, "I1_PBUD_CTRL_JTAG_TMS_VCORE"), + MTK_FUNCTION(7, "O_CLKM1_A") + ), + MTK_PIN( + 103, "GPIO103", + MTK_EINT_FUNCTION(0, 103), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO103"), + MTK_FUNCTION(1, "O_CMVREF2"), + MTK_FUNCTION(2, "O_UTXD0"), + MTK_FUNCTION(4, "O_CLKM2_B"), + MTK_FUNCTION(5, "I1_U4CP_JTAG_TDI"), + MTK_FUNCTION(6, "I1_PBUD_CTRL_JTAG_TDI_VCORE"), + MTK_FUNCTION(7, "O_CLKM2_A") + ), + MTK_PIN( + 104, "GPIO104", + MTK_EINT_FUNCTION(0, 104), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO104"), + MTK_FUNCTION(1, "O_CMVREF3"), + MTK_FUNCTION(2, "I1_URXD0"), + MTK_FUNCTION(4, "O_CLKM3_B"), + MTK_FUNCTION(5, "O_U4CP_JTAG_TDO"), + MTK_FUNCTION(6, "O_PBUD_CTRL_JTAG_TDO_VCORE"), + MTK_FUNCTION(7, "O_CLKM3_A") + ), + MTK_PIN( + 105, "GPIO105", + MTK_EINT_FUNCTION(0, 105), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO105"), + MTK_FUNCTION(2, "O_CMFLASH3"), + MTK_FUNCTION(4, "O_CLKM0_B"), + MTK_FUNCTION(5, "I0_U4CP_JTAG_TRSTN"), + MTK_FUNCTION(6, "I0_PBUD_CTRL_JTAG_TRSTN_VCORE"), + MTK_FUNCTION(7, "O_PMSR_SMAP") + ), + MTK_PIN( + 106, "GPIO106", + MTK_EINT_FUNCTION(0, 106), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO106"), + MTK_FUNCTION(1, "B0_SPINOR_CK") + ), + MTK_PIN( + 107, "GPIO107", + MTK_EINT_FUNCTION(0, 107), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO107"), + MTK_FUNCTION(1, "B0_SPINOR_IO0") + ), + MTK_PIN( + 108, "GPIO108", + MTK_EINT_FUNCTION(0, 108), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO108"), + MTK_FUNCTION(1, "B0_SPINOR_IO1") + ), + MTK_PIN( + 109, "GPIO109", + MTK_EINT_FUNCTION(0, 109), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO109"), + MTK_FUNCTION(1, "B0_SPINOR_IO2") + ), + MTK_PIN( + 110, "GPIO110", + MTK_EINT_FUNCTION(0, 110), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO110"), + MTK_FUNCTION(1, "B0_SPINOR_IO3") + ), + MTK_PIN( + 111, "GPIO111", + MTK_EINT_FUNCTION(0, 111), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO111"), + MTK_FUNCTION(1, "B1_SPINOR_CS") + ), + MTK_PIN( + 112, "GPIO112", + MTK_EINT_FUNCTION(0, 112), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO112"), + MTK_FUNCTION(1, "O_SPI1_CLK"), + MTK_FUNCTION(5, "I1_HFRP_JTAG1_TCK") + ), + MTK_PIN( + 113, "GPIO113", + MTK_EINT_FUNCTION(0, 113), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO113"), + MTK_FUNCTION(1, "B0_SPI1_MI"), + MTK_FUNCTION(5, "I1_HFRP_JTAG1_TMS") + ), + MTK_PIN( + 114, "GPIO114", + MTK_EINT_FUNCTION(0, 114), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO114"), + MTK_FUNCTION(1, "B0_SPI1_MO"), + MTK_FUNCTION(5, "I1_HFRP_JTAG1_TDI") + ), + MTK_PIN( + 115, "GPIO115", + MTK_EINT_FUNCTION(0, 115), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO115"), + MTK_FUNCTION(1, "O_SPI1_CSB0"), + MTK_FUNCTION(5, "O_HFRP_JTAG1_TDO") + ), + MTK_PIN( + 116, "GPIO116", + MTK_EINT_FUNCTION(0, 116), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO116"), + MTK_FUNCTION(1, "O_SPI1_CSB1"), + MTK_FUNCTION(5, "I0_HFRP_JTAG1_TRSTN") + ), + MTK_PIN( + 117, "GPIO117", + MTK_EINT_FUNCTION(0, 117), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO117"), + MTK_FUNCTION(1, "B1_I2C_SCL1"), + MTK_FUNCTION(2, "B1_OSROOT_SCL"), + MTK_FUNCTION(4, "O_SPI_CS_S_MON0"), + MTK_FUNCTION(5, "I1_USB4_L_PAR_SCL"), + MTK_FUNCTION(6, "I1_CKM_SCL"), + MTK_FUNCTION(7, "I1_USB4_R_PAR_SCL") + ), + MTK_PIN( + 118, "GPIO118", + MTK_EINT_FUNCTION(0, 118), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO118"), + MTK_FUNCTION(1, "B1_I2C_SDA1"), + MTK_FUNCTION(2, "B1_OSROOT_SDA"), + MTK_FUNCTION(4, "O_SPI_SCL_S_MON0"), + MTK_FUNCTION(5, "B1_USB4_L_PAR_SDA"), + MTK_FUNCTION(6, "B1_CKM_SDA"), + MTK_FUNCTION(7, "B1_USB4_R_PAR_SDA") + ), + MTK_PIN( + 119, "GPIO119", + MTK_EINT_FUNCTION(0, 119), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO119"), + MTK_FUNCTION(1, "B1_I2C_SCL2"), + MTK_FUNCTION(4, "I1_HDMITX_DBG_I2C_SCL"), + MTK_FUNCTION(5, "O_CLUSTER0_MBISTREADEN_TRIGGER"), + MTK_FUNCTION(6, "O_CLUSTER1_MBISTREADEN_TRIGGER"), + MTK_FUNCTION(7, "O_PMSR_SMAP_MAX") + ), + MTK_PIN( + 120, "GPIO120", + MTK_EINT_FUNCTION(0, 120), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO120"), + MTK_FUNCTION(1, "B1_I2C_SDA2+J130_S133"), + MTK_FUNCTION(4, "B1_HDMITX_DBG_I2C_SDA"), + MTK_FUNCTION(5, "O_CLUSTER0_MBISTWRITEEN_TRIGGER"), + MTK_FUNCTION(6, "O_CLUSTER1_MBISTWRITEEN_TRIGGER"), + MTK_FUNCTION(7, "O_PMSR_SMAP_MAX_W") + ), + MTK_PIN( + 121, "GPIO121", + MTK_EINT_FUNCTION(0, 121), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO121"), + MTK_FUNCTION(1, "B1_I3C_SCL0"), + MTK_FUNCTION(2, "B1_I3C_HCI_0_AO_SCL"), + MTK_FUNCTION(4, "I1_PCIE5_PAR_SCL"), + MTK_FUNCTION(5, "O_CLUSTER0_AD_ILDO_DTEST0"), + MTK_FUNCTION(6, "O_CLUSTER1_AD_ILDO_DTEST0"), + MTK_FUNCTION(7, "I1_USB4_R_PAR_SCL") + ), + MTK_PIN( + 122, "GPIO122", + MTK_EINT_FUNCTION(0, 122), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO122"), + MTK_FUNCTION(1, "B1_I3C_SDA0"), + MTK_FUNCTION(2, "B1_I3C_HCI_0_AO_SDA"), + MTK_FUNCTION(4, "B1_PCIE5_PAR_SDA"), + MTK_FUNCTION(5, "O_CLUSTER0_AD_ILDO_DTEST1"), + MTK_FUNCTION(6, "O_CLUSTER1_AD_ILDO_DTEST1"), + MTK_FUNCTION(7, "B1_USB4_R_PAR_SDA") + ), + MTK_PIN( + 123, "GPIO123", + MTK_EINT_FUNCTION(0, 123), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO123"), + MTK_FUNCTION(1, "B1_I3C_SCL1"), + MTK_FUNCTION(2, "B1_I3C_HCI_1_AO_SCL"), + MTK_FUNCTION(4, "I1_PCIE4_USB3_PAR_SCL"), + MTK_FUNCTION(5, "O_CLUSTER0_AD_ILDO_DTEST2"), + MTK_FUNCTION(6, "O_CLUSTER1_AD_ILDO_DTEST2"), + MTK_FUNCTION(7, "O_VADSP_UTXD0") + ), + MTK_PIN( + 124, "GPIO124", + MTK_EINT_FUNCTION(0, 124), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO124"), + MTK_FUNCTION(1, "B1_I3C_SDA1"), + MTK_FUNCTION(2, "B1_I3C_HCI_1_AO_SDA"), + MTK_FUNCTION(4, "B1_PCIE4_USB3_PAR_SDA"), + MTK_FUNCTION(5, "O_CLUSTER0_AD_ILDO_DTEST3"), + MTK_FUNCTION(6, "O_CLUSTER1_AD_ILDO_DTEST3"), + MTK_FUNCTION(7, "I1_VADSP_URXD0") + ), + MTK_PIN( + 125, "GPIO125", + MTK_EINT_FUNCTION(0, 125), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO125"), + MTK_FUNCTION(1, "O_I2SIN_1_MCK"), + MTK_FUNCTION(4, "I0_TSFDC_SCK"), + MTK_FUNCTION(6, "O_CLKM2_C") + ), + MTK_PIN( + 126, "GPIO126", + MTK_EINT_FUNCTION(0, 126), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO126"), + MTK_FUNCTION(1, "O_SOUNDWIRE1_CK"), + MTK_FUNCTION(2, "O_I2SIN_1_BCK"), + MTK_FUNCTION(4, "O_CLUSTER0_AD_ILDO_DTEST4"), + MTK_FUNCTION(5, "O_CLUSTER1_AD_ILDO_DTEST4"), + MTK_FUNCTION(6, "I0_ADSP_JTAG0_TCK"), + MTK_FUNCTION(7, "I1_HFRP_JTAG0_TCK") + ), + MTK_PIN( + 127, "GPIO127", + MTK_EINT_FUNCTION(0, 127), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO127"), + MTK_FUNCTION(1, "B0_SOUNDWIRE1_D0"), + MTK_FUNCTION(2, "O_I2SOUT1_DO"), + MTK_FUNCTION(4, "O_CLUSTER0_AD_ILDO_DTEST5"), + MTK_FUNCTION(5, "O_CLUSTER1_AD_ILDO_DTEST5"), + MTK_FUNCTION(6, "I1_ADSP_JTAG0_TMS"), + MTK_FUNCTION(7, "B1_HFRP_JTAG0_TMS") + ), + MTK_PIN( + 128, "GPIO128", + MTK_EINT_FUNCTION(0, 128), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO128"), + MTK_FUNCTION(1, "B0_SOUNDWIRE1_D1"), + MTK_FUNCTION(2, "I0_I2SIN_1_DI"), + MTK_FUNCTION(4, "O_CLUSTER0_AD_ILDO_DTEST6"), + MTK_FUNCTION(5, "O_CLUSTER1_AD_ILDO_DTEST6"), + MTK_FUNCTION(6, "I1_ADSP_JTAG0_TDI"), + MTK_FUNCTION(7, "I1_HFRP_JTAG0_TDI") + ), + MTK_PIN( + 129, "GPIO129", + MTK_EINT_FUNCTION(0, 129), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO129"), + MTK_FUNCTION(1, "B0_SOUNDWIRE1_D2"), + MTK_FUNCTION(2, "O_I2SIN_1_LRCK"), + MTK_FUNCTION(4, "O_CLUSTER0_AD_ILDO_DTEST7"), + MTK_FUNCTION(5, "O_CLUSTER1_AD_ILDO_DTEST7"), + MTK_FUNCTION(6, "O_ADSP_JTAG0_TDO"), + MTK_FUNCTION(7, "O_HFRP_JTAG0_TDO") + ), + MTK_PIN( + 130, "GPIO130", + MTK_EINT_FUNCTION(0, 130), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO130"), + MTK_FUNCTION(1, "B0_I2SIN0_BCK"), + MTK_FUNCTION(3, "O_DISP_CLKM0"), + MTK_FUNCTION(4, "I0_TSFDC_26M"), + MTK_FUNCTION(5, "I0_RG_TSFDC_LDO_EN"), + MTK_FUNCTION(6, "O_CCU0_URTS"), + MTK_FUNCTION(7, "O_DBG_MON_B4") + ), + MTK_PIN( + 131, "GPIO131", + MTK_EINT_FUNCTION(0, 131), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO131"), + MTK_FUNCTION(1, "I0_I2SIN0_DI"), + MTK_FUNCTION(3, "O_DISP_CLKM1"), + MTK_FUNCTION(4, "O_TSFDC_FOUT"), + MTK_FUNCTION(5, "I0_DA_TSFDC_LDO_MODE"), + MTK_FUNCTION(6, "I1_CCU0_UCTS"), + MTK_FUNCTION(7, "O_DBG_MON_B5") + ), + MTK_PIN( + 132, "GPIO132", + MTK_EINT_FUNCTION(0, 132), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO132"), + MTK_FUNCTION(1, "O_I2SOUT0_DO"), + MTK_FUNCTION(3, "O_DISP_CLKM2"), + MTK_FUNCTION(4, "O_TSFDC_SDO"), + MTK_FUNCTION(5, "I0_RG_TSFDC_LDO_REFSEL1"), + MTK_FUNCTION(6, "O_CCU1_URTS"), + MTK_FUNCTION(7, "O_DBG_MON_B6") + ), + MTK_PIN( + 133, "GPIO133", + MTK_EINT_FUNCTION(0, 133), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO133"), + MTK_FUNCTION(1, "B0_I2SIN0_LRCK"), + MTK_FUNCTION(3, "O_DISP_CLKM3"), + MTK_FUNCTION(4, "I0_TSFDC_SCF"), + MTK_FUNCTION(5, "I0_RG_TSFDC_LDO_REFSEL0"), + MTK_FUNCTION(6, "I1_CCU1_UCTS"), + MTK_FUNCTION(7, "O_DBG_MON_B7") + ), + MTK_PIN( + 134, "GPIO134", + MTK_EINT_FUNCTION(0, 134), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO134"), + MTK_FUNCTION(1, "B1_SCP_SCL1"), + MTK_FUNCTION(2, "B1_VADSP_SCL0"), + MTK_FUNCTION(3, "B1_SROOT_SCL"), + MTK_FUNCTION(4, "O_SSPM_UTXD_AO_VLP"), + MTK_FUNCTION(5, "O_SPI_HID_IRQ_S_MON0"), + MTK_FUNCTION(6, "I1_ADSP_JTAG0_TRSTN"), + MTK_FUNCTION(7, "O_DBG_MON_B8") + ), + MTK_PIN( + 135, "GPIO135", + MTK_EINT_FUNCTION(0, 135), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO135"), + MTK_FUNCTION(1, "B1_SCP_SDA1"), + MTK_FUNCTION(2, "B1_VADSP_SDA0"), + MTK_FUNCTION(3, "B1_SROOT_SDA"), + MTK_FUNCTION(4, "I1_SSPM_URXD_AO_VLP"), + MTK_FUNCTION(7, "O_DBG_MON_B9") + ), + MTK_PIN( + 136, "GPIO136", + MTK_EINT_FUNCTION(0, 136), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO136"), + MTK_FUNCTION(1, "O_CMVREF2"), + MTK_FUNCTION(2, "B1_SCP_SCL3"), + MTK_FUNCTION(3, "B1_VADSP_SCL0"), + MTK_FUNCTION(4, "B1_SCP_SCL1"), + MTK_FUNCTION(5, "I1_MD32_8_RXD"), + MTK_FUNCTION(6, "I1_MD32_9_RXD"), + MTK_FUNCTION(7, "O_DBG_MON_B10") + ), + MTK_PIN( + 137, "GPIO137", + MTK_EINT_FUNCTION(0, 137), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO137"), + MTK_FUNCTION(1, "O_CMVREF3"), + MTK_FUNCTION(2, "B1_SCP_SDA3"), + MTK_FUNCTION(3, "B1_VADSP_SDA0"), + MTK_FUNCTION(4, "B1_SCP_SDA1"), + MTK_FUNCTION(5, "O_MD32_8_GPIO0"), + MTK_FUNCTION(6, "O_MD32_9_GPIO0"), + MTK_FUNCTION(7, "O_DBG_MON_B11") + ), + MTK_PIN( + 138, "GPIO138", + MTK_EINT_FUNCTION(0, 138), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO138"), + MTK_FUNCTION(1, "B0_DISP_GPIO_N3"), + MTK_FUNCTION(2, "I0_DISP_LSPII"), + MTK_FUNCTION(4, "O_CLKM1_B"), + MTK_FUNCTION(5, "O_MD32_8_TXD"), + MTK_FUNCTION(6, "O_MD32_9_TXD"), + MTK_FUNCTION(7, "O_DBG_MON_B12") + ), + MTK_PIN( + 139, "GPIO139", + MTK_EINT_FUNCTION(0, 139), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO139"), + MTK_FUNCTION(1, "B0_DISP_GPIO_N4"), + MTK_FUNCTION(2, "O_DISP_HSYNC0"), + MTK_FUNCTION(3, "O_DISP_HSYNC1"), + MTK_FUNCTION(4, "O_CLKM0_A"), + MTK_FUNCTION(5, "O_CLKM0_B"), + MTK_FUNCTION(6, "O_CLKM0_C"), + MTK_FUNCTION(7, "O_DBG_MON_B13") + ), + MTK_PIN( + 140, "GPIO140", + MTK_EINT_FUNCTION(0, 140), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO140"), + MTK_FUNCTION(1, "B0_DISP_GPIO_N5"), + MTK_FUNCTION(2, "O_DISP_VSYNC0"), + MTK_FUNCTION(3, "O_DISP_VSYNC1"), + MTK_FUNCTION(4, "O_CLKM1_A"), + MTK_FUNCTION(5, "O_CLKM1_B"), + MTK_FUNCTION(6, "O_CLKM1_C"), + MTK_FUNCTION(7, "O_DBG_MON_A14") + ), + MTK_PIN( + 141, "GPIO141", + MTK_EINT_FUNCTION(0, 141), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO141"), + MTK_FUNCTION(1, "B0_DISP_GPIO_N6"), + MTK_FUNCTION(2, "O_DISP_HSYNC2"), + MTK_FUNCTION(3, "O_DISP_HSYNC3"), + MTK_FUNCTION(4, "O_CLKM2_A"), + MTK_FUNCTION(5, "O_CLKM2_B"), + MTK_FUNCTION(6, "O_CLKM2_C"), + MTK_FUNCTION(7, "O_DBG_MON_A15") + ), + MTK_PIN( + 142, "GPIO142", + MTK_EINT_FUNCTION(0, 142), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO142"), + MTK_FUNCTION(1, "B0_DISP_GPIO_N7"), + MTK_FUNCTION(2, "O_DISP_VSYNC2"), + MTK_FUNCTION(3, "O_DISP_VSYNC3"), + MTK_FUNCTION(4, "O_CLKM3_A"), + MTK_FUNCTION(5, "O_CLKM3_B"), + MTK_FUNCTION(6, "O_CLKM3_C"), + MTK_FUNCTION(7, "O_DBG_MON_A16") + ), + MTK_PIN( + 143, "GPIO143", + MTK_EINT_FUNCTION(0, 143), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO143"), + MTK_FUNCTION(1, "I0_MCU_M_PMIC_POC_I"), + MTK_FUNCTION(2, "I0_JTCK_SEL1"), + MTK_FUNCTION(3, "O_JTAGAP_JTCK"), + MTK_FUNCTION(4, "I0_ADSP_JTAG1_TCK"), + MTK_FUNCTION(5, "I0_ADSP_JTAG0_TCK"), + MTK_FUNCTION(6, "I0_CLUSTER0_UDI_TCK"), + MTK_FUNCTION(7, "I0_CLUSTER1_UDI_TCK") + ), + MTK_PIN( + 144, "GPIO144", + MTK_EINT_FUNCTION(0, 144), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO144"), + MTK_FUNCTION(1, "I0_MCU_B_PMIC_POC_I"), + MTK_FUNCTION(2, "B1_JTMS_SEL1"), + MTK_FUNCTION(3, "O_JTAGAP_JTMS"), + MTK_FUNCTION(4, "I1_ADSP_JTAG1_TMS"), + MTK_FUNCTION(5, "I1_ADSP_JTAG0_TMS"), + MTK_FUNCTION(6, "I0_CLUSTER0_UDI_TMS"), + MTK_FUNCTION(7, "I0_CLUSTER1_UDI_TMS") + ), + MTK_PIN( + 145, "GPIO145", + MTK_EINT_FUNCTION(0, 145), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO145"), + MTK_FUNCTION(1, "I1_UCTS0"), + MTK_FUNCTION(2, "I1_JTDI_SEL1"), + MTK_FUNCTION(3, "O_JTAGAP_JTDI"), + MTK_FUNCTION(4, "I1_ADSP_JTAG1_TDI"), + MTK_FUNCTION(5, "I1_ADSP_JTAG0_TDI"), + MTK_FUNCTION(6, "I0_CLUSTER0_UDI_TDI_0"), + MTK_FUNCTION(7, "I0_CLUSTER1_UDI_TDI_0") + ), + MTK_PIN( + 146, "GPIO146", + MTK_EINT_FUNCTION(0, 146), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO146"), + MTK_FUNCTION(1, "O_URTS0"), + MTK_FUNCTION(2, "O_JTDO_SEL1"), + MTK_FUNCTION(3, "I0_JTAGAP_JTDO"), + MTK_FUNCTION(4, "O_ADSP_JTAG1_TDO"), + MTK_FUNCTION(5, "O_ADSP_JTAG0_TDO"), + MTK_FUNCTION(6, "O_CLUSTER0_UDI_TDO_0"), + MTK_FUNCTION(7, "O_CLUSTER1_UDI_TDO_0") + ), + MTK_PIN( + 147, "GPIO147", + MTK_EINT_FUNCTION(0, 147), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO147"), + MTK_FUNCTION(2, "I1_JTRSTn_SEL1"), + MTK_FUNCTION(3, "O_JTAGAP_JTRSTn"), + MTK_FUNCTION(4, "I1_ADSP_JTAG1_TRSTN"), + MTK_FUNCTION(5, "I1_ADSP_JTAG0_TRSTN"), + MTK_FUNCTION(6, "I0_CLUSTER0_UDI_NTRST"), + MTK_FUNCTION(7, "I0_CLUSTER1_UDI_NTRST") + ), + MTK_PIN( + 148, "GPIO148", + MTK_EINT_FUNCTION(0, 148), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO148"), + MTK_FUNCTION(1, "O_SRCLKENA0") + ), + MTK_PIN( + 149, "GPIO149", + MTK_EINT_FUNCTION(0, 149), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO149"), + MTK_FUNCTION(1, "O_SRCLKENA1") + ), + MTK_PIN( + 150, "GPIO150", + MTK_EINT_FUNCTION(0, 150), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO150"), + MTK_FUNCTION(2, "O_NVJTAG_SEL"), + MTK_FUNCTION(4, "I0_TSFDC_SDI"), + MTK_FUNCTION(5, "O_TSFDC_BG_COMP"), + MTK_FUNCTION(6, "O_CLKM3_C"), + MTK_FUNCTION(7, "I0_HFRP_JTAG0_TRSTN") + ), + MTK_PIN( + 151, "GPIO151", + MTK_EINT_FUNCTION(0, 151), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO151"), + MTK_FUNCTION(1, "I1_JTRSTn_SEL1"), + MTK_FUNCTION(2, "O_JTAGAP_JTRSTn"), + MTK_FUNCTION(6, "I0_HFRP_JTAG1_TRSTN"), + MTK_FUNCTION(7, "I1_ADSP_JTAG1_TRSTN") + ), + MTK_PIN( + 152, "GPIO152", + MTK_EINT_FUNCTION(0, 152), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO152"), + MTK_FUNCTION(1, "B1_JTMS_SEL1"), + MTK_FUNCTION(2, "O_JTAGAP_JTMS"), + MTK_FUNCTION(6, "I1_HFRP_JTAG1_TMS"), + MTK_FUNCTION(7, "I1_ADSP_JTAG1_TMS") + ), + MTK_PIN( + 153, "GPIO153", + MTK_EINT_FUNCTION(0, 153), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO153"), + MTK_FUNCTION(1, "O_JTDO_SEL1"), + MTK_FUNCTION(2, "I0_JTAGAP_JTDO"), + MTK_FUNCTION(6, "O_HFRP_JTAG1_TDO"), + MTK_FUNCTION(7, "O_ADSP_JTAG1_TDO") + ), + MTK_PIN( + 154, "GPIO154", + MTK_EINT_FUNCTION(0, 154), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO154"), + MTK_FUNCTION(1, "I0_JTCK_SEL1"), + MTK_FUNCTION(2, "O_JTAGAP_JTCK"), + MTK_FUNCTION(6, "I1_HFRP_JTAG1_TCK"), + MTK_FUNCTION(7, "I0_ADSP_JTAG1_TCK") + ), + MTK_PIN( + 155, "GPIO155", + MTK_EINT_FUNCTION(0, 155), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO155"), + MTK_FUNCTION(1, "I1_JTDI_SEL1"), + MTK_FUNCTION(2, "O_JTAGAP_JTDI"), + MTK_FUNCTION(6, "I1_HFRP_JTAG1_TDI"), + MTK_FUNCTION(7, "I1_ADSP_JTAG1_TDI") + ), + MTK_PIN( + 156, "GPIO156", + MTK_EINT_FUNCTION(0, 156), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO156"), + MTK_FUNCTION(1, "O_TP_UTXD0_VLP"), + MTK_FUNCTION(2, "O_UTXD3"), + MTK_FUNCTION(3, "O_VADSP_UTXD1"), + MTK_FUNCTION(4, "O_SPI_SIO0_S_MON0"), + MTK_FUNCTION(5, "O_ADSP_UTXD1"), + MTK_FUNCTION(6, "I0_SROOT_TCK"), + MTK_FUNCTION(7, "O_OSROOT_UTX") + ), + MTK_PIN( + 157, "GPIO157", + MTK_EINT_FUNCTION(0, 157), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO157"), + MTK_FUNCTION(1, "I1_TP_URXD0_VLP"), + MTK_FUNCTION(2, "I1_URXD3"), + MTK_FUNCTION(3, "I1_VADSP_URXD1"), + MTK_FUNCTION(4, "O_SPI_SIO1_S_MON0"), + MTK_FUNCTION(5, "I1_ADSP_URXD1"), + MTK_FUNCTION(6, "I0_SROOT_TDI"), + MTK_FUNCTION(7, "I1_OSROOT_URX") + ), + MTK_PIN( + 158, "GPIO158", + MTK_EINT_FUNCTION(0, 158), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO158"), + MTK_FUNCTION(1, "O_TP_URTS0_VLP"), + MTK_FUNCTION(2, "O_URTS3"), + MTK_FUNCTION(3, "O_VADSP_URTS1"), + MTK_FUNCTION(4, "O_SPI_SIO2_S_MON0"), + MTK_FUNCTION(5, "O_ADSP_URTX1"), + MTK_FUNCTION(6, "O_SROOT_TDO"), + MTK_FUNCTION(7, "O_PBUD_CTRL_UTXD_AO_VLP") + ), + MTK_PIN( + 159, "GPIO159", + MTK_EINT_FUNCTION(0, 159), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO159"), + MTK_FUNCTION(1, "I1_TP_UCTS0_VLP"), + MTK_FUNCTION(2, "I1_UCTS3"), + MTK_FUNCTION(3, "I1_VADSP_UCTS1"), + MTK_FUNCTION(4, "O_SPI_SIO3_S_MON0"), + MTK_FUNCTION(5, "I1_ADSP_UCTS1"), + MTK_FUNCTION(6, "I0_SROOT_TMS"), + MTK_FUNCTION(7, "I1_PBUD_CTRL_URXD_AO_VLP") + ), + MTK_PIN( + 160, "GPIO160", + MTK_EINT_FUNCTION(0, 160), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO160"), + MTK_FUNCTION(1, "O_UTXD1"), + MTK_FUNCTION(2, "O_ADSP_UTXD1"), + MTK_FUNCTION(3, "O_HFRP_UTXD1"), + MTK_FUNCTION(4, "O_CCU1_UTXD"), + MTK_FUNCTION(5, "O_PBUD_CTRL_UTXD_AO_VCORE"), + MTK_FUNCTION(6, "O_U4CP_UTXD"), + MTK_FUNCTION(7, "O_DBG_MON_B14") + ), + MTK_PIN( + 161, "GPIO161", + MTK_EINT_FUNCTION(0, 161), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO161"), + MTK_FUNCTION(1, "I1_URXD1"), + MTK_FUNCTION(2, "I1_ADSP_URXD1"), + MTK_FUNCTION(3, "I1_HFRP_URXD1"), + MTK_FUNCTION(4, "I1_CCU1_URXD"), + MTK_FUNCTION(5, "I1_PBUD_CTRL_URXD_AO_VCORE"), + MTK_FUNCTION(6, "I1_U4CP_URXD"), + MTK_FUNCTION(7, "O_DBG_MON_B15") + ), + MTK_PIN( + 162, "GPIO162", + MTK_EINT_FUNCTION(0, 162), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO162"), + MTK_FUNCTION(1, "O_URTS1"), + MTK_FUNCTION(2, "O_ADSP_URTX1"), + MTK_FUNCTION(3, "O_HFRP_URTS1"), + MTK_FUNCTION(4, "O_CCU0_UTXD"), + MTK_FUNCTION(5, "O_PBUD_CTRL_UTXD_AO_VCORE"), + MTK_FUNCTION(6, "O_U4CP_URTS"), + MTK_FUNCTION(7, "O_DBG_MON_B16") + ), + MTK_PIN( + 163, "GPIO163", + MTK_EINT_FUNCTION(0, 163), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO163"), + MTK_FUNCTION(1, "I1_UCTS1"), + MTK_FUNCTION(2, "I1_ADSP_UCTS1"), + MTK_FUNCTION(3, "I1_HFRP_UCTS1"), + MTK_FUNCTION(4, "I1_CCU0_URXD"), + MTK_FUNCTION(5, "I1_PBUD_CTRL_URXD_AO_VCORE"), + MTK_FUNCTION(6, "I1_U4CP_UCTS"), + MTK_FUNCTION(7, "O_DBG_MON_B17") + ), + MTK_PIN( + 164, "GPIO164", + MTK_EINT_FUNCTION(0, 164), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO164"), + MTK_FUNCTION(1, "O_HDMITX_DC_CTRL"), + MTK_FUNCTION(6, "I0_SROOT_NTRST"), + MTK_FUNCTION(7, "O_DBG_MON_B18") + ), + MTK_PIN( + 165, "GPIO165", + MTK_EINT_FUNCTION(0, 165), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO165"), + MTK_FUNCTION(1, "B1_DISP_SCL0"), + MTK_FUNCTION(4, "I1_EDP0_SCL"), + MTK_FUNCTION(5, "O_CCU0_URTS"), + MTK_FUNCTION(6, "O_CCU1_URTS"), + MTK_FUNCTION(7, "I1_MD32_6_RXD") + ), + MTK_PIN( + 166, "GPIO166", + MTK_EINT_FUNCTION(0, 166), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO166"), + MTK_FUNCTION(1, "B1_DISP_SDA0"), + MTK_FUNCTION(4, "B1_EDP0_SDA"), + MTK_FUNCTION(5, "I1_CCU0_UCTS"), + MTK_FUNCTION(6, "I1_CCU1_UCTS"), + MTK_FUNCTION(7, "O_MD32_6_GPIO0") + ), + MTK_PIN( + 167, "GPIO167", + MTK_EINT_FUNCTION(0, 167), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO167"), + MTK_FUNCTION(1, "B0_DISP_GPIO_N0"), + MTK_FUNCTION(5, "O_CLUSTER0_AD_ILDO_DTEST0"), + MTK_FUNCTION(6, "O_CLUSTER1_AD_ILDO_DTEST0"), + MTK_FUNCTION(7, "O_DBG_MON_B19") + ), + MTK_PIN( + 168, "GPIO168", + MTK_EINT_FUNCTION(0, 168), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO168"), + MTK_FUNCTION(1, "I0_DPAUX_HPD_IN_0"), + MTK_FUNCTION(5, "O_CLUSTER0_AD_ILDO_DTEST1"), + MTK_FUNCTION(6, "O_CLUSTER1_AD_ILDO_DTEST1"), + MTK_FUNCTION(7, "O_DBG_MON_B20") + ), + MTK_PIN( + 169, "GPIO169", + MTK_EINT_FUNCTION(0, 169), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO169"), + MTK_FUNCTION(1, "I0_DPAUX_HPD_IN_1"), + MTK_FUNCTION(5, "O_CLUSTER0_AD_ILDO_DTEST3"), + MTK_FUNCTION(6, "O_CLUSTER1_AD_ILDO_DTEST3"), + MTK_FUNCTION(7, "O_DBG_MON_B21") + ), + MTK_PIN( + 170, "GPIO170", + MTK_EINT_FUNCTION(0, 170), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO170"), + MTK_FUNCTION(1, "O_USB4_R_TCPC_RESET"), + MTK_FUNCTION(5, "O_CLUSTER0_AD_ILDO_DTEST2"), + MTK_FUNCTION(6, "O_CLUSTER1_AD_ILDO_DTEST2"), + MTK_FUNCTION(7, "O_DBG_MON_A17") + ), + MTK_PIN( + 171, "GPIO171", + MTK_EINT_FUNCTION(0, 171), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO171"), + MTK_FUNCTION(1, "O_PCIE_PERSTN"), + MTK_FUNCTION(4, "O_MD32_14_GPIO0"), + MTK_FUNCTION(7, "O_DBG_MON_B22") + ), + MTK_PIN( + 172, "GPIO172", + MTK_EINT_FUNCTION(0, 172), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO172"), + MTK_FUNCTION(1, "B1_PCIE_CLKREQN_0P"), + MTK_FUNCTION(2, "I1_PCIE_PRSNT_0P"), + MTK_FUNCTION(5, "O_HFRP_UTXD1"), + MTK_FUNCTION(6, "O_CCU0_UTXD"), + MTK_FUNCTION(7, "O_DBG_MON_B23") + ), + MTK_PIN( + 173, "GPIO173", + MTK_EINT_FUNCTION(0, 173), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO173"), + MTK_FUNCTION(1, "B1_PCIE_CLKREQN_1P"), + MTK_FUNCTION(2, "I1_PCIE_PRSNT_1P"), + MTK_FUNCTION(5, "I1_HFRP_URXD1"), + MTK_FUNCTION(6, "I1_CCU0_URXD"), + MTK_FUNCTION(7, "O_DBG_MON_B24") + ), + MTK_PIN( + 174, "GPIO174", + MTK_EINT_FUNCTION(0, 174), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO174"), + MTK_FUNCTION(1, "B1_PCIE_CLKREQN_2P"), + MTK_FUNCTION(2, "I1_PCIE_PRSNT_2P"), + MTK_FUNCTION(4, "O_MD32PCM_UTXD_AO_VLP"), + MTK_FUNCTION(5, "O_HFRP_URTS1"), + MTK_FUNCTION(6, "O_CCU1_UTXD"), + MTK_FUNCTION(7, "O_DBG_MON_B25") + ), + MTK_PIN( + 175, "GPIO175", + MTK_EINT_FUNCTION(0, 175), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO175"), + MTK_FUNCTION(1, "B1_PCIE_CLKREQN_3P"), + MTK_FUNCTION(2, "I1_PCIE_PRSNT_3P"), + MTK_FUNCTION(4, "I1_MD32PCM_URXD_AO_VLP"), + MTK_FUNCTION(5, "I1_HFRP_UCTS1"), + MTK_FUNCTION(6, "I1_CCU1_URXD"), + MTK_FUNCTION(7, "O_DBG_MON_B26") + ), + MTK_PIN( + 176, "GPIO176", + MTK_EINT_FUNCTION(0, 176), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO176"), + MTK_FUNCTION(1, "B1_PCIE_CLKREQN_4P"), + MTK_FUNCTION(2, "I1_PCIE_PRSNT_4P"), + MTK_FUNCTION(4, "O_MD32_14_TXD"), + MTK_FUNCTION(7, "O_DBG_MON_B27") + ), + MTK_PIN( + 177, "GPIO177", + MTK_EINT_FUNCTION(0, 177), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO177"), + MTK_FUNCTION(1, "B1_PCIE_CLKREQN_5P"), + MTK_FUNCTION(2, "I1_PCIE_PRSNT_5P"), + MTK_FUNCTION(4, "I1_MD32_14_RXD"), + MTK_FUNCTION(7, "O_DBG_MON_B28") + ), + MTK_PIN( + 178, "GPIO178", + MTK_EINT_FUNCTION(0, 178), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO178"), + MTK_FUNCTION(1, "B1_PCIE_CLKREQN_6P"), + MTK_FUNCTION(2, "I1_PCIE_PRSNT_6P"), + MTK_FUNCTION(4, "O_MD32_15_TXD"), + MTK_FUNCTION(7, "O_DBG_MON_B29") + ), + MTK_PIN( + 179, "GPIO179", + MTK_EINT_FUNCTION(0, 179), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO179"), + MTK_FUNCTION(1, "B1_PCIE_CLKREQN_7P"), + MTK_FUNCTION(2, "I1_PCIE_PRSNT_7P"), + MTK_FUNCTION(4, "I1_MD32_15_RXD"), + MTK_FUNCTION(7, "O_DBG_MON_B30") + ), + MTK_PIN( + 180, "GPIO180", + MTK_EINT_FUNCTION(0, 180), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO180"), + MTK_FUNCTION(1, "I1_PCIE_WAKEN"), + MTK_FUNCTION(4, "O_MD32_15_GPIO0"), + MTK_FUNCTION(7, "O_DBG_MON_B31") + ), + MTK_PIN( + 181, "GPIO181", + MTK_EINT_FUNCTION(0, 181), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO181"), + MTK_FUNCTION(1, "O_GPU_PWRGOOD") + ), +}; + +static struct mtk_eint_pin eint_pins_mt8901[] = { + MTK_EINT_PIN(0, 0, 16, 0), + MTK_EINT_PIN(1, 0, 17, 0), + MTK_EINT_PIN(2, 0, 18, 0), + MTK_EINT_PIN(3, 0, 19, 0), + MTK_EINT_PIN(4, 0, 20, 0), + MTK_EINT_PIN(5, 0, 21, 0), + MTK_EINT_PIN(6, 0, 22, 0), + MTK_EINT_PIN(7, 0, 23, 0), + MTK_EINT_PIN(8, 0, 24, 0), + MTK_EINT_PIN(9, 0, 25, 0), + MTK_EINT_PIN(10, 0, 26, 0), + MTK_EINT_PIN(11, 0, 27, 0), + MTK_EINT_PIN(12, INVALID_BASE, 0, 0), + MTK_EINT_PIN(13, INVALID_BASE, 0, 0), + MTK_EINT_PIN(14, 0, 28, 0), + MTK_EINT_PIN(15, 0, 29, 0), + MTK_EINT_PIN(16, 0, 30, 0), + MTK_EINT_PIN(17, INVALID_BASE, 0, 0), + MTK_EINT_PIN(18, INVALID_BASE, 0, 0), + MTK_EINT_PIN(19, 0, 31, 0), + MTK_EINT_PIN(20, 0, 0, 1), + MTK_EINT_PIN(21, 0, 1, 1), + MTK_EINT_PIN(22, INVALID_BASE, 0, 0), + MTK_EINT_PIN(23, 0, 2, 1), + MTK_EINT_PIN(24, 0, 3, 1), + MTK_EINT_PIN(25, 0, 4, 1), + MTK_EINT_PIN(26, 0, 5, 1), + MTK_EINT_PIN(27, 0, 6, 1), + MTK_EINT_PIN(28, 0, 7, 1), + MTK_EINT_PIN(29, 0, 8, 1), + MTK_EINT_PIN(30, 0, 9, 1), + MTK_EINT_PIN(31, 0, 10, 1), + MTK_EINT_PIN(32, 1, 0, 1), + MTK_EINT_PIN(33, 0, 32, 0), + MTK_EINT_PIN(34, 0, 33, 0), + MTK_EINT_PIN(35, 0, 34, 0), + MTK_EINT_PIN(36, 1, 1, 1), + MTK_EINT_PIN(37, 0, 11, 1), + MTK_EINT_PIN(38, 0, 12, 1), + MTK_EINT_PIN(39, 0, 35, 0), + MTK_EINT_PIN(40, 0, 36, 0), + MTK_EINT_PIN(41, 0, 37, 0), + MTK_EINT_PIN(42, 0, 13, 1), + MTK_EINT_PIN(43, 0, 14, 1), + MTK_EINT_PIN(44, 0, 38, 0), + MTK_EINT_PIN(45, 0, 39, 0), + MTK_EINT_PIN(46, 0, 40, 0), + MTK_EINT_PIN(47, 0, 15, 1), + MTK_EINT_PIN(48, 0, 41, 0), + MTK_EINT_PIN(49, 0, 42, 0), + MTK_EINT_PIN(50, 0, 43, 0), + MTK_EINT_PIN(51, 0, 44, 0), + MTK_EINT_PIN(52, 0, 45, 0), + MTK_EINT_PIN(53, 0, 46, 0), + MTK_EINT_PIN(54, 2, 13, 0), + MTK_EINT_PIN(55, 2, 14, 0), + MTK_EINT_PIN(56, INVALID_BASE, 0, 0), + MTK_EINT_PIN(57, INVALID_BASE, 0, 0), + MTK_EINT_PIN(58, INVALID_BASE, 0, 0), + MTK_EINT_PIN(59, INVALID_BASE, 0, 0), + MTK_EINT_PIN(60, 1, 3, 0), + MTK_EINT_PIN(61, 1, 4, 0), + MTK_EINT_PIN(62, 1, 5, 0), + MTK_EINT_PIN(63, 1, 6, 0), + MTK_EINT_PIN(64, 1, 7, 0), + MTK_EINT_PIN(65, 1, 8, 0), + MTK_EINT_PIN(66, 2, 15, 0), + MTK_EINT_PIN(67, 2, 16, 0), + MTK_EINT_PIN(68, 1, 2, 1), + MTK_EINT_PIN(69, INVALID_BASE, 0, 0), + MTK_EINT_PIN(70, 2, 17, 0), + MTK_EINT_PIN(71, 2, 18, 0), + MTK_EINT_PIN(72, 2, 19, 0), + MTK_EINT_PIN(73, 2, 20, 0), + MTK_EINT_PIN(74, 2, 21, 0), + MTK_EINT_PIN(75, 2, 22, 0), + MTK_EINT_PIN(76, 2, 23, 0), + MTK_EINT_PIN(77, 2, 24, 0), + MTK_EINT_PIN(78, 2, 25, 0), + MTK_EINT_PIN(79, 2, 26, 0), + MTK_EINT_PIN(80, 2, 27, 0), + MTK_EINT_PIN(81, 2, 28, 0), + MTK_EINT_PIN(82, 2, 29, 1), + MTK_EINT_PIN(83, 2, 30, 1), + MTK_EINT_PIN(84, 2, 31, 1), + MTK_EINT_PIN(85, 2, 32, 1), + MTK_EINT_PIN(86, 2, 33, 0), + MTK_EINT_PIN(87, 2, 34, 0), + MTK_EINT_PIN(88, 2, 35, 0), + MTK_EINT_PIN(89, 2, 36, 0), + MTK_EINT_PIN(90, 2, 37, 0), + MTK_EINT_PIN(91, 2, 38, 0), + MTK_EINT_PIN(92, 2, 39, 0), + MTK_EINT_PIN(93, 2, 40, 0), + MTK_EINT_PIN(94, 2, 0, 1), + MTK_EINT_PIN(95, 2, 1, 1), + MTK_EINT_PIN(96, 2, 2, 1), + MTK_EINT_PIN(97, 2, 3, 1), + MTK_EINT_PIN(98, 2, 41, 0), + MTK_EINT_PIN(99, 2, 4, 1), + MTK_EINT_PIN(100, 2, 5, 1), + MTK_EINT_PIN(101, 2, 6, 1), + MTK_EINT_PIN(102, 2, 7, 1), + MTK_EINT_PIN(103, 2, 8, 1), + MTK_EINT_PIN(104, 2, 9, 1), + MTK_EINT_PIN(105, 2, 10, 1), + MTK_EINT_PIN(106, 0, 47, 0), + MTK_EINT_PIN(107, 0, 48, 0), + MTK_EINT_PIN(108, 0, 49, 0), + MTK_EINT_PIN(109, 0, 50, 0), + MTK_EINT_PIN(110, 0, 51, 0), + MTK_EINT_PIN(111, 0, 52, 0), + MTK_EINT_PIN(112, 2, 42, 0), + MTK_EINT_PIN(113, 2, 43, 0), + MTK_EINT_PIN(114, 2, 44, 0), + MTK_EINT_PIN(115, 2, 45, 0), + MTK_EINT_PIN(116, 2, 46, 0), + MTK_EINT_PIN(117, 2, 47, 0), + MTK_EINT_PIN(118, 2, 48, 0), + MTK_EINT_PIN(119, 2, 49, 0), + MTK_EINT_PIN(120, 2, 50, 0), + MTK_EINT_PIN(121, 2, 51, 0), + MTK_EINT_PIN(122, 2, 52, 0), + MTK_EINT_PIN(123, 2, 53, 0), + MTK_EINT_PIN(124, 2, 54, 0), + MTK_EINT_PIN(125, 2, 55, 0), + MTK_EINT_PIN(126, 2, 56, 0), + MTK_EINT_PIN(127, 2, 57, 0), + MTK_EINT_PIN(128, 2, 58, 0), + MTK_EINT_PIN(129, 2, 59, 0), + MTK_EINT_PIN(130, 2, 60, 0), + MTK_EINT_PIN(131, 2, 61, 0), + MTK_EINT_PIN(132, 2, 62, 0), + MTK_EINT_PIN(133, 2, 63, 0), + MTK_EINT_PIN(134, 2, 64, 0), + MTK_EINT_PIN(135, 2, 65, 0), + MTK_EINT_PIN(136, 2, 66, 0), + MTK_EINT_PIN(137, 2, 67, 0), + MTK_EINT_PIN(138, 2, 11, 1), + MTK_EINT_PIN(139, 2, 12, 1), + MTK_EINT_PIN(140, 2, 68, 0), + MTK_EINT_PIN(141, 2, 69, 0), + MTK_EINT_PIN(142, 2, 70, 0), + MTK_EINT_PIN(143, 2, 71, 0), + MTK_EINT_PIN(144, 2, 72, 0), + MTK_EINT_PIN(145, 2, 73, 0), + MTK_EINT_PIN(146, 2, 74, 0), + MTK_EINT_PIN(147, 2, 75, 0), + MTK_EINT_PIN(148, INVALID_BASE, 0, 0), + MTK_EINT_PIN(149, INVALID_BASE, 0, 0), + MTK_EINT_PIN(150, 2, 76, 0), + MTK_EINT_PIN(151, 2, 77, 0), + MTK_EINT_PIN(152, 2, 78, 0), + MTK_EINT_PIN(153, 2, 79, 0), + MTK_EINT_PIN(154, 2, 80, 0), + MTK_EINT_PIN(155, 2, 81, 0), + MTK_EINT_PIN(156, 2, 82, 0), + MTK_EINT_PIN(157, 2, 83, 0), + MTK_EINT_PIN(158, 2, 84, 0), + MTK_EINT_PIN(159, 2, 85, 0), + MTK_EINT_PIN(160, 2, 86, 0), + MTK_EINT_PIN(161, 2, 87, 0), + MTK_EINT_PIN(162, 2, 88, 0), + MTK_EINT_PIN(163, 2, 89, 0), + MTK_EINT_PIN(164, 2, 90, 0), + MTK_EINT_PIN(165, 2, 91, 0), + MTK_EINT_PIN(166, 2, 92, 0), + MTK_EINT_PIN(167, 2, 93, 0), + MTK_EINT_PIN(168, 2, 94, 0), + MTK_EINT_PIN(169, 2, 95, 0), + MTK_EINT_PIN(170, 2, 96, 0), + MTK_EINT_PIN(171, 2, 97, 0), + MTK_EINT_PIN(172, 2, 98, 0), + MTK_EINT_PIN(173, 2, 99, 0), + MTK_EINT_PIN(174, 2, 100, 0), + MTK_EINT_PIN(175, 2, 101, 0), + MTK_EINT_PIN(176, 2, 102, 0), + MTK_EINT_PIN(177, 2, 103, 0), + MTK_EINT_PIN(178, 2, 104, 0), + MTK_EINT_PIN(179, 2, 105, 0), + MTK_EINT_PIN(180, 2, 106, 0), + MTK_EINT_PIN(181, 3, 0, 0), + MTK_EINT_PIN(182, 3, 1, 0), + MTK_EINT_PIN(183, 3, 2, 0), + MTK_EINT_PIN(184, 3, 3, 0), + MTK_EINT_PIN(185, 3, 4, 0), + MTK_EINT_PIN(186, 3, 5, 0), + MTK_EINT_PIN(187, 3, 6, 0), + MTK_EINT_PIN(188, 3, 7, 0), + MTK_EINT_PIN(189, 3, 8, 0), + MTK_EINT_PIN(190, 3, 9, 0), + MTK_EINT_PIN(191, 3, 10, 0), + MTK_EINT_PIN(192, 3, 11, 0), + MTK_EINT_PIN(193, 3, 12, 0), + MTK_EINT_PIN(194, 3, 13, 0), + MTK_EINT_PIN(195, 3, 14, 0), + MTK_EINT_PIN(196, 3, 15, 0), + MTK_EINT_PIN(197, 3, 16, 0), + MTK_EINT_PIN(198, 3, 17, 0), + MTK_EINT_PIN(199, 3, 18, 0), + MTK_EINT_PIN(200, 3, 19, 0), + MTK_EINT_PIN(201, 3, 20, 0), + MTK_EINT_PIN(202, 3, 21, 0), + MTK_EINT_PIN(203, 3, 22, 0), + MTK_EINT_PIN(204, 3, 23, 0), + MTK_EINT_PIN(205, 3, 24, 0), + MTK_EINT_PIN(206, 3, 25, 0), + MTK_EINT_PIN(207, 3, 26, 0), + MTK_EINT_PIN(208, 3, 27, 0), +}; + +#endif /* __PINCTRL__MTK_MT8901_H */ diff --git a/drivers/pinctrl/mediatek/pinctrl-paris.c b/drivers/pinctrl/mediatek/pinctrl-paris.c index 6bf37d8085fae..f74221acba8cb 100644 --- a/drivers/pinctrl/mediatek/pinctrl-paris.c +++ b/drivers/pinctrl/mediatek/pinctrl-paris.c @@ -3,7 +3,7 @@ * MediaTek Pinctrl Paris Driver, which implement the vendor per-pin * bindings for MediaTek SoC. * - * Copyright (C) 2018 MediaTek Inc. + * Copyright (C) 2018-2025 MediaTek Inc. * Author: Sean Wang * Zhiyong Tao * Hongzhou.Yang @@ -936,6 +936,15 @@ static int mtk_gpio_set_config(struct gpio_chip *chip, unsigned int offset, return mtk_eint_set_debounce(hw->eint, desc->eint.eint_n, debounce); } +static void mtk_pinctrl_gpio_range_init(struct mtk_pinctrl *hw, struct gpio_chip *chip) +{ + hw->range.name = "mtk_pinctrl_gpio_range"; + hw->range.id = 0; + hw->range.pin_base = 0; + hw->range.base = chip->base; + hw->range.npins = hw->soc->npins; +} + static int mtk_build_gpiochip(struct mtk_pinctrl *hw) { struct gpio_chip *chip = &hw->chip; @@ -959,6 +968,8 @@ static int mtk_build_gpiochip(struct mtk_pinctrl *hw) if (ret < 0) return ret; + mtk_pinctrl_gpio_range_init(hw, chip); + return 0; } @@ -997,6 +1008,7 @@ int mtk_paris_pinctrl_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct pinctrl_pin_desc *pins; struct mtk_pinctrl *hw; + struct fwnode_handle *fwnode = dev_fwnode(&pdev->dev); int err, i; hw = devm_kzalloc(&pdev->dev, sizeof(*hw), GFP_KERNEL); @@ -1021,16 +1033,20 @@ int mtk_paris_pinctrl_probe(struct platform_device *pdev) return -ENOMEM; for (i = 0; i < hw->soc->nbase_names; i++) { - hw->base[i] = devm_platform_ioremap_resource_byname(pdev, - hw->soc->base_names[i]); + hw->base[i] = is_of_node(fwnode) + ? devm_platform_ioremap_resource_byname(pdev, hw->soc->base_names[i]) + : devm_platform_get_and_ioremap_resource(pdev, i, NULL); if (IS_ERR(hw->base[i])) return PTR_ERR(hw->base[i]); } hw->nbase = hw->soc->nbase_names; - hw->rsel_si_unit = of_property_read_bool(hw->dev->of_node, + if (is_of_node(fwnode)) + hw->rsel_si_unit = of_property_read_bool(hw->dev->of_node, "mediatek,rsel-resistance-in-si-unit"); + else + hw->rsel_si_unit = false; spin_lock_init(&hw->lock); @@ -1077,6 +1093,8 @@ int mtk_paris_pinctrl_probe(struct platform_device *pdev) if (err) return dev_err_probe(dev, err, "Failed to add gpio_chip\n"); + pinctrl_add_gpio_range(hw->pctrl, &hw->range); + platform_set_drvdata(pdev, hw); return 0; diff --git a/drivers/platform/arm64/Kconfig b/drivers/platform/arm64/Kconfig index c1ca1d78eeb86..4bbd8eca38a87 100644 --- a/drivers/platform/arm64/Kconfig +++ b/drivers/platform/arm64/Kconfig @@ -102,4 +102,19 @@ config EC_LENOVO_YOGA_SLIM7X mute button, and reporting device suspend to the EC so it can take appropriate actions. +config NVIDIA_FFA_EC + tristate "NVIDIA FFA EC services driver" + depends on ARM_FFA_TRANSPORT || COMPILE_TEST + depends on ACPI + depends on ACPI_FFH + help + Enable NVIDIA FFA EC services. + For GB10 and other similar SOC’s, to communicate with embedded controller, a new + specification is being defined. It is currently in draft stage and maintained in + https://github.com/OpenDevicePartnership/documentation/blob/main/bookshelf/Shelf%204%20Specifications/EC%20Interface/src/secure-ec-services-overview.md + + Say M or Y here to include this support. + +source "drivers/platform/arm64/nvidia/Kconfig" + endif # ARM64_PLATFORM_DEVICES diff --git a/drivers/platform/arm64/Makefile b/drivers/platform/arm64/Makefile index c135a895a3ea5..baa8d477a2cd7 100644 --- a/drivers/platform/arm64/Makefile +++ b/drivers/platform/arm64/Makefile @@ -10,3 +10,5 @@ obj-$(CONFIG_EC_HUAWEI_GAOKUN) += huawei-gaokun-ec.o obj-$(CONFIG_EC_LENOVO_YOGA_C630) += lenovo-yoga-c630.o obj-$(CONFIG_EC_LENOVO_THINKPAD_T14S) += lenovo-thinkpad-t14s.o obj-$(CONFIG_EC_LENOVO_YOGA_SLIM7X) += lenovo-yoga-slim7x.o +obj-$(CONFIG_NVIDIA_FFA_EC) += nvidia-ffa-ec.o +obj-y += nvidia/ diff --git a/drivers/platform/arm64/nvidia-ffa-ec.c b/drivers/platform/arm64/nvidia-ffa-ec.c new file mode 100644 index 0000000000000..7a259ba64956c --- /dev/null +++ b/drivers/platform/arm64/nvidia-ffa-ec.c @@ -0,0 +1,829 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved + */ + +#include +#include +#include +#include +#include +#include + +#define DRV_NAME "nvidia-ffa-ec" + +/* platform device for FFA ACPI device (HID MSFT000C) */ +static struct platform_device *ffa_pdev; + +/* FFA device for EC notification service */ +static struct ffa_device *notify_ffa_dev; + +static const uuid_t nvidia_ec_notify_service_uuid = + UUID_INIT(0xb510b3a3, 0x59f6, 0x4054, 0xba, 0x7a, 0xff, 0x2e, 0xb1, 0xea, 0xc7, 0x65); + +static const uuid_t nvidia_ec_managment_service_uuid = + UUID_INIT(0x330c1273, 0xfde5, 0x4757, 0x98, 0x19, 0x5b, 0x65, 0x39, 0x03, 0x75, 0x02); + +static const uuid_t nvidia_ec_power_service_uuid = + UUID_INIT(0x7157addf, 0x2fbe, 0x4c63, 0xae, 0x95, 0xef, 0xac, 0x16, 0xe3, 0xb0, 0x1c); + +static const uuid_t nvidia_ec_battery_service_uuid = + UUID_INIT(0x25cb5207, 0xac36, 0x427d, 0xaa, 0xef, 0x3a, 0xa7, 0x88, 0x77, 0xd2, 0x7e); + +static const uuid_t nvidia_ec_thermal_service_uuid = + UUID_INIT(0x31f56da7, 0x593c, 0x4d72, 0xa4, 0xb3, 0x8f, 0xc7, 0x17, 0x1a, 0xc0, 0x73); + +static const uuid_t nvidia_ec_fan_service_uuid = + UUID_INIT(0x7697530c, 0xd079, 0x4ec1, 0xa4, 0xc4, 0xcf, 0x0d, 0x2b, 0xdc, 0x93, 0xfa); + +static const uuid_t nvidia_ec_ucsi_service_uuid = + UUID_INIT(0x65467f50, 0x827f, 0x4e4f, 0x87, 0x70, 0xdb, 0xf4, 0xc3, 0xf7, 0x7f, 0x45); + +static const uuid_t nvidia_ec_input_service_uuid = + UUID_INIT(0xe3168a99, 0x4a57, 0x4a2b, 0x8c, 0x5e, 0x11, 0xbc, 0xfe, 0xc7, 0x34, 0x06); + +static const uuid_t nvidia_ec_time_alarm_service_uuid = + UUID_INIT(0x23ea63ed, 0xb593, 0x46ea, 0xb0, 0x27, 0x89, 0x24, 0xdf, 0x88, 0xe9, 0x2f); + +static const guid_t nvidia_notify_bind_guid = + GUID_INIT(0xdaffd814, 0x6eba, 0x4d8c, 0x8a, 0x91, 0xbc, 0x9b, 0xbf, 0x4a, 0xa3, 0x01); + +static const guid_t nvidia_notify_dsm_guid = + GUID_INIT(0x7681541e, 0x8827, 0x4239, 0x8d, 0x9d, 0x36, 0xbe, 0x7f, 0xe1, 0x25, 0x42); + +#define NVIDIA_FFA_MAX_NOTIFICATIONS 64 + +/* EC service FFA device structure */ +struct nvidia_ec_ffa_device { + struct ffa_device *ffa_dev; + u8 notification_count; + u8 notification_id[NVIDIA_FFA_MAX_NOTIFICATIONS]; + struct list_head list; +}; + +/* List to contain all EC services FFA device */ +static LIST_HEAD(nvidia_ec_ffa_dev_head); + +/* Lock to serialize EC services FFA device list access */ +static DEFINE_MUTEX(nvidia_ffa_lock); + +/* EC secure services FFA packet structure sent via ACPI */ +struct nvidia_ec_ffa_packet { + u8 status; + u8 length; + u8 uuid[UUID_SIZE]; + u8 rawdata[]; +} __packed; + +/* + * ACPI ASL code uses ToUUID() macro which encodes it in mixed-endian format. + * Convert the AML UUID buffer into FFA UUID format. + */ +static uuid_t nvidia_get_uuid_from_aml_buf(const u8 *buf) +{ + return (uuid_t) {{ buf[3], buf[2], buf[1], buf[0], + buf[5], buf[4], buf[7], buf[6], + buf[8], buf[9], buf[10], buf[11], + buf[12], buf[13], buf[14], buf[15] }}; +} + +/* + * ACPI ASL code uses ToUUID() macro which encodes it in mixed-endian format. + * Convert UUID buffer to AML UUID. + */ +static void nvidia_uuid_to_aml_uuid_buf(const uuid_t *uuid, u8 *buf) +{ + const u8 *src = (u8 *)uuid; + + buf[0] = src[3]; + buf[1] = src[2]; + buf[2] = src[1]; + buf[3] = src[0]; + + buf[4] = src[5]; + buf[5] = src[4]; + buf[6] = src[7]; + buf[7] = src[6]; + + memcpy(buf + 8, src + 8, 8); +} + +static int nvidia_ffa_rescan_acpi_device(struct device *dev, void *data) +{ + struct acpi_device *adev = to_acpi_device(dev); + + if (acpi_dev_hid_uid_match(adev, data, NULL)) { + acpi_bus_scan(adev->handle); + return 1; + } + + return 0; +} + +static const char *nvidia_get_acpi_id_from_uuid(uuid_t *uuid) +{ + if (uuid_equal(uuid, &nvidia_ec_battery_service_uuid)) + return "PNP0C0A"; + + if (uuid_equal(uuid, &nvidia_ec_time_alarm_service_uuid)) + return "ACPI000E"; + + if (uuid_equal(uuid, &nvidia_ec_fan_service_uuid)) + return "PNP0C0B"; + + if (uuid_equal(uuid, &nvidia_ec_ucsi_service_uuid)) + return "PNP0CA0"; + + return NULL; +} + +/* + * Fill the virtual notification IDs array supported by the current FFA device. + * ACPI _DSD object contains notification mapping. It uses nexted package + * acpi object. + * + * From the example given in + * https://github.com/OpenDevicePartnership/documentation/blob/main/bookshelf/Shelf%204%20Specifications/EC%20Interface/src/secure-ec-services-overview.md#register-notification + * + * pkg1 Name(_DSD, Package() { + * pkg1_guid ToUUID("daffd814-6eba-4d8c-8a91-bc9bbf4aa301"), // Device Prop UUID + * pkg2 Package() { + * pkg3 Package(2) { + * pkg3_prop "arm-arml0002-ffa-ntf-bind", + * pkg4 Package() { + * pkg4_rev 1, // Revision + * pkg4_count 1, // Count of following packages + * pkg5 Package () { + * pkg5_uuid ToUUID("330c1273-fde5-4757-9819-5b6539037502"), // Service1 UUID + * pkg6 Package () { + * pkg6_notify_id[] 0x01, // Cookie1 (UINT32) + * 0x07, // Cookie2 + * } + * }, + * } + * } + * } + * }) // _DSD() + * + * The variable names in this function are according to above. + */ +static int nvidia_ffa_fill_notification_map(struct nvidia_ec_ffa_device *ec_ffa_dev) +{ + struct acpi_device *adev = ACPI_COMPANION(&ffa_pdev->dev); + struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL }; + union acpi_object *pkg1, *pkg1_guid; + union acpi_object *pkg2; + union acpi_object *pkg3, *pkg3_prop; + union acpi_object *pkg4, *pkg4_rev, *pkg4_count; + acpi_status status; + int i; + + status = acpi_evaluate_object_typed(adev->handle, "_DSD", NULL, + &output, ACPI_TYPE_PACKAGE); + if (ACPI_FAILURE(status)) { + dev_err(&ffa_pdev->dev, "ACPI _DSD object not found\n"); + return -ENODEV; + } + + pkg1 = output.pointer; + + /* + * _DSD returns a Package() with one or more pairs of elements. + * The first element of each pair is a Universal Unique Identifier (UUID). + * The second element of each pair is another Package() Data Structure. + * + * The _DSD for FFA device will have only one pair of elements so + * pkg1 elements count should be 2. + */ + if (pkg1->package.count != 2) { + kfree(output.pointer); + return -EINVAL; + } + + pkg1_guid = &pkg1->package.elements[0]; + pkg2 = &pkg1->package.elements[1]; + if (pkg1_guid->type != ACPI_TYPE_BUFFER || + pkg1_guid->buffer.length != UUID_SIZE || + pkg2->type != ACPI_TYPE_PACKAGE) { + kfree(output.pointer); + return -EINVAL; + } + + /* Check if GUID macthes with notify device prop GUID */ + if (!guid_equal((guid_t *)pkg1_guid->buffer.pointer, + &nvidia_notify_bind_guid)) { + kfree(output.pointer); + return -EINVAL; + } + + /* pkg3 should conatin 1 element with package type */ + if (pkg2->package.count != 1) { + kfree(output.pointer); + return -EINVAL; + } + + pkg3 = &pkg2->package.elements[0]; + if (pkg3->type != ACPI_TYPE_PACKAGE) { + kfree(output.pointer); + return -EINVAL; + } + + pkg3_prop = &pkg3->package.elements[0]; + if (pkg3_prop->type != ACPI_TYPE_STRING || + strncmp(pkg3_prop->string.pointer, + "arm-arml0002-ffa-ntf-bind", + pkg3_prop->string.length)) { + kfree(output.pointer); + return -EINVAL; + } + + pkg4 = &pkg3->package.elements[1]; + /* + * pkg4 should have minimum 3 elements (revision, count and minimum + * one notification map package) + */ + if (pkg4->type != ACPI_TYPE_PACKAGE || + pkg4->package.count < 3) { + kfree(output.pointer); + return -EINVAL; + } + + pkg4_rev = &pkg4->package.elements[0]; + pkg4_count = &pkg4->package.elements[1]; + + /* Check if revision is 1 */ + if (pkg4_rev->type != ACPI_TYPE_INTEGER || + pkg4_rev->integer.value != 1) { + kfree(output.pointer); + return -EINVAL; + } + + /* + * The pkg4_count represents the count of following packages. + * pkg4_count + 1 (for revision) + 1 (for pkg4_count itself) should + * match total number of elements in pkg4. + */ + if (pkg4_count->type != ACPI_TYPE_INTEGER || + (pkg4_count->integer.value + 2) != pkg4->package.count) { + kfree(output.pointer); + return -EINVAL; + } + + /* + * Traverse the array of notification map packages. + * Each notification map package contains 2 elements, UUID + * and notification ID array package. Check if there is a notification + * map for the FFA device by comparing UUID and update the + * notification_id[] and notification_count. + */ + for (i = 2; i < pkg4->package.count; i++) { + union acpi_object *pkg5_uuid, *pkg5 = &pkg4->package.elements[2]; + union acpi_object *pkg6; + uuid_t uuid; + int j; + + if (pkg5->type != ACPI_TYPE_PACKAGE && + pkg5->package.count != 2) { + kfree(output.pointer); + return -EINVAL; + } + + pkg5_uuid = &pkg5->package.elements[0]; + pkg6 = &pkg5->package.elements[1]; + if (pkg5_uuid->type != ACPI_TYPE_BUFFER || + pkg5_uuid->buffer.length != UUID_SIZE || + pkg6->type != ACPI_TYPE_PACKAGE) { + kfree(output.pointer); + return -EINVAL; + } + + uuid = nvidia_get_uuid_from_aml_buf(pkg5_uuid->buffer.pointer); + if (!uuid_equal(&uuid, &ec_ffa_dev->ffa_dev->uuid)) + continue; + + for (j = 0; j < pkg6->package.count; j++) { + union acpi_object *pkg6_notify_id = &pkg6->package.elements[j]; + + if (pkg6_notify_id->type != ACPI_TYPE_INTEGER) { + kfree(output.pointer); + return -EINVAL; + } + + ec_ffa_dev->notification_id[j] = pkg6_notify_id->integer.value; + } + + ec_ffa_dev->notification_count = pkg6->package.count; + kfree(output.pointer); + return 0; + } + + kfree(output.pointer); + return 0; +} + +/* + * Notification EC service callback. + * Get the ffa device from callback data and invoke notification _DSM with + * notify_id. + * + * The details regarding _DSM is documented in + * https://github.com/OpenDevicePartnership/documentation/tree/main/bookshelf/Shelf%204%20Specifications#notification-events + */ +static void nvidia_ffa_ec_service_notif_callback(int notify_id, void *cb_data) +{ + struct acpi_device *adev = ACPI_COMPANION(&ffa_pdev->dev); + struct ffa_device *ffa_dev = (struct ffa_device *)cb_data; + union acpi_object args[2], input_pkg; + union acpi_object *output; + u8 uuid[UUID_SIZE]; + + nvidia_uuid_to_aml_uuid_buf(&ffa_dev->uuid, uuid); + + args[0].type = ACPI_TYPE_BUFFER; + args[0].buffer.length = sizeof(uuid); + args[0].buffer.pointer = uuid; + + args[1].type = ACPI_TYPE_INTEGER; + args[1].integer.value = notify_id; + + input_pkg.type = ACPI_TYPE_PACKAGE; + input_pkg.package.count = 2; + input_pkg.package.elements = args; + + output = acpi_evaluate_dsm(adev->handle, &nvidia_notify_dsm_guid, + 1, 1, &input_pkg); + if (!output) + dev_err(&ffa_pdev->dev, "Failed to execute notify\n"); + else + ACPI_FREE(output); +} + +/* + * Create notification setup for the notification_id. + * + * The details regarding notification setup is documented in + * https://github.com/OpenDevicePartnership/documentation/tree/main/bookshelf/Shelf%204%20Specifications#register-notification + * + * This function setup 1:1 mapping between hardware notification ID and + * virtual notification ID. + */ +static int nvidia_ffa_notification_setup(struct nvidia_ec_ffa_device *ec_ffa_dev, + u8 notification_id) +{ + struct ffa_send_direct_data2 ffa_data = { 0 }; + u8 *uuid = (u8 *)&ec_ffa_dev->ffa_dev->uuid; + int ret; + + /* X4 register, function 1 */ + ffa_data.data[0] = 1; + + BUILD_BUG_ON(UUID_SIZE != 16); + BUILD_BUG_ON(sizeof(ffa_data.data[1]) < 8); + + /* X5 and X6 registers contain UUID */ + memcpy(&ffa_data.data[1], uuid, 8); + memcpy(&ffa_data.data[2], uuid + 8, 8); + + /* X7 register, the number of notification mappings */ + ffa_data.data[3] = 1; + + /* X7 register, notification ID and notification bitmap bit number */ + ffa_data.data[4] = ((u64)notification_id << 32) | notification_id; + + if (!notify_ffa_dev->ops || + !notify_ffa_dev->ops->msg_ops || + !notify_ffa_dev->ops->msg_ops->sync_send_receive2) { + return -EINVAL; + } + + ret = notify_ffa_dev->ops->msg_ops->sync_send_receive2(notify_ffa_dev, + &ffa_data); + if (ret) { + dev_err(&ec_ffa_dev->ffa_dev->dev, + "Failed to send NOTIFY_SETUP id=%d error=%d\n", + notification_id, ret); + return ret; + } + + if (ffa_data.data[0]) { + dev_err(&ec_ffa_dev->ffa_dev->dev, + "NOTIFY_SETUP returned failure id=%d error=%ld\n", + notification_id, ffa_data.data[0]); + + /* + * TODO: destroy operation is not yet implemented in the firmware + * So, if driver is reloaded, then the previous notification + * still exists and failure will be returned. Once destroy + * is implemented in firmware, update code here to return error + */ + } + + return 0; +} + +/* Destroy notification setup for the notification_id */ +static void nvidia_ffa_notification_destroy(struct nvidia_ec_ffa_device *ec_ffa_dev, + u8 notification_id) +{ + /* + * TODO: destroy operation is not yet implemented in the firmware. + * Once implemented in firmware, update code here. + */ +} + +/* + * Create notifications for the FFA device. + * + * 1. Get notification map array for FFA device. + * 2. For each notification, setup notification with notify service and + * then invoke notify_request method to enable notification for FFA device. + */ +static int nvidia_ffa_create_notifications(struct nvidia_ec_ffa_device *ec_ffa_dev) +{ + int i, ret = 0; + + if (!ec_ffa_dev->ffa_dev->ops || + !ec_ffa_dev->ffa_dev->ops->notifier_ops || + !ec_ffa_dev->ffa_dev->ops->notifier_ops->notify_request || + !ec_ffa_dev->ffa_dev->ops->notifier_ops->notify_relinquish) { + return -EOPNOTSUPP; + } + + ret = nvidia_ffa_fill_notification_map(ec_ffa_dev); + if (ret) { + dev_err(&ffa_pdev->dev, "Error in filling notification map error=%d\n", ret); + return ret; + } + + for (i = 0; i < ec_ffa_dev->notification_count; i++) { + ret = nvidia_ffa_notification_setup(ec_ffa_dev, + ec_ffa_dev->notification_id[i]); + if (ret) { + dev_err(&ec_ffa_dev->ffa_dev->dev, + "Failed to setup notification id=%d error=%d\n", + ec_ffa_dev->notification_id[i], ret); + break; + } + + ret = ec_ffa_dev->ffa_dev->ops->notifier_ops->notify_request( + ec_ffa_dev->ffa_dev, false, + nvidia_ffa_ec_service_notif_callback, + ec_ffa_dev->ffa_dev, ec_ffa_dev->notification_id[i]); + if (ret) { + nvidia_ffa_notification_destroy(ec_ffa_dev, + ec_ffa_dev->notification_id[i]); + dev_err(&ec_ffa_dev->ffa_dev->dev, + "Failed to request notification id=%d error=%d\n", + ec_ffa_dev->notification_id[i], ret); + break; + } + } + + /* Remove already setup notification in case of error */ + if (ret) { + int j; + + for (j = 0; j < i; j++) { + ec_ffa_dev->ffa_dev->ops->notifier_ops->notify_relinquish( + ec_ffa_dev->ffa_dev, + ec_ffa_dev->notification_id[j]); + nvidia_ffa_notification_destroy(ec_ffa_dev, + ec_ffa_dev->notification_id[j]); + } + + ec_ffa_dev->notification_count = 0; + } + + return ret; +} + +/* Remove notifications for the FFA device. */ +static void nvidia_ffa_remove_notifications(struct nvidia_ec_ffa_device *ec_ffa_dev) +{ + int i; + + for (i = 0; i < ec_ffa_dev->notification_count; i++) { + ec_ffa_dev->ffa_dev->ops->notifier_ops->notify_relinquish( + ec_ffa_dev->ffa_dev, + ec_ffa_dev->notification_id[i]); + nvidia_ffa_notification_destroy(ec_ffa_dev, + ec_ffa_dev->notification_id[i]); + } +} + +/* + * Handler function for FFH operation region offset 4. + * When ACPI interpreter runs code with FFH operation region offset 4, + * then this data is meant for EC secure services. The FFH buffer has + * data in 'struct nvidia_ec_ffa_packet' format. In this packet, it has UUID + * for EC secure service and then the service specific raw data. + * + * 1. Extract the UUID from this packet and get ffa_device for it. + * 2. Fill raw data in 'struct ffa_send_direct_data2' and + * invoke sync_send_receive2() routine for the ffa_device. + * 3. From response, fill the data in 'struct ffa_send_direct_data2' + * and return. + */ +static int nvidia_ffh_handler(struct acpi_ffh_info *info, acpi_integer *value, void *region_context) +{ + struct ffa_send_direct_data2 ffa_data = { 0 }; + struct nvidia_ec_ffa_packet *ffa_packet = (struct nvidia_ec_ffa_packet *)value; + struct nvidia_ec_ffa_device *cur, *ec_dev = NULL; + int ret; + unsigned int ffh_copy_len; + uuid_t uuid; + + /* Only offset 4 is supported */ + if (info->offset != 4) + return -EOPNOTSUPP; + + /* Length should not be less than header length */ + if (info->length < offsetof(struct nvidia_ec_ffa_packet, rawdata)) + return -EINVAL; + + /* Length should not be less than actual packet length */ + if (info->length < + ffa_packet->length + offsetof(struct nvidia_ec_ffa_packet, rawdata)) { + ffa_packet->status = 1; + return -EINVAL; + } + + /* Packet length should not greater than FFA supported data length */ + if (ffa_packet->length > sizeof(ffa_data.data)) { + ffa_packet->status = 1; + return -EINVAL; + } + + /* Convert AML UUID to FFA UUID */ + uuid = nvidia_get_uuid_from_aml_buf((u8 *)ffa_packet->uuid); + + mutex_lock(&nvidia_ffa_lock); + /* Get nvidia_ec_ffa_device for the current UUID */ + list_for_each_entry(cur, &nvidia_ec_ffa_dev_head, list) { + if (uuid_equal(&uuid, &cur->ffa_dev->uuid)) { + ec_dev = cur; + break; + } + } + mutex_unlock(&nvidia_ffa_lock); + + if (!ec_dev) { + ffa_packet->status = 1; + return -EINVAL; + } + + /* Copy the ACPI FFH packet data into FFA data */ + memcpy(ffa_data.data, ffa_packet->rawdata, ffa_packet->length); + + if (!ec_dev->ffa_dev->ops || + !ec_dev->ffa_dev->ops->msg_ops || + !ec_dev->ffa_dev->ops->msg_ops->sync_send_receive2) { + return -EINVAL; + } + + ret = ec_dev->ffa_dev->ops->msg_ops->sync_send_receive2(ec_dev->ffa_dev, + &ffa_data); + if (ret) { + dev_err(&ec_dev->ffa_dev->dev, + "Failed to send FFA messages error=%d\n", ret); + ffa_packet->status = 1; + return ret; + } + + /* Set the status as success */ + ffa_packet->status = 0; + + /* + * Copy the ACPI FFA data back into ACPI FFH packet. + * + * ACPI FFH packet raw data length can't be fetched here, so copy + * all bytes from ffa_data.data + */ + ffh_copy_len = min(sizeof(ffa_data.data), + info->length - offsetof(struct nvidia_ec_ffa_packet, rawdata)); + + memcpy(ffa_packet->rawdata, ffa_data.data, ffh_copy_len); + return 0; +} + +static int nvidia_ffa_ec_service_probe(struct ffa_device *ffa_dev) +{ + struct nvidia_ec_ffa_device *nvidia_ec_ffa_dev; + const char *acpi_id = NULL; + int ret; + + if (!ffa_pdev || !notify_ffa_dev) { + dev_err(&ffa_dev->dev, "nvidia ffa or notify device not available\n"); + return -ENODEV; + } + + nvidia_ec_ffa_dev = devm_kzalloc(&ffa_dev->dev, + sizeof(*nvidia_ec_ffa_dev), + GFP_KERNEL); + if (!nvidia_ec_ffa_dev) { + dev_err(&ffa_dev->dev, "Failed to allocate memory\n"); + return -ENOMEM; + } + + nvidia_ec_ffa_dev->ffa_dev = ffa_dev; + INIT_LIST_HEAD(&nvidia_ec_ffa_dev->list); + + ret = nvidia_ffa_create_notifications(nvidia_ec_ffa_dev); + if (ret) { + dev_info(&ffa_dev->dev, + "Failed to create ffa notifications error=%d\n", + ret); + devm_kfree(&ffa_dev->dev, nvidia_ec_ffa_dev); + return ret; + } + + mutex_lock(&nvidia_ffa_lock); + list_add(&nvidia_ec_ffa_dev->list, &nvidia_ec_ffa_dev_head); + mutex_unlock(&nvidia_ffa_lock); + + /* + * When acpi subsystem probe all ACPI devices, then it execute _STA + * method for each device. The _STA method fails at that time since + * custom FFA driver won't be ready. Get ACPI ID from UUID and + * rescan the device again. + */ + acpi_id = nvidia_get_acpi_id_from_uuid(&ffa_dev->uuid); + if (acpi_id) { + acpi_bus_for_each_dev(nvidia_ffa_rescan_acpi_device, + (void *)acpi_id); + } + + return 0; +} + +static void nvidia_ffa_ec_service_remove(struct ffa_device *ffa_dev) +{ + struct nvidia_ec_ffa_device *cur, *tmp; + + mutex_lock(&nvidia_ffa_lock); + list_for_each_entry_safe(cur, tmp, &nvidia_ec_ffa_dev_head, list) { + if (cur->ffa_dev == ffa_dev) { + list_del(&cur->list); + nvidia_ffa_remove_notifications(cur); + devm_kfree(&ffa_dev->dev, cur); + break; + } + } + mutex_unlock(&nvidia_ffa_lock); +} + +static const struct ffa_device_id nvidia_ffa_ec_service_ids[] = { + { nvidia_ec_managment_service_uuid }, + { nvidia_ec_power_service_uuid }, + { nvidia_ec_battery_service_uuid }, + { nvidia_ec_thermal_service_uuid }, + { nvidia_ec_fan_service_uuid }, + { nvidia_ec_ucsi_service_uuid }, + { nvidia_ec_input_service_uuid }, + { nvidia_ec_time_alarm_service_uuid }, + {} +}; + +static struct ffa_driver nvidia_ffa_ec_service_driver = { + .name = DRV_NAME, + .probe = nvidia_ffa_ec_service_probe, + .remove = nvidia_ffa_ec_service_remove, + .id_table = nvidia_ffa_ec_service_ids, +}; + +static int nvidia_ffa_notify_service_probe(struct ffa_device *ffa_dev) +{ + int ret; + + if (!ffa_pdev) { + dev_err(&ffa_dev->dev, "nvidia ffa device not available\n"); + return -ENODEV; + } + + notify_ffa_dev = ffa_dev; + + ret = ffa_driver_register(&nvidia_ffa_ec_service_driver, THIS_MODULE, DRV_NAME); + if (ret) { + dev_err(&ffa_dev->dev, + "Failed to register ec service driver error=%d\n", ret); + notify_ffa_dev = NULL; + return ret; + } + + return 0; +} + +static void nvidia_ffa_notify_service_remove(struct ffa_device *ffa_dev) +{ + ffa_driver_unregister(&nvidia_ffa_ec_service_driver); + notify_ffa_dev = NULL; +} + +static const struct ffa_device_id nvidia_ffa_notify_service_ids[] = { + { nvidia_ec_notify_service_uuid }, + {} +}; + +static struct ffa_driver nvidia_ffa_notify_service_driver = { + .name = "nvidia-ffa-notify", + .probe = nvidia_ffa_notify_service_probe, + .remove = nvidia_ffa_notify_service_remove, + .id_table = nvidia_ffa_notify_service_ids, +}; + +static const struct acpi_device_id nvidia_ffa_device_ids[] = { + /* + * Please refer + * https://github.com/OpenDevicePartnership/documentation/blob/main/bookshelf/Shelf%204%20Specifications/EC%20Interface/src/secure-ec-services-overview.md#hid-definition + * where MSFT000C is documented. + * + * The _HID 'MSFT000C' is reserved for FFA device which uses + * FFA interface for secure EC communication. + */ + {"MSFT000C", 0}, + {"", 0}, +}; + +MODULE_DEVICE_TABLE(acpi, nvidia_ffa_device_ids); + +static int nvidia_ffa_probe(struct platform_device *pdev) +{ + struct acpi_device *adev = ACPI_COMPANION(&pdev->dev); + acpi_status status; + unsigned long long data = 0; + int ret; + + if (ffa_pdev) { + dev_err(&pdev->dev, "FFA device already registered\n"); + return -EINVAL; + } + + if (!adev) { + dev_err(&pdev->dev, "No ACPI companion found\n"); + return -ENODEV; + } + + status = acpi_evaluate_integer(adev->handle, "AVAL", NULL, &data); + if (ACPI_FAILURE(status)) { + dev_err(&pdev->dev, "Failed to execute AVAL method\n"); + return -ENODEV; + } + + if (data != 1) { + dev_err(&pdev->dev, "FFA not available\n"); + return -ENODEV; + } + + ret = acpi_arm64_ffh_update_custom_offset_handler(nvidia_ffh_handler); + if (ret) { + dev_err(&pdev->dev, + "Failed to register custom offset handler error=%d\n", ret); + return ret; + } + + ffa_pdev = pdev; + + ret = ffa_driver_register(&nvidia_ffa_notify_service_driver, THIS_MODULE, DRV_NAME); + if (ret) { + dev_err(&pdev->dev, + "Failed to register notify service driver error=%d\n", ret); + acpi_arm64_ffh_update_custom_offset_handler(NULL); + ffa_pdev = NULL; + return ret; + } + + return 0; +} + +static void nvidia_ffa_remove(struct platform_device *pdev) +{ + ffa_driver_unregister(&nvidia_ffa_notify_service_driver); + ffa_pdev = NULL; + acpi_arm64_ffh_update_custom_offset_handler(NULL); +} + +static struct platform_driver nvidia_ffa_driver = { + .probe = nvidia_ffa_probe, + .remove = nvidia_ffa_remove, + .driver = { + .name = "nvidia-ffa", + .acpi_match_table = nvidia_ffa_device_ids, + }, +}; + +static int __init nvidia_ffa_init(void) +{ + return platform_driver_register(&nvidia_ffa_driver); +} +module_init(nvidia_ffa_init); + +static void __exit nvidia_ffa_exit(void) +{ + platform_driver_unregister(&nvidia_ffa_driver); +} +module_exit(nvidia_ffa_exit); + +MODULE_SOFTDEP("pre: arm-ffa"); +MODULE_AUTHOR("NVIDIA CORPORATION"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("NVIDIA FFA EC services driver"); diff --git a/drivers/platform/arm64/nvidia/Kconfig b/drivers/platform/arm64/nvidia/Kconfig new file mode 100644 index 0000000000000..b12b290f30d4f --- /dev/null +++ b/drivers/platform/arm64/nvidia/Kconfig @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# NVIDIA ARM64 Platform-Specific Device Drivers +# + +config MTK_PCIE_HOTPLUG + tristate "CX7 PCIe Hotplug Driver" + depends on EINT_MTK + depends on PCI && ACPI + help + Say Y here to support PCIe device plug in/out detection. + It will disable PCIe link when plug out and enable + PCIe link after plug in. + + This is particularly useful for GB10 SoC. + + If unsure, say N. diff --git a/drivers/platform/arm64/nvidia/Makefile b/drivers/platform/arm64/nvidia/Makefile new file mode 100644 index 0000000000000..37cfbebb8d1af --- /dev/null +++ b/drivers/platform/arm64/nvidia/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for NVIDIA ARM64 platform-specific drivers +# +# CX7 PCIe Hotplug Driver +# Provides hotplug support for CX7 PCIe devices on GB10 SoC-based systems +# + +obj-$(CONFIG_MTK_PCIE_HOTPLUG) += mtk-pcie-hotplug.o diff --git a/drivers/platform/arm64/nvidia/mtk-pcie-hotplug.c b/drivers/platform/arm64/nvidia/mtk-pcie-hotplug.c new file mode 100644 index 0000000000000..06a84a29aa6fd --- /dev/null +++ b/drivers/platform/arm64/nvidia/mtk-pcie-hotplug.c @@ -0,0 +1,2324 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2014-2025 MediaTek Inc. + * Copyright (c) 2025-2026 NVIDIA Corporation + * + * CX7 PCIe Hotplug Driver + * + * Manages PCIe device hotplug using GPIO interrupts and ACPI resources. + * Supports cable insertion/removal detection and device power management. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HP_PORT_MAX 3 +#define HP_POLL_CNT_MAX 200 +#define MAX_VENDOR_DATA_LEN 16 +#define CX7_HP_MMIO_REGION_COUNT 5 /* TOP, PROTECT, CKM, MAC Port 0, MAC Port 1 */ +#define CX7_HP_MIN_GPIO_COUNT 4 /* Minimum required: BOOT, PRSNT, PERST, EN */ +#define PINCTRL_MAPPING_ENTRY_SIZE 5 /* dev_name, state, ctrl_dev, group, function */ +/* Indices for pinctrl mapping entry strings */ +#define PINCTRL_IDX_DEV_NAME 0 +#define PINCTRL_IDX_STATE 1 +#define PINCTRL_IDX_CTRL_DEV 2 +#define PINCTRL_IDX_GROUP 3 +#define PINCTRL_IDX_FUNCTION 4 + +/* Hardware timing requirements (in microseconds unless noted) */ +#define CX7_HP_DELAY_SHORT_US 10 /* Short delay for register writes */ +#define CX7_HP_DELAY_STANDARD_US 10000 /* Standard delay (10ms) */ +#define CX7_HP_DELAY_BUS_PROTECT_US 5000 /* Bus protection setup delay */ +#define CX7_HP_DELAY_PHY_RESET_US 3000 /* PHY reset delay */ +#define CX7_HP_DELAY_LINK_STABLE_MS 100 /* Link stabilization delay (ms) */ +#define CX7_HP_POLL_SLEEP_US 10000 /* Polling loop sleep interval */ + +#define PLUG_IN_EVT "HOTPLUG_STATE=plugin" +#define REMOVAL_EVT "HOTPLUG_STATE=removal" + +/* Bus protection stages to prevent PCIe core reset glitches */ +#define BUS_PROTECT_INIT 0 +#define BUS_PROTECT_CABLE_REMOVAL 1 +#define BUS_PROTECT_CABLE_PLUGIN 2 +#define BUS_PROTECT_CLEANUP 3 + +enum cx7_hp_state { + STATE_READY = 0, + STATE_PLUG_OUT, /* Cable plug-out */ + STATE_DEV_POWER_OFF, /* Device is powered off */ + STATE_PLUG_IN, /* Cable plug-in detected */ + STATE_DEV_POWER_ON, /* Device is powered on */ + STATE_DEV_FW_START, /* Device firmware is running */ + STATE_RESCAN, /* Device ready, can perform bus rescan */ + STATE_UNKNOWN +}; + +enum pcie_pin_index { + PCIE_PIN_BOOT = 0, /* Device boot status pin */ + PCIE_PIN_PRSNT, /* Presence detection pin */ + PCIE_PIN_PERST, /* PCIe reset pin */ + PCIE_PIN_EN, /* Power enable pin */ + PCIE_PIN_CLQ0, /* Clock request pin 0 */ + PCIE_PIN_CLQ1, /* Clock request pin 1 */ + PCIE_PIN_MAX +}; + +struct pcie_port_info { + int domain; + int bus; + int devfn; +}; + +struct rp_bus_mmio_top { + u32 ctrl; + u32 port_bits[HP_PORT_MAX]; + u32 update_bit; +}; + +struct rp_bus_mmio_protect { + u32 mode; + u32 enable; + u32 port_bits[HP_PORT_MAX]; +}; + +struct rp_bus_mmio_mac { + u32 init_ctrl; + u32 ltssm_bit; + u32 phy_rst_bit; +}; + +struct rp_bus_mmio_ckm { + u32 ctrl; + u32 disable_bit; +}; + +struct rp_bus_mmio_info { + struct rp_bus_mmio_top top; + struct rp_bus_mmio_protect protect; + struct rp_bus_mmio_mac mac; + struct rp_bus_mmio_ckm ckm; +}; + +struct gpio_acpi_context { + struct device *dev; + unsigned int debounce_timeout_us; + int pin; + int wake_capable; + int triggering; + int polarity; + unsigned long irq_flags; + int valid; + unsigned int connection_type; + char vendor_data[MAX_VENDOR_DATA_LEN + 1]; +}; + +struct cx7_hp_dev; + +/** + * struct cx7_hp_plat_data - Platform configuration data parsed from ACPI + * + * Platform-specific configuration parsed from ACPI devices: + * - RES0 device (PNP0C02): PCIe configuration and MMIO register offsets via _DSD + * - PEDE device (MTKP0001): Pinctrl mappings via _DSD + */ +struct cx7_hp_plat_data { + int port_nums; + struct pcie_port_info ports[HP_PORT_MAX]; + u32 vendor_id; + u32 device_id; + int num_devices; + struct rp_bus_mmio_info rp_bus_mmio; + u32 ltssm_reg; + u32 ltssm_l0_state; + int pin_nums; + struct pinctrl_map *parsed_pinmap; +}; + +struct cx7_hp_gpio_ctx { + struct gpio_desc *desc; + struct gpio_acpi_context *ctx; + struct cx7_hp_dev *hp_dev; +}; + +struct acpi_gpio_parse_context { + struct gpio_acpi_context *ctx; + struct cx7_hp_dev *hp_dev; +}; + +struct acpi_gpio_walk_context { + struct device *dev; + struct gpio_info { + unsigned int pin; + unsigned int connection_type; + unsigned int triggering; + unsigned int polarity; + unsigned int debounce_timeout; + unsigned int wake_capable; + char vendor_data[MAX_VENDOR_DATA_LEN + 1]; + char resource_source[16]; + unsigned int resource_source_index; + } gpios[PCIE_PIN_MAX]; + int count; +}; + +struct cx7_hp_acpi_mmio { + struct acpi_resource_fixed_memory32 + mmio_regions[CX7_HP_MMIO_REGION_COUNT]; + int count; + struct device *dev; +}; + +enum cx7_hp_debug_val { + CX7_HP_DEBUG_PLUG_OUT = 0, + CX7_HP_DEBUG_PLUG_IN, + CX7_HP_DEBUG_MAX_VAL +}; + +struct cx7_hp_mmio_runtime { + void __iomem *top_base; + void __iomem *protect_base; + void __iomem *ckm_base; + void __iomem *mac_port_base[HP_PORT_MAX]; +}; + +/** + * cx7_hp_dev - Hotplug device structure + * + * ACPI resource sources: + * - MMIO addresses: RES0 device (PNP0C02) _CRS, stored in mmio field + * - GPIO resources: PEDE device (MTKP0001) _CRS, stored in pins field + */ +struct cx7_hp_dev { + struct cx7_hp_gpio_ctx *pins; + struct cx7_hp_plat_data *pd; + struct platform_device *pdev; + enum cx7_hp_state state; + int gpio_count; + int boot_pin; + int prsnt_pin; + enum cx7_hp_debug_val debug_state; + bool hotplug_enabled; + spinlock_t lock; + struct pci_dev *cached_root_ports[HP_PORT_MAX]; + struct cx7_hp_mmio_runtime mmio; + struct gpio_device *gdev; + struct notifier_block pci_notifier; +}; + +/* ACPI _DSD device properties GUID: daffd814-6eba-4d8c-8a91-bc9bbf4aa301 */ +static const guid_t device_properties_guid = +GUID_INIT(0xdaffd814, 0x6eba, 0x4d8c, + 0x8a, 0x91, 0xbc, 0x9b, + 0xbf, 0x4a, 0xa3, 0x01); + +/** + * cx7_hp_parse_pinctrl_config_dsd - Parse pinctrl configuration from PEDE device _DSD + * @hp_dev: hotplug device + * + * Parses pin-nums and pinctrl-mappings from _DSD. + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_parse_pinctrl_config_dsd(struct cx7_hp_dev *hp_dev) +{ + struct acpi_device *adev; + struct device *dev = &hp_dev->pdev->dev; + const union acpi_object *mappings_pkg, *mapping_entry; + struct pinctrl_map *pinmap; + u32 pin_nums = 0; + int k; + const char *strings[PINCTRL_MAPPING_ENTRY_SIZE]; + + adev = ACPI_COMPANION(dev); + if (!adev) { + dev_err(dev, "Failed to get ACPI companion device\n"); + return -ENODEV; + } + + struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; + acpi_status status; + const union acpi_object *dsd_pkg, *props_pkg = NULL; + int i, j; + + status = acpi_evaluate_object_typed(adev->handle, "_DSD", NULL, &buffer, + ACPI_TYPE_PACKAGE); + if (ACPI_FAILURE(status)) { + dev_err(dev, "Failed to evaluate _DSD: %s\n", + acpi_format_exception(status)); + return -ENODEV; + } + + dsd_pkg = buffer.pointer; + if (!dsd_pkg || dsd_pkg->type != ACPI_TYPE_PACKAGE) { + dev_err(dev, "Invalid _DSD package\n"); + ACPI_FREE(buffer.pointer); + return -EINVAL; + } + /* Find Device Properties GUID package */ + for (i = 0; i + 1 < dsd_pkg->package.count; i += 2) { + const union acpi_object *guid = &dsd_pkg->package.elements[i]; + const union acpi_object *pkg = + &dsd_pkg->package.elements[i + 1]; + + /* Verify GUID matches Device Properties GUID */ + if (guid->type == ACPI_TYPE_BUFFER && guid->buffer.length == 16 && + pkg->type == ACPI_TYPE_PACKAGE && + guid_equal((guid_t *)guid->buffer.pointer, + &device_properties_guid)) { + props_pkg = pkg; + break; + } + } + + if (!props_pkg) { + dev_err(dev, + "Device Properties GUID package not found in _DSD\n"); + ACPI_FREE(buffer.pointer); + return -EINVAL; + } + + for (j = 0; j < props_pkg->package.count; j++) { + const union acpi_object *prop = &props_pkg->package.elements[j]; + + if (prop->type != ACPI_TYPE_PACKAGE || + prop->package.count != 2 || + prop->package.elements[0].type != ACPI_TYPE_STRING) + continue; + + const char *prop_name = + prop->package.elements[0].string.pointer; + const union acpi_object *prop_value = + &prop->package.elements[1]; + + if (!strcmp(prop_name, "pin-nums")) { + if (prop_value->type == ACPI_TYPE_INTEGER) { + pin_nums = prop_value->integer.value; + } + } else if (!strcmp(prop_name, "pinctrl-mappings")) { + if (prop_value->type == ACPI_TYPE_PACKAGE) + mappings_pkg = prop_value; + } + } + + if (pin_nums == 0) { + hp_dev->pd->pin_nums = 0; + ACPI_FREE(buffer.pointer); + return 0; + } + + if (!mappings_pkg) { + dev_err(dev, + "Missing required _DSD property: pinctrl-mappings\n"); + ACPI_FREE(buffer.pointer); + return -EINVAL; + } + + if (mappings_pkg->package.count != pin_nums) { + dev_err(dev, + "pinctrl-mappings count mismatch: expected %u, got %u\n", + pin_nums, mappings_pkg->package.count); + ACPI_FREE(buffer.pointer); + return -EINVAL; + } + + /* Allocate pinmap array */ + pinmap = devm_kcalloc(dev, pin_nums, sizeof(*pinmap), GFP_KERNEL); + if (!pinmap) { + ACPI_FREE(buffer.pointer); + return -ENOMEM; + } + + /* Parse each mapping entry */ + for (k = 0; k < pin_nums; k++) { + mapping_entry = &mappings_pkg->package.elements[k]; + if (mapping_entry->type != ACPI_TYPE_PACKAGE || + mapping_entry->package.count != ARRAY_SIZE(strings)) { + dev_err(dev, + "Invalid pinctrl mapping entry %d: expected Package(%zu), " + "got %s(count=%u)\n", + k, ARRAY_SIZE(strings), + mapping_entry->type == ACPI_TYPE_PACKAGE ? + "Package" : "non-Package", + mapping_entry->type == ACPI_TYPE_PACKAGE ? + mapping_entry->package.count : 0); + ACPI_FREE(buffer.pointer); + return -EINVAL; + } + + /* Extract strings: dev_name, state, ctrl_dev, group, function */ + for (int l = 0; l < ARRAY_SIZE(strings); l++) { + if (mapping_entry->package.elements[l].type != + ACPI_TYPE_STRING) { + dev_err(dev, + "Mapping entry %d element %d is not a string\n", + k, l); + ACPI_FREE(buffer.pointer); + return -EINVAL; + } + strings[l] = + mapping_entry->package.elements[l].string.pointer; + } + + /* Populate pinctrl_map structure */ + pinmap[k].dev_name = + devm_kstrdup(dev, strings[PINCTRL_IDX_DEV_NAME], + GFP_KERNEL); + pinmap[k].name = + devm_kstrdup(dev, strings[PINCTRL_IDX_STATE], GFP_KERNEL); + pinmap[k].type = PIN_MAP_TYPE_MUX_GROUP; + pinmap[k].ctrl_dev_name = + devm_kstrdup(dev, strings[PINCTRL_IDX_CTRL_DEV], + GFP_KERNEL); + pinmap[k].data.mux.group = + devm_kstrdup(dev, strings[PINCTRL_IDX_GROUP], GFP_KERNEL); + pinmap[k].data.mux.function = + devm_kstrdup(dev, strings[PINCTRL_IDX_FUNCTION], + GFP_KERNEL); + + if (!pinmap[k].dev_name || !pinmap[k].name || + !pinmap[k].ctrl_dev_name || !pinmap[k].data.mux.group || + !pinmap[k].data.mux.function) { + dev_err(dev, + "Failed to allocate memory for mapping %d\n", + k); + ACPI_FREE(buffer.pointer); + return -ENOMEM; + } + } + + hp_dev->pd->pin_nums = pin_nums; + hp_dev->pd->parsed_pinmap = pinmap; + ACPI_FREE(buffer.pointer); + dev_dbg(dev, "Successfully parsed %u pinctrl mappings from ACPI\n", + pin_nums); + return 0; +} + +/** + * cx7_hp_pinctrl_init - Register pinctrl mappings for the device + * @hp_dev: hotplug device + * + * Parses pinctrl mappings from _DSD and registers them. + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_pinctrl_init(struct cx7_hp_dev *hp_dev) +{ + int ret; + + ret = cx7_hp_parse_pinctrl_config_dsd(hp_dev); + if (ret) { + dev_err(&hp_dev->pdev->dev, + "Failed to parse pinctrl configuration from ACPI: %d\n", + ret); + return ret; + } + + if (!hp_dev->pd->pin_nums) + return 0; + + ret = + pinctrl_register_mappings(hp_dev->pd->parsed_pinmap, + hp_dev->pd->pin_nums); + if (ret) { + dev_err(&hp_dev->pdev->dev, + "Failed to register pinctrl mappings\n"); + return ret; + } + + dev_dbg(&hp_dev->pdev->dev, "Registered %u pinctrl mappings\n", + hp_dev->pd->pin_nums); + return 0; +} + +/** + * cx7_hp_pinctrl_remove - Unregister pinctrl mappings + * @hp_dev: hotplug device + */ +static void cx7_hp_pinctrl_remove(struct cx7_hp_dev *hp_dev) +{ + if (!hp_dev->pd->pin_nums) + return; + + pinctrl_unregister_mappings(hp_dev->pd->parsed_pinmap); +} + +/** + * cx7_hp_change_pinctrl_state - Change pinctrl state + * @hp_dev: hotplug device + * @new_state: new pinctrl state name + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_change_pinctrl_state(struct cx7_hp_dev *hp_dev, + const char *new_state) +{ + struct pinctrl *pinctrl; + struct pinctrl_state *state; + int ret; + + pinctrl = devm_pinctrl_get(&hp_dev->pdev->dev); + if (IS_ERR(pinctrl)) { + dev_err(&hp_dev->pdev->dev, "Failed to get pinctrl\n"); + return PTR_ERR(pinctrl); + } + + state = pinctrl_lookup_state(pinctrl, new_state); + if (IS_ERR(state)) { + dev_err(&hp_dev->pdev->dev, "Failed to lookup state:%s\n", + new_state); + return PTR_ERR(state); + } + + ret = pinctrl_select_state(pinctrl, state); + if (ret) { + dev_err(&hp_dev->pdev->dev, + "Failed to select pinctrl state:%s\n", new_state); + return ret; + } + + return 0; +} + +/** + * cx7_hp_send_uevent - Send uevent to userspace + * @hp_dev: hotplug device + * @msg: uevent message string + */ +static void cx7_hp_send_uevent(struct cx7_hp_dev *hp_dev, const char *msg) +{ + char *uevent = NULL; + char *envp[2]; + + uevent = kasprintf(GFP_KERNEL, msg); + if (!uevent) { + dev_err(&hp_dev->pdev->dev, + "Failed to allocate uevent string\n"); + return; + } + + envp[0] = uevent; + envp[1] = NULL; + + if (kobject_uevent_env(&hp_dev->pdev->dev.kobj, KOBJ_CHANGE, envp)) + dev_err(&hp_dev->pdev->dev, "Failed to send uevent\n"); + + kfree(uevent); +} + +/** + * cx7_hp_reg_update_bits - Update specific bits in a register + * @base: MMIO base address + * @offset: Register offset + * @mask: Bits to modify + * @set: true to set bits, false to clear bits + */ +static inline void cx7_hp_reg_update_bits(void __iomem *base, u32 offset, + u32 mask, bool set) +{ + u32 val = readl(base + offset); + + if (set) + val |= mask; + else + val &= ~mask; + + writel(val, base + offset); +} + +/** + * cx7_hp_toggle_update_bit - Toggle control register update bit + * @base: MMIO base address + * @ctrl_offset: Control register offset + * @bits: Bits to set/clear before toggling update + * @update_bit: Update bit mask + * @set: true to set bits, false to clear bits + * + * Performs the sequence: modify bits, clear update bit, set update bit + */ +static void cx7_hp_toggle_update_bit(void __iomem *base, u32 ctrl_offset, + u32 bits, u32 update_bit, bool set) +{ + cx7_hp_reg_update_bits(base, ctrl_offset, bits, set); + cx7_hp_reg_update_bits(base, ctrl_offset, update_bit, false); + cx7_hp_reg_update_bits(base, ctrl_offset, update_bit, true); +} + +/** + * cx7_hp_bus_protect_enable - Enable bus protection for a port + * @dev: hotplug device + * @port_idx: Port index + */ +static void cx7_hp_bus_protect_enable(struct cx7_hp_dev *dev, int port_idx) +{ + struct rp_bus_mmio_info *mmio_info = &dev->pd->rp_bus_mmio; + u32 port_bit = mmio_info->protect.port_bits[port_idx]; + + cx7_hp_reg_update_bits(dev->mmio.protect_base, + mmio_info->protect.mode, port_bit, true); + cx7_hp_reg_update_bits(dev->mmio.protect_base, + mmio_info->protect.enable, port_bit, true); +} + +/** + * cx7_hp_bus_protect_disable - Disable bus protection for a port + * @dev: hotplug device + * @port_idx: Port index + */ +static void cx7_hp_bus_protect_disable(struct cx7_hp_dev *dev, int port_idx) +{ + struct rp_bus_mmio_info *mmio_info = &dev->pd->rp_bus_mmio; + u32 port_bit = mmio_info->protect.port_bits[port_idx]; + + cx7_hp_reg_update_bits(dev->mmio.protect_base, + mmio_info->protect.enable, port_bit, false); + cx7_hp_reg_update_bits(dev->mmio.protect_base, + mmio_info->protect.mode, port_bit, false); +} + +/** + * cx7_hp_ckm_control - Control clock module + * @dev: hotplug device + * @disable: true to disable clock, false to enable + */ +static void cx7_hp_ckm_control(struct cx7_hp_dev *dev, bool disable) +{ + struct rp_bus_mmio_info *mmio_info = &dev->pd->rp_bus_mmio; + + if (!dev->mmio.ckm_base) + return; + + cx7_hp_reg_update_bits(dev->mmio.ckm_base, mmio_info->ckm.ctrl, + mmio_info->ckm.disable_bit, disable); +} + +/** + * cx7_hp_parse_mmio_resources - ACPI resource callback for parsing MMIO from _CRS + * @ares: ACPI resource being processed + * @data: pointer to cx7_hp_acpi_mmio structure + * + * Returns: AE_OK to continue iteration, AE_ERROR on error + */ +static acpi_status cx7_hp_parse_mmio_resources(struct acpi_resource *ares, + void *data) +{ + struct cx7_hp_acpi_mmio *parsed = data; + + switch (ares->type) { + case ACPI_RESOURCE_TYPE_FIXED_MEMORY32: + if (parsed->count >= CX7_HP_MMIO_REGION_COUNT) { + dev_warn(parsed->dev, + "More than %d MMIO regions found in platform configuration device, ignoring extras\n", + CX7_HP_MMIO_REGION_COUNT); + break; + } + parsed->mmio_regions[parsed->count] = ares->data.fixed_memory32; + parsed->count++; + break; + default: + break; + } + + return AE_OK; +} + +/** + * cx7_hp_find_pcie_config_device - Find PCIe configuration device by HID + * + * Finds the ACPI device that provides PCIe configuration via _DSD properties + * and MMIO resources via _CRS. + * + * Returns: acpi_device pointer on success (with reference), NULL on failure + */ +static struct acpi_device *cx7_hp_find_pcie_config_device(void) +{ + return acpi_dev_get_first_match_dev("PNP0C02", NULL, -1); +} + +/** + * cx7_hp_parse_pcie_config_dsd - Parse PCIe configuration from _DSD + * @pdev: platform device + * @pd: platform data to populate + * + * Parses PCIe MMIO register offsets, bit positions, port configuration, and PCIe device + * identification from PCIe configuration device _DSD. + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_parse_pcie_config_dsd(struct platform_device *pdev, + struct cx7_hp_plat_data *pd) +{ + struct acpi_device *config_adev; + struct device *dev = &pdev->dev; + u32 val, bit1; + + config_adev = cx7_hp_find_pcie_config_device(); + if (!config_adev) { + dev_err(dev, + "Platform configuration device (PNP0C02) not found - _DSD is required\n"); + return -ENODEV; + } + + if (!acpi_dev_has_props(config_adev)) { + dev_err(dev, + "Platform configuration device has no _DSD properties. Check DSDT.\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "mac-init-ctrl-offset", &val)) { + dev_err(dev, + "Missing required _DSD property: mac-init-ctrl-offset\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.mac.init_ctrl = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "mac-ltssm-bit", &val)) { + dev_err(dev, "Missing required _DSD property: mac-ltssm-bit\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.mac.ltssm_bit = BIT(val); + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "mac-phy-rst-bit", &val)) { + dev_err(dev, + "Missing required _DSD property: mac-phy-rst-bit\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.mac.phy_rst_bit = BIT(val); + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "top-ctrl-offset", &val)) { + dev_err(dev, + "Missing required _DSD property: top-ctrl-offset\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.top.ctrl = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "top-update-bit", &val)) { + dev_err(dev, + "Missing required _DSD property: top-update-bit\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.top.update_bit = BIT(val); + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "top-port0-bit", &val)) { + dev_err(dev, "Missing required _DSD property: top-port0-bit\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.top.port_bits[0] = BIT(val); + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "top-port1-bit", &val)) { + dev_err(dev, "Missing required _DSD property: top-port1-bit\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.top.port_bits[1] = BIT(val); + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "protect-mode-offset", &val)) { + dev_err(dev, + "Missing required _DSD property: protect-mode-offset\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.protect.mode = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "protect-enable-offset", &val)) { + dev_err(dev, + "Missing required _DSD property: protect-enable-offset\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.protect.enable = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "protect-port0-bit", &val)) { + dev_err(dev, + "Missing required _DSD property: protect-port0-bit\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.protect.port_bits[0] = BIT(val); + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "protect-port1-bit", &val)) { + dev_err(dev, + "Missing required _DSD property: protect-port1-bit\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.protect.port_bits[1] = BIT(val); + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "ckm-ctrl-offset", &val)) { + dev_err(dev, + "Missing required _DSD property: ckm-ctrl-offset\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.ckm.ctrl = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "ckm-disable-bit0", &val)) { + dev_err(dev, + "Missing required _DSD property: ckm-disable-bit0\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "ckm-disable-bit1", &bit1)) { + dev_err(dev, + "Missing required _DSD property: ckm-disable-bit1\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->rp_bus_mmio.ckm.disable_bit = BIT(val) | BIT(bit1); + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "ltssm-reg-offset", &val)) { + dev_err(dev, + "Missing required _DSD property: ltssm-reg-offset\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->ltssm_reg = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "ltssm-l0-state", &val)) { + dev_err(dev, + "Missing required _DSD property: ltssm-l0-state\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->ltssm_l0_state = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "port-nums", &val)) { + dev_err(dev, "Missing required _DSD property: port-nums\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + if (val == 0 || val > HP_PORT_MAX) { + dev_err(dev, + "Invalid _DSD property port-nums: %u (must be 1-%d)\n", + val, HP_PORT_MAX); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->port_nums = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "port0-domain", &val)) { + dev_err(dev, "Missing required _DSD property: port0-domain\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->ports[0].domain = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "port0-bus", &val)) { + dev_err(dev, "Missing required _DSD property: port0-bus\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->ports[0].bus = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "port0-devfn", &val)) { + dev_err(dev, "Missing required _DSD property: port0-devfn\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->ports[0].devfn = val; + + if (pd->port_nums >= 2) { + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "port1-domain", &val)) { + dev_err(dev, + "Missing required _DSD property: port1-domain\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->ports[1].domain = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "port1-bus", &val)) { + dev_err(dev, + "Missing required _DSD property: port1-bus\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->ports[1].bus = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "port1-devfn", &val)) { + dev_err(dev, + "Missing required _DSD property: port1-devfn\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->ports[1].devfn = val; + } + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "vendor-id", &val)) { + dev_err(dev, "Missing required _DSD property: vendor-id\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->vendor_id = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "device-id", &val)) { + dev_err(dev, "Missing required _DSD property: device-id\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->device_id = val; + + if (fwnode_property_read_u32 + (acpi_fwnode_handle(config_adev), "num-devices", &val)) { + dev_err(dev, "Missing required _DSD property: num-devices\n"); + acpi_dev_put(config_adev); + return -EINVAL; + } + pd->num_devices = val; + + dev_dbg(dev, "Successfully parsed all required _DSD properties\n"); + + acpi_dev_put(config_adev); + return 0; +} + +/** + * cx7_hp_parse_mmio_resources_from_acpi - Parse MMIO regions from _CRS + * @dev: hotplug device + * @parsed: pointer to parsed MMIO structure + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_parse_mmio_resources_from_acpi(struct cx7_hp_dev *dev, + struct cx7_hp_acpi_mmio + *parsed) +{ + struct acpi_device *config_adev; + acpi_status status; + int ret = 0; + + if (!dev || !dev->pdev) { + return -EINVAL; + } + + config_adev = cx7_hp_find_pcie_config_device(); + if (!config_adev) + return -ENODEV; + + parsed->count = 0; + memset(parsed->mmio_regions, 0, sizeof(parsed->mmio_regions)); + + status = + acpi_walk_resources(config_adev->handle, METHOD_NAME__CRS, + cx7_hp_parse_mmio_resources, parsed); + if (ACPI_FAILURE(status)) { + dev_err(&dev->pdev->dev, + "Failed to walk platform configuration resources: %s\n", + acpi_format_exception(status)); + ret = -ENODEV; + goto out; + } + + if (parsed->count < CX7_HP_MMIO_REGION_COUNT) { + dev_warn(&dev->pdev->dev, + "Expected %d MMIO regions from platform configuration device, found %d\n", + CX7_HP_MMIO_REGION_COUNT, parsed->count); + ret = -ENODEV; + goto out; + } + +out: + acpi_dev_put(config_adev); + return ret; +} + +/** + * cx7_hp_map_mmio_resources - Map all MMIO regions from ACPI _CRS + * @dev: hotplug device + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_map_mmio_resources(struct cx7_hp_dev *dev) +{ + struct platform_device *pdev = dev->pdev; + struct cx7_hp_acpi_mmio parsed = {.count = 0, .dev = &pdev->dev }; + int ret; + int i; + + ret = cx7_hp_parse_mmio_resources_from_acpi(dev, &parsed); + if (ret) { + dev_err(&pdev->dev, + "Failed to get MMIO regions from platform configuration device\n"); + return ret; + } + + dev_dbg(&pdev->dev, "Found %d MMIO regions in _CRS, mapping...\n", + parsed.count); + + int mapped_count = 0; + for (i = 0; i < parsed.count; i++) { + void __iomem *base = NULL; + u32 addr = parsed.mmio_regions[i].address; + u32 size = parsed.mmio_regions[i].address_length; + + switch (i) { + case 0: + if (dev->pd->port_nums >= 1) { + base = devm_ioremap(&pdev->dev, addr, size); + if (!base) { + dev_err(&pdev->dev, + "Failed to map MAC Port 0 region (0x%08x)\n", + addr); + return -ENOMEM; + } + dev->mmio.mac_port_base[0] = base; + mapped_count++; + } + break; + case 1: + if (dev->pd->port_nums >= 2) { + base = devm_ioremap(&pdev->dev, addr, size); + if (!base) { + dev_err(&pdev->dev, + "Failed to map MAC Port 1 region (0x%08x)\n", + addr); + return -ENOMEM; + } + dev->mmio.mac_port_base[1] = base; + mapped_count++; + } + break; + case 2: + base = devm_ioremap(&pdev->dev, addr, size); + if (!base) { + dev_err(&pdev->dev, + "Failed to map TOP region (0x%08x)\n", + addr); + return -ENOMEM; + } + dev->mmio.top_base = base; + mapped_count++; + break; + case 3: + base = devm_ioremap(&pdev->dev, addr, size); + if (!base) { + dev_err(&pdev->dev, + "Failed to map PROTECT region (0x%08x)\n", + addr); + return -ENOMEM; + } + dev->mmio.protect_base = base; + mapped_count++; + break; + case 4: + base = devm_ioremap(&pdev->dev, addr, size); + if (!base) { + dev_err(&pdev->dev, + "Failed to map CKM region (0x%08x)\n", + addr); + return -ENOMEM; + } + dev->mmio.ckm_base = base; + mapped_count++; + break; + default: + dev_warn(&pdev->dev, + "Unexpected MMIO region at 0x%08x (size 0x%x), skipping\n", + addr, size); + break; + } + } + + if (!dev->mmio.top_base || !dev->mmio.protect_base + || !dev->mmio.ckm_base || (dev->pd->port_nums >= 1 + && !dev->mmio.mac_port_base[0]) + || (dev->pd->port_nums >= 2 && !dev->mmio.mac_port_base[1])) { + dev_err(&pdev->dev, + "Required MMIO regions not mapped from ACPI _CRS (mapped %d)\n", + mapped_count); + if (!dev->mmio.top_base) + dev_err(&pdev->dev, " Missing: TOP\n"); + if (!dev->mmio.protect_base) + dev_err(&pdev->dev, " Missing: PROTECT\n"); + if (!dev->mmio.ckm_base) + dev_err(&pdev->dev, " Missing: CKM\n"); + if (dev->pd->port_nums >= 1 && !dev->mmio.mac_port_base[0]) + dev_err(&pdev->dev, + " Missing: MAC Port 0 (port_nums=%d)\n", + dev->pd->port_nums); + if (dev->pd->port_nums >= 2 && !dev->mmio.mac_port_base[1]) + dev_err(&pdev->dev, + " Missing: MAC Port 1 (port_nums=%d)\n", + dev->pd->port_nums); + dev->mmio.top_base = NULL; + dev->mmio.protect_base = NULL; + dev->mmio.ckm_base = NULL; + for (i = 0; i < HP_PORT_MAX; i++) + dev->mmio.mac_port_base[i] = NULL; + return -ENODEV; + } + + dev_dbg(&pdev->dev, + "Successfully mapped all MMIO regions from ACPI _CRS\n"); + return 0; +} + +/** + * cx7_hp_rp_bus_protect - Bus protection handler + * @dev: hotplug device + * @port_idx: port index (0-based) + * @stage: protection stage (BUS_PROTECT_INIT, BUS_PROTECT_CLEANUP, etc.) + */ +static void cx7_hp_rp_bus_protect(struct cx7_hp_dev *dev, int port_idx, + int stage) +{ + switch (stage) { + case BUS_PROTECT_INIT: + { + int ret; + + ret = cx7_hp_map_mmio_resources(dev); + if (ret) { + dev_err(&dev->pdev->dev, + "Failed to map MMIO resources during bus init: %d\n", + ret); + return; + } + } + return; + + case BUS_PROTECT_CLEANUP: + { + int i; + + for (i = 0; i < HP_PORT_MAX; i++) { + if (dev->mmio.mac_port_base[i]) + dev->mmio.mac_port_base[i] = NULL; + } + if (dev->mmio.top_base) + dev->mmio.top_base = NULL; + if (dev->mmio.protect_base) + dev->mmio.protect_base = NULL; + if (dev->mmio.ckm_base) + dev->mmio.ckm_base = NULL; + } + return; + + case BUS_PROTECT_CABLE_REMOVAL: + case BUS_PROTECT_CABLE_PLUGIN: + { + struct rp_bus_mmio_info *mmio_info = + &dev->pd->rp_bus_mmio; + void __iomem *mac_base; + + if (port_idx >= dev->pd->port_nums) + return; + + mac_base = dev->mmio.mac_port_base[port_idx]; + if (!mac_base) + return; + + if (stage == BUS_PROTECT_CABLE_REMOVAL) { + cx7_hp_reg_update_bits(mac_base, + mmio_info->mac.init_ctrl, + mmio_info->mac.ltssm_bit, + false); + cx7_hp_reg_update_bits(mac_base, + mmio_info->mac.init_ctrl, + mmio_info->mac. + phy_rst_bit, false); + return; + } + + cx7_hp_toggle_update_bit(dev->mmio.top_base, + mmio_info->top.ctrl, + mmio_info->top. + port_bits[port_idx], + mmio_info->top.update_bit, + false); + udelay(CX7_HP_DELAY_SHORT_US); + + cx7_hp_bus_protect_enable(dev, port_idx); + usleep_range(CX7_HP_DELAY_BUS_PROTECT_US, + CX7_HP_DELAY_BUS_PROTECT_US + 1000); + + cx7_hp_reg_update_bits(mac_base, + mmio_info->mac.init_ctrl, + mmio_info->mac.phy_rst_bit, + true); + cx7_hp_reg_update_bits(mac_base, + mmio_info->mac.init_ctrl, + mmio_info->mac.ltssm_bit, true); + usleep_range(CX7_HP_DELAY_PHY_RESET_US, + CX7_HP_DELAY_PHY_RESET_US + 1000); + + cx7_hp_bus_protect_disable(dev, port_idx); + + cx7_hp_toggle_update_bit(dev->mmio.top_base, + mmio_info->top.ctrl, + mmio_info->top. + port_bits[port_idx], + mmio_info->top.update_bit, + true); + } + break; + + default: + dev_warn(&dev->pdev->dev, "Unknown bus protect stage: %d\n", + stage); + break; + } +} + +/** + * retrain_pcie_link - Retrain PCIe link + * @dev: PCI device + */ +static void retrain_pcie_link(struct pci_dev *dev) +{ + u16 link_control, lnksta; + int pos, i = 0; + + pos = pci_find_capability(dev, PCI_CAP_ID_EXP); + if (!pos) { + dev_err(&dev->dev, "PCIe capability not found\n"); + return; + } + + pci_read_config_word(dev, pos + PCI_EXP_LNKCTL, &link_control); + link_control |= PCI_EXP_LNKCTL_RL; + + pci_write_config_word(dev, pos + PCI_EXP_LNKCTL, link_control); + + while (i < HP_POLL_CNT_MAX) { + i++; + pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta); + if (lnksta & PCI_EXP_LNKSTA_DLLLA) + break; + usleep_range(CX7_HP_POLL_SLEEP_US, CX7_HP_POLL_SLEEP_US + 1000); + } + + pcie_capability_write_word(dev, PCI_EXP_LNKSTA, PCI_EXP_LNKSTA_LBMS); +} + +/** + * get_port_root_port - Get PCI root port device for a port + * @hp_dev: hotplug device + * @port_idx: port index + * + * Returns cached or newly found root port, or NULL if not found. + */ +static struct pci_dev *get_port_root_port(struct cx7_hp_dev *hp_dev, + int port_idx) +{ + struct pcie_port_info *port; + + if (!hp_dev->pd || port_idx >= hp_dev->pd->port_nums) + return NULL; + + port = &hp_dev->pd->ports[port_idx]; + + if (!hp_dev->cached_root_ports[port_idx]) { + hp_dev->cached_root_ports[port_idx] = + pci_get_domain_bus_and_slot(port->domain, + port->bus, port->devfn); + if (!hp_dev->cached_root_ports[port_idx]) { + dev_warn(&hp_dev->pdev->dev, + "Root port not found for domain %d bus %d\n", + port->domain, port->bus); + return NULL; + } + } + + return hp_dev->cached_root_ports[port_idx]; +} + +/** + * remove_device - Remove PCIe devices and power down hardware + * @dev: hotplug device + */ +static void remove_device(struct cx7_hp_dev *dev) +{ + int i; + + dev_info(&dev->pdev->dev, "Cable removal\n"); + + for (i = 0; i < dev->pd->port_nums; i++) + cx7_hp_rp_bus_protect(dev, i, BUS_PROTECT_CABLE_REMOVAL); + + gpiod_set_value(dev->pins[PCIE_PIN_PERST].desc, 0); + cx7_hp_change_pinctrl_state(dev, "default"); + cx7_hp_ckm_control(dev, true); + gpiod_set_value(dev->pins[PCIE_PIN_EN].desc, 0); +} + +/** + * polling_link_to_l0 - Poll until all PCIe ports reach L0 state + * @dev: hotplug device + * + * Returns: 0 on success, negative error code on failure + */ +static int polling_link_to_l0(struct cx7_hp_dev *dev) +{ + struct pci_dev *pci_dev; + u32 ltssm_reg; + u32 l0_state; + u32 ltssm_vals[HP_PORT_MAX] = { 0 }; + int count = 0; + int i; + bool all_l0; + + ltssm_reg = dev->pd->ltssm_reg; + l0_state = dev->pd->ltssm_l0_state; + + if (!ltssm_reg || !l0_state) + return 0; /* Skip if not configured */ + + /* Poll until all ports reach L0 state */ + all_l0 = false; + while (!all_l0) { + all_l0 = true; + + for (i = 0; i < dev->pd->port_nums; i++) { + pci_dev = get_port_root_port(dev, i); + if (!pci_dev) { + all_l0 = false; + continue; + } + + pci_read_config_dword(pci_dev, ltssm_reg, + <ssm_vals[i]); + if ((ltssm_vals[i] & l0_state) != l0_state) + all_l0 = false; + } + + if (all_l0) + break; + + usleep_range(CX7_HP_POLL_SLEEP_US, CX7_HP_POLL_SLEEP_US + 1000); + count++; + + if (count > HP_POLL_CNT_MAX) { + dev_err(&dev->pdev->dev, + "Timeout waiting for link to reach L0 (reached max count)\n"); + break; + } + } + + if (count > HP_POLL_CNT_MAX) { + return -ETIMEDOUT; + } + + return 0; +} + +/** + * rescan_device - Rescan PCIe bus to discover devices + * @dev: hotplug device + * + * Returns: 0 on success, negative error code on failure + */ +static int rescan_device(struct cx7_hp_dev *dev) +{ + struct pci_dev *pci_dev; + int i, err; + + err = cx7_hp_change_pinctrl_state(dev, "clkreqn"); + if (err) + return err; + + cx7_hp_ckm_control(dev, false); + usleep_range(CX7_HP_DELAY_STANDARD_US, CX7_HP_DELAY_STANDARD_US + 1000); + + for (i = 0; i < dev->pd->port_nums; i++) { + pci_dev = get_port_root_port(dev, i); + if (!pci_dev) + continue; + + err = pm_runtime_resume_and_get(&pci_dev->dev); + if (err < 0) { + dev_err(&dev->pdev->dev, + "Runtime resume failed for %s: %d\n", + pci_name(pci_dev), err); + } + } + + gpiod_set_value(dev->pins[PCIE_PIN_PERST].desc, 1); + + for (i = 0; i < dev->pd->port_nums; i++) + cx7_hp_rp_bus_protect(dev, i, BUS_PROTECT_CABLE_PLUGIN); + + err = polling_link_to_l0(dev); + if (err) + return err; + + for (i = 0; i < dev->pd->port_nums; i++) { + pci_dev = get_port_root_port(dev, i); + if (pci_dev) + retrain_pcie_link(pci_dev); + } + + msleep(CX7_HP_DELAY_LINK_STABLE_MS); + + return 0; +} + +/** + * cx7_hp_work - Work queue handler for hotplug state machine + * @irq: interrupt number + * @dev_id: GPIO context pointer + * + * Processes hotplug state transitions based on current state. + */ +static irqreturn_t cx7_hp_work(int irq, void *dev_id) +{ + struct cx7_hp_gpio_ctx *app_ctx = dev_id; + struct cx7_hp_dev *hp_dev; + enum cx7_hp_state state; + unsigned long flags; + int ret; + + if (!app_ctx || !app_ctx->hp_dev) + return IRQ_NONE; + + hp_dev = app_ctx->hp_dev; + + spin_lock_irqsave(&hp_dev->lock, flags); + if (!hp_dev->hotplug_enabled) { + spin_unlock_irqrestore(&hp_dev->lock, flags); + return IRQ_HANDLED; + } + state = hp_dev->state; + spin_unlock_irqrestore(&hp_dev->lock, flags); + + switch (state) { + case STATE_PLUG_OUT: + remove_device(hp_dev); + break; + case STATE_PLUG_IN: + dev_info(&hp_dev->pdev->dev, "Cable plugin\n"); + gpiod_set_value(hp_dev->pins[PCIE_PIN_EN].desc, 1); + break; + case STATE_DEV_POWER_OFF: + case STATE_DEV_POWER_ON: + case STATE_DEV_FW_START: + break; + case STATE_RESCAN: + ret = rescan_device(hp_dev); + spin_lock_irqsave(&hp_dev->lock, flags); + if (ret) + dev_err(app_ctx->ctx->dev, "Rescan failed: %d\n", ret); + else + hp_dev->state = STATE_READY; + spin_unlock_irqrestore(&hp_dev->lock, flags); + break; + default: + dev_err(app_ctx->ctx->dev, "Unknown state: %d\n", state); + break; + } + + return IRQ_HANDLED; +} + +/** + * hotplug_irq_handler - GPIO interrupt handler for hotplug events + * @irq: interrupt number + * @dev_id: GPIO context pointer + * + * Handles presence detection and boot status GPIO interrupts. + */ +static irqreturn_t hotplug_irq_handler(int irq, void *dev_id) +{ + struct cx7_hp_gpio_ctx *app_ctx = dev_id; + struct cx7_hp_dev *hp_dev = app_ctx->hp_dev; + struct gpio_acpi_context *gpio_ctx = app_ctx->ctx; + unsigned long flags; + int value; + enum cx7_hp_state state; + + value = gpiod_get_value(app_ctx->desc); + + if (gpio_ctx->pin == hp_dev->prsnt_pin) { + if (value) { + cx7_hp_send_uevent(hp_dev, REMOVAL_EVT); + } else { + cx7_hp_send_uevent(hp_dev, PLUG_IN_EVT); + } + return IRQ_HANDLED; + } + + spin_lock_irqsave(&hp_dev->lock, flags); + if (!hp_dev->hotplug_enabled) { + spin_unlock_irqrestore(&hp_dev->lock, flags); + return IRQ_HANDLED; + } + state = hp_dev->state; + + if (gpio_ctx->pin == hp_dev->boot_pin) { + if (value && state == STATE_PLUG_IN) { + hp_dev->state = STATE_DEV_POWER_ON; + } else if (value && state == STATE_DEV_FW_START) { + hp_dev->state = STATE_RESCAN; + } else if (!value && state == STATE_DEV_POWER_ON) { + hp_dev->state = STATE_DEV_FW_START; + } else if (!value && state == STATE_PLUG_OUT) { + hp_dev->state = STATE_DEV_POWER_OFF; + } else { + spin_unlock_irqrestore(&hp_dev->lock, flags); + return IRQ_HANDLED; + } + spin_unlock_irqrestore(&hp_dev->lock, flags); + return IRQ_WAKE_THREAD; + } + + dev_err(gpio_ctx->dev, + "Unknown GPIO pin event: pin=%d irq=%d value=%d\n", + gpio_ctx->pin, irq, value); + spin_unlock_irqrestore(&hp_dev->lock, flags); + return IRQ_HANDLED; +} + +/** + * acpi_gpio_collect_handler - ACPI resource handler to collect all GPIO resources + * @ares: ACPI resource structure + * @context: Pointer to acpi_gpio_walk_context + * + * Returns: AE_OK to continue iteration + */ +static acpi_status acpi_gpio_collect_handler(struct acpi_resource *ares, + void *context) +{ + struct acpi_gpio_walk_context *walk_ctx = context; + struct acpi_resource_gpio *agpio; + int length; + + if (ares->type != ACPI_RESOURCE_TYPE_GPIO) + return AE_OK; + + if (walk_ctx->count >= PCIE_PIN_MAX) { + dev_warn(walk_ctx->dev, + "Too many GPIO resources, truncating at %d\n", + PCIE_PIN_MAX); + return AE_OK; + } + + agpio = &ares->data.gpio; + + if (!agpio->pin_table || agpio->pin_table_length == 0) { + dev_warn(walk_ctx->dev, "GPIO resource has no pin table\n"); + return AE_OK; + } + + walk_ctx->gpios[walk_ctx->count].pin = agpio->pin_table[0]; + walk_ctx->gpios[walk_ctx->count].connection_type = + agpio->connection_type; + walk_ctx->gpios[walk_ctx->count].triggering = agpio->triggering; + walk_ctx->gpios[walk_ctx->count].polarity = agpio->polarity; + walk_ctx->gpios[walk_ctx->count].debounce_timeout = + agpio->debounce_timeout; + walk_ctx->gpios[walk_ctx->count].wake_capable = agpio->wake_capable; + + if (agpio->vendor_length && agpio->vendor_data) { + length = min_t(int, agpio->vendor_length, MAX_VENDOR_DATA_LEN); + memcpy(walk_ctx->gpios[walk_ctx->count].vendor_data, + agpio->vendor_data, length); + walk_ctx->gpios[walk_ctx->count].vendor_data[length] = '\0'; + } else { + walk_ctx->gpios[walk_ctx->count].vendor_data[0] = '\0'; + } + + if (agpio->resource_source.string_ptr) { + length = min_t(int, agpio->resource_source.string_length, 15); + memcpy(walk_ctx->gpios[walk_ctx->count].resource_source, + agpio->resource_source.string_ptr, length); + walk_ctx->gpios[walk_ctx->count].resource_source[length] = '\0'; + } else { + walk_ctx->gpios[walk_ctx->count].resource_source[0] = '\0'; + } + walk_ctx->gpios[walk_ctx->count].resource_source_index = + agpio->resource_source.index; + walk_ctx->count++; + return AE_OK; +} + +/** + * cx7_hp_walk_acpi_gpios - Walk ACPI _CRS to collect all GPIO resources + * @pdev: Platform device + * @walk_ctx: Context structure to fill with GPIO information + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_walk_acpi_gpios(struct platform_device *pdev, + struct acpi_gpio_walk_context *walk_ctx) +{ + struct acpi_device *adev; + acpi_status status; + + adev = ACPI_COMPANION(&pdev->dev); + if (!adev) { + dev_err(&pdev->dev, "Failed to get ACPI companion device\n"); + return -ENODEV; + } + + memset(walk_ctx, 0, sizeof(*walk_ctx)); + walk_ctx->dev = &pdev->dev; + + status = acpi_walk_resources(adev->handle, METHOD_NAME__CRS, + acpi_gpio_collect_handler, walk_ctx); + if (ACPI_FAILURE(status)) { + dev_err(&pdev->dev, "Failed to walk ACPI GPIO resources: %s\n", + acpi_format_exception(status)); + return -EIO; + } + + dev_dbg(&pdev->dev, "Found %d GPIO resources via ACPI walk\n", + walk_ctx->count); + + if (walk_ctx->count == 0) { + dev_err(&pdev->dev, "No GPIO resources found in ACPI _CRS\n"); + return -ENODEV; + } + + return 0; +} + +/** + * acpi_gpio_lookup_handler - ACPI resource handler to look up a specific GPIO pin + * @ares: ACPI resource being processed + * @context: Pointer to acpi_gpio_parse_context + * + * Returns: AE_OK to continue iteration + */ +static acpi_status acpi_gpio_lookup_handler(struct acpi_resource *ares, + void *context) +{ + struct acpi_gpio_parse_context *parse_ctx = context; + struct gpio_acpi_context *ctx = parse_ctx->ctx; + struct cx7_hp_dev *hp_dev = parse_ctx->hp_dev; + struct acpi_resource_gpio *agpio; + int length; + + if (ares->type != ACPI_RESOURCE_TYPE_GPIO) + return AE_OK; + + agpio = &ares->data.gpio; + + if (ctx->pin != agpio->pin_table[0]) + return AE_OK; + + ctx->valid = 1; + ctx->debounce_timeout_us = agpio->debounce_timeout * 10; + ctx->wake_capable = agpio->wake_capable; + ctx->triggering = agpio->triggering; + ctx->polarity = agpio->polarity; + ctx->connection_type = agpio->connection_type; + + if (agpio->vendor_length && agpio->vendor_data && hp_dev) { + length = min_t(int, agpio->vendor_length, MAX_VENDOR_DATA_LEN); + memcpy(&ctx->vendor_data[0], agpio->vendor_data, length); + ctx->vendor_data[length] = '\0'; + + if (!strncmp("BOOT", ctx->vendor_data, strlen("BOOT"))) + hp_dev->boot_pin = ctx->pin; + else if (!strncmp("PRSNT", ctx->vendor_data, strlen("PRSNT"))) + hp_dev->prsnt_pin = ctx->pin; + } + + if (agpio->triggering == ACPI_EDGE_SENSITIVE) { + if (agpio->polarity == ACPI_ACTIVE_LOW) + ctx->irq_flags = IRQF_TRIGGER_FALLING; + else if (agpio->polarity == ACPI_ACTIVE_HIGH) + ctx->irq_flags = IRQF_TRIGGER_RISING; + else + ctx->irq_flags = + (IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING); + } else { + if (agpio->polarity == ACPI_ACTIVE_LOW) + ctx->irq_flags = IRQF_TRIGGER_LOW; + else + ctx->irq_flags = IRQF_TRIGGER_HIGH; + } + + return AE_OK; +} + +/** + * pci_devices_present_on_domain() - Check if PCI devices exist on a domain + * @domain: PCI domain number to check + * + * Returns: true if any PCI devices are present on the specified domain, + * false otherwise. This is used as a safety check before hardware shutdown. + */ +static bool pci_devices_present_on_domain(int domain) +{ + struct pci_bus *bus; + struct pci_dev *dev; + bool has_endpoint_devices = false; + + bus = pci_find_bus(domain, 1); + if (!bus) + return false; + + list_for_each_entry(dev, &bus->devices, bus_list) { + has_endpoint_devices = true; + break; + } + + return has_endpoint_devices; +} + +static ssize_t debug_state_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cx7_hp_dev *hp_dev = dev_get_drvdata(dev); + + if (!hp_dev) + return -EINVAL; + + return scnprintf(buf, PAGE_SIZE, "%d\n", hp_dev->debug_state); +} + +static ssize_t debug_state_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct cx7_hp_dev *hp_dev = dev_get_drvdata(dev); + unsigned long val, flags; + int err, i; + + if (!hp_dev || !hp_dev->pd) + return -EINVAL; + + err = kstrtoul(buf, 10, &val); + if (err) + return err; + + spin_lock_irqsave(&hp_dev->lock, flags); + if (!hp_dev->hotplug_enabled) { + spin_unlock_irqrestore(&hp_dev->lock, flags); + dev_info(dev, "Hotplug is disabled.\n"); + return -EPERM; + } + spin_unlock_irqrestore(&hp_dev->lock, flags); + + switch (val) { + case CX7_HP_DEBUG_PLUG_OUT: + /* Safety check: Verify no devices on the bus before hardware shutdown. */ + for (i = 0; i < hp_dev->pd->port_nums; i++) { + if (pci_devices_present_on_domain + (hp_dev->pd->ports[i].domain)) { + dev_err(dev, + "PCI devices still present, remove them first\n"); + return -EBUSY; + } + } + + spin_lock_irqsave(&hp_dev->lock, flags); + hp_dev->state = STATE_PLUG_OUT; + hp_dev->debug_state = val; + spin_unlock_irqrestore(&hp_dev->lock, flags); + remove_device(hp_dev); + return count; + + case CX7_HP_DEBUG_PLUG_IN: + for (i = 0; i < hp_dev->pd->port_nums; i++) { + if (pci_devices_present_on_domain + (hp_dev->pd->ports[i].domain)) { + dev_err(dev, + "PCI devices already present, cannot reinitialize hardware\n"); + return -EBUSY; + } + } + + spin_lock_irqsave(&hp_dev->lock, flags); + hp_dev->state = STATE_PLUG_IN; + hp_dev->debug_state = val; + spin_unlock_irqrestore(&hp_dev->lock, flags); + dev_info(dev, "Cable plugin\n"); + gpiod_set_value(hp_dev->pins[PCIE_PIN_EN].desc, 1); + return count; + + default: + return -EINVAL; + } + + return count; +} + +DEVICE_ATTR_RW(debug_state); + +static ssize_t hotplug_enabled_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cx7_hp_dev *hp_dev = dev_get_drvdata(dev); + + if (!hp_dev) + return -EINVAL; + + return scnprintf(buf, PAGE_SIZE, "%d\n", hp_dev->hotplug_enabled ? 1 : 0); +} + +static ssize_t hotplug_enabled_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct cx7_hp_dev *hp_dev = dev_get_drvdata(dev); + unsigned long val; + int err; + + if (!hp_dev) + return -EINVAL; + + err = kstrtoul(buf, 10, &val); + if (err) + return err; + + hp_dev->hotplug_enabled = (val != 0); + dev_info(dev, "Hotplug %s\n", hp_dev->hotplug_enabled ? "enabled" : "disabled"); + + return count; +} + +DEVICE_ATTR_RW(hotplug_enabled); + +static struct attribute *cx7_hp_attrs[] = { + &dev_attr_debug_state.attr, + &dev_attr_hotplug_enabled.attr, + NULL +}; + +static const struct attribute_group cx7_hp_attr_group = { + .name = "pcie_hotplug", + .attrs = cx7_hp_attrs +}; + +/** + * gpio_acpi_setup - Setup GPIO ACPI context from _CRS + * @pdev: platform device + * @desc: GPIO descriptor + * @hp_dev: hotplug device + * @gpio_index: GPIO index + * + * Returns: GPIO ACPI context on success, NULL on failure + */ +static struct gpio_acpi_context *gpio_acpi_setup(struct platform_device *pdev, + struct gpio_desc *desc, + struct cx7_hp_dev *hp_dev, + int gpio_index) +{ + struct acpi_gpio_parse_context parse_ctx; + struct gpio_acpi_context *ctx; + struct acpi_device *adev; + acpi_status status; + + adev = ACPI_COMPANION(&pdev->dev); + if (!adev) { + dev_err(&pdev->dev, "Failed to get ACPI companion device\n"); + return NULL; + } + + ctx = devm_kzalloc(&pdev->dev, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return NULL; + + ctx->pin = + desc_to_gpio(desc) - + gpio_device_get_base(gpiod_to_gpio_device(desc)); + ctx->dev = &pdev->dev; + + parse_ctx.ctx = ctx; + parse_ctx.hp_dev = hp_dev; + + status = acpi_walk_resources(adev->handle, METHOD_NAME__CRS, + acpi_gpio_lookup_handler, &parse_ctx); + if (ACPI_FAILURE(status)) { + devm_kfree(&pdev->dev, ctx); + return NULL; + } + + if (ctx->valid) { + if (gpio_index == PCIE_PIN_BOOT && hp_dev->boot_pin == -1) { + hp_dev->boot_pin = ctx->pin; + } else if (gpio_index == PCIE_PIN_PRSNT + && hp_dev->prsnt_pin == -1) { + hp_dev->prsnt_pin = ctx->pin; + } + return ctx; + } + + devm_kfree(&pdev->dev, ctx); + return NULL; +} + +/** + * cx7_hp_setup_irq - Setup IRQ for GPIO + * @app_ctx: GPIO context + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_setup_irq(struct cx7_hp_gpio_ctx *app_ctx) +{ + struct gpio_acpi_context *ctx = app_ctx->ctx; + int irq, ret; + + irq = gpiod_to_irq(app_ctx->desc); + if (irq < 0) { + dev_err(ctx->dev, "Failed to get IRQ for GPIO\n"); + return irq; + } + + if (ctx->wake_capable) + enable_irq_wake(irq); + + ret = devm_request_threaded_irq(ctx->dev, irq, + hotplug_irq_handler, cx7_hp_work, + ctx->irq_flags | IRQF_ONESHOT, + "pcie_hotplug", app_ctx); + if (ret) + dev_err(ctx->dev, "Failed to request IRQ %d: %d\n", irq, ret); + + return ret; +} + +/** + * cx7_hp_put_gpio_device - Release GPIO device reference + * @data: GPIO device pointer + */ +static void cx7_hp_put_gpio_device(void *data) +{ + struct gpio_device *gdev = data; + + gpio_device_put(gdev); +} + +/** + * cx7_hp_discover_pcie_devices - Discover existing PCI devices on managed ports + * @pdev: platform device + * @pd: platform data + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_discover_pcie_devices(struct platform_device *pdev, + struct cx7_hp_plat_data *pd) +{ + struct pci_dev *pci_dev = NULL; + int device_count = 0; + int i; + + if (!pd->vendor_id || !pd->device_id) + return 0; + + while ((pci_dev = pci_get_device(pd->vendor_id, + pd->device_id, pci_dev)) != NULL) { + if (!pci_dev->state_saved) { + pci_dev_put(pci_dev); + return -EPROBE_DEFER; + } + + for (i = 0; i < pd->port_nums; i++) { + if (pci_domain_nr(pci_dev->bus) == pd->ports[i].domain) + break; + } + + if (i == pd->port_nums) { + dev_err(&pdev->dev, + "Device %s found on unexpected domain %d\n", + pci_name(pci_dev), pci_domain_nr(pci_dev->bus)); + pci_dev_put(pci_dev); + return -ENODEV; + } + + device_count++; + } + + if (pd->num_devices && device_count != pd->num_devices) { + dev_err(&pdev->dev, + "Required number of devices not found. Expected=%d Actual=%d\n", + pd->num_devices, device_count); + return -ENODEV; + } + + return 0; +} + +/** + * cx7_hp_init_pcie_data - Initialize PCIe data from _DSD and discover devices + * @pdev: platform device + * @pd: platform data to populate + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_init_pcie_data(struct platform_device *pdev, + struct cx7_hp_plat_data *pd) +{ + int ret; + + ret = cx7_hp_parse_pcie_config_dsd(pdev, pd); + if (ret) { + dev_err(&pdev->dev, + "Failed to parse PCIe configuration _DSD properties: %d\n", + ret); + return ret; + } + + if (pd->port_nums == 0 || pd->port_nums >= HP_PORT_MAX) { + dev_err(&pdev->dev, + "Invalid port count from _DSD: %d (must be 1-%d)\n", + pd->port_nums, HP_PORT_MAX - 1); + return -EINVAL; + } + + ret = cx7_hp_discover_pcie_devices(pdev, pd); + if (ret) { + dev_dbg(&pdev->dev, "Device discovery failed: %d\n", ret); + return ret; + } + + return 0; +} + +/** + * cx7_hp_enumerate_gpios - Enumerate GPIOs from ACPI + * @pdev: Platform device + * @hp_dev: Hotplug device structure + * + * Returns: Number of GPIOs found, or negative error code + */ +static int cx7_hp_enumerate_gpios(struct platform_device *pdev, + struct cx7_hp_dev *hp_dev) +{ + struct acpi_gpio_walk_context walk_ctx; + struct fwnode_handle *gpio_fwnode = NULL; + struct acpi_device *gpio_adev = NULL; + acpi_handle gpio_handle; + acpi_status status; + int ret, i; + + ret = cx7_hp_walk_acpi_gpios(pdev, &walk_ctx); + if (ret) { + dev_err(&pdev->dev, "Failed to walk ACPI GPIO resources: %d\n", + ret); + return ret; + } + + if (walk_ctx.count < CX7_HP_MIN_GPIO_COUNT) { + dev_err(&pdev->dev, + "Insufficient GPIOs from ACPI: required at least %d, got %d\n", + CX7_HP_MIN_GPIO_COUNT, walk_ctx.count); + return -ENODEV; + } + + /* Find GPIO device using resource_source from first GPIO */ + if (walk_ctx.count == 0 || walk_ctx.gpios[0].resource_source[0] == '\0') { + dev_err(&pdev->dev, + "No resource_source in ACPI GPIO resources\n"); + return -ENODEV; + } + + status = + acpi_get_handle(NULL, walk_ctx.gpios[0].resource_source, + &gpio_handle); + if (ACPI_FAILURE(status)) { + dev_err(&pdev->dev, + "Failed to get ACPI handle for GPIO controller %s\n", + walk_ctx.gpios[0].resource_source); + return -ENODEV; + } + + gpio_adev = acpi_fetch_acpi_dev(gpio_handle); + if (!gpio_adev) { + dev_err(&pdev->dev, + "Failed to get ACPI device for GPIO controller %s\n", + walk_ctx.gpios[0].resource_source); + return -ENODEV; + } + + gpio_fwnode = acpi_fwnode_handle(gpio_adev); + hp_dev->gdev = gpio_device_find_by_fwnode(gpio_fwnode); + if (!hp_dev->gdev) { + return dev_err_probe(&pdev->dev, -EPROBE_DEFER, + "GPIO controller not available\n"); + } + + /* Successfully found GPIO device - manage reference */ + ret = devm_add_action_or_reset(&pdev->dev, cx7_hp_put_gpio_device, + hp_dev->gdev); + if (ret) { + gpio_device_put(hp_dev->gdev); + hp_dev->gdev = NULL; + dev_err(&pdev->dev, "Failed to register GPIO device cleanup\n"); + return ret; + } + + hp_dev->gpio_count = walk_ctx.count; + + hp_dev->pins = devm_kzalloc(&pdev->dev, + sizeof(struct cx7_hp_gpio_ctx) * + hp_dev->gpio_count, GFP_KERNEL); + if (!hp_dev->pins) { + dev_err(&pdev->dev, "Failed to allocate memory for GPIOs\n"); + return -ENOMEM; + } + + for (i = 0; i < hp_dev->gpio_count; i++) { + struct cx7_hp_gpio_ctx *app_ctx = &hp_dev->pins[i]; + + app_ctx->desc = + gpio_device_get_desc(hp_dev->gdev, walk_ctx.gpios[i].pin); + if (IS_ERR(app_ctx->desc)) { + dev_err(&pdev->dev, + "Failed to get GPIO descriptor for ACPI pin %u (index %d): %ld\n", + walk_ctx.gpios[i].pin, i, + PTR_ERR(app_ctx->desc)); + return PTR_ERR(app_ctx->desc); + } + + app_ctx->hp_dev = hp_dev; + } + + return hp_dev->gpio_count; +} + +/** + * cx7_hp_pci_notifier - PCI bus notifier to configure MPS for CX7 devices + * @nb: notifier block + * @action: bus notification action + * @data: pointer to device being added/removed + * + * Returns: NOTIFY_OK on success, NOTIFY_DONE if not a CX7 device + */ +static int cx7_hp_pci_notifier(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct device *dev = data; + struct pci_dev *pdev = to_pci_dev(dev); + struct cx7_hp_dev *hp_dev; + unsigned long flags; + + if (action != BUS_NOTIFY_ADD_DEVICE) + return NOTIFY_DONE; + + hp_dev = container_of(nb, struct cx7_hp_dev, pci_notifier); + if (!hp_dev || !hp_dev->pd) + return NOTIFY_DONE; + + spin_lock_irqsave(&hp_dev->lock, flags); + if (!hp_dev->hotplug_enabled) { + spin_unlock_irqrestore(&hp_dev->lock, flags); + return NOTIFY_DONE; + } + spin_unlock_irqrestore(&hp_dev->lock, flags); + + if (!pdev || !hp_dev->pd->vendor_id || !hp_dev->pd->device_id) + return NOTIFY_DONE; + + if (pdev->vendor != hp_dev->pd->vendor_id || + pdev->device != hp_dev->pd->device_id) + return NOTIFY_DONE; + + if (pdev->bus) + pcie_bus_configure_settings(pdev->bus); + + return NOTIFY_OK; +} + +/** + * cx7_hp_probe - Platform device probe function + * @pdev: platform device + * + * Initializes the PCIe hotplug driver, parses ACPI resources, and sets up + * GPIO interrupts and sysfs interface. + * + * Returns: 0 on success, negative error code on failure + */ +static int cx7_hp_probe(struct platform_device *pdev) +{ + struct cx7_hp_plat_data *pd; + struct cx7_hp_gpio_ctx *app_ctx; + struct cx7_hp_dev *hp_dev; + int ret, i; + + pd = devm_kzalloc(&pdev->dev, sizeof(*pd), GFP_KERNEL); + if (!pd) { + dev_err(&pdev->dev, + "Failed to allocate memory for platform data\n"); + return -ENOMEM; + } + + ret = cx7_hp_init_pcie_data(pdev, pd); + if (ret) + return ret; + + hp_dev = devm_kzalloc(&pdev->dev, sizeof(*hp_dev), GFP_KERNEL); + if (!hp_dev) { + dev_err(&pdev->dev, + "Failed to allocate memory for hotplug device\n"); + return -ENOMEM; + } + + hp_dev->pdev = pdev; + hp_dev->pd = pd; + hp_dev->state = STATE_READY; + hp_dev->boot_pin = -1; + hp_dev->prsnt_pin = -1; + hp_dev->hotplug_enabled = false; + spin_lock_init(&hp_dev->lock); + + for (i = 0; i < HP_PORT_MAX; i++) + hp_dev->cached_root_ports[i] = NULL; + + ret = cx7_hp_enumerate_gpios(pdev, hp_dev); + if (ret < 0) { + dev_err(&pdev->dev, "Failed to enumerate GPIOs from ACPI: %d\n", + ret); + return ret; + } + + for (i = 0; i < hp_dev->gpio_count; i++) { + app_ctx = &hp_dev->pins[i]; + + app_ctx->ctx = gpio_acpi_setup(pdev, app_ctx->desc, hp_dev, i); + if (!app_ctx->ctx) { + dev_err(&pdev->dev, "Failed to setup GPIO %d\n", i); + return -ENODEV; + } + + gpiod_set_debounce(app_ctx->desc, + app_ctx->ctx->debounce_timeout_us); + + if (app_ctx->ctx->connection_type == + ACPI_RESOURCE_GPIO_TYPE_INT) { + ret = cx7_hp_setup_irq(app_ctx); + if (ret) { + dev_err(&pdev->dev, + "Failed to setup IRQ for GPIO %d\n", i); + return ret; + } + } + } + + platform_set_drvdata(pdev, hp_dev); + + ret = cx7_hp_pinctrl_init(hp_dev); + if (ret) { + dev_err(&pdev->dev, "Pinmux init failed, ret: %d\n", ret); + return ret; + } + + ret = sysfs_create_group(&pdev->dev.kobj, &cx7_hp_attr_group); + if (ret) { + dev_err(&pdev->dev, "Sysfs creation failed: %d\n", ret); + goto pinctrl_remove; + } + + cx7_hp_rp_bus_protect(hp_dev, 0, BUS_PROTECT_INIT); + + hp_dev->pci_notifier.notifier_call = cx7_hp_pci_notifier; + ret = bus_register_notifier(&pci_bus_type, &hp_dev->pci_notifier); + if (ret) { + dev_err(&pdev->dev, "Failed to register PCI bus notifier: %d\n", + ret); + goto sysfs_remove; + } + + if (gpiod_get_value(hp_dev->pins[PCIE_PIN_PRSNT].desc)) { + hp_dev->debug_state = CX7_HP_DEBUG_PLUG_OUT; + cx7_hp_send_uevent(hp_dev, REMOVAL_EVT); + } else { + hp_dev->debug_state = CX7_HP_DEBUG_PLUG_IN; + cx7_hp_send_uevent(hp_dev, PLUG_IN_EVT); + } + + dev_info(&pdev->dev, "PCIe hotplug driver initialized successfully\n"); + return 0; + +sysfs_remove: + sysfs_remove_group(&pdev->dev.kobj, &cx7_hp_attr_group); +pinctrl_remove: + cx7_hp_pinctrl_remove(hp_dev); + return ret; +} + +/** + * cx7_hp_remove - Platform device remove function + * @pdev: platform device + * + * Cleans up GPIO pins, pinctrl, sysfs interface, and bus protection. + */ +static void cx7_hp_remove(struct platform_device *pdev) +{ + struct cx7_hp_dev *hp_dev = platform_get_drvdata(pdev); + int i; + + if (!hp_dev) + return; + + sysfs_remove_group(&pdev->dev.kobj, &cx7_hp_attr_group); + + bus_unregister_notifier(&pci_bus_type, &hp_dev->pci_notifier); + + cx7_hp_rp_bus_protect(hp_dev, 0, BUS_PROTECT_CLEANUP); + + cx7_hp_pinctrl_remove(hp_dev); + + for (i = 0; i < hp_dev->pd->port_nums; i++) { + if (hp_dev->cached_root_ports[i]) + pci_dev_put(hp_dev->cached_root_ports[i]); + } + + platform_set_drvdata(pdev, NULL); +} + +static const struct acpi_device_id cx7_hp_acpi_match[] = { + {"MTKP0001", 0}, + {} +}; + +MODULE_DEVICE_TABLE(acpi, cx7_hp_acpi_match); + +static struct platform_driver cx7_hp_driver = { + .probe = cx7_hp_probe, + .remove = cx7_hp_remove, + .driver = { + .name = "cx7-pcie-hotplug", + .acpi_match_table = ACPI_PTR(cx7_hp_acpi_match), + }, +}; + +module_platform_driver(cx7_hp_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("CX7 PCIe Hotplug Driver for NVIDIA DGX Systems"); diff --git a/drivers/resctrl/Kconfig b/drivers/resctrl/Kconfig index c808e04703946..672abea3b03cc 100644 --- a/drivers/resctrl/Kconfig +++ b/drivers/resctrl/Kconfig @@ -1,6 +1,7 @@ menuconfig ARM64_MPAM_DRIVER bool "MPAM driver" - depends on ARM64 && ARM64_MPAM && EXPERT + depends on ARM64 && ARM64_MPAM + select ACPI_MPAM if ACPI help Memory System Resource Partitioning and Monitoring (MPAM) driver for System IP, e.g. caches and memory controllers. @@ -22,3 +23,9 @@ config MPAM_KUNIT_TEST If unsure, say N. endif + +config ARM64_MPAM_RESCTRL_FS + bool + default y if ARM64_MPAM_DRIVER && RESCTRL_FS + select RESCTRL_RMID_DEPENDS_ON_CLOSID + select RESCTRL_ASSIGN_FIXED diff --git a/drivers/resctrl/Makefile b/drivers/resctrl/Makefile index 898199dcf80d5..4f6d0e81f9b8f 100644 --- a/drivers/resctrl/Makefile +++ b/drivers/resctrl/Makefile @@ -1,4 +1,5 @@ obj-$(CONFIG_ARM64_MPAM_DRIVER) += mpam.o mpam-y += mpam_devices.o +mpam-$(CONFIG_ARM64_MPAM_RESCTRL_FS) += mpam_resctrl.o ccflags-$(CONFIG_ARM64_MPAM_DRIVER_DEBUG) += -DDEBUG diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 0666be6b0e88d..ab6af4aecd6d1 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -29,7 +29,15 @@ #include "mpam_internal.h" -DEFINE_STATIC_KEY_FALSE(mpam_enabled); /* This moves to arch code */ +/* Values for the T241 errata workaround */ +#define T241_CHIPS_MAX 4 +#define T241_CHIP_NSLICES 12 +#define T241_SPARE_REG0_OFF 0x1b0000 +#define T241_SPARE_REG1_OFF 0x1c0000 +#define T241_CHIP_ID(phys) FIELD_GET(GENMASK_ULL(44, 43), phys) +#define T241_SHADOW_REG_OFF(sidx, pid) (0x360048 + (sidx) * 0x10000 + (pid) * 8) +#define SMCCC_SOC_ID_T241 0x036b0241 +static void __iomem *t241_scratch_regs[T241_CHIPS_MAX]; /* * mpam_list_lock protects the SRCU lists when writing. Once the @@ -75,6 +83,14 @@ static DECLARE_WORK(mpam_broken_work, &mpam_disable); /* When mpam is disabled, the printed reason to aid debugging */ static char *mpam_disable_reason; +/* + * Whether resctrl has been setup. Used by cpuhp in preference to + * mpam_is_enabled(). The disable call after an error interrupt makes + * mpam_is_enabled() false before the cpuhp callbacks are made. + * Reads/writes should hold mpam_cpuhp_state_lock, (or be cpuhp callbacks). + */ +static bool mpam_resctrl_enabled; + /* * An MSC is a physical container for controls and monitors, each identified by * their RIS index. These share a base-address, interrupts and some MMIO @@ -624,6 +640,86 @@ static struct mpam_msc_ris *mpam_get_or_create_ris(struct mpam_msc *msc, return ERR_PTR(-ENOENT); } +static int mpam_enable_quirk_nvidia_t241_1(struct mpam_msc *msc, + const struct mpam_quirk *quirk) +{ + s32 soc_id = arm_smccc_get_soc_id_version(); + struct resource *r; + phys_addr_t phys; + + /* + * A mapping to a device other than the MSC is needed, check + * SOC_ID is NVIDIA T241 chip (036b:0241) + */ + if (soc_id < 0 || soc_id != SMCCC_SOC_ID_T241) + return -EINVAL; + + r = platform_get_resource(msc->pdev, IORESOURCE_MEM, 0); + if (!r) + return -EINVAL; + + /* Find the internal registers base addr from the CHIP ID */ + msc->t241_id = T241_CHIP_ID(r->start); + phys = FIELD_PREP(GENMASK_ULL(45, 44), msc->t241_id) | 0x19000000ULL; + + t241_scratch_regs[msc->t241_id] = ioremap(phys, SZ_8M); + if (WARN_ON_ONCE(!t241_scratch_regs[msc->t241_id])) + return -EINVAL; + + pr_info_once("Enabled workaround for NVIDIA T241 erratum T241-MPAM-1\n"); + + return 0; +} + +static const struct mpam_quirk mpam_quirks[] = { + { + /* NVIDIA t241 erratum T241-MPAM-1 */ + .init = mpam_enable_quirk_nvidia_t241_1, + .iidr = MPAM_IIDR_NVIDIA_T241, + .iidr_mask = MPAM_IIDR_MATCH_ONE, + .workaround = T241_SCRUB_SHADOW_REGS, + }, + { + /* NVIDIA t241 erratum T241-MPAM-4 */ + .iidr = MPAM_IIDR_NVIDIA_T241, + .iidr_mask = MPAM_IIDR_MATCH_ONE, + .workaround = T241_FORCE_MBW_MIN_TO_ONE, + }, + { + /* NVIDIA t241 erratum T241-MPAM-6 */ + .iidr = MPAM_IIDR_NVIDIA_T241, + .iidr_mask = MPAM_IIDR_MATCH_ONE, + .workaround = T241_MBW_COUNTER_SCALE_64, + }, + { + /* ARM CMN-650 CSU erratum 3642720 */ + .iidr = MPAM_IIDR_ARM_CMN_650, + .iidr_mask = MPAM_IIDR_MATCH_ONE, + .workaround = IGNORE_CSU_NRDY, + }, + { NULL } /* Sentinel */ +}; + +static void mpam_enable_quirks(struct mpam_msc *msc) +{ + const struct mpam_quirk *quirk; + + for (quirk = &mpam_quirks[0]; quirk->iidr_mask; quirk++) { + int err = 0; + + if (quirk->iidr != (msc->iidr & quirk->iidr_mask)) + continue; + + if (quirk->init) + err = quirk->init(msc, quirk); + + if (err) + continue; + + mpam_set_quirk(quirk->workaround, msc); + } +} + /* * IHI009A.a has this nugget: "If a monitor does not support automatic behaviour * of NRDY, software can use this bit for any purpose" - so hardware might not @@ -715,6 +811,13 @@ static void mpam_ris_hw_probe(struct mpam_msc_ris *ris) mpam_set_feature(mpam_feat_mbw_part, props); props->bwa_wd = FIELD_GET(MPAMF_MBW_IDR_BWA_WD, mbw_features); + + /* + * The BWA_WD field can represent 0-63, but the control fields it + * describes have a maximum of 16 bits. + */ + props->bwa_wd = min(props->bwa_wd, 16); + if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_MAX, mbw_features)) mpam_set_feature(mpam_feat_mbw_max, props); @@ -851,8 +954,11 @@ static int mpam_msc_hw_probe(struct mpam_msc *msc) /* Grab an IDR value to find out how many RIS there are */ mutex_lock(&msc->part_sel_lock); idr = mpam_msc_read_idr(msc); + msc->iidr = mpam_read_partsel_reg(msc, IIDR); mutex_unlock(&msc->part_sel_lock); + mpam_enable_quirks(msc); + msc->ris_max = FIELD_GET(MPAMF_IDR_RIS_MAX, idr); /* Use these values so partid/pmg always starts with a valid value */ @@ -903,6 +1009,7 @@ struct mon_read { enum mpam_device_features type; u64 *val; int err; + bool waited_timeout; }; static bool mpam_ris_has_mbwu_long_counter(struct mpam_msc_ris *ris) @@ -1052,7 +1159,7 @@ static void write_msmon_ctl_flt_vals(struct mon_read *m, u32 ctl_val, } } -static u64 mpam_msmon_overflow_val(enum mpam_device_features type) +static u64 __mpam_msmon_overflow_val(enum mpam_device_features type) { /* TODO: implement scaling counters */ switch (type) { @@ -1067,6 +1174,18 @@ static u64 mpam_msmon_overflow_val(enum mpam_device_features type) } } +static u64 mpam_msmon_overflow_val(enum mpam_device_features type, + struct mpam_msc *msc) +{ + u64 overflow_val = __mpam_msmon_overflow_val(type); + + if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc) && + type != mpam_feat_msmon_mbwu_63counter) + overflow_val *= 64; + + return overflow_val; +} + static void __ris_msmon_read(void *arg) { u64 now; @@ -1137,6 +1256,10 @@ static void __ris_msmon_read(void *arg) if (mpam_has_feature(mpam_feat_msmon_csu_hw_nrdy, rprops)) nrdy = now & MSMON___NRDY; now = FIELD_GET(MSMON___VALUE, now); + + if (mpam_has_quirk(IGNORE_CSU_NRDY, msc) && m->waited_timeout) + nrdy = false; + break; case mpam_feat_msmon_mbwu_31counter: case mpam_feat_msmon_mbwu_44counter: @@ -1157,13 +1280,17 @@ static void __ris_msmon_read(void *arg) now = FIELD_GET(MSMON___VALUE, now); } + if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc) && + m->type != mpam_feat_msmon_mbwu_63counter) + now *= 64; + if (nrdy) break; mbwu_state = &ris->mbwu_state[ctx->mon]; if (overflow) - mbwu_state->correction += mpam_msmon_overflow_val(m->type); + mbwu_state->correction += mpam_msmon_overflow_val(m->type, msc); /* * Include bandwidth consumed before the last hardware reset and @@ -1270,6 +1397,7 @@ int mpam_msmon_read(struct mpam_component *comp, struct mon_cfg *ctx, .ctx = ctx, .type = type, .val = val, + .waited_timeout = true, }; *val = 0; @@ -1338,6 +1466,75 @@ static void mpam_reset_msc_bitmap(struct mpam_msc *msc, u16 reg, u16 wd) __mpam_write_reg(msc, reg, bm); } +static void mpam_apply_t241_erratum(struct mpam_msc_ris *ris, u16 partid) +{ + int sidx, i, lcount = 1000; + void __iomem *regs; + u64 val0, val; + + regs = t241_scratch_regs[ris->vmsc->msc->t241_id]; + + for (i = 0; i < lcount; i++) { + /* Read the shadow register at index 0 */ + val0 = readq_relaxed(regs + T241_SHADOW_REG_OFF(0, partid)); + + /* Check if all the shadow registers have the same value */ + for (sidx = 1; sidx < T241_CHIP_NSLICES; sidx++) { + val = readq_relaxed(regs + + T241_SHADOW_REG_OFF(sidx, partid)); + if (val != val0) + break; + } + if (sidx == T241_CHIP_NSLICES) + break; + } + + if (i == lcount) + pr_warn_once("t241: inconsistent values in shadow regs"); + + /* Write a value zero to spare registers to take effect of MBW conf */ + writeq_relaxed(0, regs + T241_SPARE_REG0_OFF); + writeq_relaxed(0, regs + T241_SPARE_REG1_OFF); +} + +static void mpam_quirk_post_config_change(struct mpam_msc_ris *ris, u16 partid, + struct mpam_config *cfg) +{ + if (mpam_has_quirk(T241_SCRUB_SHADOW_REGS, ris->vmsc->msc)) + mpam_apply_t241_erratum(ris, partid); +} + +static u16 mpam_wa_t241_force_mbw_min_to_one(struct mpam_props *props) +{ + u16 max_hw_value, min_hw_granule, res0_bits; + + res0_bits = 16 - props->bwa_wd; + max_hw_value = ((1 << props->bwa_wd) - 1) << res0_bits; + min_hw_granule = ~max_hw_value; + + return min_hw_granule + 1; +} + +static u16 mpam_wa_t241_calc_min_from_max(struct mpam_props *props, + struct mpam_config *cfg) +{ + u16 val = 0; + u16 max; + u16 delta = ((5 * MPAMCFG_MBW_MAX_MAX) / 100) - 1; + + if (mpam_has_feature(mpam_feat_mbw_max, cfg)) { + max = cfg->mbw_max; + } else { + /* Resetting. Hence, use the ris specific default. */ + max = GENMASK(15, 16 - props->bwa_wd); + } + + if (max > delta) + val = max - delta; + + return val; +} + /* Called via IPI. Call while holding an SRCU reference */ static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid, struct mpam_config *cfg) @@ -1364,43 +1561,62 @@ static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid, __mpam_intpart_sel(ris->ris_idx, partid, msc); } - if (mpam_has_feature(mpam_feat_cpor_part, rprops) && - mpam_has_feature(mpam_feat_cpor_part, cfg)) { - if (cfg->reset_cpbm) - mpam_reset_msc_bitmap(msc, MPAMCFG_CPBM, rprops->cpbm_wd); - else + if (mpam_has_feature(mpam_feat_cpor_part, rprops)) { + if (mpam_has_feature(mpam_feat_cpor_part, cfg)) mpam_write_partsel_reg(msc, CPBM, cfg->cpbm); + else + mpam_reset_msc_bitmap(msc, MPAMCFG_CPBM, rprops->cpbm_wd); } - if (mpam_has_feature(mpam_feat_mbw_part, rprops) && - mpam_has_feature(mpam_feat_mbw_part, cfg)) { - if (cfg->reset_mbw_pbm) - mpam_reset_msc_bitmap(msc, MPAMCFG_MBW_PBM, rprops->mbw_pbm_bits); - else + if (mpam_has_feature(mpam_feat_mbw_part, rprops)) { + if (mpam_has_feature(mpam_feat_mbw_part, cfg)) mpam_write_partsel_reg(msc, MBW_PBM, cfg->mbw_pbm); + else + mpam_reset_msc_bitmap(msc, MPAMCFG_MBW_PBM, rprops->mbw_pbm_bits); } - if (mpam_has_feature(mpam_feat_mbw_min, rprops) && - mpam_has_feature(mpam_feat_mbw_min, cfg)) - mpam_write_partsel_reg(msc, MBW_MIN, 0); + if (mpam_has_feature(mpam_feat_mbw_min, rprops)) { + u16 val = 0; - if (mpam_has_feature(mpam_feat_mbw_max, rprops) && - mpam_has_feature(mpam_feat_mbw_max, cfg)) { - if (cfg->reset_mbw_max) - mpam_write_partsel_reg(msc, MBW_MAX, MPAMCFG_MBW_MAX_MAX); - else + if (mpam_has_quirk(T241_FORCE_MBW_MIN_TO_ONE, msc)) { + u16 min = mpam_wa_t241_force_mbw_min_to_one(rprops); + + val = mpam_wa_t241_calc_min_from_max(rprops, cfg); + val = max(val, min); + } + + mpam_write_partsel_reg(msc, MBW_MIN, val); + } + + if (mpam_has_feature(mpam_feat_mbw_max, rprops)) { + if (mpam_has_feature(mpam_feat_mbw_max, cfg)) mpam_write_partsel_reg(msc, MBW_MAX, cfg->mbw_max); + else + mpam_write_partsel_reg(msc, MBW_MAX, MPAMCFG_MBW_MAX_MAX); } - if (mpam_has_feature(mpam_feat_mbw_prop, rprops) && - mpam_has_feature(mpam_feat_mbw_prop, cfg)) + if (mpam_has_feature(mpam_feat_mbw_prop, rprops)) mpam_write_partsel_reg(msc, MBW_PROP, 0); - if (mpam_has_feature(mpam_feat_cmax_cmax, rprops)) - mpam_write_partsel_reg(msc, CMAX, cmax); + if (mpam_has_feature(mpam_feat_cmax_cmax, rprops)) { + if (mpam_has_feature(mpam_feat_cmax_cmax, cfg)) { + u32 cmax_val = cfg->cmax; - if (mpam_has_feature(mpam_feat_cmax_cmin, rprops)) - mpam_write_partsel_reg(msc, CMIN, 0); + if (cfg->cmax_softlim) + cmax_val |= MPAMCFG_CMAX_SOFTLIM; + mpam_write_partsel_reg(msc, CMAX, cmax_val); + } else { + mpam_write_partsel_reg(msc, CMAX, cmax); + } + } + + if (mpam_has_feature(mpam_feat_cmax_cmin, rprops)) { + if (mpam_has_feature(mpam_feat_cmax_cmin, cfg)) { + mpam_write_partsel_reg(msc, CMIN, cfg->cmin); + } else { + mpam_write_partsel_reg(msc, CMIN, 0); + } + } if (mpam_has_feature(mpam_feat_cmax_cassoc, rprops)) mpam_write_partsel_reg(msc, CASSOC, MPAMCFG_CASSOC_CASSOC); @@ -1421,6 +1637,8 @@ static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid, mpam_write_partsel_reg(msc, PRI, pri_val); } + mpam_quirk_post_config_change(ris, partid, cfg); + mutex_unlock(&msc->part_sel_lock); } @@ -1493,16 +1711,6 @@ static int mpam_save_mbwu_state(void *arg) return 0; } -static void mpam_init_reset_cfg(struct mpam_config *reset_cfg) -{ - *reset_cfg = (struct mpam_config) { - .reset_cpbm = true, - .reset_mbw_pbm = true, - .reset_mbw_max = true, - }; - bitmap_fill(reset_cfg->features, MPAM_FEATURE_LAST); -} - /* * Called via smp_call_on_cpu() to prevent migration, while still being * pre-emptible. Caller must hold mpam_srcu. @@ -1510,14 +1718,12 @@ static void mpam_init_reset_cfg(struct mpam_config *reset_cfg) static int mpam_reset_ris(void *arg) { u16 partid, partid_max; - struct mpam_config reset_cfg; + struct mpam_config reset_cfg = {}; struct mpam_msc_ris *ris = arg; if (ris->in_reset_state) return 0; - mpam_init_reset_cfg(&reset_cfg); - spin_lock(&partid_max_lock); partid_max = mpam_partid_max; spin_unlock(&partid_max_lock); @@ -1632,6 +1838,9 @@ static int mpam_cpu_online(unsigned int cpu) mpam_reprogram_msc(msc); } + if (mpam_resctrl_enabled) + return mpam_resctrl_online_cpu(cpu); + return 0; } @@ -1675,6 +1884,9 @@ static int mpam_cpu_offline(unsigned int cpu) { struct mpam_msc *msc; + if (mpam_resctrl_enabled) + mpam_resctrl_offline_cpu(cpu); + guard(srcu)(&mpam_srcu); list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list, srcu_read_lock_held(&mpam_srcu)) { @@ -1971,6 +2183,7 @@ static bool mpam_has_cmax_wd_feature(struct mpam_props *props) * resulting safe value must be compatible with both. When merging values in * the tree, all the aliasing resources must be handled first. * On mismatch, parent is modified. + * Quirks on an MSC will apply to all MSC in that class. */ static void __props_mismatch(struct mpam_props *parent, struct mpam_props *child, bool alias) @@ -2090,6 +2303,7 @@ static void __props_mismatch(struct mpam_props *parent, * nobble the class feature, as we can't configure all the resources. * e.g. The L3 cache is composed of two resources with 13 and 17 portion * bitmaps respectively. + * Quirks on an MSC will apply to all MSC in that class. */ static void __class_props_mismatch(struct mpam_class *class, struct mpam_vmsc *vmsc) @@ -2103,6 +2317,9 @@ __class_props_mismatch(struct mpam_class *class, struct mpam_vmsc *vmsc) dev_dbg(dev, "Merging features for class:0x%lx &= vmsc:0x%lx\n", (long)cprops->features, (long)vprops->features); + /* Merge quirks */ + class->quirks |= vmsc->msc->quirks; + /* Take the safe value for any common features */ __props_mismatch(cprops, vprops, false); } @@ -2167,6 +2384,9 @@ static void mpam_enable_merge_class_features(struct mpam_component *comp) list_for_each_entry(vmsc, &comp->vmsc, comp_list) __class_props_mismatch(class, vmsc); + + if (mpam_has_quirk(T241_FORCE_MBW_MIN_TO_ONE, class)) + mpam_clear_feature(mpam_feat_mbw_min, &class->props); } /* @@ -2409,6 +2629,8 @@ static void mpam_reset_component_cfg(struct mpam_component *comp) comp->cfg[i].mbw_pbm = GENMASK(cprops->mbw_pbm_bits - 1, 0); if (cprops->bwa_wd) comp->cfg[i].mbw_max = GENMASK(15, 16 - cprops->bwa_wd); + if (cprops->cmax_wd) + comp->cfg[i].cmax = MPAMCFG_CMAX_CMAX; } } @@ -2520,6 +2742,12 @@ static void mpam_enable_once(void) mutex_unlock(&mpam_list_lock); cpus_read_unlock(); + if (!err) { + err = mpam_resctrl_setup(); + if (err) + pr_err("Failed to initialise resctrl: %d\n", err); + } + if (err) { mpam_disable_reason = "Failed to enable."; schedule_work(&mpam_broken_work); @@ -2527,6 +2755,7 @@ static void mpam_enable_once(void) } static_branch_enable(&mpam_enabled); + mpam_resctrl_enabled = true; mpam_register_cpuhp_callbacks(mpam_cpu_online, mpam_cpu_offline, "mpam:online"); @@ -2559,7 +2788,7 @@ static void mpam_reset_component_locked(struct mpam_component *comp) } } -static void mpam_reset_class_locked(struct mpam_class *class) +void mpam_reset_class_locked(struct mpam_class *class) { struct mpam_component *comp; @@ -2586,24 +2815,39 @@ static void mpam_reset_class(struct mpam_class *class) void mpam_disable(struct work_struct *ignored) { int idx; + bool do_resctrl_exit; struct mpam_class *class; struct mpam_msc *msc, *tmp; + if (mpam_is_enabled()) + static_branch_disable(&mpam_enabled); + mutex_lock(&mpam_cpuhp_state_lock); if (mpam_cpuhp_state) { cpuhp_remove_state(mpam_cpuhp_state); mpam_cpuhp_state = 0; } + + /* + * Removing the cpuhp state called mpam_cpu_offline() and told resctrl + * all the CPUs are offline. + */ + do_resctrl_exit = mpam_resctrl_enabled; + mpam_resctrl_enabled = false; mutex_unlock(&mpam_cpuhp_state_lock); - static_branch_disable(&mpam_enabled); + if (do_resctrl_exit) + mpam_resctrl_exit(); mpam_unregister_irqs(); idx = srcu_read_lock(&mpam_srcu); list_for_each_entry_srcu(class, &mpam_classes, classes_list, - srcu_read_lock_held(&mpam_srcu)) + srcu_read_lock_held(&mpam_srcu)) { mpam_reset_class(class); + if (do_resctrl_exit) + mpam_resctrl_teardown_class(class); + } srcu_read_unlock(&mpam_srcu, idx); mutex_lock(&mpam_list_lock); @@ -2659,6 +2903,7 @@ static bool mpam_update_config(struct mpam_config *cfg, bool has_changes = false; maybe_update_config(cfg, mpam_feat_cpor_part, newcfg, cpbm, has_changes); + maybe_update_config(cfg, mpam_feat_cmax_cmax, newcfg, cmax, has_changes); maybe_update_config(cfg, mpam_feat_mbw_part, newcfg, mbw_pbm, has_changes); maybe_update_config(cfg, mpam_feat_mbw_max, newcfg, mbw_max, has_changes); @@ -2694,6 +2939,7 @@ int mpam_apply_config(struct mpam_component *comp, u16 partid, srcu_read_lock_held(&mpam_srcu)) { arg.ris = ris; mpam_touch_msc(msc, __write_config, &arg); + ris->in_reset_state = false; } mutex_unlock(&msc->cfg_lock); } diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index e8971842b124f..3a1ed201fe20c 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -12,22 +12,31 @@ #include #include #include +#include #include #include #include +#include + #define MPAM_MSC_MAX_NUM_RIS 16 struct platform_device; -DECLARE_STATIC_KEY_FALSE(mpam_enabled); - #ifdef CONFIG_MPAM_KUNIT_TEST #define PACKED_FOR_KUNIT __packed #else #define PACKED_FOR_KUNIT #endif +/* + * This 'mon' values must not alias an actual monitor, so must be larger than + * U16_MAX, but not be confused with an errno value, so smaller than + * (u32)-SZ_4K. + * USE_PRE_ALLOCATED is used to avoid confusion with an actual monitor. + */ +#define USE_PRE_ALLOCATED (U16_MAX + 1) + static inline bool mpam_is_enabled(void) { return static_branch_likely(&mpam_enabled); @@ -76,6 +85,8 @@ struct mpam_msc { u8 pmg_max; unsigned long ris_idxs; u32 ris_max; + u32 iidr; + u16 quirks; /* * error_irq_lock is taken when registering/unregistering the error @@ -119,6 +130,9 @@ struct mpam_msc { void __iomem *mapped_hwpage; size_t mapped_hwpage_sz; + /* Values only used on some platforms for quirks */ + u32 t241_id; + struct mpam_garbage garbage; }; @@ -207,6 +221,42 @@ struct mpam_props { #define mpam_set_feature(_feat, x) __set_bit(_feat, (x)->features) #define mpam_clear_feature(_feat, x) __clear_bit(_feat, (x)->features) +/* Workaround bits for msc->quirks */ +enum mpam_device_quirks { + T241_SCRUB_SHADOW_REGS, + T241_FORCE_MBW_MIN_TO_ONE, + T241_MBW_COUNTER_SCALE_64, + IGNORE_CSU_NRDY, + MPAM_QUIRK_LAST +}; + +#define mpam_has_quirk(_quirk, x) ((1 << (_quirk) & (x)->quirks)) +#define mpam_set_quirk(_quirk, x) ((x)->quirks |= (1 << (_quirk))) + +struct mpam_quirk { + int (*init)(struct mpam_msc *msc, const struct mpam_quirk *quirk); + + u32 iidr; + u32 iidr_mask; + + enum mpam_device_quirks workaround; +}; + +#define MPAM_IIDR_MATCH_ONE (FIELD_PREP_CONST(MPAMF_IIDR_PRODUCTID, 0xfff) | \ + FIELD_PREP_CONST(MPAMF_IIDR_VARIANT, 0xf) | \ + FIELD_PREP_CONST(MPAMF_IIDR_REVISION, 0xf) | \ + FIELD_PREP_CONST(MPAMF_IIDR_IMPLEMENTER, 0xfff)) + +#define MPAM_IIDR_NVIDIA_T241 (FIELD_PREP_CONST(MPAMF_IIDR_PRODUCTID, 0x241) | \ + FIELD_PREP_CONST(MPAMF_IIDR_VARIANT, 0) | \ + FIELD_PREP_CONST(MPAMF_IIDR_REVISION, 0) | \ + FIELD_PREP_CONST(MPAMF_IIDR_IMPLEMENTER, 0x36b)) + +#define MPAM_IIDR_ARM_CMN_650 (FIELD_PREP_CONST(MPAMF_IIDR_PRODUCTID, 0) | \ + FIELD_PREP_CONST(MPAMF_IIDR_VARIANT, 0) | \ + FIELD_PREP_CONST(MPAMF_IIDR_REVISION, 0) | \ + FIELD_PREP_CONST(MPAMF_IIDR_IMPLEMENTER, 0x43b)) + /* The values for MSMON_CFG_MBWU_FLT.RWBW */ enum mon_filter_options { COUNT_BOTH = 0, @@ -215,7 +265,11 @@ enum mon_filter_options { }; struct mon_cfg { - u16 mon; + /* + * mon must be large enough to hold out of range values like + * USE_PRE_ALLOCATED + */ + u32 mon; u8 pmg; bool match_pmg; bool csu_exclude_clean; @@ -246,6 +300,7 @@ struct mpam_class { struct mpam_props props; u32 nrdy_usec; + u16 quirks; u8 level; enum mpam_class_types type; @@ -265,10 +320,10 @@ struct mpam_config { u32 cpbm; u32 mbw_pbm; u16 mbw_max; + u16 cmax; + u16 cmin; - bool reset_cpbm; - bool reset_mbw_pbm; - bool reset_mbw_max; + bool cmax_softlim; struct mpam_garbage garbage; }; @@ -337,6 +392,36 @@ struct mpam_msc_ris { struct mpam_garbage garbage; }; +struct mpam_resctrl_dom { + struct mpam_component *ctrl_comp; + + /* + * There is no single mon_comp because different events may be backed + * by different class/components. mon_comp is indexed by the event + * number. + */ + struct mpam_component *mon_comp[QOS_NUM_EVENTS]; + + struct rdt_ctrl_domain resctrl_ctrl_dom; + struct rdt_l3_mon_domain resctrl_mon_dom; +}; + +struct mpam_resctrl_res { + struct mpam_class *class; + struct rdt_resource resctrl_res; + bool cdp_enabled; +}; + +struct mpam_resctrl_mon { + struct mpam_class *class; + + /* Array of allocated MBWU monitors, indexed by (closid, rmid). */ + int *mbwu_idx_to_mon; + + /* Array of assigned MBWU monitors, indexed by idx argument. */ + int *assigned_counters; +}; + static inline int mpam_alloc_csu_mon(struct mpam_class *class) { struct mpam_props *cprops = &class->props; @@ -381,6 +466,9 @@ extern u8 mpam_pmg_max; void mpam_enable(struct work_struct *work); void mpam_disable(struct work_struct *work); +/* Reset all the RIS in a class under cpus_read_lock() */ +void mpam_reset_class_locked(struct mpam_class *class); + int mpam_apply_config(struct mpam_component *comp, u16 partid, struct mpam_config *cfg); @@ -391,6 +479,20 @@ void mpam_msmon_reset_mbwu(struct mpam_component *comp, struct mon_cfg *ctx); int mpam_get_cpumask_from_cache_id(unsigned long cache_id, u32 cache_level, cpumask_t *affinity); +#ifdef CONFIG_RESCTRL_FS +int mpam_resctrl_setup(void); +void mpam_resctrl_exit(void); +int mpam_resctrl_online_cpu(unsigned int cpu); +void mpam_resctrl_offline_cpu(unsigned int cpu); +void mpam_resctrl_teardown_class(struct mpam_class *class); +#else +static inline int mpam_resctrl_setup(void) { return 0; } +static inline void mpam_resctrl_exit(void) { } +static inline int mpam_resctrl_online_cpu(unsigned int cpu) { return 0; } +static inline void mpam_resctrl_offline_cpu(unsigned int cpu) { } +static inline void mpam_resctrl_teardown_class(struct mpam_class *class) { } +#endif /* CONFIG_RESCTRL_FS */ + /* * MPAM MSCs have the following register layout. See: * Arm Memory System Resource Partitioning and Monitoring (MPAM) System diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c new file mode 100644 index 0000000000000..2b7a673fca211 --- /dev/null +++ b/drivers/resctrl/mpam_resctrl.c @@ -0,0 +1,2057 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Arm Ltd. + +#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "mpam_internal.h" + +static DECLARE_WAIT_QUEUE_HEAD(resctrl_mon_ctx_waiters); + +/* + * The classes we've picked to map to resctrl resources, wrapped + * in with their resctrl structure. + * Class pointer may be NULL. + */ +static struct mpam_resctrl_res mpam_resctrl_controls[RDT_NUM_RESOURCES]; + +#define for_each_mpam_resctrl_control(res, rid) \ + for (rid = 0, res = &mpam_resctrl_controls[rid]; \ + rid < RDT_NUM_RESOURCES; \ + rid++, res = &mpam_resctrl_controls[rid]) + +/* + * The classes we've picked to map to resctrl events. + * Resctrl believes all the worlds a Xeon, and these are all on the L3. This + * array lets us find the actual class backing the event counters. e.g. + * the only memory bandwidth counters may be on the memory controller, but to + * make use of them, we pretend they are on L3. Restrict the events considered + * to those supported by MPAM. + * Class pointer may be NULL. + */ +#define MPAM_MAX_EVENT QOS_L3_MBM_TOTAL_EVENT_ID +static struct mpam_resctrl_mon mpam_resctrl_counters[MPAM_MAX_EVENT + 1]; + +#define for_each_mpam_resctrl_mon(mon, eventid) \ + for (eventid = QOS_FIRST_EVENT, mon = &mpam_resctrl_counters[eventid]; \ + eventid <= MPAM_MAX_EVENT; \ + eventid++, mon = &mpam_resctrl_counters[eventid]) + +/* The lock for modifying resctrl's domain lists from cpuhp callbacks. */ +static DEFINE_MUTEX(domain_list_lock); + +/* + * MPAM emulates CDP by setting different PARTID in the I/D fields of MPAM0_EL1. + * This applies globally to all traffic the CPU generates. + */ +static bool cdp_enabled; + +/* + * We use cacheinfo to discover the size of the caches and their id. cacheinfo + * populates this from a device_initcall(). mpam_resctrl_setup() must wait. + */ +static bool cacheinfo_ready; +static DECLARE_WAIT_QUEUE_HEAD(wait_cacheinfo_ready); + +/* + * If resctrl_init() succeeded, resctrl_exit() can be used to remove support + * for the filesystem in the event of an error. + */ +static bool resctrl_enabled; + +static unsigned int l3_num_allocated_mbwu = ~0; + +bool resctrl_arch_alloc_capable(void) +{ + struct mpam_resctrl_res *res; + enum resctrl_res_level rid; + + for_each_mpam_resctrl_control(res, rid) { + if (res->resctrl_res.alloc_capable) + return true; + } + + return false; +} + +bool resctrl_arch_mon_capable(void) +{ + struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; + struct rdt_resource *l3 = &res->resctrl_res; + + /* All monitors are presented as being on the L3 cache */ + return l3->mon_capable; +} + +bool resctrl_arch_is_evt_configurable(enum resctrl_event_id evt) +{ + return false; +} + +void resctrl_arch_mon_event_config_read(void *info) +{ +} + +void resctrl_arch_mon_event_config_write(void *info) +{ +} + +void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_l3_mon_domain *d) +{ +} + +void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + u32 closid, u32 rmid, enum resctrl_event_id eventid) +{ +} + +bool resctrl_arch_mbm_cntr_assign_enabled(struct rdt_resource *r) +{ + return (r == &mpam_resctrl_controls[RDT_RESOURCE_L3].resctrl_res); +} + +int resctrl_arch_mbm_cntr_assign_set(struct rdt_resource *r, bool enable) +{ + return -EINVAL; +} + +int resctrl_arch_io_alloc_enable(struct rdt_resource *r, bool enable) +{ + return -EOPNOTSUPP; +} + +bool resctrl_arch_get_io_alloc_enabled(struct rdt_resource *r) +{ + return false; +} + +void resctrl_arch_pre_mount(void) +{ +} + +bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level rid) +{ + return mpam_resctrl_controls[rid].cdp_enabled; +} + +/** + * resctrl_reset_task_closids() - Reset the PARTID/PMG values for all tasks. + * + * At boot, all existing tasks use partid zero for D and I. + * To enable/disable CDP emulation, all these tasks need relabelling. + */ +static void resctrl_reset_task_closids(void) +{ + struct task_struct *p, *t; + + read_lock(&tasklist_lock); + for_each_process_thread(p, t) { + resctrl_arch_set_closid_rmid(t, RESCTRL_RESERVED_CLOSID, + RESCTRL_RESERVED_RMID); + } + read_unlock(&tasklist_lock); +} + +static void mpam_resctrl_monitor_sync_abmc_vals(struct rdt_resource *l3) +{ + l3->mon.num_mbm_cntrs = l3_num_allocated_mbwu; + if (cdp_enabled) + l3->mon.num_mbm_cntrs /= 2; + + /* + * Continue as normal even if enabling cdp causes there to be + * zero counters. This avoid giving resctrl mixed messages. + */ +} + +int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable) +{ + u32 partid_i = RESCTRL_RESERVED_CLOSID, partid_d = RESCTRL_RESERVED_CLOSID; + struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; + struct rdt_resource *l3 = &res->resctrl_res; + int cpu; + + if (!IS_ENABLED(CONFIG_EXPERT) && enable) { + /* + * If the resctrl fs is mounted more than once, sequentially, + * then CDP can lead to the use of out of range PARTIDs. + */ + pr_warn("CDP not supported\n"); + return -EOPNOTSUPP; + } + + if (enable) + pr_warn("CDP is an expert feature and may cause MPAM to malfunction.\n"); + + /* + * resctrl_arch_set_cdp_enabled() is only called with enable set to + * false on error and unmount. + */ + cdp_enabled = enable; + mpam_resctrl_controls[rid].cdp_enabled = enable; + + if (enable) + l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx() / 2; + else + l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx(); + + /* The mbw_max feature can't hide cdp as it's a per-partid maximum. */ + if (cdp_enabled && !mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled) + mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = false; + + /* + * If resctrl has attempted to enable CDP on MBA, re-enable MBA as two + * configurations will be provided so there is no aliasing problem. + */ + if (mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled && + mpam_resctrl_controls[RDT_RESOURCE_MBA].class) + mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = true; + + /* On unmount when CDP is disabled, re-enable MBA */ + if (!cdp_enabled && mpam_resctrl_controls[RDT_RESOURCE_MBA].class) + mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = true; + + if (enable) { + if (mpam_partid_max < 1) + return -EINVAL; + + partid_d = resctrl_get_config_index(RESCTRL_RESERVED_CLOSID, CDP_DATA); + partid_i = resctrl_get_config_index(RESCTRL_RESERVED_CLOSID, CDP_CODE); + } + + mpam_set_task_partid_pmg(current, partid_d, partid_i, 0, 0); + WRITE_ONCE(arm64_mpam_global_default, mpam_get_regval(current)); + + resctrl_reset_task_closids(); + mpam_resctrl_monitor_sync_abmc_vals(l3); + + for_each_possible_cpu(cpu) + mpam_set_cpu_defaults(cpu, partid_d, partid_i, 0, 0); + on_each_cpu(resctrl_arch_sync_cpu_closid_rmid, NULL, 1); + + return 0; +} + +static bool mpam_resctrl_hide_cdp(enum resctrl_res_level rid) +{ + return cdp_enabled && !resctrl_arch_get_cdp_enabled(rid); +} + +/* + * MSC may raise an error interrupt if it sees an out or range partid/pmg, + * and go on to truncate the value. Regardless of what the hardware supports, + * only the system wide safe value is safe to use. + */ +u32 resctrl_arch_get_num_closid(struct rdt_resource *ignored) +{ + return mpam_partid_max + 1; +} + +u32 resctrl_arch_system_num_rmid_idx(void) +{ + return (mpam_pmg_max + 1) * (mpam_partid_max + 1); +} + +u32 resctrl_arch_rmid_idx_encode(u32 closid, u32 rmid) +{ + return closid * (mpam_pmg_max + 1) + rmid; +} + +void resctrl_arch_rmid_idx_decode(u32 idx, u32 *closid, u32 *rmid) +{ + *closid = idx / (mpam_pmg_max + 1); + *rmid = idx % (mpam_pmg_max + 1); +} + +void resctrl_arch_sched_in(struct task_struct *tsk) +{ + lockdep_assert_preemption_disabled(); + + mpam_thread_switch(tsk); +} + +void resctrl_arch_set_cpu_default_closid_rmid(int cpu, u32 closid, u32 rmid) +{ + WARN_ON_ONCE(closid > U16_MAX); + WARN_ON_ONCE(rmid > U8_MAX); + + if (!cdp_enabled) { + mpam_set_cpu_defaults(cpu, closid, closid, rmid, rmid); + } else { + /* + * When CDP is enabled, resctrl halves the closid range and we + * use odd/even partid for one closid. + */ + u32 partid_d = resctrl_get_config_index(closid, CDP_DATA); + u32 partid_i = resctrl_get_config_index(closid, CDP_CODE); + + mpam_set_cpu_defaults(cpu, partid_d, partid_i, rmid, rmid); + } +} + +void resctrl_arch_sync_cpu_closid_rmid(void *info) +{ + struct resctrl_cpu_defaults *r = info; + + lockdep_assert_preemption_disabled(); + + if (r) { + resctrl_arch_set_cpu_default_closid_rmid(smp_processor_id(), + r->closid, r->rmid); + } + + resctrl_arch_sched_in(current); +} + +void resctrl_arch_set_closid_rmid(struct task_struct *tsk, u32 closid, u32 rmid) +{ + WARN_ON_ONCE(closid > U16_MAX); + WARN_ON_ONCE(rmid > U8_MAX); + + if (!cdp_enabled) { + mpam_set_task_partid_pmg(tsk, closid, closid, rmid, rmid); + } else { + u32 partid_d = resctrl_get_config_index(closid, CDP_DATA); + u32 partid_i = resctrl_get_config_index(closid, CDP_CODE); + + mpam_set_task_partid_pmg(tsk, partid_d, partid_i, rmid, rmid); + } +} + +bool resctrl_arch_match_closid(struct task_struct *tsk, u32 closid) +{ + u64 regval = mpam_get_regval(tsk); + u32 tsk_closid = FIELD_GET(MPAM0_EL1_PARTID_D, regval); + + if (cdp_enabled) + tsk_closid >>= 1; + + return tsk_closid == closid; +} + +/* The task's pmg is not unique, the partid must be considered too */ +bool resctrl_arch_match_rmid(struct task_struct *tsk, u32 closid, u32 rmid) +{ + u64 regval = mpam_get_regval(tsk); + u32 tsk_closid = FIELD_GET(MPAM0_EL1_PARTID_D, regval); + u32 tsk_rmid = FIELD_GET(MPAM0_EL1_PMG_D, regval); + + if (cdp_enabled) + tsk_closid >>= 1; + + return (tsk_closid == closid) && (tsk_rmid == rmid); +} + +struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l) +{ + if (l >= RDT_NUM_RESOURCES) + return NULL; + + return &mpam_resctrl_controls[l].resctrl_res; +} + +static int resctrl_arch_mon_ctx_alloc_no_wait(enum resctrl_event_id evtid) +{ + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid]; + + if (!mpam_is_enabled()) + return -EINVAL; + + if (!mon->class) + return -EINVAL; + + switch (evtid) { + case QOS_L3_OCCUP_EVENT_ID: + /* With CDP, one monitor gets used for both code/data reads */ + return mpam_alloc_csu_mon(mon->class); + case QOS_L3_MBM_LOCAL_EVENT_ID: + case QOS_L3_MBM_TOTAL_EVENT_ID: + return USE_PRE_ALLOCATED; + default: + return -EOPNOTSUPP; + } +} + +void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r, + enum resctrl_event_id evtid) +{ + DEFINE_WAIT(wait); + int *ret; + + ret = kmalloc_obj(*ret); + if (!ret) + return ERR_PTR(-ENOMEM); + + do { + prepare_to_wait(&resctrl_mon_ctx_waiters, &wait, + TASK_INTERRUPTIBLE); + *ret = resctrl_arch_mon_ctx_alloc_no_wait(evtid); + if (*ret == -ENOSPC) + schedule(); + } while (*ret == -ENOSPC && !signal_pending(current)); + finish_wait(&resctrl_mon_ctx_waiters, &wait); + + return ret; +} + +static void resctrl_arch_mon_ctx_free_no_wait(enum resctrl_event_id evtid, + u32 mon_idx) +{ + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid]; + + if (!mpam_is_enabled()) + return; + + if (!mon->class) + return; + + if (evtid == QOS_L3_OCCUP_EVENT_ID) + mpam_free_csu_mon(mon->class, mon_idx); + + wake_up(&resctrl_mon_ctx_waiters); +} + +void resctrl_arch_mon_ctx_free(struct rdt_resource *r, + enum resctrl_event_id evtid, void *arch_mon_ctx) +{ + u32 mon_idx = *(u32 *)arch_mon_ctx; + + kfree(arch_mon_ctx); + + resctrl_arch_mon_ctx_free_no_wait(evtid, mon_idx); +} + +static bool __resctrl_arch_mon_can_overflow(enum resctrl_event_id eventid) +{ + struct mpam_props *cprops; + struct mpam_class *class = mpam_resctrl_counters[eventid].class; + + if (!class) + return false; + + /* No need to worry about a 63 bit counter overflowing */ + cprops = &class->props; + return !mpam_has_feature(mpam_feat_msmon_mbwu_63counter, cprops); +} + +bool resctrl_arch_mon_can_overflow(void) +{ + return __resctrl_arch_mon_can_overflow(QOS_L3_MBM_TOTAL_EVENT_ID); +} + +static int __read_mon(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp, + enum mpam_device_features mon_type, + int mon_idx, + enum resctrl_conf_type cdp_type, u32 closid, u32 rmid, u64 *val) +{ + struct mon_cfg cfg; + + if (!mpam_is_enabled()) + return -EINVAL; + + /* Shift closid to account for CDP */ + closid = resctrl_get_config_index(closid, cdp_type); + + if (mon_idx == USE_PRE_ALLOCATED) { + int mbwu_idx = resctrl_arch_rmid_idx_encode(closid, rmid); + + mon_idx = mon->mbwu_idx_to_mon[mbwu_idx]; + if (mon_idx == -1) + return -ENOENT; + } + + if (irqs_disabled()) { + /* Check if we can access this domain without an IPI */ + return -EIO; + } + + cfg = (struct mon_cfg) { + .mon = mon_idx, + .match_pmg = true, + .partid = closid, + .pmg = rmid, + }; + + return mpam_msmon_read(mon_comp, &cfg, mon_type, val); +} + +static int read_mon_cdp_safe(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp, + enum mpam_device_features mon_type, + int mon_idx, u32 closid, u32 rmid, u64 *val) +{ + if (cdp_enabled) { + u64 code_val = 0, data_val = 0; + int err; + + err = __read_mon(mon, mon_comp, mon_type, mon_idx, + CDP_CODE, closid, rmid, &code_val); + if (err) + return err; + + err = __read_mon(mon, mon_comp, mon_type, mon_idx, + CDP_DATA, closid, rmid, &data_val); + if (err) + return err; + + *val += code_val + data_val; + return 0; + } + + return __read_mon(mon, mon_comp, mon_type, mon_idx, + CDP_NONE, closid, rmid, val); +} + +/* MBWU when not in ABMC mode (not supported), and CSU counters. */ +int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain_hdr *hdr, + u32 closid, u32 rmid, enum resctrl_event_id eventid, + void *arch_priv, u64 *val, void *arch_mon_ctx) +{ + struct mpam_resctrl_dom *l3_dom; + struct mpam_component *mon_comp; + u32 mon_idx = *(u32 *)arch_mon_ctx; + enum mpam_device_features mon_type; + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[eventid]; + + resctrl_arch_rmid_read_context_check(); + + if (!mpam_is_enabled()) + return -EINVAL; + + if (eventid >= QOS_NUM_EVENTS || !mon->class) + return -EINVAL; + + l3_dom = container_of(hdr, struct mpam_resctrl_dom, resctrl_mon_dom.hdr); + mon_comp = l3_dom->mon_comp[eventid]; + + if (eventid != QOS_L3_OCCUP_EVENT_ID) + return -EINVAL; + + mon_type = mpam_feat_msmon_csu; + + return read_mon_cdp_safe(mon, mon_comp, mon_type, mon_idx, + closid, rmid, val); +} + +/* MBWU counters when in ABMC mode */ +int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + u32 closid, u32 rmid, int mon_idx, + enum resctrl_event_id eventid, u64 *val) +{ + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[eventid]; + struct mpam_resctrl_dom *l3_dom; + struct mpam_component *mon_comp; + + if (!mpam_is_enabled()) + return -EINVAL; + + if (eventid == QOS_L3_OCCUP_EVENT_ID || !mon->class) + return -EINVAL; + + l3_dom = container_of(d, struct mpam_resctrl_dom, resctrl_mon_dom); + mon_comp = l3_dom->mon_comp[eventid]; + + return read_mon_cdp_safe(mon, mon_comp, mpam_feat_msmon_mbwu, + USE_PRE_ALLOCATED, closid, rmid, val); +} + +static void __reset_mon(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp, + int mon_idx, + enum resctrl_conf_type cdp_type, u32 closid, u32 rmid) +{ + struct mon_cfg cfg = { }; + + if (!mpam_is_enabled()) + return; + + /* Shift closid to account for CDP */ + closid = resctrl_get_config_index(closid, cdp_type); + + if (mon_idx == USE_PRE_ALLOCATED) { + int mbwu_idx = resctrl_arch_rmid_idx_encode(closid, rmid); + + mon_idx = mon->mbwu_idx_to_mon[mbwu_idx]; + } + + if (mon_idx == -1) + return; + cfg.mon = mon_idx; + mpam_msmon_reset_mbwu(mon_comp, &cfg); +} + +static void reset_mon_cdp_safe(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp, + int mon_idx, u32 closid, u32 rmid) +{ + if (cdp_enabled) { + __reset_mon(mon, mon_comp, mon_idx, CDP_CODE, closid, rmid); + __reset_mon(mon, mon_comp, mon_idx, CDP_DATA, closid, rmid); + } else { + __reset_mon(mon, mon_comp, mon_idx, CDP_NONE, closid, rmid); + } +} + +/* Reset an assigned counter */ +void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + u32 closid, u32 rmid, int cntr_id, + enum resctrl_event_id eventid) +{ + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[eventid]; + struct mpam_resctrl_dom *l3_dom; + struct mpam_component *mon_comp; + + if (!mpam_is_enabled()) + return; + + if (eventid == QOS_L3_OCCUP_EVENT_ID || !mon->class) + return; + + l3_dom = container_of(d, struct mpam_resctrl_dom, resctrl_mon_dom); + mon_comp = l3_dom->mon_comp[eventid]; + + reset_mon_cdp_safe(mon, mon_comp, USE_PRE_ALLOCATED, closid, rmid); +} + +/* + * The rmid realloc threshold should be for the smallest cache exposed to + * resctrl. + */ +static int update_rmid_limits(struct mpam_class *class) +{ + u32 num_unique_pmg = resctrl_arch_system_num_rmid_idx(); + struct mpam_props *cprops = &class->props; + struct cacheinfo *ci; + + lockdep_assert_cpus_held(); + + if (!mpam_has_feature(mpam_feat_msmon_csu, cprops)) + return 0; + + /* + * Assume cache levels are the same size for all CPUs... + * The check just requires any online CPU and it can't go offline as we + * hold the cpu lock. + */ + ci = get_cpu_cacheinfo_level(raw_smp_processor_id(), class->level); + if (!ci || ci->size == 0) { + pr_debug("Could not read cache size for class %u\n", + class->level); + return -EINVAL; + } + + if (!resctrl_rmid_realloc_limit || + ci->size < resctrl_rmid_realloc_limit) { + resctrl_rmid_realloc_limit = ci->size; + resctrl_rmid_realloc_threshold = ci->size / num_unique_pmg; + } + + return 0; +} + +static bool cache_has_usable_cpor(struct mpam_class *class) +{ + struct mpam_props *cprops = &class->props; + + if (!mpam_has_feature(mpam_feat_cpor_part, cprops)) + return false; + + /* resctrl uses u32 for all bitmap configurations */ + return class->props.cpbm_wd <= 32; +} + +static bool cache_has_usable_cmax(struct mpam_class *class) +{ + struct mpam_props *cprops = &class->props; + + return mpam_has_feature(mpam_feat_cmax_cmax, cprops); +} + +static bool mba_class_use_mbw_max(struct mpam_props *cprops) +{ + return (mpam_has_feature(mpam_feat_mbw_max, cprops) && + cprops->bwa_wd); +} + +static bool class_has_usable_mba(struct mpam_props *cprops) +{ + return mba_class_use_mbw_max(cprops); +} + +static bool cache_has_usable_csu(struct mpam_class *class) +{ + struct mpam_props *cprops; + + if (!class) + return false; + + cprops = &class->props; + + if (!mpam_has_feature(mpam_feat_msmon_csu, cprops)) + return false; + + /* + * CSU counters settle on the value, so we can get away with + * having only one. + */ + if (!cprops->num_csu_mon) + return false; + + return true; +} + +static bool class_has_usable_mbwu(struct mpam_class *class) +{ + struct mpam_props *cprops = &class->props; + + if (!mpam_has_feature(mpam_feat_msmon_mbwu, cprops)) + return false; + + if (!cprops->num_mbwu_mon) + return false; + + return true; +} + +/* + * Calculate the worst-case percentage change from each implemented step + * in the control. + */ +static u32 get_mba_granularity(struct mpam_props *cprops) +{ + if (!mba_class_use_mbw_max(cprops)) + return 0; + + /* + * bwa_wd is the number of bits implemented in the 0.xxx + * fixed point fraction. 1 bit is 50%, 2 is 25% etc. + */ + return DIV_ROUND_UP(MAX_MBA_BW, 1 << cprops->bwa_wd); +} + +/* + * Each fixed-point hardware value architecturally represents a range + * of values: the full range 0% - 100% is split contiguously into + * (1 << cprops->bwa_wd) equal bands. + * + * Although the bwa_bwd fields have 6 bits the maximum valid value is 16 + * as it reports the width of fields that are at most 16 bits. When + * fewer than 16 bits are valid the least significant bits are + * ignored. The implied binary point is kept between bits 15 and 16 and + * so the valid bits are leftmost. + * + * See ARM IHI0099B.a "MPAM system component specification", Section 9.3, + * "The fixed-point fractional format" for more information. + * + * Find the nearest percentage value to the upper bound of the selected band: + */ +static u32 fract16_to_percent(u16 fract, u8 wd) +{ + u32 val = fract; + + val >>= 16 - wd; + val += 1; + val *= MAX_MBA_BW; + val = DIV_ROUND_CLOSEST(val, 1 << wd); + + return val; +} + +/* + * Find the band whose upper bound is closest to the specified percentage. + * + * A round-to-nearest policy is followed here as a balanced compromise + * between unexpected under-commit of the resource (where the total of + * a set of resource allocations after conversion is less than the + * expected total, due to rounding of the individual converted + * percentages) and over-commit (where the total of the converted + * allocations is greater than expected). + */ +static u16 percent_to_fract16(u8 pc, u8 wd) +{ + u32 val = pc; + + val <<= wd; + val = DIV_ROUND_CLOSEST(val, MAX_MBA_BW); + val = max(val, 1) - 1; + val <<= 16 - wd; + + return val; +} + +static u32 mbw_max_to_percent(u16 mbw_max, struct mpam_props *cprops) +{ + return fract16_to_percent(mbw_max, cprops->bwa_wd); +} + +static u16 percent_to_mbw_max(u8 pc, struct mpam_props *cprops) +{ + return percent_to_fract16(pc, cprops->bwa_wd); +} + +static u16 percent_to_cmax(u8 pc, struct mpam_props *cprops) +{ + return percent_to_fract16(pc, cprops->cmax_wd); +} + +static u32 get_mba_min(struct mpam_props *cprops) +{ + if (!mba_class_use_mbw_max(cprops)) { + WARN_ON_ONCE(1); + return 0; + } + + return mbw_max_to_percent(0, cprops); +} + +/* Find the L3 cache that has affinity with this CPU */ +static int find_l3_equivalent_bitmask(int cpu, cpumask_var_t tmp_cpumask) +{ + u32 cache_id = get_cpu_cacheinfo_id(cpu, 3); + + lockdep_assert_cpus_held(); + + return mpam_get_cpumask_from_cache_id(cache_id, 3, tmp_cpumask); +} + +/* + * topology_matches_l3() - Is the provided class the same shape as L3 + * @victim: The class we'd like to pretend is L3. + * + * resctrl expects all the world's a Xeon, and all counters are on the + * L3. We allow some mapping counters on other classes. This requires + * that the CPU->domain mapping is the same kind of shape. + * + * Using cacheinfo directly would make this work even if resctrl can't + * use the L3 - but cacheinfo can't tell us anything about offline CPUs. + * Using the L3 resctrl domain list also depends on CPUs being online. + * Using the mpam_class we picked for L3 so we can use its domain list + * assumes that there are MPAM controls on the L3. + * Instead, this path eventually uses the mpam_get_cpumask_from_cache_id() + * helper which can tell us about offline CPUs ... but getting the cache_id + * to start with relies on at least one CPU per L3 cache being online at + * boot. + * + * Walk the victim component list and compare the affinity mask with the + * corresponding L3. The topology matches if each victim:component's affinity + * mask is the same as the CPU's corresponding L3's. These lists/masks are + * computed from firmware tables so don't change at runtime. + */ +static bool topology_matches_l3(struct mpam_class *victim) +{ + int cpu, err; + struct mpam_component *victim_iter; + + lockdep_assert_cpus_held(); + + cpumask_var_t __free(free_cpumask_var) tmp_cpumask = CPUMASK_VAR_NULL; + if (!alloc_cpumask_var(&tmp_cpumask, GFP_KERNEL)) + return false; + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(victim_iter, &victim->components, class_list, + srcu_read_lock_held(&mpam_srcu)) { + if (cpumask_empty(&victim_iter->affinity)) { + pr_debug("class %u has CPU-less component %u - can't match L3!\n", + victim->level, victim_iter->comp_id); + return false; + } + + cpu = cpumask_any_and(&victim_iter->affinity, cpu_online_mask); + if (WARN_ON_ONCE(cpu >= nr_cpu_ids)) + return false; + + cpumask_clear(tmp_cpumask); + err = find_l3_equivalent_bitmask(cpu, tmp_cpumask); + if (err) { + pr_debug("Failed to find L3's equivalent component to class %u component %u\n", + victim->level, victim_iter->comp_id); + return false; + } + + /* Any differing bits in the affinity mask? */ + if (!cpumask_equal(tmp_cpumask, &victim_iter->affinity)) { + pr_debug("class %u component %u has Mismatched CPU mask with L3 equivalent\n" + "L3:%*pbl != victim:%*pbl\n", + victim->level, victim_iter->comp_id, + cpumask_pr_args(tmp_cpumask), + cpumask_pr_args(&victim_iter->affinity)); + + return false; + } + } + + return true; +} + +/* + * Test if the traffic for a class matches that at egress from the L3. For + * MSC at memory controllers this is only possible if there is a single L3 + * as otherwise the counters at the memory can include bandwidth from the + * non-local L3. + */ +static bool traffic_matches_l3(struct mpam_class *class) +{ + int err, cpu; + + lockdep_assert_cpus_held(); + + if (class->type == MPAM_CLASS_CACHE && class->level == 3) + return true; + + if (class->type == MPAM_CLASS_CACHE && class->level != 3) { + pr_debug("class %u is a different cache from L3\n", class->level); + return false; + } + + if (class->type != MPAM_CLASS_MEMORY) { + pr_debug("class %u is neither of type cache or memory\n", class->level); + return false; + } + + cpumask_var_t __free(free_cpumask_var) tmp_cpumask = CPUMASK_VAR_NULL; + if (!alloc_cpumask_var(&tmp_cpumask, GFP_KERNEL)) { + pr_debug("cpumask allocation failed\n"); + return false; + } + + cpu = cpumask_any_and(&class->affinity, cpu_online_mask); + err = find_l3_equivalent_bitmask(cpu, tmp_cpumask); + if (err) { + pr_debug("Failed to find L3 downstream to cpu %d\n", cpu); + return false; + } + + if (!cpumask_equal(tmp_cpumask, cpu_possible_mask)) { + pr_debug("There is more than one L3\n"); + return false; + } + + /* Be strict; the traffic might stop in the intermediate cache. */ + if (get_cpu_cacheinfo_id(cpu, 4) != -1) { + pr_debug("L3 isn't the last level of cache\n"); + return false; + } + + if (num_possible_nodes() > 1) { + pr_debug("There is more than one numa node\n"); + return false; + } + +#ifdef CONFIG_HMEM_REPORTING + if (node_devices[cpu_to_node(cpu)]->cache_dev) { + pr_debug("There is a memory side cache\n"); + return false; + } +#endif + + return true; +} + +/* Test whether we can export MPAM_CLASS_CACHE:{2,3}? */ +static void mpam_resctrl_pick_caches(void) +{ + bool has_cpor, has_cmax; + struct mpam_class *class; + struct mpam_resctrl_res *res; + + lockdep_assert_cpus_held(); + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(class, &mpam_classes, classes_list, + srcu_read_lock_held(&mpam_srcu)) { + if (class->type != MPAM_CLASS_CACHE) { + pr_debug("class %u is not a cache\n", class->level); + continue; + } + + if (class->level != 2 && class->level != 3) { + pr_debug("class %u is not L2 or L3\n", class->level); + continue; + } + + has_cpor = cache_has_usable_cpor(class); + has_cmax = cache_has_usable_cmax(class); + if (!has_cpor && !has_cmax) { + pr_debug("class %u cache misses CPOR\n", class->level); + continue; + } + + if (!cpumask_equal(&class->affinity, cpu_possible_mask)) { + pr_debug("class %u has missing CPUs, mask %*pb != %*pb\n", class->level, + cpumask_pr_args(&class->affinity), + cpumask_pr_args(cpu_possible_mask)); + continue; + } + if (has_cpor) { + pr_debug("pick_caches: Class has CPOR\n"); + if (class->level == 2) + res = &mpam_resctrl_controls[RDT_RESOURCE_L2]; + else + res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; + res->class = class; + } + if (has_cmax) { + pr_debug("pick_caches: Class has CMAX\n"); + if (class->level == 2) + res = &mpam_resctrl_controls[RDT_RESOURCE_L2_MAX]; + else + res = &mpam_resctrl_controls[RDT_RESOURCE_L3_MAX]; + res->class = class; + } + } +} + +static void mpam_resctrl_pick_mba(void) +{ + struct mpam_class *class, *candidate_class = NULL; + struct mpam_resctrl_res *res; + + lockdep_assert_cpus_held(); + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(class, &mpam_classes, classes_list, + srcu_read_lock_held(&mpam_srcu)) { + struct mpam_props *cprops = &class->props; + + if (class->level != 3 && class->type == MPAM_CLASS_CACHE) { + pr_debug("class %u is a cache but not the L3\n", class->level); + continue; + } + + if (!class_has_usable_mba(cprops)) { + pr_debug("class %u has no bandwidth control\n", + class->level); + continue; + } + + if (!cpumask_equal(&class->affinity, cpu_possible_mask)) { + pr_debug("class %u has missing CPUs\n", class->level); + continue; + } + + if (!topology_matches_l3(class)) { + pr_debug("class %u topology doesn't match L3\n", + class->level); + continue; + } + + if (!traffic_matches_l3(class)) { + pr_debug("class %u traffic doesn't match L3 egress\n", + class->level); + continue; + } + + /* + * Pick a resource to be MBA that as close as possible to + * the L3. mbm_total counts the bandwidth leaving the L3 + * cache and MBA should correspond as closely as possible + * for proper operation of mba_sc. + */ + if (!candidate_class || class->level < candidate_class->level) + candidate_class = class; + } + + if (candidate_class) { + pr_debug("selected class %u to back MBA\n", + candidate_class->level); + res = &mpam_resctrl_controls[RDT_RESOURCE_MBA]; + res->class = candidate_class; + } +} + +static void __free_mbwu_mon(struct mpam_class *class, int *array, + u16 num_mbwu_mon) +{ + for (int i = 0; i < num_mbwu_mon; i++) { + if (array[i] < 0) + continue; + + mpam_free_mbwu_mon(class, array[i]); + array[i] = ~0; + } +} + +static int __alloc_mbwu_mon(struct mpam_class *class, int *array, + u16 num_mbwu_mon) +{ + for (int i = 0; i < num_mbwu_mon; i++) { + int mbwu_mon = mpam_alloc_mbwu_mon(class); + + if (mbwu_mon < 0) { + __free_mbwu_mon(class, array, num_mbwu_mon); + return mbwu_mon; + } + array[i] = mbwu_mon; + } + + l3_num_allocated_mbwu = min(l3_num_allocated_mbwu, num_mbwu_mon); + + return 0; +} + +static int *__alloc_mbwu_array(struct mpam_class *class, u16 num_mbwu_mon) +{ + int err; + + int *array __free(kvfree) = kvmalloc_objs(*array, num_mbwu_mon); + if (!array) + return ERR_PTR(-ENOMEM); + + memset(array, -1, num_mbwu_mon * sizeof(*array)); + + err = __alloc_mbwu_mon(class, array, num_mbwu_mon); + if (err) + return ERR_PTR(err); + return_ptr(array); +} + +static void counter_update_class(enum resctrl_event_id evt_id, + struct mpam_class *class) +{ + struct mpam_class *existing_class = mpam_resctrl_counters[evt_id].class; + + if (existing_class) { + if (class->level == 3) { + pr_debug("Existing class is L3 - L3 wins\n"); + return; + } + + if (existing_class->level < class->level) { + pr_debug("Existing class is closer to L3, %u versus %u - closer is better\n", + existing_class->level, class->level); + return; + } + } + + mpam_resctrl_counters[evt_id].class = class; +} + +static void mpam_resctrl_pick_counters(void) +{ + struct mpam_class *class; + + lockdep_assert_cpus_held(); + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(class, &mpam_classes, classes_list, + srcu_read_lock_held(&mpam_srcu)) { + /* The name of the resource is L3... */ + if (class->type == MPAM_CLASS_CACHE && class->level != 3) { + pr_debug("class %u is a cache but not the L3", class->level); + continue; + } + + if (!cpumask_equal(&class->affinity, cpu_possible_mask)) { + pr_debug("class %u does not cover all CPUs", + class->level); + continue; + } + + if (cache_has_usable_csu(class)) { + pr_debug("class %u has usable CSU", + class->level); + + /* CSU counters only make sense on a cache. */ + switch (class->type) { + case MPAM_CLASS_CACHE: + if (update_rmid_limits(class)) + break; + + counter_update_class(QOS_L3_OCCUP_EVENT_ID, class); + break; + default: + break; + } + } + + if (class_has_usable_mbwu(class) && + topology_matches_l3(class) && + traffic_matches_l3(class)) { + pr_debug("class %u has usable MBWU, and matches L3 topology and traffic\n", + class->level); + + /* + * We can't distinguish traffic by destination so + * we don't know if it's staying on the same NUMA + * node. Hence, we can't calculate mbm_local except + * when we only have one L3 and it's equivalent to + * mbm_total and so always use mbm_total. + */ + counter_update_class(QOS_L3_MBM_TOTAL_EVENT_ID, class); + } + } +} + +static void __config_cntr(struct mpam_resctrl_mon *mon, u32 cntr_id, + enum resctrl_conf_type cdp_type, u32 closid, u32 rmid, + bool assign) +{ + u32 mbwu_idx, mon_idx = resctrl_get_config_index(cntr_id, cdp_type); + + WARN_ON_ONCE(mon_idx >= l3_num_allocated_mbwu); + + closid = resctrl_get_config_index(closid, cdp_type); + mbwu_idx = resctrl_arch_rmid_idx_encode(closid, rmid); + + if (assign) + mon->mbwu_idx_to_mon[mbwu_idx] = mon->assigned_counters[mon_idx]; + else + mon->mbwu_idx_to_mon[mbwu_idx] = -1; +} + +void resctrl_arch_config_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + enum resctrl_event_id evtid, u32 rmid, u32 closid, + u32 cntr_id, bool assign) +{ + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid]; + + if (!mon->mbwu_idx_to_mon || !mon->assigned_counters) { + pr_debug("monitor arrays not allocated\n"); + return; + } + + if (cdp_enabled) { + __config_cntr(mon, cntr_id, CDP_CODE, closid, rmid, assign); + __config_cntr(mon, cntr_id, CDP_DATA, closid, rmid, assign); + } else { + __config_cntr(mon, cntr_id, CDP_NONE, closid, rmid, assign); + } + + resctrl_arch_reset_rmid(r, d, closid, rmid, evtid); +} + +static int mpam_resctrl_control_init(struct mpam_resctrl_res *res) +{ + struct mpam_class *class = res->class; + struct mpam_props *cprops = &class->props; + struct rdt_resource *r = &res->resctrl_res; + + switch (r->rid) { + case RDT_RESOURCE_L2: + case RDT_RESOURCE_L3: + r->schema_fmt = RESCTRL_SCHEMA_BITMAP; + r->cache.arch_has_sparse_bitmasks = true; + + r->cache.cbm_len = class->props.cpbm_wd; + /* mpam_devices will reject empty bitmaps */ + r->cache.min_cbm_bits = 1; + + if (r->rid == RDT_RESOURCE_L2) { + r->name = "L2"; + r->ctrl_scope = RESCTRL_L2_CACHE; + r->cdp_capable = true; + } else { + r->name = "L3"; + r->ctrl_scope = RESCTRL_L3_CACHE; + r->cdp_capable = true; + } + + /* + * Which bits are shared with other ...things... Unknown + * devices use partid-0 which uses all the bitmap fields. Until + * we have configured the SMMU and GIC not to do this 'all the + * bits' is the correct answer here. + */ + r->cache.shareable_bits = resctrl_get_resource_default_ctrl(r); + r->alloc_capable = true; + break; + case RDT_RESOURCE_L2_MAX: + case RDT_RESOURCE_L3_MAX: + r->alloc_capable = true; + r->schema_fmt = RESCTRL_SCHEMA_PERCENT; + r->membw.min_bw = max(100 / (1 << cprops->cmax_wd), 1); + r->membw.bw_gran = max(100 / (1 << cprops->cmax_wd), 1); + r->membw.max_bw = 100; + + if (r->rid == RDT_RESOURCE_L2_MAX) { + r->name = "L2_MAX"; + r->ctrl_scope = RESCTRL_L2_CACHE; + } else { + r->name = "L3_MAX"; + r->ctrl_scope = RESCTRL_L3_CACHE; + } + + break; + case RDT_RESOURCE_MBA: + r->schema_fmt = RESCTRL_SCHEMA_RANGE; + r->ctrl_scope = RESCTRL_L3_CACHE; + + r->mba.delay_linear = true; + r->mba.throttle_mode = THREAD_THROTTLE_UNDEFINED; + r->membw.min_bw = get_mba_min(cprops); + r->membw.max_bw = MAX_MBA_BW; + r->membw.bw_gran = get_mba_granularity(cprops); + + r->name = "MB"; + r->alloc_capable = true; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int mpam_resctrl_pick_domain_id(int cpu, struct mpam_component *comp) +{ + struct mpam_class *class = comp->class; + + if (class->type == MPAM_CLASS_CACHE) + return comp->comp_id; + + if (topology_matches_l3(class)) { + /* Use the corresponding L3 component ID as the domain ID */ + int id = get_cpu_cacheinfo_id(cpu, 3); + + /* Implies topology_matches_l3() made a mistake */ + if (WARN_ON_ONCE(id == -1)) + return comp->comp_id; + + return id; + } + + /* Otherwise, expose the ID used by the firmware table code. */ + return comp->comp_id; +} + +/* + * This must run after all event counters have been picked so that any free + * running counters have already been allocated. + */ +static int mpam_resctrl_monitor_init_abmc(struct mpam_resctrl_mon *mon) +{ + struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; + size_t num_rmid = resctrl_arch_system_num_rmid_idx(); + struct rdt_resource *l3 = &res->resctrl_res; + struct mpam_class *class = mon->class; + u16 num_mbwu_mon; + int *cntrs; + + int *rmid_array __free(kvfree) = kvmalloc_objs(*rmid_array, num_rmid); + if (!rmid_array) { + pr_debug("Failed to allocate RMID array\n"); + return -ENOMEM; + } + memset(rmid_array, -1, num_rmid * sizeof(*rmid_array)); + + num_mbwu_mon = class->props.num_mbwu_mon; + cntrs = __alloc_mbwu_array(mon->class, num_mbwu_mon); + if (IS_ERR(cntrs)) + return PTR_ERR(cntrs); + mon->assigned_counters = cntrs; + mon->mbwu_idx_to_mon = no_free_ptr(rmid_array); + + l3->mon.mbm_cntr_assignable = true; + l3->mon.mbm_assign_on_mkdir = true; + l3->mon.mbm_cntr_configurable = false; + l3->mon.mbm_cntr_assign_fixed = true; + + mpam_resctrl_monitor_sync_abmc_vals(l3); + + return 0; +} + +static int mpam_resctrl_monitor_init(struct mpam_resctrl_mon *mon, + enum resctrl_event_id type) +{ + struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; + struct rdt_resource *l3 = &res->resctrl_res; + + lockdep_assert_cpus_held(); + + /* + * There also needs to be an L3 cache present. + * The check just requires any online CPU and it can't go offline as we + * hold the cpu lock. + */ + if (get_cpu_cacheinfo_id(raw_smp_processor_id(), 3) == -1) + return 0; + + /* + * If there are no MPAM resources on L3, force it into existence. + * topology_matches_l3() already ensures this looks like the L3. + * The domain-ids will be fixed up by mpam_resctrl_domain_hdr_init(). + */ + if (!res->class) { + pr_warn_once("Faking L3 MSC to enable counters.\n"); + res->class = mpam_resctrl_counters[type].class; + } + + /* + * Called multiple times!, once per event type that has a + * monitoring class. + * Setting name is necessary on monitor only platforms. + */ + l3->name = "L3"; + l3->mon_scope = RESCTRL_L3_CACHE; + + /* + * num-rmid is the upper bound for the number of monitoring groups that + * can exist simultaneously, including the default monitoring group for + * each control group. Hence, advertise the whole rmid_idx space even + * though each control group has its own pmg/rmid space. Unfortunately, + * this does mean userspace needs to know the architecture to correctly + * interpret this value. + */ + l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx(); + + if (type == QOS_L3_MBM_TOTAL_EVENT_ID) { + int err; + + err = mpam_resctrl_monitor_init_abmc(mon); + if (err) + return err; + + static_assert(MAX_EVT_CONFIG_BITS == 0x7f); + l3->mon.mbm_cfg_mask = MAX_EVT_CONFIG_BITS; + } + + if (!resctrl_enable_mon_event(type, false, 0, NULL)) + return -EINVAL; + + l3->mon_capable = true; + + return 0; +} + +u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d, + u32 closid, enum resctrl_conf_type type) +{ + u32 partid; + struct mpam_config *cfg; + struct mpam_props *cprops; + struct mpam_resctrl_res *res; + struct mpam_resctrl_dom *dom; + enum mpam_device_features configured_by; + + lockdep_assert_cpus_held(); + + if (!mpam_is_enabled()) + return resctrl_get_resource_default_ctrl(r); + + res = container_of(r, struct mpam_resctrl_res, resctrl_res); + dom = container_of(d, struct mpam_resctrl_dom, resctrl_ctrl_dom); + cprops = &res->class->props; + + /* + * When CDP is enabled, but the resource doesn't support it, + * the control is cloned across both partids. + * Pick one at random to read: + */ + if (mpam_resctrl_hide_cdp(r->rid)) + type = CDP_DATA; + + partid = resctrl_get_config_index(closid, type); + cfg = &dom->ctrl_comp->cfg[partid]; + + switch (r->rid) { + case RDT_RESOURCE_L2: + case RDT_RESOURCE_L3: + configured_by = mpam_feat_cpor_part; + break; + case RDT_RESOURCE_L2_MAX: + case RDT_RESOURCE_L3_MAX: + configured_by = mpam_feat_cmax_cmax; + break; + case RDT_RESOURCE_MBA: + if (mpam_has_feature(mpam_feat_mbw_max, cprops)) { + configured_by = mpam_feat_mbw_max; + break; + } + fallthrough; + default: + return resctrl_get_resource_default_ctrl(r); + } + + if (!r->alloc_capable || partid >= resctrl_arch_get_num_closid(r) || + !mpam_has_feature(configured_by, cfg)) + return resctrl_get_resource_default_ctrl(r); + + switch (configured_by) { + case mpam_feat_cpor_part: + return cfg->cpbm; + case mpam_feat_cmax_cmax: + return fract16_to_percent(cfg->cmax, cprops->cmax_wd); + case mpam_feat_mbw_max: + return mbw_max_to_percent(cfg->mbw_max, cprops); + default: + return resctrl_get_resource_default_ctrl(r); + } +} + +int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d, + u32 closid, enum resctrl_conf_type t, u32 cfg_val) +{ + int err; + u32 partid; + struct mpam_config cfg; + struct mpam_props *cprops; + struct mpam_resctrl_res *res; + struct mpam_resctrl_dom *dom; + + lockdep_assert_cpus_held(); + lockdep_assert_irqs_enabled(); + + if (!mpam_is_enabled()) + return -EINVAL; + + /* + * No need to check the CPU as mpam_apply_config() doesn't care, and + * resctrl_arch_update_domains() relies on this. + */ + res = container_of(r, struct mpam_resctrl_res, resctrl_res); + dom = container_of(d, struct mpam_resctrl_dom, resctrl_ctrl_dom); + cprops = &res->class->props; + + if (mpam_resctrl_hide_cdp(r->rid)) + t = CDP_DATA; + + partid = resctrl_get_config_index(closid, t); + if (!r->alloc_capable || partid >= resctrl_arch_get_num_closid(r)) { + pr_debug("Not alloc capable or computed PARTID out of range\n"); + return -EINVAL; + } + + /* + * Copy the current config to avoid clearing other resources when the + * same component is exposed multiple times through resctrl. + */ + cfg = dom->ctrl_comp->cfg[partid]; + + switch (r->rid) { + case RDT_RESOURCE_L2: + case RDT_RESOURCE_L3: + cfg.cpbm = cfg_val; + mpam_set_feature(mpam_feat_cpor_part, &cfg); + break; + case RDT_RESOURCE_L2_MAX: + case RDT_RESOURCE_L3_MAX: + cfg.cmax = percent_to_cmax(cfg_val, cprops); + mpam_set_feature(mpam_feat_cmax_cmax, &cfg); + break; + case RDT_RESOURCE_MBA: + if (mpam_has_feature(mpam_feat_mbw_max, cprops)) { + cfg.mbw_max = percent_to_mbw_max(cfg_val, cprops); + mpam_set_feature(mpam_feat_mbw_max, &cfg); + break; + } + fallthrough; + default: + return -EINVAL; + } + + /* + * When CDP is enabled, but the resource doesn't support it, we need to + * apply the same configuration to the other partid. + */ + if (mpam_resctrl_hide_cdp(r->rid)) { + partid = resctrl_get_config_index(closid, CDP_CODE); + err = mpam_apply_config(dom->ctrl_comp, partid, &cfg); + if (err) + return err; + + partid = resctrl_get_config_index(closid, CDP_DATA); + return mpam_apply_config(dom->ctrl_comp, partid, &cfg); + } + + return mpam_apply_config(dom->ctrl_comp, partid, &cfg); +} + +int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid) +{ + int err; + struct rdt_ctrl_domain *d; + + lockdep_assert_cpus_held(); + lockdep_assert_irqs_enabled(); + + if (!mpam_is_enabled()) + return -EINVAL; + + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list) { + for (enum resctrl_conf_type t = 0; t < CDP_NUM_TYPES; t++) { + struct resctrl_staged_config *cfg = &d->staged_config[t]; + + if (!cfg->have_new_ctrl) + continue; + + err = resctrl_arch_update_one(r, d, closid, t, + cfg->new_ctrl); + if (err) + return err; + } + } + + return 0; +} + +void resctrl_arch_reset_all_ctrls(struct rdt_resource *r) +{ + struct mpam_resctrl_res *res; + + lockdep_assert_cpus_held(); + + if (!mpam_is_enabled()) + return; + + res = container_of(r, struct mpam_resctrl_res, resctrl_res); + mpam_reset_class_locked(res->class); +} + +static void mpam_resctrl_domain_hdr_init(int cpu, struct mpam_component *comp, + enum resctrl_res_level rid, + struct rdt_domain_hdr *hdr) +{ + lockdep_assert_cpus_held(); + + INIT_LIST_HEAD(&hdr->list); + hdr->id = mpam_resctrl_pick_domain_id(cpu, comp); + hdr->rid = rid; + cpumask_set_cpu(cpu, &hdr->cpu_mask); +} + +static void mpam_resctrl_online_domain_hdr(unsigned int cpu, + struct rdt_domain_hdr *hdr) +{ + lockdep_assert_cpus_held(); + + cpumask_set_cpu(cpu, &hdr->cpu_mask); +} + +/** + * mpam_resctrl_offline_domain_hdr() - Update the domain header to remove a CPU. + * @cpu: The CPU to remove from the domain. + * @hdr: The domain's header. + * + * Removes @cpu from the header mask. If this was the last CPU in the domain, + * the domain header is removed from its parent list and true is returned, + * indicating the parent structure can be freed. + * If there are other CPUs in the domain, returns false. + */ +static bool mpam_resctrl_offline_domain_hdr(unsigned int cpu, + struct rdt_domain_hdr *hdr) +{ + lockdep_assert_held(&domain_list_lock); + + cpumask_clear_cpu(cpu, &hdr->cpu_mask); + if (cpumask_empty(&hdr->cpu_mask)) { + list_del_rcu(&hdr->list); + synchronize_rcu(); + return true; + } + + return false; +} + +static void mpam_resctrl_domain_insert(struct list_head *list, + struct rdt_domain_hdr *new) +{ + struct rdt_domain_hdr *err; + struct list_head *pos = NULL; + + lockdep_assert_held(&domain_list_lock); + + err = resctrl_find_domain(list, new->id, &pos); + if (WARN_ON_ONCE(err)) + return; + + list_add_tail_rcu(&new->list, pos); +} + +static struct mpam_component *find_component(struct mpam_class *class, int cpu) +{ + struct mpam_component *comp; + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(comp, &class->components, class_list, + srcu_read_lock_held(&mpam_srcu)) { + if (cpumask_test_cpu(cpu, &comp->affinity)) + return comp; + } + + return NULL; +} + +static struct mpam_resctrl_dom * +mpam_resctrl_alloc_domain(unsigned int cpu, struct mpam_resctrl_res *res) +{ + int err; + struct mpam_resctrl_dom *dom; + struct rdt_l3_mon_domain *mon_d; + struct rdt_ctrl_domain *ctrl_d; + struct mpam_class *class = res->class; + struct mpam_component *comp_iter, *ctrl_comp; + struct rdt_resource *r = &res->resctrl_res; + + lockdep_assert_held(&domain_list_lock); + + ctrl_comp = NULL; + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(comp_iter, &class->components, class_list, + srcu_read_lock_held(&mpam_srcu)) { + if (cpumask_test_cpu(cpu, &comp_iter->affinity)) { + ctrl_comp = comp_iter; + break; + } + } + + /* class has no component for this CPU */ + if (WARN_ON_ONCE(!ctrl_comp)) + return ERR_PTR(-EINVAL); + + dom = kzalloc_node(sizeof(*dom), GFP_KERNEL, cpu_to_node(cpu)); + if (!dom) + return ERR_PTR(-ENOMEM); + + if (r->alloc_capable) { + dom->ctrl_comp = ctrl_comp; + + ctrl_d = &dom->resctrl_ctrl_dom; + mpam_resctrl_domain_hdr_init(cpu, ctrl_comp, r->rid, &ctrl_d->hdr); + ctrl_d->hdr.type = RESCTRL_CTRL_DOMAIN; + err = resctrl_online_ctrl_domain(r, ctrl_d); + if (err) + goto free_domain; + + mpam_resctrl_domain_insert(&r->ctrl_domains, &ctrl_d->hdr); + } else { + pr_debug("Skipped control domain online - no controls\n"); + } + + if (r->mon_capable) { + struct mpam_component *any_mon_comp = NULL; + struct mpam_resctrl_mon *mon; + enum resctrl_event_id eventid; + + /* + * Even if the monitor domain is backed by a different + * component, the L3 component IDs need to be used... only + * there may be no ctrl_comp for the L3. + * Search each event's class list for a component with + * overlapping CPUs and set up the dom->mon_comp array. + */ + + for_each_mpam_resctrl_mon(mon, eventid) { + struct mpam_component *mon_comp; + + if (!mon->class) + continue; // dummy resource + + mon_comp = find_component(mon->class, cpu); + dom->mon_comp[eventid] = mon_comp; + if (mon_comp) + any_mon_comp = mon_comp; + } + if (!any_mon_comp) { + WARN_ON_ONCE(0); + err = -EFAULT; + goto offline_ctrl_domain; + } + + mon_d = &dom->resctrl_mon_dom; + mpam_resctrl_domain_hdr_init(cpu, any_mon_comp, r->rid, &mon_d->hdr); + mon_d->hdr.type = RESCTRL_MON_DOMAIN; + err = resctrl_online_mon_domain(r, &mon_d->hdr); + if (err) + goto offline_ctrl_domain; + + mpam_resctrl_domain_insert(&r->mon_domains, &mon_d->hdr); + } else { + pr_debug("Skipped monitor domain online - no monitors\n"); + } + + return dom; + +offline_ctrl_domain: + if (r->alloc_capable) { + mpam_resctrl_offline_domain_hdr(cpu, &ctrl_d->hdr); + resctrl_offline_ctrl_domain(r, ctrl_d); + } +free_domain: + kfree(dom); + dom = ERR_PTR(err); + + return dom; +} + +/* + * We know all the monitors are associated with the L3, even if there are no + * controls and therefore no control component. Find the cache-id for the CPU + * and use that to search for existing resctrl domains. + * This relies on mpam_resctrl_pick_domain_id() using the L3 cache-id + * for anything that is not a cache. + */ +static struct mpam_resctrl_dom *mpam_resctrl_get_mon_domain_from_cpu(int cpu) +{ + int cache_id; + struct mpam_resctrl_dom *dom; + struct mpam_resctrl_res *l3 = &mpam_resctrl_controls[RDT_RESOURCE_L3]; + + lockdep_assert_cpus_held(); + + if (!l3->class) + return NULL; + cache_id = get_cpu_cacheinfo_id(cpu, 3); + if (cache_id < 0) + return NULL; + + list_for_each_entry_rcu(dom, &l3->resctrl_res.mon_domains, resctrl_mon_dom.hdr.list) { + if (dom->resctrl_mon_dom.hdr.id == cache_id) + return dom; + } + + return NULL; +} + +static struct mpam_resctrl_dom * +mpam_resctrl_get_domain_from_cpu(int cpu, struct mpam_resctrl_res *res) +{ + struct mpam_resctrl_dom *dom; + struct rdt_resource *r = &res->resctrl_res; + + lockdep_assert_cpus_held(); + + list_for_each_entry_rcu(dom, &r->ctrl_domains, resctrl_ctrl_dom.hdr.list) { + if (cpumask_test_cpu(cpu, &dom->ctrl_comp->affinity)) + return dom; + } + + if (r->rid != RDT_RESOURCE_L3) + return NULL; + + /* Search the mon domain list too - needed on monitor only platforms. */ + return mpam_resctrl_get_mon_domain_from_cpu(cpu); +} + +int mpam_resctrl_online_cpu(unsigned int cpu) +{ + struct mpam_resctrl_res *res; + enum resctrl_res_level rid; + + guard(mutex)(&domain_list_lock); + for_each_mpam_resctrl_control(res, rid) { + struct mpam_resctrl_dom *dom; + struct rdt_resource *r = &res->resctrl_res; + + if (!res->class) + continue; // dummy_resource; + + dom = mpam_resctrl_get_domain_from_cpu(cpu, res); + if (!dom) { + dom = mpam_resctrl_alloc_domain(cpu, res); + if (IS_ERR(dom)) + return PTR_ERR(dom); + } else { + if (r->alloc_capable) { + struct rdt_ctrl_domain *ctrl_d = &dom->resctrl_ctrl_dom; + + mpam_resctrl_online_domain_hdr(cpu, &ctrl_d->hdr); + } + if (r->mon_capable) { + struct rdt_l3_mon_domain *mon_d = &dom->resctrl_mon_dom; + + mpam_resctrl_online_domain_hdr(cpu, &mon_d->hdr); + } + } + } + + resctrl_online_cpu(cpu); + + return 0; +} + +void mpam_resctrl_offline_cpu(unsigned int cpu) +{ + struct mpam_resctrl_res *res; + enum resctrl_res_level rid; + + resctrl_offline_cpu(cpu); + + guard(mutex)(&domain_list_lock); + for_each_mpam_resctrl_control(res, rid) { + struct mpam_resctrl_dom *dom; + struct rdt_l3_mon_domain *mon_d; + struct rdt_ctrl_domain *ctrl_d; + bool ctrl_dom_empty, mon_dom_empty; + struct rdt_resource *r = &res->resctrl_res; + + if (!res->class) + continue; // dummy resource + + dom = mpam_resctrl_get_domain_from_cpu(cpu, res); + if (WARN_ON_ONCE(!dom)) + continue; + + if (r->alloc_capable) { + ctrl_d = &dom->resctrl_ctrl_dom; + ctrl_dom_empty = mpam_resctrl_offline_domain_hdr(cpu, &ctrl_d->hdr); + if (ctrl_dom_empty) + resctrl_offline_ctrl_domain(&res->resctrl_res, ctrl_d); + } else { + ctrl_dom_empty = true; + } + + if (r->mon_capable) { + mon_d = &dom->resctrl_mon_dom; + mon_dom_empty = mpam_resctrl_offline_domain_hdr(cpu, &mon_d->hdr); + if (mon_dom_empty) + resctrl_offline_mon_domain(&res->resctrl_res, &mon_d->hdr); + } else { + mon_dom_empty = true; + } + + if (ctrl_dom_empty && mon_dom_empty) + kfree(dom); + } +} + +int mpam_resctrl_setup(void) +{ + int err = 0; + struct mpam_resctrl_res *res; + enum resctrl_res_level rid; + struct mpam_resctrl_mon *mon; + enum resctrl_event_id eventid; + + wait_event(wait_cacheinfo_ready, cacheinfo_ready); + + cpus_read_lock(); + for_each_mpam_resctrl_control(res, rid) { + INIT_LIST_HEAD_RCU(&res->resctrl_res.ctrl_domains); + INIT_LIST_HEAD_RCU(&res->resctrl_res.mon_domains); + res->resctrl_res.rid = rid; + } + + /* Find some classes to use for controls */ + mpam_resctrl_pick_caches(); + mpam_resctrl_pick_mba(); + + /* Initialise the resctrl structures from the classes */ + for_each_mpam_resctrl_control(res, rid) { + if (!res->class) + continue; // dummy resource + + err = mpam_resctrl_control_init(res); + if (err) { + pr_debug("Failed to initialise rid %u\n", rid); + goto internal_error; + } + } + + /* Find some classes to use for monitors */ + mpam_resctrl_pick_counters(); + + for_each_mpam_resctrl_mon(mon, eventid) { + if (!mon->class) + continue; // dummy resource + + err = mpam_resctrl_monitor_init(mon, eventid); + if (err) { + pr_debug("Failed to initialise event %u\n", eventid); + goto internal_error; + } + } + + cpus_read_unlock(); + + if (!resctrl_arch_alloc_capable() && !resctrl_arch_mon_capable()) { + pr_debug("No alloc(%u) or monitor(%u) found - resctrl not supported\n", + resctrl_arch_alloc_capable(), resctrl_arch_mon_capable()); + return -EOPNOTSUPP; + } + + err = resctrl_init(); + if (err) + return err; + + WRITE_ONCE(resctrl_enabled, true); + + return 0; + +internal_error: + cpus_read_unlock(); + pr_debug("Internal error %d - resctrl not supported\n", err); + return err; +} + +void mpam_resctrl_exit(void) +{ + if (!READ_ONCE(resctrl_enabled)) + return; + + WRITE_ONCE(resctrl_enabled, false); + resctrl_exit(); +} + +static void mpam_resctrl_teardown_mon(struct mpam_resctrl_mon *mon, struct mpam_class *class) +{ + u32 num_mbwu_mon = l3_num_allocated_mbwu; + + if (!mon->mbwu_idx_to_mon) + return; + + if (mon->assigned_counters) { + __free_mbwu_mon(class, mon->assigned_counters, num_mbwu_mon); + kvfree(mon->assigned_counters); + mon->assigned_counters = NULL; + } + + kvfree(mon->mbwu_idx_to_mon); + mon->mbwu_idx_to_mon = NULL; +} + +/* + * The driver is detaching an MSC from this class, if resctrl was using it, + * pull on resctrl_exit(). + */ +void mpam_resctrl_teardown_class(struct mpam_class *class) +{ + struct mpam_resctrl_res *res; + enum resctrl_res_level rid; + struct mpam_resctrl_mon *mon; + enum resctrl_event_id eventid; + + might_sleep(); + + for_each_mpam_resctrl_control(res, rid) { + if (res->class == class) { + res->class = NULL; + break; + } + } + for_each_mpam_resctrl_mon(mon, eventid) { + if (mon->class == class) { + mon->class = NULL; + + mpam_resctrl_teardown_mon(mon, class); + break; + } + } +} + +static int __init __cacheinfo_ready(void) +{ + cacheinfo_ready = true; + wake_up(&wait_cacheinfo_ready); + + return 0; +} +device_initcall_sync(__cacheinfo_ready); + +#ifdef CONFIG_MPAM_KUNIT_TEST +#include "test_mpam_resctrl.c" +#endif diff --git a/drivers/resctrl/test_mpam_resctrl.c b/drivers/resctrl/test_mpam_resctrl.c new file mode 100644 index 0000000000000..71d8edf1f7d90 --- /dev/null +++ b/drivers/resctrl/test_mpam_resctrl.c @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Arm Ltd. +/* This file is intended to be included into mpam_resctrl.c */ + +#include +#include +#include +#include +#include + +struct percent_value_case { + u8 pc; + u8 width; + u16 value; +}; + +/* + * Mysterious inscriptions taken from the union of ARM DDI 0598D.b, + * "Arm Architecture Reference Manual Supplement - Memory System + * Resource Partitioning and Monitoring (MPAM), for A-profile + * architecture", Section 9.8, "About the fixed-point fractional + * format" (exact percentage entries only) and ARM IHI0099B.a + * "MPAM system component specification", Section 9.3, + * "The fixed-point fractional format": + */ +static const struct percent_value_case percent_value_cases[] = { + /* Architectural cases: */ + { 1, 8, 1 }, { 1, 12, 0x27 }, { 1, 16, 0x28e }, + { 25, 8, 0x3f }, { 25, 12, 0x3ff }, { 25, 16, 0x3fff }, + { 33, 8, 0x53 }, { 33, 12, 0x546 }, { 33, 16, 0x5479 }, + { 35, 8, 0x58 }, { 35, 12, 0x598 }, { 35, 16, 0x5998 }, + { 45, 8, 0x72 }, { 45, 12, 0x732 }, { 45, 16, 0x7332 }, + { 50, 8, 0x7f }, { 50, 12, 0x7ff }, { 50, 16, 0x7fff }, + { 52, 8, 0x84 }, { 52, 12, 0x850 }, { 52, 16, 0x851d }, + { 55, 8, 0x8b }, { 55, 12, 0x8cb }, { 55, 16, 0x8ccb }, + { 58, 8, 0x93 }, { 58, 12, 0x946 }, { 58, 16, 0x9479 }, + { 75, 8, 0xbf }, { 75, 12, 0xbff }, { 75, 16, 0xbfff }, + { 80, 8, 0xcb }, { 80, 12, 0xccb }, { 80, 16, 0xcccb }, + { 88, 8, 0xe0 }, { 88, 12, 0xe13 }, { 88, 16, 0xe146 }, + { 95, 8, 0xf2 }, { 95, 12, 0xf32 }, { 95, 16, 0xf332 }, + { 100, 8, 0xff }, { 100, 12, 0xfff }, { 100, 16, 0xffff }, +}; + +static void test_percent_value_desc(const struct percent_value_case *param, + char *desc) +{ + snprintf(desc, KUNIT_PARAM_DESC_SIZE, + "pc=%d, width=%d, value=0x%.*x\n", + param->pc, param->width, + DIV_ROUND_UP(param->width, 4), param->value); +} + +KUNIT_ARRAY_PARAM(test_percent_value, percent_value_cases, + test_percent_value_desc); + +struct percent_value_test_info { + u32 pc; /* result of value-to-percent conversion */ + u32 value; /* result of percent-to-value conversion */ + u32 max_value; /* maximum raw value allowed by test params */ + unsigned int shift; /* promotes raw testcase value to 16 bits */ +}; + +/* + * Convert a reference percentage to a fixed-point MAX value and + * vice-versa, based on param (not test->param_value!) + */ +static void __prepare_percent_value_test(struct kunit *test, + struct percent_value_test_info *res, + const struct percent_value_case *param) +{ + struct mpam_props fake_props = { }; + + /* Reject bogus test parameters that would break the tests: */ + KUNIT_ASSERT_GE(test, param->width, 1); + KUNIT_ASSERT_LE(test, param->width, 16); + KUNIT_ASSERT_LT(test, param->value, 1 << param->width); + + mpam_set_feature(mpam_feat_mbw_max, &fake_props); + fake_props.bwa_wd = param->width; + + res->shift = 16 - param->width; + res->max_value = GENMASK_U32(param->width - 1, 0); + res->value = percent_to_mbw_max(param->pc, &fake_props); + res->pc = mbw_max_to_percent(param->value << res->shift, &fake_props); +} + +static void test_get_mba_granularity(struct kunit *test) +{ + int ret; + struct mpam_props fake_props = { }; + + /* Use MBW_MAX */ + mpam_set_feature(mpam_feat_mbw_max, &fake_props); + + fake_props.bwa_wd = 0; + KUNIT_EXPECT_FALSE(test, mba_class_use_mbw_max(&fake_props)); + + fake_props.bwa_wd = 1; + KUNIT_EXPECT_TRUE(test, mba_class_use_mbw_max(&fake_props)); + + /* Architectural maximum: */ + fake_props.bwa_wd = 16; + KUNIT_EXPECT_TRUE(test, mba_class_use_mbw_max(&fake_props)); + + /* No usable control... */ + fake_props.bwa_wd = 0; + ret = get_mba_granularity(&fake_props); + KUNIT_EXPECT_EQ(test, ret, 0); + + fake_props.bwa_wd = 1; + ret = get_mba_granularity(&fake_props); + KUNIT_EXPECT_EQ(test, ret, 50); /* DIV_ROUND_UP(100, 1 << 1)% = 50% */ + + fake_props.bwa_wd = 2; + ret = get_mba_granularity(&fake_props); + KUNIT_EXPECT_EQ(test, ret, 25); /* DIV_ROUND_UP(100, 1 << 2)% = 25% */ + + fake_props.bwa_wd = 3; + ret = get_mba_granularity(&fake_props); + KUNIT_EXPECT_EQ(test, ret, 13); /* DIV_ROUND_UP(100, 1 << 3)% = 13% */ + + fake_props.bwa_wd = 6; + ret = get_mba_granularity(&fake_props); + KUNIT_EXPECT_EQ(test, ret, 2); /* DIV_ROUND_UP(100, 1 << 6)% = 2% */ + + fake_props.bwa_wd = 7; + ret = get_mba_granularity(&fake_props); + KUNIT_EXPECT_EQ(test, ret, 1); /* DIV_ROUND_UP(100, 1 << 7)% = 1% */ + + /* Granularity saturates at 1% */ + fake_props.bwa_wd = 16; /* architectural maximum */ + ret = get_mba_granularity(&fake_props); + KUNIT_EXPECT_EQ(test, ret, 1); /* DIV_ROUND_UP(100, 1 << 16)% = 1% */ +} + +static void test_fract16_to_percent(struct kunit *test) +{ + const struct percent_value_case *param = test->param_value; + struct percent_value_test_info res; + + /* + * Since the reference values in percent_value_cases[] all + * correspond to exact percentages, round-to-nearest will + * always give the exact percentage back when the MPAM max + * value has precision of 0.5% or finer. (Always true for the + * reference data, since they all specify 8 bits or more of + * precision. + * + * So, keep it simple and demand an exact match: + */ + __prepare_percent_value_test(test, &res, param); + KUNIT_EXPECT_EQ(test, res.pc, param->pc); +} + +static void test_percent_to_mbw_max(struct kunit *test) +{ + const struct percent_value_case *param = test->param_value; + struct percent_value_test_info res; + + __prepare_percent_value_test(test, &res, param); + + KUNIT_EXPECT_GE(test, res.value, param->value << res.shift); + KUNIT_EXPECT_LE(test, res.value, (param->value + 1) << res.shift); + KUNIT_EXPECT_LE(test, res.value, res.max_value << res.shift); + + /* No flexibility allowed for 0% and 100%! */ + + if (param->pc == 0) + KUNIT_EXPECT_EQ(test, res.value, 0); + + if (param->pc == 100) + KUNIT_EXPECT_EQ(test, res.value, res.max_value << res.shift); +} + +static const void *test_all_bwa_wd_gen_params(struct kunit *test, const void *prev, + char *desc) +{ + uintptr_t param = (uintptr_t)prev; + + if (param > 15) + return NULL; + + param++; + + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "wd=%u\n", (unsigned int)param); + + return (void *)param; +} + +static unsigned int test_get_bwa_wd(struct kunit *test) +{ + uintptr_t param = (uintptr_t)test->param_value; + + KUNIT_ASSERT_GE(test, param, 1); + KUNIT_ASSERT_LE(test, param, 16); + + return param; +} + +static void test_mbw_max_to_percent_limits(struct kunit *test) +{ + struct mpam_props fake_props = {0}; + u32 max_value; + + mpam_set_feature(mpam_feat_mbw_max, &fake_props); + fake_props.bwa_wd = test_get_bwa_wd(test); + max_value = GENMASK(15, 16 - fake_props.bwa_wd); + + KUNIT_EXPECT_EQ(test, mbw_max_to_percent(max_value, &fake_props), + MAX_MBA_BW); + KUNIT_EXPECT_EQ(test, mbw_max_to_percent(0, &fake_props), + get_mba_min(&fake_props)); + + /* + * Rounding policy dependent 0% sanity-check: + * With round-to-nearest, the minimum mbw_max value really + * should map to 0% if there are at least 200 steps. + * (100 steps may be enough for some other rounding policies.) + */ + if (fake_props.bwa_wd >= 8) + KUNIT_EXPECT_EQ(test, mbw_max_to_percent(0, &fake_props), 0); + + if (fake_props.bwa_wd < 8 && + mbw_max_to_percent(0, &fake_props) == 0) + kunit_warn(test, "wd=%d: Testsuite/driver Rounding policy mismatch?", + fake_props.bwa_wd); +} + +/* + * Check that converting a percentage to mbw_max and back again (or, as + * appropriate, vice-versa) always restores the original value: + */ +static void test_percent_max_roundtrip_stability(struct kunit *test) +{ + struct mpam_props fake_props = {0}; + unsigned int shift; + u32 pc, max, pc2, max2; + + mpam_set_feature(mpam_feat_mbw_max, &fake_props); + fake_props.bwa_wd = test_get_bwa_wd(test); + shift = 16 - fake_props.bwa_wd; + + /* + * Converting a valid value from the coarser scale to the finer + * scale and back again must yield the original value: + */ + if (fake_props.bwa_wd >= 7) { + /* More than 100 steps: only test exact pc values: */ + for (pc = get_mba_min(&fake_props); pc <= MAX_MBA_BW; pc++) { + max = percent_to_mbw_max(pc, &fake_props); + pc2 = mbw_max_to_percent(max, &fake_props); + KUNIT_EXPECT_EQ(test, pc2, pc); + } + } else { + /* Fewer than 100 steps: only test exact mbw_max values: */ + for (max = 0; max < 1 << 16; max += 1 << shift) { + pc = mbw_max_to_percent(max, &fake_props); + max2 = percent_to_mbw_max(pc, &fake_props); + KUNIT_EXPECT_EQ(test, max2, max); + } + } +} + +static void test_percent_to_max_rounding(struct kunit *test) +{ + const struct percent_value_case *param = test->param_value; + unsigned int num_rounded_up = 0, total = 0; + struct percent_value_test_info res; + + for (param = percent_value_cases, total = 0; + param < &percent_value_cases[ARRAY_SIZE(percent_value_cases)]; + param++, total++) { + __prepare_percent_value_test(test, &res, param); + if (res.value > param->value << res.shift) + num_rounded_up++; + } + + /* + * The MPAM driver applies a round-to-nearest policy, whereas a + * round-down policy seems to have been applied in the + * reference table from which the test vectors were selected. + * + * For a large and well-distributed suite of test vectors, + * about half should be rounded up and half down compared with + * the reference table. The actual test vectors are few in + * number and probably not very well distributed however, so + * tolerate a round-up rate of between 1/4 and 3/4 before + * crying foul: + */ + + kunit_info(test, "Round-up rate: %u%% (%u/%u)\n", + DIV_ROUND_CLOSEST(num_rounded_up * 100, total), + num_rounded_up, total); + + KUNIT_EXPECT_GE(test, 4 * num_rounded_up, 1 * total); + KUNIT_EXPECT_LE(test, 4 * num_rounded_up, 3 * total); +} + +static struct kunit_case mpam_resctrl_test_cases[] = { + KUNIT_CASE(test_get_mba_granularity), + KUNIT_CASE_PARAM(test_fract16_to_percent, test_percent_value_gen_params), + KUNIT_CASE_PARAM(test_percent_to_mbw_max, test_percent_value_gen_params), + KUNIT_CASE_PARAM(test_mbw_max_to_percent_limits, test_all_bwa_wd_gen_params), + KUNIT_CASE(test_percent_to_max_rounding), + KUNIT_CASE_PARAM(test_percent_max_roundtrip_stability, + test_all_bwa_wd_gen_params), + {} +}; + +static struct kunit_suite mpam_resctrl_test_suite = { + .name = "mpam_resctrl_test_suite", + .test_cases = mpam_resctrl_test_cases, +}; + +kunit_test_suites(&mpam_resctrl_test_suite); diff --git a/drivers/tty/serial/8250/8250_mtk.c b/drivers/tty/serial/8250/8250_mtk.c index 5875a7b9b4b10..e6a56cf54ae0c 100644 --- a/drivers/tty/serial/8250/8250_mtk.c +++ b/drivers/tty/serial/8250/8250_mtk.c @@ -19,6 +19,7 @@ #include #include #include +#include #include "8250.h" @@ -521,6 +522,7 @@ static int mtk8250_probe(struct platform_device *pdev) struct mtk8250_data *data; struct resource *regs; int irq, err; + struct fwnode_handle *fwnode = dev_fwnode(&pdev->dev); irq = platform_get_irq(pdev, 0); if (irq < 0) @@ -543,12 +545,13 @@ static int mtk8250_probe(struct platform_device *pdev) data->clk_count = 0; - if (pdev->dev.of_node) { + if (is_of_node(fwnode)) { err = mtk8250_probe_of(pdev, &uart.port, data); if (err) return err; - } else + } else if (!fwnode) { return -ENODEV; + } spin_lock_init(&uart.port.lock); uart.port.mapbase = regs->start; @@ -564,14 +567,18 @@ static int mtk8250_probe(struct platform_device *pdev) uart.port.startup = mtk8250_startup; uart.port.set_termios = mtk8250_set_termios; uart.port.uartclk = clk_get_rate(data->uart_clk); + if (!uart.port.uartclk) + uart.port.uartclk = 26 * HZ_PER_MHZ; #ifdef CONFIG_SERIAL_8250_DMA if (data->dma) uart.dma = data->dma; #endif - /* Disable Rate Fix function */ - writel(0x0, uart.port.membase + + if (is_of_node(fwnode)) { + /* Disable Rate Fix function */ + writel(0x0, uart.port.membase + (MTK_UART_RATE_FIX << uart.port.regshift)); + } platform_set_drvdata(pdev, data); @@ -649,11 +656,19 @@ static const struct of_device_id mtk8250_of_match[] = { }; MODULE_DEVICE_TABLE(of, mtk8250_of_match); +static const struct acpi_device_id mtk8250_acpi_match[] = { + { "MTKI0511" }, + { "NVDA0240" }, + {} +}; +MODULE_DEVICE_TABLE(acpi, mtk8250_acpi_match); + static struct platform_driver mtk8250_platform_driver = { .driver = { .name = "mt6577-uart", .pm = &mtk8250_pm_ops, .of_match_table = mtk8250_of_match, + .acpi_match_table = mtk8250_acpi_match, }, .probe = mtk8250_probe, .remove = mtk8250_remove, diff --git a/drivers/usb/host/xhci-hub.c b/drivers/usb/host/xhci-hub.c index 04cc3d681495f..09a92ab1537b2 100644 --- a/drivers/usb/host/xhci-hub.c +++ b/drivers/usb/host/xhci-hub.c @@ -1290,6 +1290,17 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, } port_li = readl(&port->port_reg->portli); status = xhci_get_ext_port_status(temp, port_li); + + /* + * In MT8901 USB host controller, the lane count can be wrongly set + * to 2 instead of 1 for USB Gen2x1 devices due to a HW Bug. As a SW + * WAR, check if port speed is 0x5 (SuperSpeedPlus Gen2x1) in + * PORTSC register and update the lane count as 1. + */ + if ((xhci->quirks & XHCI_NVIDIA_MT8901_HOST) && + DEV_SUPERSPEEDPLUS(temp)) + status &= ~0xff00; + put_unaligned_le32(status, &buf[4]); } break; diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index 074d9c731639f..771836ab20ef0 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -277,6 +277,9 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s if (device_property_read_bool(tmpdev, "xhci-skip-phy-init-quirk")) xhci->quirks |= XHCI_SKIP_PHY_INIT; + if (device_property_read_bool(tmpdev, "xhci-nvidia-mediatek-host")) + xhci->quirks |= XHCI_NVIDIA_MT8901_HOST; + device_property_read_u32(tmpdev, "imod-interval-ns", &xhci->imod_interval); device_property_read_u16(tmpdev, "num-hc-interrupters", diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index 8d8f0865fc121..35bd19073b685 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -3731,6 +3731,11 @@ static int xhci_alloc_streams(struct usb_hcd *hcd, struct usb_device *udev, if (ret < 0) goto cleanup; + if (xhci->quirks & XHCI_NVIDIA_MT8901_HOST) { + xhci_hub_control(hcd, SetPortFeature, USB_PORT_FEAT_U1_TIMEOUT, 0, NULL, 0); + xhci_hub_control(hcd, SetPortFeature, USB_PORT_FEAT_U2_TIMEOUT, 0, NULL, 0); + } + spin_lock_irqsave(&xhci->lock, flags); for (i = 0; i < num_eps; i++) { ep_index = xhci_get_endpoint_index(&eps[i]->desc); diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index 2b0796f6d00ea..7421a806a6eae 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1644,6 +1644,7 @@ struct xhci_hcd { #define XHCI_CDNS_SCTX_QUIRK BIT_ULL(48) #define XHCI_ETRON_HOST BIT_ULL(49) #define XHCI_LIMIT_ENDPOINT_INTERVAL_9 BIT_ULL(50) +#define XHCI_NVIDIA_MT8901_HOST BIT_ULL(51) unsigned int num_active_eps; unsigned int limit_active_eps; diff --git a/drivers/vfio/pci/nvgrace-gpu/Kconfig b/drivers/vfio/pci/nvgrace-gpu/Kconfig index a7f624b37e410..d5773bbd22f5e 100644 --- a/drivers/vfio/pci/nvgrace-gpu/Kconfig +++ b/drivers/vfio/pci/nvgrace-gpu/Kconfig @@ -1,8 +1,19 @@ # SPDX-License-Identifier: GPL-2.0-only +config NVGRACE_EGM + tristate "EGM driver for NVIDIA Grace Hopper and Blackwell Superchip" + depends on ARM64 || (COMPILE_TEST && 64BIT) + help + Extended GPU Memory (EGM) support for the GPU in the NVIDIA Grace + based chips required to avail the CPU memory as additional + cross-node/cross-socket memory for GPU using KVM/qemu. + + If you don't know what to do here, say N. + config NVGRACE_GPU_VFIO_PCI tristate "VFIO support for the GPU in the NVIDIA Grace Hopper Superchip" depends on ARM64 || (COMPILE_TEST && 64BIT) select VFIO_PCI_CORE + select NVGRACE_EGM help VFIO support for the GPU in the NVIDIA Grace Hopper Superchip is required to assign the GPU device to userspace using KVM/qemu/etc. diff --git a/drivers/vfio/pci/nvgrace-gpu/Makefile b/drivers/vfio/pci/nvgrace-gpu/Makefile index 3ca8c187897a9..c99b04a94e770 100644 --- a/drivers/vfio/pci/nvgrace-gpu/Makefile +++ b/drivers/vfio/pci/nvgrace-gpu/Makefile @@ -1,3 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_NVGRACE_GPU_VFIO_PCI) += nvgrace-gpu-vfio-pci.o nvgrace-gpu-vfio-pci-y := main.o + +obj-$(CONFIG_NVGRACE_EGM) += nvgrace-egm.o +nvgrace-egm-y := egm.o diff --git a/drivers/vfio/pci/nvgrace-gpu/egm.c b/drivers/vfio/pci/nvgrace-gpu/egm.c new file mode 100644 index 0000000000000..a2e4c05a83c34 --- /dev/null +++ b/drivers/vfio/pci/nvgrace-gpu/egm.c @@ -0,0 +1,590 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved + */ + +#include +#include +#include +#include +#include +#include + +#define MAX_EGM_NODES 256 + +struct gpu_node { + struct list_head list; + struct pci_dev *pdev; +}; + +struct egm_region { + struct list_head list; + int egmpxm; + atomic_t open_count; + phys_addr_t egmphys; + size_t egmlength; + struct device device; + struct cdev cdev; + struct list_head gpus; + DECLARE_HASHTABLE(htbl, 0x10); + struct pfn_address_space pfn_address_space; +}; + +struct h_node { + unsigned long mem_offset; + struct hlist_node node; +}; + +static dev_t dev; +static struct class *class; +static struct list_head egm_list; + +static int pfn_memregion_offset(struct egm_region *region, + unsigned long pfn, + pgoff_t *pfn_offset_in_region) +{ + unsigned long start_pfn, num_pages; + + start_pfn = PHYS_PFN(region->egmphys); + num_pages = region->egmlength >> PAGE_SHIFT; + + if (pfn < start_pfn || pfn >= start_pfn + num_pages) + return -EFAULT; + + *pfn_offset_in_region = pfn - start_pfn; + + return 0; +} + +static int track_ecc_offset(struct egm_region *region, + unsigned long mem_offset) +{ + struct h_node *cur_page, *ecc_page; + unsigned long bkt; + + hash_for_each(region->htbl, bkt, cur_page, node) { + if (cur_page->mem_offset == mem_offset) + return 0; + } + + ecc_page = (struct h_node *)(vzalloc(sizeof(struct h_node))); + if (!ecc_page) + return -ENOMEM; + + ecc_page->mem_offset = mem_offset; + + hash_add(region->htbl, &ecc_page->node, ecc_page->mem_offset); + + return 0; +} + +static int nvgrace_egm_pfn_to_vma_pgoff(struct vm_area_struct *vma, + unsigned long pfn, + pgoff_t *pgoff) +{ + struct egm_region *region = vma->vm_file->private_data; + pgoff_t vma_offset_in_region = vma->vm_pgoff & + ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); + pgoff_t pfn_offset_in_region; + int ret; + + ret = pfn_memregion_offset(region, pfn, &pfn_offset_in_region); + if (ret) + return ret; + + /* Ensure PFN is not before VMA's start within the region */ + if (pfn_offset_in_region < vma_offset_in_region) + return -EFAULT; + + /* Calculate offset from VMA start */ + *pgoff = vma->vm_pgoff + + (pfn_offset_in_region - vma_offset_in_region); + + /* Track and save the poisoned offset */ + return track_ecc_offset(region, *pgoff << PAGE_SHIFT); +} + +static int +nvgrace_egm_vfio_pci_register_pfn_range(struct inode *inode, + struct egm_region *region) +{ + int ret; + unsigned long pfn, nr_pages; + + pfn = PHYS_PFN(region->egmphys); + nr_pages = region->egmlength >> PAGE_SHIFT; + + region->pfn_address_space.node.start = pfn; + region->pfn_address_space.node.last = pfn + nr_pages - 1; + region->pfn_address_space.mapping = inode->i_mapping; + region->pfn_address_space.pfn_to_vma_pgoff = nvgrace_egm_pfn_to_vma_pgoff; + + ret = register_pfn_address_space(®ion->pfn_address_space); + + return ret; +} + +static int nvgrace_egm_open(struct inode *inode, struct file *file) +{ + void *memaddr; + struct egm_region *region = container_of(inode->i_cdev, + struct egm_region, cdev); + int ret; + + if (atomic_inc_return(®ion->open_count) > 1) + return 0; + + memaddr = memremap(region->egmphys, region->egmlength, MEMREMAP_WB); + if (!memaddr) { + atomic_dec(®ion->open_count); + return -EINVAL; + } + + { + size_t remaining = region->egmlength; + u8 *chunk_addr = (u8 *)memaddr; + size_t chunk_size; + + while (remaining > 0) { + chunk_size = min(remaining, SZ_1G); + memset(chunk_addr, 0, chunk_size); + cond_resched(); + chunk_addr += chunk_size; + remaining -= chunk_size; + } + } + + memunmap(memaddr); + file->private_data = region; + + ret = nvgrace_egm_vfio_pci_register_pfn_range(inode, region); + if (ret && ret != -EOPNOTSUPP) { + file->private_data = NULL; + return ret; + } + + return 0; +} + +static int nvgrace_egm_release(struct inode *inode, struct file *file) +{ + struct egm_region *region = container_of(inode->i_cdev, + struct egm_region, cdev); + + if (atomic_dec_and_test(®ion->open_count)) { + unregister_pfn_address_space(®ion->pfn_address_space); + + file->private_data = NULL; + } + + return 0; +} + +static int nvgrace_egm_mmap(struct file *file, struct vm_area_struct *vma) +{ + int ret = 0; + struct egm_region *region = file->private_data; + + if (!region) + return -EINVAL; + + ret = remap_pfn_range(vma, vma->vm_start, + PHYS_PFN(region->egmphys), + (vma->vm_end - vma->vm_start), + vma->vm_page_prot); + return ret; +} + +static long nvgrace_egm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + unsigned long minsz = offsetofend(struct egm_bad_pages_list, count); + struct egm_bad_pages_list info; + void __user *uarg = (void __user *)arg; + struct egm_region *region = file->private_data; + + if (copy_from_user(&info, uarg, minsz)) + return -EFAULT; + + if (info.argsz < minsz) + return -EINVAL; + + if (!region) + return -EINVAL; + + switch (cmd) { + case EGM_BAD_PAGES_LIST: + { + int ret; + unsigned long bad_page_struct_size = sizeof(struct egm_bad_pages_info); + struct egm_bad_pages_info tmp; + struct h_node *cur_page; + struct hlist_node *tmp_node; + unsigned long bkt; + int count = 0, index = 0; + + hash_for_each_safe(region->htbl, bkt, tmp_node, cur_page, node) + count++; + + if (info.argsz < (minsz + count * bad_page_struct_size)) { + info.argsz = minsz + count * bad_page_struct_size; + info.count = 0; + goto done; + } else { + hash_for_each_safe(region->htbl, bkt, tmp_node, cur_page, node) { + /* + * This check fails if there was an ECC error + * after the usermode app read the count of + * bad pages through this ioctl. + */ + if (minsz + index * bad_page_struct_size >= info.argsz) { + info.argsz = minsz + index * bad_page_struct_size; + info.count = index; + goto done; + } + + tmp.offset = cur_page->mem_offset; + tmp.size = PAGE_SIZE; + + ret = copy_to_user(uarg + minsz + + index * bad_page_struct_size, + &tmp, bad_page_struct_size); + if (ret) + return -EFAULT; + index++; + } + + info.count = index; + } + break; + } + default: + return -EINVAL; + } + +done: + return copy_to_user(uarg, &info, minsz) ? -EFAULT : 0; +} + +static const struct file_operations file_ops = { + .owner = THIS_MODULE, + .open = nvgrace_egm_open, + .release = nvgrace_egm_release, + .mmap = nvgrace_egm_mmap, + .unlocked_ioctl = nvgrace_egm_ioctl, +}; + +static int setup_egm_chardev(struct egm_region *region) +{ + int ret = 0; + + device_initialize(®ion->device); + + /* + * Use the proximity domain number as the device minor + * number. So the EGM corresponding to node X would be + * /dev/egmX. + */ + region->device.devt = MKDEV(MAJOR(dev), region->egmpxm); + region->device.class = class; + cdev_init(®ion->cdev, &file_ops); + region->cdev.owner = THIS_MODULE; + + ret = dev_set_name(®ion->device, "egm%d", region->egmpxm); + if (ret) + return ret; + + ret = cdev_device_add(®ion->cdev, ®ion->device); + + return ret; +} + +static void destroy_egm_chardev(struct egm_region *region) +{ + cdev_device_del(®ion->cdev, ®ion->device); +} + +static int +nvgrace_gpu_fetch_egm_property(struct pci_dev *pdev, u64 *pegmphys, + u64 *pegmlength, u64 *pegmpxm) +{ + int ret; + + /* + * The memory information is present in the system ACPI tables as DSD + * properties nvidia,egm-base-pa and nvidia,egmm-size. + */ + ret = device_property_read_u64(&pdev->dev, "nvidia,egm-size", + pegmlength); + if (ret) + return ret; + + if (overflows_type(*pegmlength, size_t)) + return -EOVERFLOW; + + ret = device_property_read_u64(&pdev->dev, "nvidia,egm-base-pa", + pegmphys); + if (ret) + return ret; + + if (overflows_type(*pegmphys, phys_addr_t)) + return -EOVERFLOW; + + ret = device_property_read_u64(&pdev->dev, "nvidia,egm-pxm", + pegmpxm); + if (ret) + return ret; + + if (overflows_type(*pegmpxm, int)) + return -EOVERFLOW; + + return 0; +} + +static void nvgrace_egm_fetch_bad_pages(struct pci_dev *pdev, + struct egm_region *region) +{ + u64 retiredpagesphys, count; + void *memaddr; + int index; + + if (device_property_read_u64(&pdev->dev, + "nvidia,egm-retired-pages-data-base", + &retiredpagesphys)) + return; + + /* Catch firmware bug and avoid a crash */ + if (WARN_ON_ONCE(retiredpagesphys == 0)) + return; + + memaddr = memremap(retiredpagesphys, PAGE_SIZE, MEMREMAP_WB); + if (!memaddr) + return; + + count = *(u64 *)memaddr; + + for (index = 0; index < count; index++) { + struct h_node *retired_page; + + /* + * Since the EGM is linearly mapped, the offset in the + * carveout is the same offset in the VM system memory. + * + * Calculate the offset to communicate to the usermode + * apps. + */ + retired_page = (struct h_node *)(vzalloc(sizeof(struct h_node))); + if (!retired_page) + continue; /* Skip this entry on allocation failure */ + retired_page->mem_offset = *((u64 *)memaddr + index + 1) - + region->egmphys; + hash_add(region->htbl, &retired_page->node, retired_page->mem_offset); + } + + memunmap(memaddr); +} + +static ssize_t gpu_devices_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct egm_region *region = + container_of(dev, struct egm_region, device); + struct gpu_node *node, *temp_node; + int len = 0; + + list_for_each_entry_safe(node, temp_node, ®ion->gpus, list) { + struct pci_dev *pdev = node->pdev; + + len += sysfs_emit_at(buf, len, "%04x:%02x:%02x.%x\n", + pci_domain_nr(pdev->bus), + pdev->bus->number, + PCI_SLOT(pdev->devfn), + PCI_FUNC(pdev->devfn)); + } + + return len; +} + +static DEVICE_ATTR_RO(gpu_devices); + +static ssize_t egm_size_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct egm_region *region = + container_of(dev, struct egm_region, device); + return sysfs_emit(buf, "0x%lx\n", region->egmlength); +} + +static DEVICE_ATTR_RO(egm_size); + +static struct attribute *attrs[] = { + &dev_attr_gpu_devices.attr, + &dev_attr_egm_size.attr, + NULL, +}; + +static struct attribute_group attr_group = { + .attrs = attrs, +}; + +static int add_gpu(struct egm_region *region, struct pci_dev *pdev) +{ + struct gpu_node *node; + + node = kvzalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return -ENOMEM; + + node->pdev = pdev; + + list_add_tail(&node->list, ®ion->gpus); + return 0; +} + +static void remove_gpu(struct egm_region *region, struct pci_dev *pdev) +{ + struct gpu_node *node, *tmp; + + list_for_each_entry_safe(node, tmp, ®ion->gpus, list) { + if (node->pdev == pdev) { + list_del(&node->list); + kvfree(node); + } + } +} + +int register_egm_node(struct pci_dev *pdev) +{ + struct egm_region *region = NULL; + u64 egmphys, egmlength, egmpxm; + int ret; + + ret = nvgrace_gpu_fetch_egm_property(pdev, &egmphys, &egmlength, &egmpxm); + if (ret) + return ret; + + /* Check if region already exists */ + list_for_each_entry(region, &egm_list, list) { + if (region->egmphys == egmphys) { + /* Add GPU to existing region */ + return add_gpu(region, pdev); + } + } + + /* Create new region */ + region = kvzalloc(sizeof(*region), GFP_KERNEL); + if (!region) + return -ENOMEM; + + region->egmphys = egmphys; + region->egmlength = egmlength; + region->egmpxm = egmpxm; + + hash_init(region->htbl); + INIT_LIST_HEAD(®ion->gpus); + + atomic_set(®ion->open_count, 0); + + nvgrace_egm_fetch_bad_pages(pdev, region); + + ret = setup_egm_chardev(region); + if (ret) + goto err_free_region; + + list_add_tail(®ion->list, &egm_list); + + ret = sysfs_create_group(®ion->device.kobj, &attr_group); + if (ret) + goto err_remove_from_list; + + ret = add_gpu(region, pdev); + if (ret) + goto err_remove_sysfs; + + return 0; + +err_remove_sysfs: + sysfs_remove_group(®ion->device.kobj, &attr_group); +err_remove_from_list: + list_del(®ion->list); + destroy_egm_chardev(region); +err_free_region: + kfree(region); + return ret; +} +EXPORT_SYMBOL_GPL(register_egm_node); + +void unregister_egm_node(struct pci_dev *pdev) +{ + struct egm_region *region, *temp_region; + struct h_node *cur_page; + unsigned long bkt; + struct hlist_node *temp_node; + u64 egmphys, egmlength, egmpxm; + int ret; + + ret = nvgrace_gpu_fetch_egm_property(pdev, &egmphys, &egmlength, &egmpxm); + if (ret) + return; + + list_for_each_entry_safe(region, temp_region, &egm_list, list) { + if (egmpxm == region->egmpxm) { + remove_gpu(region, pdev); + if (!list_empty(®ion->gpus)) + break; + + hash_for_each_safe(region->htbl, bkt, temp_node, cur_page, node) { + hash_del(&cur_page->node); + vfree(cur_page); + } + + sysfs_remove_group(®ion->device.kobj, &attr_group); + destroy_egm_chardev(region); + list_del(®ion->list); + kfree(region); + } + } +} +EXPORT_SYMBOL_GPL(unregister_egm_node); + +static char *egm_devnode(const struct device *device, umode_t *mode) +{ + if (mode) + *mode = 0600; + + return NULL; +} + +static int __init nvgrace_egm_init(void) +{ + int ret; + + ret = alloc_chrdev_region(&dev, + 0, MAX_EGM_NODES, "egm"); + if (ret < 0) + return ret; + + class = class_create("egm"); + if (IS_ERR(class)) { + unregister_chrdev_region(dev, MAX_EGM_NODES); + return PTR_ERR(class); + } + + class->devnode = egm_devnode; + + INIT_LIST_HEAD(&egm_list); + + return 0; +} + +static void __exit nvgrace_egm_cleanup(void) +{ + class_destroy(class); + unregister_chrdev_region(dev, MAX_EGM_NODES); +} + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Ankit Agrawal "); +MODULE_DESCRIPTION("NVGRACE EGM - Helper module of NVGRACE GPU to support Extended GPU Memory"); + +module_init(nvgrace_egm_init); +module_exit(nvgrace_egm_cleanup); diff --git a/drivers/vfio/pci/nvgrace-gpu/main.c b/drivers/vfio/pci/nvgrace-gpu/main.c index fa056b69f899a..967313b9e0029 100644 --- a/drivers/vfio/pci/nvgrace-gpu/main.c +++ b/drivers/vfio/pci/nvgrace-gpu/main.c @@ -10,6 +10,7 @@ #include #include #include +#include /* * The device memory usable to the workloads running in the VM is cached @@ -64,8 +65,11 @@ struct nvgrace_gpu_pci_core_device { bool has_mig_hw_bug; /* GPU has just been reset */ bool reset_done; + int egm_node; }; +static bool egm_enabled; + static void nvgrace_gpu_init_fake_bar_emu_regs(struct vfio_device *core_vdev) { struct nvgrace_gpu_pci_core_device *nvdev = @@ -991,7 +995,7 @@ nvgrace_gpu_fetch_memory_property(struct pci_dev *pdev, if (ret) return ret; - if (*pmemphys > type_max(phys_addr_t)) + if (overflows_type(*pmemphys, phys_addr_t)) return -EOVERFLOW; ret = device_property_read_u64(&pdev->dev, "nvidia,gpu-mem-size", @@ -999,7 +1003,7 @@ nvgrace_gpu_fetch_memory_property(struct pci_dev *pdev, if (ret) return ret; - if (*pmemlength > type_max(size_t)) + if (overflows_type(*pmemlength, size_t)) return -EOVERFLOW; /* @@ -1012,6 +1016,13 @@ nvgrace_gpu_fetch_memory_property(struct pci_dev *pdev, return ret; } +static int +nvgrace_gpu_has_egm_property(struct pci_dev *pdev, u64 *pegmpxm) +{ + return device_property_read_u64(&pdev->dev, "nvidia,egm-pxm", + pegmpxm); +} + static int nvgrace_gpu_init_nvdev_struct(struct pci_dev *pdev, struct nvgrace_gpu_pci_core_device *nvdev, @@ -1181,6 +1192,7 @@ static int nvgrace_gpu_probe(struct pci_dev *pdev, const struct vfio_device_ops *ops = &nvgrace_gpu_pci_core_ops; struct nvgrace_gpu_pci_core_device *nvdev; u64 memphys, memlength; + u64 egmpxm; int ret; ret = nvgrace_gpu_probe_check_device_ready(pdev); @@ -1188,9 +1200,14 @@ static int nvgrace_gpu_probe(struct pci_dev *pdev, return ret; ret = nvgrace_gpu_fetch_memory_property(pdev, &memphys, &memlength); - if (!ret) + if (!ret) { ops = &nvgrace_gpu_pci_ops; + ret = nvgrace_gpu_has_egm_property(pdev, &egmpxm); + if (!ret) + egm_enabled = true; + } + nvdev = vfio_alloc_device(nvgrace_gpu_pci_core_device, core_device.vdev, &pdev->dev, ops); if (IS_ERR(nvdev)) @@ -1210,16 +1227,28 @@ static int nvgrace_gpu_probe(struct pci_dev *pdev, if (ret) goto out_put_vdev; nvdev->core_device.pci_ops = &nvgrace_gpu_pci_dev_ops; + + if (egm_enabled) { + ret = register_egm_node(pdev); + if (ret) + goto out_put_vdev; + + nvdev->egm_node = egmpxm; + } + } else { nvdev->core_device.pci_ops = &nvgrace_gpu_pci_dev_core_ops; } ret = vfio_pci_core_register_device(&nvdev->core_device); if (ret) - goto out_put_vdev; + goto out_egm_unreg; return ret; +out_egm_unreg: + if (egm_enabled) + unregister_egm_node(pdev); out_put_vdev: vfio_put_device(&nvdev->core_device.vdev); return ret; @@ -1228,6 +1257,12 @@ static int nvgrace_gpu_probe(struct pci_dev *pdev, static void nvgrace_gpu_remove(struct pci_dev *pdev) { struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev); + struct nvgrace_gpu_pci_core_device *nvdev = + container_of(core_device, struct nvgrace_gpu_pci_core_device, + core_device); + + if (egm_enabled) + unregister_egm_node(pdev); vfio_pci_core_unregister_device(core_device); vfio_put_device(&core_device->vdev); diff --git a/drivers/virt/coco/Kconfig b/drivers/virt/coco/Kconfig index df1cfaf26c658..ae7d390831d9d 100644 --- a/drivers/virt/coco/Kconfig +++ b/drivers/virt/coco/Kconfig @@ -14,6 +14,8 @@ source "drivers/virt/coco/tdx-guest/Kconfig" source "drivers/virt/coco/arm-cca-guest/Kconfig" +source "drivers/virt/coco/nvidia-firme/Kconfig" + source "drivers/virt/coco/guest/Kconfig" endif diff --git a/drivers/virt/coco/Makefile b/drivers/virt/coco/Makefile index cb52021912b34..16983136fbae0 100644 --- a/drivers/virt/coco/Makefile +++ b/drivers/virt/coco/Makefile @@ -7,5 +7,6 @@ obj-$(CONFIG_ARM_PKVM_GUEST) += pkvm-guest/ obj-$(CONFIG_SEV_GUEST) += sev-guest/ obj-$(CONFIG_INTEL_TDX_GUEST) += tdx-guest/ obj-$(CONFIG_ARM_CCA_GUEST) += arm-cca-guest/ +obj-$(CONFIG_NVIDIA_FIRME_GUEST) += nvidia-firme/ obj-$(CONFIG_TSM) += tsm-core.o obj-$(CONFIG_TSM_GUEST) += guest/ diff --git a/drivers/virt/coco/nvidia-firme/Kconfig b/drivers/virt/coco/nvidia-firme/Kconfig new file mode 100644 index 0000000000000..04aad6e4882af --- /dev/null +++ b/drivers/virt/coco/nvidia-firme/Kconfig @@ -0,0 +1,16 @@ +config NVIDIA_FIRME_GUEST + tristate "NVIDIA FIRME attestation driver" + depends on ARM64 + default m + select TSM_REPORTS + select TSM_MEASUREMENTS + help + TSM report and measurement register backend for ARM FIRME (DEN0149) + on NVIDIA Grace (TH500) platforms. After a DRTM Secure Launch: + + - FIRME_ATTEST_PAT_GET retrieves attestation tokens from PSC, + exposed via configfs at /sys/kernel/config/tsm/report/. + - FIRME_ATTEST_EXTEND extends measurement registers in PSC, + exposed via sysfs for GPU/device evidence binding. + + If you choose 'M' here, this module will be called nvidia-firme. diff --git a/drivers/virt/coco/nvidia-firme/Makefile b/drivers/virt/coco/nvidia-firme/Makefile new file mode 100644 index 0000000000000..235b156c4f142 --- /dev/null +++ b/drivers/virt/coco/nvidia-firme/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_NVIDIA_FIRME_GUEST) += nvidia-firme.o diff --git a/drivers/virt/coco/nvidia-firme/nvidia-firme.c b/drivers/virt/coco/nvidia-firme/nvidia-firme.c new file mode 100644 index 0000000000000..a543541580550 --- /dev/null +++ b/drivers/virt/coco/nvidia-firme/nvidia-firme.c @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * NVIDIA FIRME Attestation Driver + * + * TSM report backend and TSM measurement register backend for ARM FIRME + * (DEN0149) on NVIDIA Grace (TH500) platforms. + * + * - TSM Reports: invokes FIRME_ATTEST_PAT_GET (0xC4000408) to retrieve + * platform attestation tokens from EL3/PSC via configfs. + * - TSM MR: invokes FIRME_ATTEST_EXT_CLAIMS (0xC400040B) to extend + * measurement registers and submit BMDR device reports to PSC + * via sysfs. + * + * Copyright (c) 2025-2026, NVIDIA Corporation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* FIRME SMC Function IDs — DEN0149, SMC64, Fast call, OEN=4 (Std Svc) */ +#define FIRME_FID(fn) (0xC4000000UL | (fn)) +#define FIRME_SERVICE_VERSION FIRME_FID(0x0400) +#define FIRME_ATTEST_PAT_GET FIRME_FID(0x0408) +#define FIRME_ATTEST_EXTEND FIRME_FID(0x040B) + +/* FIRME return codes (DEN0149 Table 8.24) */ +#define FIRME_SUCCESS 0L +#define FIRME_NOT_SUPPORTED (-1L) +#define FIRME_INVALID_PARAMETERS (-2L) +#define FIRME_ABORTED (-3L) +#define FIRME_INCOMPLETE (-4L) +#define FIRME_DENIED (-5L) +#define FIRME_RETRY (-6L) + +/* SHA-384 digest size */ +#define SHA384_DIGEST_SIZE 48 + +/* + * BMDR (Baremetal Device Report) entry size for slot 0. + * Layout: [48 identity_digest][48 mexchange_digest][1 gpu_id][3 reserved] + */ +#define FIRME_BMDR_SIZE 100 + +/* + * Shared buffer sizing. ATF computes: size = (page_count + 1) * 4KB. + * page_count=3 → 16KB buffer, sufficient for most tokens. + */ +#define FIRME_BUF_PAGE_COUNT 3 +#define FIRME_MIN_SHARED_BUF_SZ SZ_4K +#define FIRME_BUF_SIZE ((FIRME_BUF_PAGE_COUNT + 1) * \ + FIRME_MIN_SHARED_BUF_SZ) +#define FIRME_MAX_TOKEN_SIZE SZ_32K +#define FIRME_MAX_RETRIES 5 + +static bool nvidia_firme_available(void) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_invoke(FIRME_SERVICE_VERSION, 0, 0, 0, 0, &res); + if ((long)res.a0 == FIRME_NOT_SUPPORTED) + return false; + + pr_info("nvidia-firme: FIRME service version %lu.%lu\n", + res.a0 >> 16, res.a0 & 0xffff); + return true; +} + +/** + * nvidia_firme_report_new - Generate an attestation report via FIRME SMC. + * @report: TSM report structure; inblob is used as the challenge nonce. + * @data: unused private data. + * + * Allocates a physically contiguous NS buffer, places the optional + * challenge at offset 0, then calls FIRME_ATTEST_PAT_GET in a loop + * to retrieve the full platform attestation token. + * + * Return: 0 on success, negative errno on failure. + */ +static int nvidia_firme_report_new(struct tsm_report *report, void *data) +{ + struct tsm_report_desc *desc = &report->desc; + struct arm_smccc_res res; + void *buf; + phys_addr_t buf_phys; + u8 *token __free(kvfree) = NULL; + u64 offset = 0; + u64 challenge_sz = 0; + size_t token_size = 0; + int retries; + + buf = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, + get_order(FIRME_BUF_SIZE)); + if (!buf) + return -ENOMEM; + + buf_phys = virt_to_phys(buf); + + /* Place challenge/nonce at offset 0 of the shared buffer */ + if (desc->inblob_len > 0) { + if (desc->inblob_len > FIRME_BUF_SIZE) { + free_pages((unsigned long)buf, get_order(FIRME_BUF_SIZE)); + return -EINVAL; + } + memcpy(buf, desc->inblob, desc->inblob_len); + challenge_sz = desc->inblob_len; + } + + token = kvzalloc(FIRME_MAX_TOKEN_SIZE, GFP_KERNEL); + if (!token) { + free_pages((unsigned long)buf, get_order(FIRME_BUF_SIZE)); + return -ENOMEM; + } + + /* Chunked retrieval loop per DEN0149 Section 8.12 */ + do { + retries = 0; +retry: + pr_info("nvidia-firme: FIRME_ATTEST_PAT_GET SMC: buf=0x%llx offset=%llu page_count=%d challenge_sz=%llu\n", + (u64)buf_phys, offset, FIRME_BUF_PAGE_COUNT, + challenge_sz); + + arm_smccc_1_1_invoke(FIRME_ATTEST_PAT_GET, + buf_phys, + offset, + FIRME_BUF_PAGE_COUNT, + challenge_sz, + &res); + + pr_info("nvidia-firme: SMC returned: status=%ld written=%lu remaining=%lu\n", + (long)res.a0, res.a1, res.a2); + + if ((long)res.a0 == FIRME_RETRY) { + pr_warn("nvidia-firme: ATF returned RETRY (%d/%d)\n", + retries + 1, FIRME_MAX_RETRIES); + if (++retries > FIRME_MAX_RETRIES) { + pr_err("nvidia-firme: too many RETRYs\n"); + free_pages((unsigned long)buf, + get_order(FIRME_BUF_SIZE)); + return -EAGAIN; + } + cond_resched(); + goto retry; + } + + if ((long)res.a0 == FIRME_ABORTED) { + pr_err("nvidia-firme: ATF returned ABORTED (-3). PSC token not available (PSC task may not be running)\n"); + free_pages((unsigned long)buf, get_order(FIRME_BUF_SIZE)); + return -ENODATA; + } + + if ((long)res.a0 != FIRME_SUCCESS && + (long)res.a0 != FIRME_INCOMPLETE) { + pr_err("nvidia-firme: SMC failed, status=%ld (NOT_SUPPORTED=%ld, INVALID_PARAMS=%ld, DENIED=%ld)\n", + (long)res.a0, FIRME_NOT_SUPPORTED, + FIRME_INVALID_PARAMETERS, (long)-5); + free_pages((unsigned long)buf, get_order(FIRME_BUF_SIZE)); + return -EIO; + } + + /* res.a1 = bytes written this call, starting at offset in buf */ + if (token_size + res.a1 > FIRME_MAX_TOKEN_SIZE) { + pr_err("nvidia-firme: token exceeds max size\n"); + free_pages((unsigned long)buf, get_order(FIRME_BUF_SIZE)); + return -ENOSPC; + } + + memcpy(&token[token_size], buf + offset, res.a1); + token_size += res.a1; + offset += res.a1; + + /* Clear challenge after first call */ + challenge_sz = 0; + } while (res.a2 > 0); /* res.a2 = remaining bytes */ + + free_pages((unsigned long)buf, get_order(FIRME_BUF_SIZE)); + + report->outblob = no_free_ptr(token); + report->outblob_len = token_size; + + pr_info("nvidia-firme: attestation token retrieved, %zu bytes\n", + token_size); + return 0; +} + +static const struct tsm_report_ops nvidia_firme_tsm_ops = { + .name = KBUILD_MODNAME, + .report_new = nvidia_firme_report_new, +}; + +/* ================================================================ + * TSM Measurement Registers (MR) — FIRME_ATTEST_EXT_CLAIMS backend + * + * Exposes PSC measurement slots via sysfs. Writing to a slot sends + * data to PSC by calling FIRME_ATTEST_EXT_CLAIMS SMC (0xC400040B). + * + * Slot 0 (bmdr): 100-byte BMDR device report per GPU + * [0-47] identity_digest (SHA-384 of device cert chain) + * [48-95] mexchange_digest (SHA-384 of SPDM measurement exchange) + * [96] gpu_device_id + * [97-99] reserved (0) + * + * Slot 1+ (rem0-rem2): 48-byte SHA-384 extensible measurement slots + * ================================================================ */ + +#define FIRME_MR_NUM_SLOTS 4 + +static u8 firme_mr_bmdr_value[FIRME_BMDR_SIZE]; +static u8 firme_mr_rem_values[FIRME_MR_NUM_SLOTS - 1][SHA384_DIGEST_SIZE]; + +/** + * firme_mr_extend - Submit data to PSC via FIRME_ATTEST_EXT_CLAIMS SMC. + * + * For slot 0 (bmdr): sends a 100-byte BMDR device report per GPU. + * For slot 1+ (rem): sends a 48-byte SHA-384 digest for REM extension. + */ +static int firme_mr_extend(const struct tsm_measurements *tm, + const struct tsm_measurement_register *mr, + const u8 *data) +{ + struct arm_smccc_res res; + unsigned int slot_index = mr - tm->mrs; + void *buf; + phys_addr_t buf_phys; + + buf = (void *)get_zeroed_page(GFP_KERNEL); + if (!buf) + return -ENOMEM; + + memcpy(buf, data, mr->mr_size); + buf_phys = virt_to_phys(buf); + + pr_info("nvidia-firme: EXT_CLAIMS SMC: slot=%u buf=0x%llx size=%u\n", + slot_index, (u64)buf_phys, mr->mr_size); + + arm_smccc_1_1_invoke(FIRME_ATTEST_EXTEND, + buf_phys, + mr->mr_size, + slot_index, + 0, + &res); + + free_page((unsigned long)buf); + + pr_info("nvidia-firme: EXT_CLAIMS returned status=%ld\n", (long)res.a0); + + if ((long)res.a0 == FIRME_SUCCESS) { + memcpy(mr->mr_value, data, mr->mr_size); + return 0; + } + + if ((long)res.a0 == FIRME_INVALID_PARAMETERS) { + pr_err("nvidia-firme: EXT_CLAIMS failed: invalid parameters (slot=%u size=%u)\n", + slot_index, mr->mr_size); + return -EINVAL; + } + + if ((long)res.a0 == FIRME_DENIED) { + pr_err("nvidia-firme: EXT_CLAIMS failed: denied (slot %u)\n", + slot_index); + return -EACCES; + } + + pr_err("nvidia-firme: EXT_CLAIMS failed: status=%ld\n", (long)res.a0); + return -EIO; +} + +static int firme_mr_refresh(const struct tsm_measurements *tm) +{ + /* MR values are cached locally after each extend call. + * PSC doesn't provide a read-back API, so refresh is a no-op. */ + return 0; +} + +static struct tsm_measurement_register firme_mrs[FIRME_MR_NUM_SLOTS] = { + { + .mr_name = "bmdr", + .mr_value = firme_mr_bmdr_value, + .mr_size = FIRME_BMDR_SIZE, + .mr_flags = TSM_MR_F_READABLE | TSM_MR_F_WRITABLE + | TSM_MR_F_NOHASH, + .mr_hash = 0, + }, + { + .mr_name = "rem0", + .mr_value = firme_mr_rem_values[0], + .mr_size = SHA384_DIGEST_SIZE, + .mr_flags = TSM_MR_F_READABLE | TSM_MR_F_WRITABLE, + .mr_hash = HASH_ALGO_SHA384, + }, + { + .mr_name = "rem1", + .mr_value = firme_mr_rem_values[1], + .mr_size = SHA384_DIGEST_SIZE, + .mr_flags = TSM_MR_F_READABLE | TSM_MR_F_WRITABLE, + .mr_hash = HASH_ALGO_SHA384, + }, + { + .mr_name = "rem2", + .mr_value = firme_mr_rem_values[2], + .mr_size = SHA384_DIGEST_SIZE, + .mr_flags = TSM_MR_F_READABLE | TSM_MR_F_WRITABLE, + .mr_hash = HASH_ALGO_SHA384, + }, +}; + +static struct tsm_measurements firme_measurements = { + .mrs = firme_mrs, + .nr_mrs = FIRME_MR_NUM_SLOTS, + .refresh = firme_mr_refresh, + .write = firme_mr_extend, +}; + +static const struct attribute_group *firme_mr_grp; +static struct kobject *firme_kobj; + +/* ================================================================ + * Module init/exit + * ================================================================ */ + +static int __init nvidia_firme_init(void) +{ + int ret; + + if (!nvidia_firme_available()) { + pr_info("nvidia-firme: FIRME service not available\n"); + return -ENODEV; + } + + ret = tsm_report_register(&nvidia_firme_tsm_ops, NULL); + if (ret < 0) { + pr_err("nvidia-firme: failed to register TSM reports (%d)\n", ret); + return ret; + } + + firme_mr_grp = tsm_mr_create_attribute_group(&firme_measurements); + if (IS_ERR(firme_mr_grp)) { + pr_err("nvidia-firme: failed to create TSM MR group (%ld)\n", + PTR_ERR(firme_mr_grp)); + firme_mr_grp = NULL; + } else { + firme_kobj = kobject_create_and_add("nvidia-firme", + firmware_kobj); + if (!firme_kobj) { + pr_err("nvidia-firme: failed to create sysfs kobject\n"); + tsm_mr_free_attribute_group(firme_mr_grp); + firme_mr_grp = NULL; + } else { + ret = sysfs_create_group(firme_kobj, firme_mr_grp); + if (ret) { + pr_err("nvidia-firme: failed to create MR sysfs group (%d)\n", ret); + kobject_put(firme_kobj); + firme_kobj = NULL; + tsm_mr_free_attribute_group(firme_mr_grp); + firme_mr_grp = NULL; + } else { + pr_info("nvidia-firme: measurement registers at /sys/firmware/nvidia-firme/\n"); + } + } + } + + pr_info("nvidia-firme: registered with TSM framework\n"); + return 0; +} +module_init(nvidia_firme_init); + +static void __exit nvidia_firme_exit(void) +{ + if (firme_mr_grp && firme_kobj) + sysfs_remove_group(firme_kobj, firme_mr_grp); + if (firme_kobj) + kobject_put(firme_kobj); + if (firme_mr_grp) + tsm_mr_free_attribute_group(firme_mr_grp); + tsm_report_unregister(&nvidia_firme_tsm_ops); + pr_info("nvidia-firme: unregistered from TSM\n"); +} +module_exit(nvidia_firme_exit); + +MODULE_AUTHOR("Hyder Ali "); +MODULE_DESCRIPTION("NVIDIA FIRME (DEN0149) Attestation and Measurement Driver"); +MODULE_LICENSE("GPL"); diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index aae657fd56c0e..57a4d6254bc98 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -12,6 +12,7 @@ #include #include #include +#include static bool fuse_use_readdirplus(struct inode *dir, struct dir_context *ctx) { @@ -335,6 +336,43 @@ static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file, return 0; } +static struct page **fuse_readdir_alloc_buf(struct fuse_args_pages *ap, size_t *bufsize) +{ + unsigned int i, nr_alloc, nr_pages = DIV_ROUND_UP(*bufsize, PAGE_SIZE); + struct page **pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL); + + if (!pages) + return NULL; + + nr_alloc = alloc_pages_bulk(GFP_KERNEL, nr_pages, pages); + if (!nr_alloc) + goto free_array; + + if (nr_alloc < nr_pages) { + nr_pages = nr_alloc; + *bufsize = (size_t) nr_pages << PAGE_SHIFT; + } + + ap->folios = fuse_folios_alloc(nr_pages, GFP_KERNEL, &ap->descs); + if (!ap->folios) + goto release_pages; + + for (i = 0; i < nr_pages; i++) { + ap->folios[i] = page_folio(pages[i]); + ap->descs[i].length = min_t(size_t, *bufsize - (size_t)i * PAGE_SIZE, PAGE_SIZE); + } + ap->num_folios = nr_pages; + ap->args.out_pages = true; + + return pages; + +release_pages: + release_pages(pages, nr_pages); +free_array: + kfree(pages); + return NULL; +} + static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) { int plus; @@ -343,18 +381,16 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) struct fuse_mount *fm = get_fuse_mount(inode); struct fuse_conn *fc = fm->fc; struct fuse_io_args ia = {}; - struct fuse_args *args = &ia.ap.args; + struct fuse_args_pages *ap = &ia.ap; void *buf; size_t bufsize = clamp((unsigned int) ctx->count, PAGE_SIZE, fc->max_pages << PAGE_SHIFT); u64 attr_version = 0, evict_ctr = 0; bool locked; + struct page **pages = fuse_readdir_alloc_buf(ap, &bufsize); - buf = kvmalloc(bufsize, GFP_KERNEL); - if (!buf) + if (!pages) return -ENOMEM; - args->out_args[0].value = buf; - plus = fuse_use_readdirplus(inode, ctx); if (plus) { attr_version = fuse_get_attr_version(fm->fc); @@ -364,24 +400,37 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) fuse_read_args_fill(&ia, file, ctx->pos, bufsize, FUSE_READDIR); } locked = fuse_lock_inode(inode); - res = fuse_simple_request(fm, args); + res = fuse_simple_request(fm, &ap->args); fuse_unlock_inode(inode, locked); - if (res >= 0) { - if (!res) { - struct fuse_file *ff = file->private_data; - - if (ff->open_flags & FOPEN_CACHE_DIR) - fuse_readdir_cache_end(file, ctx->pos); - } else if (plus) { - res = parse_dirplusfile(buf, res, file, ctx, attr_version, - evict_ctr); - } else { + if (res < 0) + goto out; + + if (!res) { + struct fuse_file *ff = file->private_data; + + if (ff->open_flags & FOPEN_CACHE_DIR) + fuse_readdir_cache_end(file, ctx->pos); + goto out; + } + + buf = vm_map_ram(pages, ap->num_folios, -1); + if (!buf) { + res = -ENOMEM; + } else { + if (plus) + res = parse_dirplusfile(buf, res, file, ctx, attr_version, evict_ctr); + else res = parse_dirfile(buf, res, file, ctx); - } + + vm_unmap_ram(buf, ap->num_folios); } +out: + kfree(ap->folios); + release_pages(pages, ap->num_folios); + kfree(pages); - kvfree(buf); fuse_invalidate_atime(inode); + return res; } diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c index 037df47fa9f3a..272ea39e3c999 100644 --- a/fs/ntfs3/fslog.c +++ b/fs/ntfs3/fslog.c @@ -1172,7 +1172,7 @@ static int read_log_page(struct ntfs_log *log, u32 vbo, goto out; if (page_buf->rhdr.sign != NTFS_FFFF_SIGNATURE) - ntfs_fix_post_read(&page_buf->rhdr, PAGE_SIZE, false); + ntfs_fix_post_read(&page_buf->rhdr, log->page_size, false); if (page_buf != *buffer) memcpy(*buffer, Add2Ptr(page_buf, page_off), bytes); @@ -3796,11 +3796,7 @@ int log_replay(struct ntfs_inode *ni, bool *initialized) log->l_size = log->orig_file_size = ni->vfs_inode.i_size; /* Get the size of page. NOTE: To replay we can use default page. */ -#if PAGE_SIZE >= DefaultLogPageSize && PAGE_SIZE <= DefaultLogPageSize * 2 log->page_size = norm_file_page(PAGE_SIZE, &log->l_size, true); -#else - log->page_size = norm_file_page(PAGE_SIZE, &log->l_size, false); -#endif if (!log->page_size) { err = -EINVAL; goto out; diff --git a/fs/overlayfs/readdir.c b/fs/overlayfs/readdir.c index 1dcc75b3a90f9..e7fe29cb6028b 100644 --- a/fs/overlayfs/readdir.c +++ b/fs/overlayfs/readdir.c @@ -838,15 +838,14 @@ static int ovl_iterate_merged(struct file *file, struct dir_context *ctx) struct ovl_dir_file *od = file->private_data; struct dentry *dentry = file->f_path.dentry; struct ovl_cache_entry *p; - int err = 0; + int err; if (!od->cache) { struct ovl_dir_cache *cache; cache = ovl_cache_get(dentry); - err = PTR_ERR(cache); if (IS_ERR(cache)) - return err; + return PTR_ERR(cache); od->cache = cache; ovl_seek_cursor(od, ctx->pos); @@ -869,7 +868,7 @@ static int ovl_iterate_merged(struct file *file, struct dir_context *ctx) od->cursor = p->l_node.next; ctx->pos++; } - return err; + return 0; } static bool ovl_need_adjust_d_ino(struct file *file) diff --git a/fs/resctrl/ctrlmondata.c b/fs/resctrl/ctrlmondata.c index cc4237c57cbe4..ec925ce6c8773 100644 --- a/fs/resctrl/ctrlmondata.c +++ b/fs/resctrl/ctrlmondata.c @@ -40,19 +40,11 @@ typedef int (ctrlval_parser_t)(struct rdt_parse_data *data, * hardware. The allocated bandwidth percentage is rounded to the next * control step available on the hardware. */ -static bool bw_validate(char *buf, u32 *data, struct rdt_resource *r) +static bool bw_validate(char *buf, u32 *data, struct resctrl_schema *s) { int ret; u32 bw; - /* - * Only linear delay values is supported for current Intel SKUs. - */ - if (!r->membw.delay_linear && r->membw.arch_needs_linear) { - rdt_last_cmd_puts("No support for non-linear MB domains\n"); - return false; - } - ret = kstrtou32(buf, 10, &bw); if (ret) { rdt_last_cmd_printf("Invalid MB value %s\n", buf); @@ -60,18 +52,18 @@ static bool bw_validate(char *buf, u32 *data, struct rdt_resource *r) } /* Nothing else to do if software controller is enabled. */ - if (is_mba_sc(r)) { + if (is_mba_sc(s->res)) { *data = bw; return true; } - if (bw < r->membw.min_bw || bw > r->membw.max_bw) { + if (bw < s->membw.min_bw || bw > s->membw.max_bw) { rdt_last_cmd_printf("MB value %u out of range [%d,%d]\n", - bw, r->membw.min_bw, r->membw.max_bw); + bw, s->membw.min_bw, s->membw.max_bw); return false; } - *data = roundup(bw, (unsigned long)r->membw.bw_gran); + *data = resctrl_arch_round_bw(bw, s); return true; } @@ -83,13 +75,7 @@ static int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s, u32 closid = data->closid; u32 bw_val; - cfg = &d->staged_config[s->conf_type]; - if (cfg->have_new_ctrl) { - rdt_last_cmd_printf("Duplicate domain %d\n", d->hdr.id); - return -EINVAL; - } - - if (!bw_validate(data->buf, &bw_val, r)) + if (!bw_validate(data->buf, &bw_val, s)) return -EINVAL; if (is_mba_sc(r)) { @@ -97,6 +83,7 @@ static int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s, return 0; } + cfg = &d->staged_config[s->conf_type]; cfg->new_ctrl = bw_val; cfg->have_new_ctrl = true; @@ -164,12 +151,6 @@ static int parse_cbm(struct rdt_parse_data *data, struct resctrl_schema *s, u32 closid = data->closid; u32 cbm_val; - cfg = &d->staged_config[s->conf_type]; - if (cfg->have_new_ctrl) { - rdt_last_cmd_printf("Duplicate domain %d\n", d->hdr.id); - return -EINVAL; - } - /* * Cannot set up more than one pseudo-locked region in a cache * hierarchy. @@ -206,6 +187,7 @@ static int parse_cbm(struct rdt_parse_data *data, struct resctrl_schema *s, } } + cfg = &d->staged_config[s->conf_type]; cfg->new_ctrl = cbm_val; cfg->have_new_ctrl = true; @@ -233,11 +215,14 @@ static int parse_line(char *line, struct resctrl_schema *s, /* Walking r->domains, ensure it can't race with cpuhp */ lockdep_assert_cpus_held(); - switch (r->schema_fmt) { + switch (s->schema_fmt) { case RESCTRL_SCHEMA_BITMAP: parse_ctrlval = &parse_cbm; break; case RESCTRL_SCHEMA_RANGE: + case RESCTRL_SCHEMA_PERCENT: + case RESCTRL_SCHEMA_MBPS: + case RESCTRL_SCHEMA__AMD_MBA: parse_ctrlval = &parse_bw; break; } @@ -251,6 +236,15 @@ static int parse_line(char *line, struct resctrl_schema *s, return -EINVAL; } + /* + * Only linear delay values is supported for current Intel SKUs. + */ + if (r->rid == RDT_RESOURCE_MBA && + !r->mba.delay_linear && r->mba.arch_needs_linear) { + rdt_last_cmd_puts("No support for non-linear MB domains\n"); + return -EINVAL; + } + next: if (!line || line[0] == '\0') return 0; @@ -263,13 +257,18 @@ static int parse_line(char *line, struct resctrl_schema *s, dom = strim(dom); list_for_each_entry(d, &r->ctrl_domains, hdr.list) { if (d->hdr.id == dom_id) { + cfg = &d->staged_config[t]; + if (cfg->have_new_ctrl) { + rdt_last_cmd_printf("Duplicate domain %d\n", d->hdr.id); + return -EINVAL; + } + data.buf = dom; data.closid = rdtgrp->closid; data.mode = rdtgrp->mode; if (parse_ctrlval(&data, s, d)) return -EINVAL; if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { - cfg = &d->staged_config[t]; /* * In pseudo-locking setup mode and just * parsed a valid CBM that should be @@ -954,25 +953,34 @@ static int resctrl_io_alloc_parse_line(char *line, struct rdt_resource *r, struct resctrl_schema *s, u32 closid) { enum resctrl_conf_type peer_type; + unsigned long dom_id = ULONG_MAX; struct rdt_parse_data data; struct rdt_ctrl_domain *d; + bool update_all = false; char *dom = NULL, *id; - unsigned long dom_id; next: if (!line || line[0] == '\0') return 0; + if (update_all) { + rdt_last_cmd_puts("Configurations after global '*'\n"); + return -EINVAL; + } + dom = strsep(&line, ";"); id = strsep(&dom, "="); - if (!dom || kstrtoul(id, 10, &dom_id)) { + + if (dom && !strcmp(id, "*")) { + update_all = true; + } else if (!dom || kstrtoul(id, 10, &dom_id)) { rdt_last_cmd_puts("Missing '=' or non-numeric domain\n"); return -EINVAL; } dom = strim(dom); list_for_each_entry(d, &r->ctrl_domains, hdr.list) { - if (d->hdr.id == dom_id) { + if (update_all || d->hdr.id == dom_id) { data.buf = dom; data.mode = RDT_MODE_SHAREABLE; data.closid = closid; @@ -988,10 +996,15 @@ static int resctrl_io_alloc_parse_line(char *line, struct rdt_resource *r, &d->staged_config[s->conf_type], sizeof(d->staged_config[0])); } - goto next; + if (!update_all) + goto next; } } + if (update_all) + goto next; + + rdt_last_cmd_printf("Invalid domain %lu\n", dom_id); return -EINVAL; } diff --git a/fs/resctrl/internal.h b/fs/resctrl/internal.h index 1a9b29119f88f..ec7c968059291 100644 --- a/fs/resctrl/internal.h +++ b/fs/resctrl/internal.h @@ -251,6 +251,7 @@ struct rdtgroup { #define RFTYPE_TOP BIT(6) +/* files that are specific to a type of resource, e.g. throttle_mode */ #define RFTYPE_RES_CACHE BIT(8) #define RFTYPE_RES_MB BIT(9) @@ -261,6 +262,11 @@ struct rdtgroup { #define RFTYPE_RES_PERF_PKG BIT(12) +/* files that are specific to a type of control, e.g. percent_min */ +#define RFTYPE_SCHEMA_BITMAP BIT(13) +#define RFTYPE_SCHEMA_PERCENT BIT(14) +#define RFTYPE_SCHEMA_MBPS BIT(15) + #define RFTYPE_CTRL_INFO (RFTYPE_INFO | RFTYPE_CTRL) #define RFTYPE_MON_INFO (RFTYPE_INFO | RFTYPE_MON) @@ -408,6 +414,8 @@ void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free); void resctrl_file_fflags_init(const char *config, unsigned long fflags); +void resctrl_file_mode_init(const char *config, umode_t mode); + void rdt_staged_configs_clear(void); bool closid_allocated(unsigned int closid); diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c index 49f3f6b846b27..2378017fb3b7e 100644 --- a/fs/resctrl/monitor.c +++ b/fs/resctrl/monitor.c @@ -234,6 +234,8 @@ static struct rmid_entry *resctrl_find_free_rmid(u32 closid) * * When the CLOSID and RMID are independent numbers, the first free CLOSID will * be returned. + * + * Return: Free CLOSID on success, < 0 on failure. */ int resctrl_find_cleanest_closid(void) { @@ -893,8 +895,10 @@ void mbm_setup_overflow_handler(struct rdt_l3_mon_domain *dom, unsigned long del /* * When a domain comes online there is no guarantee the filesystem is * mounted. If not, there is no need to catch counter overflow. + * Some architecture may have ~64bit counters, and can ignore overflow. */ - if (!resctrl_mounted || !resctrl_arch_mon_capable()) + if (!resctrl_mounted || !resctrl_arch_mon_capable() || + !resctrl_arch_mon_can_overflow()) return; cpu = cpumask_any_housekeeping(&dom->hdr.cpu_mask, exclude_cpu); dom->mbm_work_cpu = cpu; @@ -1209,9 +1213,10 @@ static int rdtgroup_alloc_assign_cntr(struct rdt_resource *r, struct rdt_l3_mon_ * NULL; otherwise, assign the counter to the specified domain @d. * * If all counters in a domain are already in use, rdtgroup_alloc_assign_cntr() - * will fail. The assignment process will abort at the first failure encountered - * during domain traversal, which may result in the event being only partially - * assigned. + * will fail. When attempting to assign counters to all domains, carry on trying + * to assign counters after a failure since only some domains may have counters + * and the goal is to assign counters where possible. If any counter assignment + * fails, return the error from the last failing assignment. * * Return: * 0 on success, < 0 on failure. @@ -1224,9 +1229,11 @@ static int rdtgroup_assign_cntr_event(struct rdt_l3_mon_domain *d, struct rdtgro if (!d) { list_for_each_entry(d, &r->mon_domains, hdr.list) { - ret = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt); - if (ret) - return ret; + int err; + + err = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt); + if (err) + ret = err; } } else { ret = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt); @@ -1420,6 +1427,11 @@ ssize_t event_filter_write(struct kernfs_open_file *of, char *buf, size_t nbytes ret = -EINVAL; goto out_unlock; } + if (!r->mon.mbm_cntr_configurable) { + rdt_last_cmd_puts("event_filter is not configurable\n"); + ret = -EPERM; + goto out_unlock; + } ret = resctrl_parse_mem_transactions(buf, &evt_cfg); if (!ret && mevt->evt_cfg != evt_cfg) { @@ -1449,7 +1461,7 @@ int resctrl_mbm_assign_mode_show(struct kernfs_open_file *of, else seq_puts(s, "[default]\n"); - if (!IS_ENABLED(CONFIG_RESCTRL_ASSIGN_FIXED)) { + if (!r->mon.mbm_cntr_assign_fixed) { if (enabled) seq_puts(s, "default\n"); else @@ -1500,6 +1512,12 @@ ssize_t resctrl_mbm_assign_mode_write(struct kernfs_open_file *of, char *buf, } if (enable != resctrl_arch_mbm_cntr_assign_enabled(r)) { + if (r->mon.mbm_cntr_assign_fixed) { + ret = -EINVAL; + rdt_last_cmd_puts("Counter assignment mode is not configurable\n"); + goto out_unlock; + } + ret = resctrl_arch_mbm_cntr_assign_set(r, enable); if (ret) goto out_unlock; @@ -1884,6 +1902,8 @@ int resctrl_l3_mon_resource_init(void) resctrl_file_fflags_init("available_mbm_cntrs", RFTYPE_MON_INFO | RFTYPE_RES_CACHE); resctrl_file_fflags_init("event_filter", RFTYPE_ASSIGN_CONFIG); + if (r->mon.mbm_cntr_configurable) + resctrl_file_mode_init("event_filter", 0644); resctrl_file_fflags_init("mbm_assign_on_mkdir", RFTYPE_MON_INFO | RFTYPE_RES_CACHE); resctrl_file_fflags_init("mbm_L3_assignments", RFTYPE_MON_BASE); diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index 5da305bd36c96..f7848f20599a1 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -1004,9 +1004,8 @@ static int rdt_default_ctrl_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); - struct rdt_resource *r = s->res; - seq_printf(seq, "%x\n", resctrl_get_default_ctrl(r)); + seq_printf(seq, "%x\n", resctrl_get_schema_default_ctrl(s)); return 0; } @@ -1147,9 +1146,8 @@ static int rdt_min_bw_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); - struct rdt_resource *r = s->res; - seq_printf(seq, "%u\n", r->membw.min_bw); + seq_printf(seq, "%u\n", s->membw.min_bw); return 0; } @@ -1185,9 +1183,8 @@ static int rdt_bw_gran_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); - struct rdt_resource *r = s->res; - seq_printf(seq, "%u\n", r->membw.bw_gran); + seq_printf(seq, "%u\n", s->membw.bw_gran); return 0; } @@ -1197,7 +1194,7 @@ static int rdt_delay_linear_show(struct kernfs_open_file *of, struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); struct rdt_resource *r = s->res; - seq_printf(seq, "%u\n", r->membw.delay_linear); + seq_printf(seq, "%u\n", r->mba.delay_linear); return 0; } @@ -1215,7 +1212,7 @@ static int rdt_thread_throttle_mode_show(struct kernfs_open_file *of, struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); struct rdt_resource *r = s->res; - switch (r->membw.throttle_mode) { + switch (r->mba.throttle_mode) { case THREAD_THROTTLE_PER_THREAD: seq_puts(seq, "per-thread\n"); return 0; @@ -1519,6 +1516,8 @@ static ssize_t rdtgroup_mode_write(struct kernfs_open_file *of, * * @cbm is unsigned long, even if only 32 bits are used to make the * bitmap functions work correctly. + * + * Return: Size (in bytes) of cache portion represented by CBM, 0 on failure. */ unsigned int rdtgroup_cbm_to_size(struct rdt_resource *r, struct rdt_ctrl_domain *d, unsigned long cbm) @@ -1550,7 +1549,7 @@ bool is_mba_sc(struct rdt_resource *r) if (r->rid != RDT_RESOURCE_MBA) return false; - return r->membw.mba_sc; + return r->mba.mba_sc; } /* @@ -1688,6 +1687,33 @@ static int mbm_local_bytes_config_show(struct kernfs_open_file *of, return 0; } +static int resctrl_schema_format_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); + + switch (s->schema_fmt) { + case RESCTRL_SCHEMA_BITMAP: + seq_puts(seq, "bitmap\n"); + break; + case RESCTRL_SCHEMA_PERCENT: + seq_puts(seq, "percentage\n"); + break; + case RESCTRL_SCHEMA_MBPS: + seq_puts(seq, "mbps\n"); + break; + case RESCTRL_SCHEMA_RANGE: + seq_puts(seq, "range\n"); + break; + /* The way these schema behave isn't discoverable from resctrl */ + case RESCTRL_SCHEMA__AMD_MBA: + seq_puts(seq, "platform\n"); + break; + } + + return 0; +} + static void mbm_config_write_domain(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 evtid, u32 val) { @@ -1929,6 +1955,13 @@ static struct rftype res_common_files[] = { .kf_ops = &rdtgroup_kf_single_ops, .seq_show = resctrl_num_mbm_cntrs_show, }, + { + .name = "bitmap_mask", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_default_ctrl_show, + .fflags = RFTYPE_CTRL_INFO | RFTYPE_SCHEMA_BITMAP, + }, { .name = "min_cbm_bits", .mode = 0444, @@ -1936,6 +1969,13 @@ static struct rftype res_common_files[] = { .seq_show = rdt_min_cbm_bits_show, .fflags = RFTYPE_CTRL_INFO | RFTYPE_RES_CACHE, }, + { + .name = "bitmaps_min_bits", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_min_cbm_bits_show, + .fflags = RFTYPE_CTRL_INFO | RFTYPE_SCHEMA_BITMAP, + }, { .name = "shareable_bits", .mode = 0444, @@ -1957,6 +1997,13 @@ static struct rftype res_common_files[] = { .seq_show = rdt_min_bw_show, .fflags = RFTYPE_CTRL_INFO | RFTYPE_RES_MB, }, + { + .name = "percent_min", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_min_bw_show, + .fflags = RFTYPE_CTRL_INFO | RFTYPE_SCHEMA_PERCENT, + }, { .name = "bandwidth_gran", .mode = 0444, @@ -1964,6 +2011,13 @@ static struct rftype res_common_files[] = { .seq_show = rdt_bw_gran_show, .fflags = RFTYPE_CTRL_INFO | RFTYPE_RES_MB, }, + { + .name = "percent_gran", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_bw_gran_show, + .fflags = RFTYPE_CTRL_INFO | RFTYPE_SCHEMA_PERCENT, + }, { .name = "delay_linear", .mode = 0444, @@ -2020,7 +2074,7 @@ static struct rftype res_common_files[] = { }, { .name = "event_filter", - .mode = 0644, + .mode = 0444, .kf_ops = &rdtgroup_kf_single_ops, .seq_show = event_filter_show, .write = event_filter_write, @@ -2116,6 +2170,14 @@ static struct rftype res_common_files[] = { .seq_show = rdtgroup_closid_show, .fflags = RFTYPE_CTRL_BASE | RFTYPE_DEBUG, }, + { + .name = "schema_format", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = resctrl_schema_format_show, + .fflags = RFTYPE_CTRL_INFO, + }, + }; static int rdtgroup_add_files(struct kernfs_node *kn, unsigned long fflags) @@ -2172,13 +2234,13 @@ static void thread_throttle_mode_init(void) r_mba = resctrl_arch_get_resource(RDT_RESOURCE_MBA); if (r_mba->alloc_capable && - r_mba->membw.throttle_mode != THREAD_THROTTLE_UNDEFINED) - throttle_mode = r_mba->membw.throttle_mode; + r_mba->mba.throttle_mode != THREAD_THROTTLE_UNDEFINED) + throttle_mode = r_mba->mba.throttle_mode; r_smba = resctrl_arch_get_resource(RDT_RESOURCE_SMBA); if (r_smba->alloc_capable && - r_smba->membw.throttle_mode != THREAD_THROTTLE_UNDEFINED) - throttle_mode = r_smba->membw.throttle_mode; + r_smba->mba.throttle_mode != THREAD_THROTTLE_UNDEFINED) + throttle_mode = r_smba->mba.throttle_mode; if (throttle_mode == THREAD_THROTTLE_UNDEFINED) return; @@ -2213,6 +2275,15 @@ void resctrl_file_fflags_init(const char *config, unsigned long fflags) rft->fflags = fflags; } +void resctrl_file_mode_init(const char *config, umode_t mode) +{ + struct rftype *rft; + + rft = rdtgroup_get_rftype_by_name(config); + if (rft) + rft->mode = mode; +} + /** * rdtgroup_kn_mode_restrict - Restrict user access to named resctrl file * @r: The resource group with which the file is associated. @@ -2329,22 +2400,19 @@ static int resctrl_mkdir_event_configs(struct rdt_resource *r, struct kernfs_nod continue; kn_subdir2 = kernfs_create_dir(kn_subdir, mevt->name, kn_subdir->mode, mevt); - if (IS_ERR(kn_subdir2)) { - ret = PTR_ERR(kn_subdir2); - goto out; - } + if (IS_ERR(kn_subdir2)) + return PTR_ERR(kn_subdir2); ret = rdtgroup_kn_set_ugid(kn_subdir2); if (ret) - goto out; + return ret; ret = rdtgroup_add_files(kn_subdir2, RFTYPE_ASSIGN_CONFIG); if (ret) - break; + return ret; } -out: - return ret; + return 0; } static int rdtgroup_mkdir_info_resdir(void *priv, char *name, @@ -2400,7 +2468,35 @@ static unsigned long fflags_from_resource(struct rdt_resource *r) return RFTYPE_RES_PERF_PKG; } - return WARN_ON_ONCE(1); + return 0; +} + +static u32 fflags_from_schema(struct resctrl_schema *s) +{ + struct rdt_resource *r = s->res; + u32 fflags = 0; + + /* Some resources are configured purely from their rid */ + fflags |= fflags_from_resource(r); + if (fflags) + return fflags; + + switch (s->schema_fmt) { + case RESCTRL_SCHEMA_BITMAP: + fflags |= RFTYPE_SCHEMA_BITMAP; + break; + case RESCTRL_SCHEMA_PERCENT: + fflags |= RFTYPE_SCHEMA_PERCENT; + break; + case RESCTRL_SCHEMA_MBPS: + fflags |= RFTYPE_SCHEMA_MBPS; + break; + case RESCTRL_SCHEMA__AMD_MBA: + /* No standard files are exposed */ + break; + } + + return fflags; } static int rdtgroup_create_info_dir(struct kernfs_node *parent_kn) @@ -2423,7 +2519,7 @@ static int rdtgroup_create_info_dir(struct kernfs_node *parent_kn) /* loop over enabled controls, these are all alloc_capable */ list_for_each_entry(s, &resctrl_schema_all, list) { r = s->res; - fflags = fflags_from_resource(r) | RFTYPE_CTRL_INFO; + fflags = fflags_from_schema(s) | RFTYPE_CTRL_INFO; ret = rdtgroup_mkdir_info_resdir(s, s->name, fflags); if (ret) goto out_destroy; @@ -2480,7 +2576,7 @@ mongroup_create_dir(struct kernfs_node *parent_kn, struct rdtgroup *prgrp, static inline bool is_mba_linear(void) { - return resctrl_arch_get_resource(RDT_RESOURCE_MBA)->membw.delay_linear; + return resctrl_arch_get_resource(RDT_RESOURCE_MBA)->mba.delay_linear; } static int mba_sc_domain_allocate(struct rdt_resource *r, struct rdt_ctrl_domain *d) @@ -2508,10 +2604,13 @@ static void mba_sc_domain_destroy(struct rdt_resource *r, } /* - * MBA software controller is supported only if - * MBM is supported and MBA is in linear scale, - * and the MBM monitor scope is the same as MBA - * control scope. + * The MBA software controller is supported only if MBM is supported and MBA is + * in linear scale, and the MBM monitor scope is the same as MBA control scope. + * + * The software controller cannot be supported when the MBM counters are + * assignable. There is no guarantee that MBM counters are assigned to the + * event backing the software controller in all monitoring domains of all + * monitoring groups. */ static bool supports_mba_mbps(void) { @@ -2520,7 +2619,8 @@ static bool supports_mba_mbps(void) return (resctrl_is_mbm_enabled() && r->alloc_capable && is_mba_linear() && - r->ctrl_scope == rmbm->mon_scope); + r->ctrl_scope == rmbm->mon_scope && + !rmbm->mon.mbm_cntr_assignable); } /* @@ -2538,7 +2638,7 @@ static int set_mba_sc(bool mba_sc) if (!supports_mba_mbps() || mba_sc == is_mba_sc(r)) return -EINVAL; - r->membw.mba_sc = mba_sc; + r->mba.mba_sc = mba_sc; rdtgroup_default.mba_mbps_event = mba_mbps_default_event; @@ -2727,11 +2827,29 @@ static int schemata_list_add(struct rdt_resource *r, enum resctrl_conf_type type if (cl > max_name_width) max_name_width = cl; - switch (r->schema_fmt) { + s->schema_fmt = r->schema_fmt; + s->membw = r->membw; + + /* + * When mba_sc() is enabled the format used by user space is different + * to that expected by hardware. The conversion is done by + * update_mba_bw(). + */ + if (is_mba_sc(r)) { + s->schema_fmt = RESCTRL_SCHEMA_RANGE; + s->membw.min_bw = 0; + s->membw.max_bw = MBA_MAX_MBPS; + s->membw.bw_gran = 1; + } + + switch (s->schema_fmt) { case RESCTRL_SCHEMA_BITMAP: s->fmt_str = "%d=%x"; break; case RESCTRL_SCHEMA_RANGE: + case RESCTRL_SCHEMA_PERCENT: + case RESCTRL_SCHEMA_MBPS: + case RESCTRL_SCHEMA__AMD_MBA: s->fmt_str = "%d=%u"; break; } @@ -2935,7 +3053,7 @@ static int rdt_parse_param(struct fs_context *fc, struct fs_parameter *param) ctx->enable_cdpl2 = true; return 0; case Opt_mba_mbps: - msg = "mba_MBps requires MBM and linear scale MBA at L3 scope"; + msg = "mba_MBps requires MBM (mbm_event mode not supported) and linear scale MBA at L3 scope"; if (!supports_mba_mbps()) return invalfc(fc, msg); ctx->enable_mba_mbps = true; @@ -3102,6 +3220,8 @@ static void rmdir_all_sub(void) * @mevt: The type of event file being created. * @do_sum: Whether SNC summing monitors are being created. Only set * when @rid == RDT_RESOURCE_L3. + * + * Return: Pointer to mon_data private data of the event, NULL on failure. */ static struct mon_data *mon_get_kn_priv(enum resctrl_res_level rid, int domid, struct mon_evt *mevt, @@ -3496,6 +3616,8 @@ static int mkdir_mondata_all(struct kernfs_node *parent_kn, * resource group is initialized. The user can follow this with a * modification to the CBM if the default does not satisfy the * requirements. + * + * Return: A CBM that is valid for resource @r. */ static u32 cbm_ensure_valid(u32 _val, struct rdt_resource *r) { @@ -3626,7 +3748,7 @@ static void rdtgroup_init_mba(struct rdt_resource *r, u32 closid) } cfg = &d->staged_config[CDP_NONE]; - cfg->new_ctrl = resctrl_get_default_ctrl(r); + cfg->new_ctrl = resctrl_get_resource_default_ctrl(r); cfg->have_new_ctrl = true; } } diff --git a/fs/smb/client/cifs_spnego.c b/fs/smb/client/cifs_spnego.c index 3a41bbada04c7..44c4072756804 100644 --- a/fs/smb/client/cifs_spnego.c +++ b/fs/smb/client/cifs_spnego.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -40,12 +41,27 @@ cifs_spnego_key_destroy(struct key *key) kfree(key->payload.data[0]); } +static int +cifs_spnego_key_vet_description(const char *description) +{ + /* + * cifs.spnego descriptions are authority-bearing inputs to cifs.upcall. + * They are only valid when produced by CIFS while using the private + * spnego_cred installed below. Do not let userspace create this type + * of key through request_key(2)/add_key(2), since the helper treats + * pid/uid/creduid/upcall_target as kernel-originating fields. + */ + if (current_cred() != spnego_cred) + return -EPERM; + return 0; +} /* * keytype for CIFS spnego keys */ struct key_type cifs_spnego_key_type = { .name = "cifs.spnego", + .vet_description = cifs_spnego_key_vet_description, .instantiate = cifs_spnego_key_instantiate, .destroy = cifs_spnego_key_destroy, .describe = user_describe, diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c index 4ec204d2c7742..fcf135b6ff808 100644 --- a/fs/smb/client/cifsacl.c +++ b/fs/smb/client/cifsacl.c @@ -1264,6 +1264,17 @@ static int parse_sid(struct smb_sid *psid, char *end_of_acl) return 0; } +static bool dacl_offset_valid(unsigned int acl_len, __u32 dacloffset) +{ + if (acl_len < sizeof(struct smb_acl)) + return false; + + if (dacloffset < sizeof(struct smb_ntsd)) + return false; + + return dacloffset <= acl_len - sizeof(struct smb_acl); +} + /* Convert CIFS ACL to POSIX form */ static int parse_sec_desc(struct cifs_sb_info *cifs_sb, @@ -1284,7 +1295,6 @@ static int parse_sec_desc(struct cifs_sb_info *cifs_sb, group_sid_ptr = (struct smb_sid *)((char *)pntsd + le32_to_cpu(pntsd->gsidoffset)); dacloffset = le32_to_cpu(pntsd->dacloffset); - dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); cifs_dbg(NOISY, "revision %d type 0x%x ooffset 0x%x goffset 0x%x sacloffset 0x%x dacloffset 0x%x\n", pntsd->revision, pntsd->type, le32_to_cpu(pntsd->osidoffset), le32_to_cpu(pntsd->gsidoffset), @@ -1315,11 +1325,18 @@ static int parse_sec_desc(struct cifs_sb_info *cifs_sb, return rc; } - if (dacloffset) + if (dacloffset) { + if (!dacl_offset_valid(acl_len, dacloffset)) { + cifs_dbg(VFS, "Server returned illegal DACL offset\n"); + return -EINVAL; + } + + dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); parse_dacl(dacl_ptr, end_of_acl, owner_sid_ptr, group_sid_ptr, fattr, get_mode_from_special_sid); - else + } else { cifs_dbg(FYI, "no ACL\n"); /* BB grant all or default perms? */ + } return rc; } @@ -1342,6 +1359,11 @@ static int build_sec_desc(struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd, dacloffset = le32_to_cpu(pntsd->dacloffset); if (dacloffset) { + if (!dacl_offset_valid(secdesclen, dacloffset)) { + cifs_dbg(VFS, "Server returned illegal DACL offset\n"); + return -EINVAL; + } + dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); rc = validate_dacl(dacl_ptr, end_of_acl); if (rc) @@ -1710,6 +1732,12 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, nsecdesclen = sizeof(struct smb_ntsd) + (sizeof(struct smb_sid) * 2); dacloffset = le32_to_cpu(pntsd->dacloffset); if (dacloffset) { + if (!dacl_offset_valid(secdesclen, dacloffset)) { + cifs_dbg(VFS, "Server returned illegal DACL offset\n"); + rc = -EINVAL; + goto id_mode_to_cifs_acl_exit; + } + dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); rc = validate_dacl(dacl_ptr, (char *)pntsd + secdesclen); if (rc) { @@ -1752,6 +1780,7 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, rc = ops->set_acl(pnntsd, nsecdesclen, inode, path, aclflag); cifs_dbg(NOISY, "set_cifs_acl rc: %d\n", rc); } +id_mode_to_cifs_acl_exit: cifs_put_tlink(tlink); kfree(pnntsd); diff --git a/fs/smb/client/smb2inode.c b/fs/smb/client/smb2inode.c index fe1c9d7765806..3b09cf8ab0f27 100644 --- a/fs/smb/client/smb2inode.c +++ b/fs/smb/client/smb2inode.c @@ -111,7 +111,7 @@ static int check_wsl_eas(struct kvec *rsp_iov) u32 outlen, next; u16 vlen; u8 nlen; - u8 *end; + u8 *ea_end, *iov_end; outlen = le32_to_cpu(rsp->OutputBufferLength); if (outlen < SMB2_WSL_MIN_QUERY_EA_RESP_SIZE || @@ -120,15 +120,19 @@ static int check_wsl_eas(struct kvec *rsp_iov) ea = (void *)((u8 *)rsp_iov->iov_base + le16_to_cpu(rsp->OutputBufferOffset)); - end = (u8 *)rsp_iov->iov_base + rsp_iov->iov_len; + ea_end = (u8 *)ea + outlen; + iov_end = (u8 *)rsp_iov->iov_base + rsp_iov->iov_len; + if (ea_end > iov_end) + return -EINVAL; + for (;;) { - if ((u8 *)ea > end - sizeof(*ea)) + if ((u8 *)ea > ea_end - sizeof(*ea)) return -EINVAL; nlen = ea->ea_name_length; vlen = le16_to_cpu(ea->ea_value_length); if (nlen != SMB2_WSL_XATTR_NAME_LEN || - (u8 *)ea->ea_data + nlen + 1 + vlen > end) + (u8 *)ea->ea_data + nlen + 1 + vlen > ea_end) return -EINVAL; switch (vlen) { diff --git a/fs/smb/client/smb2misc.c b/fs/smb/client/smb2misc.c index 973fce3c959c4..2a7355ce1a078 100644 --- a/fs/smb/client/smb2misc.c +++ b/fs/smb/client/smb2misc.c @@ -241,7 +241,8 @@ smb2_check_message(char *buf, unsigned int pdu_len, unsigned int len, if (len != calc_len) { /* create failed on symlink */ if (command == SMB2_CREATE_HE && - shdr->Status == STATUS_STOPPED_ON_SYMLINK) + shdr->Status == STATUS_STOPPED_ON_SYMLINK && + len > calc_len) return 0; /* Windows 7 server returns 24 bytes more */ if (calc_len + 24 == len && command == SMB2_OPLOCK_BREAK_HE) diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h index 4d644f03098e3..d8e405becdc31 100644 --- a/include/acpi/cppc_acpi.h +++ b/include/acpi/cppc_acpi.h @@ -42,6 +42,11 @@ #define CPPC_EPP_PERFORMANCE_PREF 0x00 #define CPPC_EPP_ENERGY_EFFICIENCY_PREF 0xFF +#define CPPC_PERF_LIMITED_DESIRED_EXCURSION BIT(0) +#define CPPC_PERF_LIMITED_MINIMUM_EXCURSION BIT(1) +#define CPPC_PERF_LIMITED_MASK (CPPC_PERF_LIMITED_DESIRED_EXCURSION | \ + CPPC_PERF_LIMITED_MINIMUM_EXCURSION) + /* Each register has the folowing format. */ struct cpc_reg { u8 descriptor; @@ -116,6 +121,7 @@ struct cppc_perf_caps { u32 guaranteed_perf; u32 highest_perf; u32 nominal_perf; + u32 reference_perf; u32 lowest_perf; u32 lowest_nonlinear_perf; u32 lowest_freq; @@ -133,7 +139,6 @@ struct cppc_perf_ctrls { struct cppc_perf_fb_ctrs { u64 reference; u64 delivered; - u64 reference_perf; u64 wraparound_time; }; @@ -151,6 +156,7 @@ extern int cppc_get_desired_perf(int cpunum, u64 *desired_perf); extern int cppc_get_nominal_perf(int cpunum, u64 *nominal_perf); extern int cppc_get_highest_perf(int cpunum, u64 *highest_perf); extern int cppc_get_perf_ctrs(int cpu, struct cppc_perf_fb_ctrs *perf_fb_ctrs); +extern int cppc_get_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls); extern int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls); extern int cppc_set_enable(int cpu, bool enable); extern int cppc_get_perf_caps(int cpu, struct cppc_perf_caps *caps); @@ -173,6 +179,8 @@ extern int cppc_get_auto_act_window(int cpu, u64 *auto_act_window); extern int cppc_set_auto_act_window(int cpu, u64 auto_act_window); extern int cppc_get_auto_sel(int cpu, bool *enable); extern int cppc_set_auto_sel(int cpu, bool enable); +extern int cppc_get_perf_limited(int cpu, u64 *perf_limited); +extern int cppc_set_perf_limited(int cpu, u64 bits_to_clear); extern int amd_get_highest_perf(unsigned int cpu, u32 *highest_perf); extern int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator); extern int amd_detect_prefcore(bool *detected); @@ -193,6 +201,10 @@ static inline int cppc_get_perf_ctrs(int cpu, struct cppc_perf_fb_ctrs *perf_fb_ { return -EOPNOTSUPP; } +static inline int cppc_get_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) +{ + return -EOPNOTSUPP; +} static inline int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) { return -EOPNOTSUPP; @@ -265,6 +277,14 @@ static inline int cppc_set_auto_sel(int cpu, bool enable) { return -EOPNOTSUPP; } +static inline int cppc_get_perf_limited(int cpu, u64 *perf_limited) +{ + return -EOPNOTSUPP; +} +static inline int cppc_set_perf_limited(int cpu, u64 bits_to_clear) +{ + return -EOPNOTSUPP; +} static inline int amd_get_highest_perf(unsigned int cpu, u32 *highest_perf) { return -ENODEV; diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 4d2f0bed7a06d..ee3aa5604512a 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1596,10 +1596,26 @@ extern int acpi_ffh_address_space_arch_setup(void *handler_ctxt, void **region_ctxt); extern int acpi_ffh_address_space_arch_handler(acpi_integer *value, void *region_context); +int acpi_ffh_address_space_arch_update_custom_offset_handler( + int (*handler)(struct acpi_ffh_info *info, acpi_integer *value, + void *region_context)); #else static inline void acpi_init_ffh(void) { } #endif +#if defined(CONFIG_ACPI_FFH) && defined(CONFIG_ARM64) +int acpi_arm64_ffh_update_custom_offset_handler( + int (*handler)(struct acpi_ffh_info *info, acpi_integer *value, + void *region_context)); +#else +static inline int acpi_arm64_ffh_update_custom_offset_handler( + int (*handler)(struct acpi_ffh_info *info, acpi_integer *value, + void *region_context)) +{ + return -EOPNOTSUPP; +} +#endif + #ifdef CONFIG_ACPI extern void acpi_device_notify(struct device *dev); extern void acpi_device_notify_remove(struct device *dev); diff --git a/include/linux/arm_mpam.h b/include/linux/arm_mpam.h index 7f00c5285a326..3aed6fe510151 100644 --- a/include/linux/arm_mpam.h +++ b/include/linux/arm_mpam.h @@ -5,6 +5,8 @@ #define __LINUX_ARM_MPAM_H #include +#include +#include #include struct mpam_msc; @@ -49,6 +51,49 @@ static inline int mpam_ris_create(struct mpam_msc *msc, u8 ris_idx, } #endif +bool resctrl_arch_alloc_capable(void); +bool resctrl_arch_mon_capable(void); +bool resctrl_arch_mon_can_overflow(void); + +void resctrl_arch_set_cpu_default_closid(int cpu, u32 closid); +void resctrl_arch_set_closid_rmid(struct task_struct *tsk, u32 closid, u32 rmid); +void resctrl_arch_set_cpu_default_closid_rmid(int cpu, u32 closid, u32 rmid); +void resctrl_arch_sched_in(struct task_struct *tsk); +bool resctrl_arch_match_closid(struct task_struct *tsk, u32 closid); +bool resctrl_arch_match_rmid(struct task_struct *tsk, u32 closid, u32 rmid); +u32 resctrl_arch_rmid_idx_encode(u32 closid, u32 rmid); +void resctrl_arch_rmid_idx_decode(u32 idx, u32 *closid, u32 *rmid); +u32 resctrl_arch_system_num_rmid_idx(void); + +struct rdt_resource; +void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r, enum resctrl_event_id evtid); +void resctrl_arch_mon_ctx_free(struct rdt_resource *r, enum resctrl_event_id evtid, void *ctx); + +/* + * The CPU configuration for MPAM is cheap to write, and is only written if it + * has changed. No need for fine grained enables. + */ +static inline void resctrl_arch_enable_mon(void) { } +static inline void resctrl_arch_disable_mon(void) { } +static inline void resctrl_arch_enable_alloc(void) { } +static inline void resctrl_arch_disable_alloc(void) { } + +struct resctrl_schema; +static inline u32 resctrl_arch_round_bw(u32 val, + const struct resctrl_schema *s __always_unused) +{ + /* + * Do nothing: for MPAM, resctrl_arch_update_one() has the necessary + * context to round the incoming value correctly. + */ + return val; +} + +static inline unsigned int resctrl_arch_round_mon_val(unsigned int val) +{ + return val; +} + /** * mpam_register_requestor() - Register a requestor with the MPAM driver * @partid_max: The maximum PARTID value the requestor can generate. diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index cc894fc389710..ae3f15fd4e6e2 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -79,8 +79,9 @@ struct cpufreq_policy { * called, but you're in IRQ context */ struct freq_constraints constraints; - struct freq_qos_request *min_freq_req; - struct freq_qos_request *max_freq_req; + struct freq_qos_request min_freq_req; + struct freq_qos_request max_freq_req; + struct freq_qos_request boost_freq_req; struct cpufreq_frequency_table *freq_table; enum cpufreq_table_sorting freq_table_sorted; diff --git a/include/linux/efi.h b/include/linux/efi.h index 635f7c1d5d66d..88f00b06bbce2 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -1146,8 +1146,10 @@ static inline bool efi_capsule_pending(int *reset_type) { return false; } #ifdef CONFIG_EFI extern bool efi_runtime_disabled(void); +void efi_disable_runtime(void); #else static inline bool efi_runtime_disabled(void) { return true; } +static inline void efi_disable_runtime(void) { } #endif extern void efi_call_virt_check_flags(unsigned long flags, const void *caller); diff --git a/include/linux/nvgrace-egm.h b/include/linux/nvgrace-egm.h new file mode 100644 index 0000000000000..4bbd383a02732 --- /dev/null +++ b/include/linux/nvgrace-egm.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved + */ + +#ifndef _NVGRACE_EGM_H +#define _NVGRACE_EGM_H + +int register_egm_node(struct pci_dev *pdev); +void unregister_egm_node(struct pci_dev *pdev); + +#endif /* _NVGRACE_EGM_H */ diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 6cdc23fb7e093..f929a7dd324ec 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1797,22 +1797,8 @@ static inline int perf_is_paranoid(void) } extern int perf_allow_kernel(void); - -static inline int perf_allow_cpu(void) -{ - if (sysctl_perf_event_paranoid > 0 && !perfmon_capable()) - return -EACCES; - - return security_perf_event_open(PERF_SECURITY_CPU); -} - -static inline int perf_allow_tracepoint(void) -{ - if (sysctl_perf_event_paranoid > -1 && !perfmon_capable()) - return -EPERM; - - return security_perf_event_open(PERF_SECURITY_TRACEPOINT); -} +extern int perf_allow_cpu(void); +extern int perf_allow_tracepoint(void); extern int perf_exclude_event(struct perf_event *event, struct pt_regs *regs); @@ -2029,6 +2015,19 @@ perf_event_pause(struct perf_event *event, bool reset) { return 0; } static inline int perf_exclude_event(struct perf_event *event, struct pt_regs *regs) { return 0; } +static inline int perf_allow_kernel(void) +{ + return perfmon_capable() ? 0 : -EACCES; +} +static inline int perf_allow_cpu(void) +{ + return perfmon_capable() ? 0 : -EACCES; +} +static inline int perf_allow_tracepoint(void) +{ + return perfmon_capable() ? 0 : -EPERM; +} + #endif /* !CONFIG_PERF_EVENTS */ #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL) diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h index 006e57fd7ca58..010897e1b91cc 100644 --- a/include/linux/resctrl.h +++ b/include/linux/resctrl.h @@ -54,6 +54,8 @@ enum resctrl_res_level { RDT_RESOURCE_MBA, RDT_RESOURCE_SMBA, RDT_RESOURCE_PERF_PKG, + RDT_RESOURCE_L3_MAX, + RDT_RESOURCE_L2_MAX, /* Must be the last */ RDT_NUM_RESOURCES, @@ -247,22 +249,28 @@ enum membw_throttle_mode { * @min_bw: Minimum memory bandwidth percentage user can request * @max_bw: Maximum memory bandwidth value, used as the reset value * @bw_gran: Granularity at which the memory bandwidth is allocated - * @delay_linear: True if memory B/W delay is in linear scale - * @arch_needs_linear: True if we can't configure non-linear resources - * @throttle_mode: Bandwidth throttling mode when threads request - * different memory bandwidths - * @mba_sc: True if MBA software controller(mba_sc) is enabled - * @mb_map: Mapping of memory B/W percentage to memory B/W delay */ struct resctrl_membw { u32 min_bw; u32 max_bw; u32 bw_gran; - u32 delay_linear; - bool arch_needs_linear; - enum membw_throttle_mode throttle_mode; +}; + +/** + * struct resctrl_mba - Resource properties that are specific to the MBA resource + * @mba_sc: True if MBA software controller(mba_sc) is enabled + * @mb_map: Mapping of memory B/W percentage to memory B/W delay + * @delay_linear: True if control is in linear scale + * @arch_needs_linear: True if we can't configure non-linear resources + * @throttle_mode: Mode when threads request different control values + */ +struct resctrl_mba { bool mba_sc; u32 *mb_map; + bool delay_linear; + bool arch_needs_linear; + enum membw_throttle_mode throttle_mode; + }; struct resctrl_schema; @@ -278,21 +286,29 @@ enum resctrl_scope { * enum resctrl_schema_fmt - The format user-space provides for a schema. * @RESCTRL_SCHEMA_BITMAP: The schema is a bitmap in hex. * @RESCTRL_SCHEMA_RANGE: The schema is a decimal number. + * @RESCTRL_SCHEMA_PERCENT: The schema is a percentage. + * @RESCTRL_SCHEMA_MBPS: The schema ia a MBps value. + * @RESCTRL_SCHEMA__AMD_MBA: The schema value is MBA for AMD platforms. */ enum resctrl_schema_fmt { RESCTRL_SCHEMA_BITMAP, RESCTRL_SCHEMA_RANGE, + RESCTRL_SCHEMA_PERCENT, + RESCTRL_SCHEMA_MBPS, + RESCTRL_SCHEMA__AMD_MBA, }; /** * struct resctrl_mon - Monitoring related data of a resctrl resource. - * @num_rmid: Number of RMIDs available. - * @mbm_cfg_mask: Memory transactions that can be tracked when bandwidth - * monitoring events can be configured. - * @num_mbm_cntrs: Number of assignable counters. - * @mbm_cntr_assignable:Is system capable of supporting counter assignment? - * @mbm_assign_on_mkdir:True if counters should automatically be assigned to MBM - * events of monitor groups created via mkdir. + * @num_rmid: Number of RMIDs available. + * @mbm_cfg_mask: Memory transactions that can be tracked when + * bandwidth monitoring events can be configured. + * @num_mbm_cntrs: Number of assignable counters. + * @mbm_cntr_assignable: Is system capable of supporting counter assignment? + * @mbm_assign_on_mkdir: True if counters should automatically be assigned to MBM + * events of monitor groups created via mkdir. + * @mbm_cntr_configurable: True if assignable counters are configurable. + * @mbm_cntr_assign_fixed: True if the counter assignment mode is fixed. */ struct resctrl_mon { u32 num_rmid; @@ -300,6 +316,8 @@ struct resctrl_mon { int num_mbm_cntrs; bool mbm_cntr_assignable; bool mbm_assign_on_mkdir; + bool mbm_cntr_configurable; + bool mbm_cntr_assign_fixed; }; /** @@ -314,8 +332,12 @@ struct resctrl_mon { * @mon: Monitoring related data. * @ctrl_domains: RCU list of all control domains for this resource * @mon_domains: RCU list of all monitor domains for this resource + * @mba: Properties of the MBA resource * @name: Name to use in "schemata" file. - * @schema_fmt: Which format string and parser is used for this schema. + * @schema_fmt: Which format control parameters should be in for this resource. + * @evt_list: List of monitoring events + * @mbm_cfg_mask: Bandwidth sources that can be tracked when bandwidth + * monitoring events can be configured. * @cdp_capable: Is the CDP feature available on this resource */ struct rdt_resource { @@ -327,6 +349,7 @@ struct rdt_resource { struct resctrl_cache cache; struct resctrl_membw membw; struct resctrl_mon mon; + struct resctrl_mba mba; struct list_head ctrl_domains; struct list_head mon_domains; char *name; @@ -347,9 +370,12 @@ struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l); * @list: Member of resctrl_schema_all. * @name: The name to use in the "schemata" file. * @fmt_str: Format string to show domain value. + * @schema_fmt: Which format string and parser is used for this schema. * @conf_type: Whether this schema is specific to code/data. * @res: The resource structure exported by the architecture to describe * the hardware that is configured by this schema. + * @membw The properties of the schema which may be different to the format + * that was specified by the resource, * @num_closid: The number of closid that can be used with this schema. When * features like CDP are enabled, this will be lower than the * hardware supports for the resource. @@ -358,8 +384,10 @@ struct resctrl_schema { struct list_head list; char name[8]; const char *fmt_str; + enum resctrl_schema_fmt schema_fmt; enum resctrl_conf_type conf_type; struct rdt_resource *res; + struct resctrl_membw membw; u32 num_closid; }; @@ -393,22 +421,45 @@ struct resctrl_mon_config_info { void resctrl_arch_sync_cpu_closid_rmid(void *info); /** - * resctrl_get_default_ctrl() - Return the default control value for this - * resource. - * @r: The resource whose default control type is queried. + * resctrl_get_resource_default_ctrl() - Return the default control value for + * this resource. + * @r: The resource whose default control value is queried. */ -static inline u32 resctrl_get_default_ctrl(struct rdt_resource *r) +static inline u32 resctrl_get_resource_default_ctrl(struct rdt_resource *r) { switch (r->schema_fmt) { case RESCTRL_SCHEMA_BITMAP: return BIT_MASK(r->cache.cbm_len) - 1; case RESCTRL_SCHEMA_RANGE: + case RESCTRL_SCHEMA_PERCENT: + case RESCTRL_SCHEMA_MBPS: + case RESCTRL_SCHEMA__AMD_MBA: return r->membw.max_bw; } return WARN_ON_ONCE(1); } +/** + * resctrl_get_schema_default_ctrl() - Return the default control value for + * this schema. + * @s: The schema whose default control value is queried. + */ +static inline u32 resctrl_get_schema_default_ctrl(struct resctrl_schema *s) +{ + switch (s->schema_fmt) { + case RESCTRL_SCHEMA_BITMAP: + return resctrl_get_resource_default_ctrl(s->res); + case RESCTRL_SCHEMA_RANGE: + case RESCTRL_SCHEMA_PERCENT: + case RESCTRL_SCHEMA_MBPS: + case RESCTRL_SCHEMA__AMD_MBA: + return s->membw.max_bw; + } + + return WARN_ON_ONCE(1); +} + /* The number of closid supported by this resource regardless of CDP */ u32 resctrl_arch_get_num_closid(struct rdt_resource *r); u32 resctrl_arch_system_num_rmid_idx(void); @@ -500,6 +551,8 @@ bool resctrl_arch_mbm_cntr_assign_enabled(struct rdt_resource *r); */ int resctrl_arch_mbm_cntr_assign_set(struct rdt_resource *r, bool enable); +u32 resctrl_arch_round_bw(u32 val, const struct resctrl_schema *s); + /* * Update the ctrl_val and apply this config right now. * Must be called on one of the domain's CPUs. diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h index a4749f56398fd..50bdb7e30d35f 100644 --- a/include/linux/workqueue.h +++ b/include/linux/workqueue.h @@ -131,8 +131,9 @@ struct rcu_work { enum wq_affn_scope { WQ_AFFN_DFL, /* use system default */ WQ_AFFN_CPU, /* one pod per CPU */ - WQ_AFFN_SMT, /* one pod poer SMT */ + WQ_AFFN_SMT, /* one pod per SMT */ WQ_AFFN_CACHE, /* one pod per LLC */ + WQ_AFFN_CACHE_SHARD, /* synthetic sub-LLC shards */ WQ_AFFN_NUMA, /* one pod per NUMA node */ WQ_AFFN_SYSTEM, /* one pod across the whole system */ diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h index 766f4fb25e266..03e16b07a8bac 100644 --- a/include/net/mana/gdma.h +++ b/include/net/mana/gdma.h @@ -411,6 +411,7 @@ struct gdma_context { u32 test_event_eq_id; bool is_pf; + bool is_pf2; bool in_service; phys_addr_t bar0_pa; @@ -560,6 +561,7 @@ struct gdma_eqe { #define GDMA_SRIOV_REG_CFG_BASE_OFF 0x108 #define MANA_PF_DEVICE_ID 0x00B9 +#define MANA_PF2_DEVICE_ID 0x00C1 #define MANA_VF_DEVICE_ID 0x00BA struct gdma_posted_wqe_info { diff --git a/include/uapi/linux/egm.h b/include/uapi/linux/egm.h new file mode 100644 index 0000000000000..8a808e45c2052 --- /dev/null +++ b/include/uapi/linux/egm.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved + */ + +#ifndef _UAPIEGM_H +#define _UAPIEGM_H + +#define EGM_TYPE ('E') + +struct egm_bad_pages_info { + __aligned_u64 offset; + __aligned_u64 size; +}; + +struct egm_bad_pages_list { + __u32 argsz; + /* out */ + __u32 count; + /* out */ + struct egm_bad_pages_info bad_pages[]; +}; + +#define EGM_BAD_PAGES_LIST _IO(EGM_TYPE, 100) + +#endif /* _UAPIEGM_H */ diff --git a/kernel/events/core.c b/kernel/events/core.c index b46f849d726dc..eaad93d4493f5 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -14700,6 +14700,24 @@ int perf_allow_kernel(void) } EXPORT_SYMBOL_GPL(perf_allow_kernel); +int perf_allow_cpu(void) +{ + if (sysctl_perf_event_paranoid > 0 && !perfmon_capable()) + return -EACCES; + + return security_perf_event_open(PERF_SECURITY_CPU); +} +EXPORT_SYMBOL_GPL(perf_allow_cpu); + +int perf_allow_tracepoint(void) +{ + if (sysctl_perf_event_paranoid > -1 && !perfmon_capable()) + return -EPERM; + + return security_perf_event_open(PERF_SECURITY_TRACEPOINT); +} +EXPORT_SYMBOL_GPL(perf_allow_tracepoint); + /* * Inherit an event from parent task to child task. * diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ab4114712be74..82714027a6564 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -7762,6 +7762,22 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool return idle_cpu; } +/* + * Idle-capacity scan ranks transformed util_fits_cpu() outcomes; lower values + * are more preferred (see select_idle_capacity()). + */ +enum asym_fits_state { + /* In descending order of preference */ + ASYM_IDLE_CORE_UCLAMP_MISFIT = -4, + ASYM_IDLE_CORE_COMPLETE_MISFIT, + ASYM_IDLE_THREAD_FITS, + ASYM_IDLE_THREAD_UCLAMP_MISFIT, + ASYM_IDLE_COMPLETE_MISFIT, + + /* asym_fits_cpu() bias for an idle core. */ + ASYM_IDLE_CORE_BIAS = -3, +}; + /* * Scan the asym_capacity domain for idle CPUs; pick the first idle one on which * the task fits. If no CPU is big enough, but there are idle ones, try to @@ -7770,10 +7786,12 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool static int select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) { + bool prefers_idle_core = sched_smt_active() && test_idle_cores(target); unsigned long task_util, util_min, util_max, best_cap = 0; - int fits, best_fits = 0; + int fits, best_fits = ASYM_IDLE_COMPLETE_MISFIT; int cpu, best_cpu = -1; struct cpumask *cpus; + int nr = INT_MAX; cpus = this_cpu_cpumask_var_ptr(select_rq_mask); cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); @@ -7782,16 +7800,35 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) util_min = uclamp_eff_value(p, UCLAMP_MIN); util_max = uclamp_eff_value(p, UCLAMP_MAX); + if (sched_feat(SIS_UTIL) && sd->shared) { + /* + * Same nr_idle_scan hint as select_idle_cpu(), nr only limits + * the scan when not preferring an idle core. + */ + nr = READ_ONCE(sd->shared->nr_idle_scan) + 1; + /* overloaded domain is unlikely to have idle cpu/core */ + if (nr == 1) + return -1; + } + for_each_cpu_wrap(cpu, cpus, target) { + bool preferred_core = !prefers_idle_core || is_core_idle(cpu); unsigned long cpu_cap = capacity_of(cpu); + /* + * Good-enough early exit (mirrors select_idle_cpu() logic). + */ + if (!prefers_idle_core && + --nr <= 0 && best_fits == ASYM_IDLE_CORE_UCLAMP_MISFIT) + return best_cpu; + if (!available_idle_cpu(cpu) && !sched_idle_cpu(cpu)) continue; fits = util_fits_cpu(task_util, util_min, util_max, cpu); /* This CPU fits with all requirements */ - if (fits > 0) + if (fits > 0 && preferred_core) return cpu; /* * Only the min performance hint (i.e. uclamp_min) doesn't fit. @@ -7799,9 +7836,33 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) */ else if (fits < 0) cpu_cap = get_actual_cpu_capacity(cpu); + /* + * fits > 0 implies we are not on a preferred core + * but the util fits CPU capacity. Set fits to ASYM_IDLE_THREAD_FITS + * so the effective range becomes + * [ASYM_IDLE_THREAD_FITS, ASYM_IDLE_COMPLETE_MISFIT] where: + * ASYM_IDLE_COMPLETE_MISFIT - does not fit + * ASYM_IDLE_THREAD_UCLAMP_MISFIT - fits with the exception of UCLAMP_MIN + * ASYM_IDLE_THREAD_FITS - fits with the exception of preferred_core + */ + else if (fits > 0) + fits = ASYM_IDLE_THREAD_FITS; /* - * First, select CPU which fits better (-1 being better than 0). + * If we are on a preferred core, translate the range of fits + * of [ASYM_IDLE_THREAD_UCLAMP_MISFIT, ASYM_IDLE_COMPLETE_MISFIT] to + * [ASYM_IDLE_CORE_UCLAMP_MISFIT, ASYM_IDLE_CORE_COMPLETE_MISFIT]. + * This ensures that an idle core is always given priority over + * (partially) busy core. + * + * A fully fitting idle core would have returned early and hence + * fits > 0 for preferred_core need not be dealt with. + */ + if (preferred_core) + fits += ASYM_IDLE_CORE_BIAS; + + /* + * First, select CPU which fits better (lower is more preferred). * Then, select the one with best capacity at same level. */ if ((fits < best_fits) || @@ -7812,6 +7873,19 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) } } + /* + * A value in the [ASYM_IDLE_CORE_UCLAMP_MISFIT, ASYM_IDLE_CORE_BIAS] + * range means the chosen CPU is in a fully idle SMT core. Values above + * ASYM_IDLE_CORE_BIAS mean we never ranked such a CPU best. + * + * The asym-capacity wakeup path returns from select_idle_sibling() + * after this function and never runs select_idle_cpu(), so the usual + * select_idle_cpu() tail that clears idle cores must live here when the + * idle-core preference did not win. + */ + if (prefers_idle_core && best_fits > ASYM_IDLE_CORE_BIAS) + set_idle_cores(target, false); + return best_cpu; } @@ -7820,12 +7894,17 @@ static inline bool asym_fits_cpu(unsigned long util, unsigned long util_max, int cpu) { - if (sched_asym_cpucap_active()) + if (sched_asym_cpucap_active()) { /* * Return true only if the cpu fully fits the task requirements * which include the utilization and the performance hints. + * + * When SMT is active, also require that the core has no busy + * siblings. */ - return (util_fits_cpu(util, util_min, util_max, cpu) > 0); + return (!sched_smt_active() || is_core_idle(cpu)) && + (util_fits_cpu(util, util_min, util_max, cpu) > 0); + } return true; } @@ -9330,6 +9409,7 @@ struct lb_env { int dst_cpu; struct rq *dst_rq; + bool dst_core_idle; struct cpumask *dst_grpmask; int new_dst_cpu; @@ -10575,10 +10655,16 @@ static bool update_sd_pick_busiest(struct lb_env *env, * We can use max_capacity here as reduction in capacity on some * CPUs in the group should either be possible to resolve * internally or be covered by avg_load imbalance (eventually). + * + * When SMT is active, only pull a misfit to dst_cpu if it is on a + * fully idle core; otherwise the effective capacity of the core is + * reduced and we may not actually provide more capacity than the + * source. */ if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && (sgs->group_type == group_misfit_task) && - (!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) || + (!env->dst_core_idle || + !capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) || sds->local_stat.group_type != group_has_spare)) return false; @@ -11144,6 +11230,8 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd unsigned long sum_util = 0; bool sg_overloaded = 0, sg_overutilized = 0; + env->dst_core_idle = !sched_smt_active() || is_core_idle(env->dst_cpu); + do { struct sg_lb_stats *sgs = &tmp_sgs; int local_group; @@ -12617,7 +12705,8 @@ static void set_cpu_sd_state_busy(int cpu) goto unlock; sd->nohz_idle = 0; - atomic_inc(&sd->shared->nr_busy_cpus); + if (sd->shared) + atomic_inc(&sd->shared->nr_busy_cpus); unlock: rcu_read_unlock(); } @@ -12646,7 +12735,8 @@ static void set_cpu_sd_state_idle(int cpu) goto unlock; sd->nohz_idle = 1; - atomic_dec(&sd->shared->nr_busy_cpus); + if (sd->shared) + atomic_dec(&sd->shared->nr_busy_cpus); unlock: rcu_read_unlock(); } diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 32dcddaead82d..7bc2d13b3bf57 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -680,16 +680,38 @@ static void update_top_cache_domain(int cpu) int id = cpu; int size = 1; + sd = lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY_FULL); + /* + * The shared object is attached to sd_asym_cpucapacity only when the + * asym domain is non-overlapping (i.e., not built from SD_NUMA). + * On overlapping (NUMA) asym domains we fall back to letting the + * SD_SHARE_LLC path own the shared object, so sd->shared may be NULL + * here. + */ + if (sd && sd->shared) + sds = sd->shared; + + rcu_assign_pointer(per_cpu(sd_asym_cpucapacity, cpu), sd); + sd = highest_flag_domain(cpu, SD_SHARE_LLC); if (sd) { id = cpumask_first(sched_domain_span(sd)); size = cpumask_weight(sched_domain_span(sd)); - sds = sd->shared; + + /* + * If sd_asym_cpucapacity didn't claim the shared object, + * sd_llc must have one linked. + */ + if (!sds) { + WARN_ON_ONCE(!sd->shared); + sds = sd->shared; + } } rcu_assign_pointer(per_cpu(sd_llc, cpu), sd); per_cpu(sd_llc_size, cpu) = size; per_cpu(sd_llc_id, cpu) = id; + rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds); sd = lowest_flag_domain(cpu, SD_CLUSTER); @@ -708,9 +730,6 @@ static void update_top_cache_domain(int cpu) sd = highest_flag_domain(cpu, SD_ASYM_PACKING); rcu_assign_pointer(per_cpu(sd_asym_packing, cpu), sd); - - sd = lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY_FULL); - rcu_assign_pointer(per_cpu(sd_asym_cpucapacity, cpu), sd); } /* @@ -1640,7 +1659,7 @@ sd_init(struct sched_domain_topology_level *tl, { struct sd_data *sdd = &tl->data; struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu); - int sd_id, sd_weight, sd_flags = 0; + int sd_weight, sd_flags = 0; struct cpumask *sd_span; sd_weight = cpumask_weight(tl->mask(tl, cpu)); @@ -1688,7 +1707,6 @@ sd_init(struct sched_domain_topology_level *tl, sd_span = sched_domain_span(sd); cpumask_and(sd_span, cpu_map, tl->mask(tl, cpu)); - sd_id = cpumask_first(sd_span); sd->flags |= asym_cpu_capacity_classify(sd_span, cpu_map); @@ -1727,16 +1745,6 @@ sd_init(struct sched_domain_topology_level *tl, sd->cache_nice_tries = 1; } - /* - * For all levels sharing cache; connect a sched_domain_shared - * instance. - */ - if (sd->flags & SD_SHARE_LLC) { - sd->shared = *per_cpu_ptr(sdd->sds, sd_id); - atomic_inc(&sd->shared->ref); - atomic_set(&sd->shared->nr_busy_cpus, sd_weight); - } - sd->private = sdd; return sd; @@ -2548,6 +2556,16 @@ static bool topology_span_sane(const struct cpumask *cpu_map) return true; } +static void init_sched_domain_shared(struct sched_domain *sd) +{ + struct sd_data *sdd = sd->private; + int sd_id = cpumask_first(sched_domain_span(sd)); + + sd->shared = *per_cpu_ptr(sdd->sds, sd_id); + atomic_set(&sd->shared->nr_busy_cpus, sd->span_weight); + atomic_inc(&sd->shared->ref); +} + /* * Build sched domains for a given set of CPUs and attach the sched domains * to the individual CPUs @@ -2605,6 +2623,57 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att } } + for_each_cpu(i, cpu_map) { + struct sched_domain *sd_asym = NULL; + bool asym_claimed = false; + + sd = *per_cpu_ptr(d.sd, i); + if (!sd) + continue; + + /* + * In case of ASYM_CPUCAPACITY, attach sd->shared to + * sd_asym_cpucapacity for wakeup stat tracking. + * + * Caveats: + * + * 1) has_asym is system-wide, but a given CPU may still + * lack an SD_ASYM_CPUCAPACITY_FULL ancestor (e.g., an + * exclusive cpuset carving out a symmetric capacity island). + * Such CPUs must fall through to the LLC seeding path below. + * + * 2) Skip the asym attach if the asym ancestor is an + * overlapping domain (SD_NUMA). On those topologies let the + * LLC path own the shared object instead. + * + * XXX: This assumes SD_ASYM_CPUCAPACITY_FULL domain + * always has more than one group else it is prone to + * degeneration. + */ + sd_asym = sd; + while (sd_asym && !(sd_asym->flags & SD_ASYM_CPUCAPACITY_FULL)) + sd_asym = sd_asym->parent; + + if (sd_asym && !(sd_asym->flags & SD_NUMA)) { + init_sched_domain_shared(sd_asym); + asym_claimed = true; + } + + /* First, find the topmost SD_SHARE_LLC domain */ + sd = *per_cpu_ptr(d.sd, i); + while (sd->parent && (sd->parent->flags & SD_SHARE_LLC)) + sd = sd->parent; + + if (sd->flags & SD_SHARE_LLC) { + /* + * Initialize the sd->shared for SD_SHARE_LLC unless + * the asym path above already claimed it. + */ + if (!asym_claimed) + init_sched_domain_shared(sd); + } + } + /* * Calculate an allowed NUMA imbalance such that LLCs do not get * imbalanced. diff --git a/kernel/workqueue.c b/kernel/workqueue.c index c6ea96d5b7167..a3309f41e4106 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -130,6 +130,14 @@ enum wq_internal_consts { WORKER_ID_LEN = 10 + WQ_NAME_LEN, /* "kworker/R-" + WQ_NAME_LEN */ }; +/* Layout of shards within one LLC pod */ +struct llc_shard_layout { + int nr_large_shards; /* number of large shards (cores_per_shard + 1) */ + int cores_per_shard; /* base number of cores per default shard */ + int nr_shards; /* total number of shards */ + /* nr_default shards = (nr_shards - nr_large_shards) */ +}; + /* * We don't want to trap softirq for too long. See MAX_SOFTIRQ_TIME and * MAX_SOFTIRQ_RESTART in kernel/softirq.c. These are macros because @@ -404,11 +412,12 @@ struct work_offq_data { u32 flags; }; -static const char *wq_affn_names[WQ_AFFN_NR_TYPES] = { +static const char * const wq_affn_names[WQ_AFFN_NR_TYPES] = { [WQ_AFFN_DFL] = "default", [WQ_AFFN_CPU] = "cpu", [WQ_AFFN_SMT] = "smt", [WQ_AFFN_CACHE] = "cache", + [WQ_AFFN_CACHE_SHARD] = "cache_shard", [WQ_AFFN_NUMA] = "numa", [WQ_AFFN_SYSTEM] = "system", }; @@ -431,13 +440,16 @@ module_param_named(cpu_intensive_warning_thresh, wq_cpu_intensive_warning_thresh static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT); module_param_named(power_efficient, wq_power_efficient, bool, 0444); +static unsigned int wq_cache_shard_size = 8; +module_param_named(cache_shard_size, wq_cache_shard_size, uint, 0444); + static bool wq_online; /* can kworkers be created yet? */ static bool wq_topo_initialized __read_mostly = false; static struct kmem_cache *pwq_cache; static struct wq_pod_type wq_pod_types[WQ_AFFN_NR_TYPES]; -static enum wq_affn_scope wq_affn_dfl = WQ_AFFN_CACHE; +static enum wq_affn_scope wq_affn_dfl = WQ_AFFN_CACHE_SHARD; /* buf for wq_update_unbound_pod_attrs(), protected by CPU hotplug exclusion */ static struct workqueue_attrs *unbound_wq_update_pwq_attrs_buf; @@ -7078,13 +7090,7 @@ int workqueue_unbound_housekeeping_update(const struct cpumask *hk) static int parse_affn_scope(const char *val) { - int i; - - for (i = 0; i < ARRAY_SIZE(wq_affn_names); i++) { - if (!strncasecmp(val, wq_affn_names[i], strlen(wq_affn_names[i]))) - return i; - } - return -EINVAL; + return sysfs_match_string(wq_affn_names, val); } static int wq_affn_dfl_set(const char *val, const struct kernel_param *kp) @@ -8144,6 +8150,186 @@ static bool __init cpus_share_numa(int cpu0, int cpu1) return cpu_to_node(cpu0) == cpu_to_node(cpu1); } +/* Maps each CPU to its shard index within the LLC pod it belongs to */ +static int cpu_shard_id[NR_CPUS] __initdata; + +/** + * llc_count_cores - count distinct cores (SMT groups) within an LLC pod + * @pod_cpus: the cpumask of CPUs in the LLC pod + * @smt_pods: the SMT pod type, used to identify sibling groups + * + * A core is represented by the lowest-numbered CPU in its SMT group. Returns + * the number of distinct cores found in @pod_cpus. + */ +static int __init llc_count_cores(const struct cpumask *pod_cpus, + struct wq_pod_type *smt_pods) +{ + const struct cpumask *sibling_cpus; + int nr_cores = 0, c; + + /* + * Count distinct cores by only counting the first CPU in each + * SMT sibling group. + */ + for_each_cpu(c, pod_cpus) { + sibling_cpus = smt_pods->pod_cpus[smt_pods->cpu_pod[c]]; + if (cpumask_first(sibling_cpus) == c) + nr_cores++; + } + + return nr_cores; +} + +/* + * llc_shard_size - number of cores in a given shard + * + * Cores are spread as evenly as possible. The first @nr_large_shards shards are + * "large shards" with (cores_per_shard + 1) cores; the rest are "default + * shards" with cores_per_shard cores. + */ +static int __init llc_shard_size(int shard_id, int cores_per_shard, int nr_large_shards) +{ + /* The first @nr_large_shards shards are large shards */ + if (shard_id < nr_large_shards) + return cores_per_shard + 1; + + /* The remaining shards are default shards */ + return cores_per_shard; +} + +/* + * llc_calc_shard_layout - compute the shard layout for an LLC pod + * @nr_cores: number of distinct cores in the LLC pod + * + * Chooses the number of shards that keeps average shard size closest to + * wq_cache_shard_size. Returns a struct describing the total number of shards, + * the base size of each, and how many are large shards. + */ +static struct llc_shard_layout __init llc_calc_shard_layout(int nr_cores) +{ + struct llc_shard_layout layout; + + /* Ensure at least one shard; pick the count closest to the target size */ + layout.nr_shards = max(1, DIV_ROUND_CLOSEST(nr_cores, wq_cache_shard_size)); + layout.cores_per_shard = nr_cores / layout.nr_shards; + layout.nr_large_shards = nr_cores % layout.nr_shards; + + return layout; +} + +/* + * llc_shard_is_full - check whether a shard has reached its core capacity + * @cores_in_shard: number of cores already assigned to this shard + * @shard_id: index of the shard being checked + * @layout: the shard layout computed by llc_calc_shard_layout() + * + * Returns true if @cores_in_shard equals the expected size for @shard_id. + */ +static bool __init llc_shard_is_full(int cores_in_shard, int shard_id, + const struct llc_shard_layout *layout) +{ + return cores_in_shard == llc_shard_size(shard_id, layout->cores_per_shard, + layout->nr_large_shards); +} + +/** + * llc_populate_cpu_shard_id - populate cpu_shard_id[] for each CPU in an LLC pod + * @pod_cpus: the cpumask of CPUs in the LLC pod + * @smt_pods: the SMT pod type, used to identify sibling groups + * @nr_cores: number of distinct cores in @pod_cpus (from llc_count_cores()) + * + * Walks @pod_cpus in order. At each SMT group leader, advances to the next + * shard once the current shard is full. Results are written to cpu_shard_id[]. + */ +static void __init llc_populate_cpu_shard_id(const struct cpumask *pod_cpus, + struct wq_pod_type *smt_pods, + int nr_cores) +{ + struct llc_shard_layout layout = llc_calc_shard_layout(nr_cores); + const struct cpumask *sibling_cpus; + /* Count the number of cores in the current shard_id */ + int cores_in_shard = 0; + unsigned int leader; + /* This is a cursor for the shards. Go from zero to nr_shards - 1*/ + int shard_id = 0; + int c; + + /* Iterate at every CPU for a given LLC pod, and assign it a shard */ + for_each_cpu(c, pod_cpus) { + sibling_cpus = smt_pods->pod_cpus[smt_pods->cpu_pod[c]]; + if (cpumask_first(sibling_cpus) == c) { + /* This is the CPU leader for the siblings */ + if (llc_shard_is_full(cores_in_shard, shard_id, &layout)) { + shard_id++; + cores_in_shard = 0; + } + cores_in_shard++; + cpu_shard_id[c] = shard_id; + } else { + /* + * The siblings' shard MUST be the same as the leader. + * never split threads in the same core. + */ + leader = cpumask_first(sibling_cpus); + + /* + * This check silences a Warray-bounds warning on UP + * configs where NR_CPUS=1 makes cpu_shard_id[] + * a single-element array, and the compiler can't + * prove the index is always 0. + */ + if (WARN_ON_ONCE(leader >= nr_cpu_ids)) + continue; + cpu_shard_id[c] = cpu_shard_id[leader]; + } + } + + WARN_ON_ONCE(shard_id != (layout.nr_shards - 1)); +} + +/** + * precompute_cache_shard_ids - assign each CPU its shard index within its LLC + * + * Iterates over all LLC pods. For each pod, counts distinct cores then assigns + * shard indices to all CPUs in the pod. Must be called after WQ_AFFN_CACHE and + * WQ_AFFN_SMT have been initialized. + */ +static void __init precompute_cache_shard_ids(void) +{ + struct wq_pod_type *llc_pods = &wq_pod_types[WQ_AFFN_CACHE]; + struct wq_pod_type *smt_pods = &wq_pod_types[WQ_AFFN_SMT]; + const struct cpumask *cpus_sharing_llc; + int nr_cores; + int pod; + + if (!wq_cache_shard_size) { + pr_warn("workqueue: cache_shard_size must be > 0, setting to 1\n"); + wq_cache_shard_size = 1; + } + + for (pod = 0; pod < llc_pods->nr_pods; pod++) { + cpus_sharing_llc = llc_pods->pod_cpus[pod]; + + /* Number of cores in this given LLC */ + nr_cores = llc_count_cores(cpus_sharing_llc, smt_pods); + llc_populate_cpu_shard_id(cpus_sharing_llc, smt_pods, nr_cores); + } +} + +/* + * cpus_share_cache_shard - test whether two CPUs belong to the same cache shard + * + * Two CPUs share a cache shard if they are in the same LLC and have the same + * shard index. Used as the pod affinity callback for WQ_AFFN_CACHE_SHARD. + */ +static bool __init cpus_share_cache_shard(int cpu0, int cpu1) +{ + if (!cpus_share_cache(cpu0, cpu1)) + return false; + + return cpu_shard_id[cpu0] == cpu_shard_id[cpu1]; +} + /** * workqueue_init_topology - initialize CPU pods for unbound workqueues * @@ -8159,6 +8345,8 @@ void __init workqueue_init_topology(void) init_pod_type(&wq_pod_types[WQ_AFFN_CPU], cpus_dont_share); init_pod_type(&wq_pod_types[WQ_AFFN_SMT], cpus_share_smt); init_pod_type(&wq_pod_types[WQ_AFFN_CACHE], cpus_share_cache); + precompute_cache_shard_ids(); + init_pod_type(&wq_pod_types[WQ_AFFN_CACHE_SHARD], cpus_share_cache_shard); init_pod_type(&wq_pod_types[WQ_AFFN_NUMA], cpus_share_numa); wq_topo_initialized = true; diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 93f356d2b3d95..38bee649697f3 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2628,6 +2628,16 @@ config TEST_VMALLOC If unsure, say N. +config TEST_WORKQUEUE + tristate "Test module for stress/performance analysis of workqueue" + default n + help + This builds the "test_workqueue" module for benchmarking + workqueue throughput under contention. Useful for evaluating + affinity scope changes (e.g., cache_shard vs cache). + + If unsure, say N. + config TEST_BPF tristate "Test BPF filter functionality" depends on m && NET diff --git a/lib/Makefile b/lib/Makefile index 1b9ee167517f3..ea660cca04f40 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -79,6 +79,7 @@ UBSAN_SANITIZE_test_ubsan.o := y obj-$(CONFIG_TEST_KSTRTOX) += test-kstrtox.o obj-$(CONFIG_TEST_LKM) += test_module.o obj-$(CONFIG_TEST_VMALLOC) += test_vmalloc.o +obj-$(CONFIG_TEST_WORKQUEUE) += test_workqueue.o obj-$(CONFIG_TEST_RHASHTABLE) += test_rhashtable.o obj-$(CONFIG_TEST_STATIC_KEYS) += test_static_keys.o obj-$(CONFIG_TEST_STATIC_KEYS) += test_static_key_base.o diff --git a/lib/scatterlist.c b/lib/scatterlist.c index d773720d11bf2..befdc4b9c11d3 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -1247,7 +1247,7 @@ static ssize_t extract_kvec_to_sg(struct iov_iter *iter, else page = virt_to_page((void *)kaddr); - sg_set_page(sg, page, len, off); + sg_set_page(sg, page, seg, off); sgtable->nents++; sg++; sg_max--; @@ -1256,6 +1256,7 @@ static ssize_t extract_kvec_to_sg(struct iov_iter *iter, kaddr += PAGE_SIZE; off = 0; } while (len > 0 && sg_max > 0); + ret -= len; if (maxsize <= 0 || sg_max == 0) break; @@ -1409,7 +1410,7 @@ ssize_t extract_iter_to_sg(struct iov_iter *iter, size_t maxsize, struct sg_table *sgtable, unsigned int sg_max, iov_iter_extraction_t extraction_flags) { - if (maxsize == 0) + if (maxsize == 0 || sg_max == 0) return 0; switch (iov_iter_type(iter)) { diff --git a/lib/test_workqueue.c b/lib/test_workqueue.c new file mode 100644 index 0000000000000..99e160bd5ad17 --- /dev/null +++ b/lib/test_workqueue.c @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Test module for stress and performance analysis of workqueue. + * + * Benchmarks queue_work() throughput on an unbound workqueue to measure + * pool->lock contention under different affinity scope configurations + * (e.g., cache vs cache_shard). + * + * The affinity scope is changed between runs via the workqueue's sysfs + * affinity_scope attribute (WQ_SYSFS). + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates + * Copyright (c) 2026 Breno Leitao + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define WQ_NAME "bench_wq" +#define SCOPE_PATH "/sys/bus/workqueue/devices/" WQ_NAME "/affinity_scope" + +static int nr_threads; +module_param(nr_threads, int, 0444); +MODULE_PARM_DESC(nr_threads, + "Number of threads to spawn (default: 0 = num_online_cpus())"); + +static int wq_items = 50000; +module_param(wq_items, int, 0444); +MODULE_PARM_DESC(wq_items, + "Number of work items each thread queues (default: 50000)"); + +static struct workqueue_struct *bench_wq; +static atomic_t threads_done; +static DECLARE_COMPLETION(start_comp); +static DECLARE_COMPLETION(all_done_comp); + +struct thread_ctx { + struct completion work_done; + struct work_struct work; + u64 *latencies; + int cpu; + int items; +}; + +static void bench_work_fn(struct work_struct *work) +{ + struct thread_ctx *ctx = container_of(work, struct thread_ctx, work); + + complete(&ctx->work_done); +} + +static int bench_kthread_fn(void *data) +{ + struct thread_ctx *ctx = data; + ktime_t t_start, t_end; + int i; + + /* Wait for all threads to be ready */ + wait_for_completion(&start_comp); + + if (kthread_should_stop()) + return 0; + + for (i = 0; i < ctx->items; i++) { + reinit_completion(&ctx->work_done); + INIT_WORK(&ctx->work, bench_work_fn); + + t_start = ktime_get(); + queue_work(bench_wq, &ctx->work); + t_end = ktime_get(); + + ctx->latencies[i] = ktime_to_ns(ktime_sub(t_end, t_start)); + wait_for_completion(&ctx->work_done); + } + + if (atomic_dec_and_test(&threads_done)) + complete(&all_done_comp); + + /* + * Wait for kthread_stop() so the module text isn't freed + * while we're still executing. + */ + while (!kthread_should_stop()) + schedule(); + + return 0; +} + +static int cmp_u64(const void *a, const void *b) +{ + u64 va = *(const u64 *)a; + u64 vb = *(const u64 *)b; + + if (va < vb) + return -1; + if (va > vb) + return 1; + return 0; +} + +static int __init set_affn_scope(const char *scope) +{ + struct file *f; + loff_t pos = 0; + ssize_t ret; + + f = filp_open(SCOPE_PATH, O_WRONLY, 0); + if (IS_ERR(f)) { + pr_err("test_workqueue: open %s failed: %ld\n", + SCOPE_PATH, PTR_ERR(f)); + return PTR_ERR(f); + } + + ret = kernel_write(f, scope, strlen(scope), &pos); + filp_close(f, NULL); + + if (ret < 0) { + pr_err("test_workqueue: write '%s' failed: %zd\n", scope, ret); + return ret; + } + + return 0; +} + +static int __init run_bench(int n_threads, const char *scope, const char *label) +{ + struct task_struct **tasks; + unsigned long total_items; + struct thread_ctx *ctxs; + u64 *all_latencies; + ktime_t start, end; + int cpu, i, j, ret; + s64 elapsed_us; + + ret = set_affn_scope(scope); + if (ret) + return ret; + + ctxs = kcalloc(n_threads, sizeof(*ctxs), GFP_KERNEL); + if (!ctxs) + return -ENOMEM; + + tasks = kcalloc(n_threads, sizeof(*tasks), GFP_KERNEL); + if (!tasks) { + kfree(ctxs); + return -ENOMEM; + } + + total_items = (unsigned long)n_threads * wq_items; + all_latencies = kvmalloc_array(total_items, sizeof(u64), GFP_KERNEL); + if (!all_latencies) { + kfree(tasks); + kfree(ctxs); + return -ENOMEM; + } + + /* Allocate per-thread latency arrays */ + for (i = 0; i < n_threads; i++) { + ctxs[i].latencies = kvmalloc_array(wq_items, sizeof(u64), + GFP_KERNEL); + if (!ctxs[i].latencies) { + while (--i >= 0) + kvfree(ctxs[i].latencies); + kvfree(all_latencies); + kfree(tasks); + kfree(ctxs); + return -ENOMEM; + } + } + + atomic_set(&threads_done, n_threads); + reinit_completion(&all_done_comp); + reinit_completion(&start_comp); + + /* Create kthreads, each bound to a different online CPU */ + i = 0; + for_each_online_cpu(cpu) { + if (i >= n_threads) + break; + + ctxs[i].cpu = cpu; + ctxs[i].items = wq_items; + init_completion(&ctxs[i].work_done); + + tasks[i] = kthread_create(bench_kthread_fn, &ctxs[i], + "wq_bench/%d", cpu); + if (IS_ERR(tasks[i])) { + ret = PTR_ERR(tasks[i]); + pr_err("test_workqueue: failed to create kthread %d: %d\n", + i, ret); + /* Unblock threads waiting on start_comp before stopping them */ + complete_all(&start_comp); + while (--i >= 0) + kthread_stop(tasks[i]); + goto out_free; + } + + kthread_bind(tasks[i], cpu); + wake_up_process(tasks[i]); + i++; + } + + /* Start timing and release all threads */ + start = ktime_get(); + complete_all(&start_comp); + + /* Wait for all threads to finish the benchmark */ + wait_for_completion(&all_done_comp); + + /* Drain any remaining work */ + flush_workqueue(bench_wq); + + /* Ensure all kthreads have fully exited before module memory is freed */ + for (i = 0; i < n_threads; i++) + kthread_stop(tasks[i]); + + end = ktime_get(); + elapsed_us = ktime_us_delta(end, start); + + /* Merge all per-thread latencies and sort for percentile calculation */ + j = 0; + for (i = 0; i < n_threads; i++) { + memcpy(&all_latencies[j], ctxs[i].latencies, + wq_items * sizeof(u64)); + j += wq_items; + } + + sort(all_latencies, total_items, sizeof(u64), cmp_u64, NULL); + + pr_info("test_workqueue: %-16s %llu items/sec\tp50=%llu\tp90=%llu\tp95=%llu ns\n", + label, + elapsed_us ? div_u64(total_items * 1000000ULL, elapsed_us) : 0, + all_latencies[total_items * 50 / 100], + all_latencies[total_items * 90 / 100], + all_latencies[total_items * 95 / 100]); + + ret = 0; +out_free: + for (i = 0; i < n_threads; i++) + kvfree(ctxs[i].latencies); + kvfree(all_latencies); + kfree(tasks); + kfree(ctxs); + + return ret; +} + +static const char * const bench_scopes[] = { + "cpu", "smt", "cache_shard", "cache", "numa", "system", +}; + +static int __init test_workqueue_init(void) +{ + int n_threads = min(nr_threads ?: num_online_cpus(), num_online_cpus()); + int i; + + if (wq_items <= 0) { + pr_err("test_workqueue: wq_items must be > 0\n"); + return -EINVAL; + } + + bench_wq = alloc_workqueue(WQ_NAME, WQ_UNBOUND | WQ_SYSFS, 0); + if (!bench_wq) + return -ENOMEM; + + pr_info("test_workqueue: running %d threads, %d items/thread\n", + n_threads, wq_items); + + for (i = 0; i < ARRAY_SIZE(bench_scopes); i++) + run_bench(n_threads, bench_scopes[i], bench_scopes[i]); + + destroy_workqueue(bench_wq); + + /* Return -EAGAIN so the module doesn't stay loaded after the benchmark */ + return -EAGAIN; +} + +module_init(test_workqueue_init); +MODULE_AUTHOR("Breno Leitao "); +MODULE_DESCRIPTION("Stress/performance benchmark for workqueue subsystem"); +MODULE_LICENSE("GPL"); diff --git a/net/ceph/auth.c b/net/ceph/auth.c index 3314705e59146..17660bde896be 100644 --- a/net/ceph/auth.c +++ b/net/ceph/auth.c @@ -257,7 +257,7 @@ int ceph_handle_auth_reply(struct ceph_auth_client *ac, ac->negotiating = false; } - if (result) { + if (result < 0) { pr_err("auth protocol '%s' mauth authentication failed: %d\n", ceph_auth_proto_name(ac->protocol), result); ret = result; diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c index d5080530ce0cc..d2cdc8ee31551 100644 --- a/net/ceph/mon_client.c +++ b/net/ceph/mon_client.c @@ -174,6 +174,8 @@ int ceph_monmap_contains(struct ceph_monmap *m, struct ceph_entity_addr *addr) */ static void __send_prepared_auth_request(struct ceph_mon_client *monc, int len) { + BUG_ON(len > monc->m_auth->front_alloc_len); + monc->pending_auth = 1; monc->m_auth->front.iov_len = len; monc->m_auth->hdr.front_len = cpu_to_le32(len); diff --git a/net/ipv4/nexthop.c b/net/ipv4/nexthop.c index 2c9036c719b68..34a5005fee55e 100644 --- a/net/ipv4/nexthop.c +++ b/net/ipv4/nexthop.c @@ -20,8 +20,8 @@ #define NH_RES_DEFAULT_IDLE_TIMER (120 * HZ) #define NH_RES_DEFAULT_UNBALANCED_TIMER 0 /* No forced rebalancing. */ -static void remove_nexthop(struct net *net, struct nexthop *nh, - struct nl_info *nlinfo); +static bool __must_check remove_nexthop(struct net *net, struct nexthop *nh, + struct nl_info *nlinfo); #define NH_DEV_HASHBITS 8 #define NH_DEV_HASHSIZE (1U << NH_DEV_HASHBITS) @@ -2016,9 +2016,9 @@ static void nh_hthr_group_rebalance(struct nh_group *nhg) } } -static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge, - struct nl_info *nlinfo, - struct list_head *deferred_free) +static bool __must_check +remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge, + struct nl_info *nlinfo, struct list_head *deferred_free) { struct nh_grp_entry *nhges, *new_nhges; struct nexthop *nhp = nhge->nh_parent; @@ -2033,10 +2033,8 @@ static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge, newg = nhg->spare; /* last entry, keep it visible and remove the parent */ - if (nhg->num_nh == 1) { - remove_nexthop(net, nhp, nlinfo); - return; - } + if (nhg->num_nh == 1) + return remove_nexthop(net, nhp, nlinfo); newg->has_v4 = false; newg->is_multipath = nhg->is_multipath; @@ -2093,22 +2091,27 @@ static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge, if (nlinfo) nexthop_notify(RTM_NEWNEXTHOP, nhp, nlinfo); + + return false; } -static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh, - struct nl_info *nlinfo) +static bool __must_check +remove_nexthop_from_groups(struct net *net, struct nexthop *nh, + struct nl_info *nlinfo) { struct nh_grp_entry *nhge, *tmp; LIST_HEAD(deferred_free); + bool need_flush = false; /* If there is nothing to do, let's avoid the costly call to * synchronize_net() */ if (list_empty(&nh->grp_list)) - return; + return false; list_for_each_entry_safe(nhge, tmp, &nh->grp_list, nh_list) - remove_nh_grp_entry(net, nhge, nlinfo, &deferred_free); + need_flush |= remove_nh_grp_entry(net, nhge, nlinfo, + &deferred_free); /* make sure all see the newly published array before releasing rtnl */ synchronize_net(); @@ -2118,6 +2121,8 @@ static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh, list_del(&nhge->nh_list); free_percpu(nhge->stats); } + + return need_flush; } static void remove_nexthop_group(struct nexthop *nh, struct nl_info *nlinfo) @@ -2142,18 +2147,15 @@ static void remove_nexthop_group(struct nexthop *nh, struct nl_info *nlinfo) } /* not called for nexthop replace */ -static void __remove_nexthop_fib(struct net *net, struct nexthop *nh) +static bool __must_check __remove_nexthop_fib(struct net *net, + struct nexthop *nh) { + bool need_flush = !list_empty(&nh->fi_list); struct fib6_info *f6i; - bool do_flush = false; struct fib_info *fi; - list_for_each_entry(fi, &nh->fi_list, nh_list) { + list_for_each_entry(fi, &nh->fi_list, nh_list) fi->fib_flags |= RTNH_F_DEAD; - do_flush = true; - } - if (do_flush) - fib_flush(net); spin_lock_bh(&nh->lock); @@ -2173,12 +2175,14 @@ static void __remove_nexthop_fib(struct net *net, struct nexthop *nh) } spin_unlock_bh(&nh->lock); + + return need_flush; } -static void __remove_nexthop(struct net *net, struct nexthop *nh, - struct nl_info *nlinfo) +static bool __must_check __remove_nexthop(struct net *net, struct nexthop *nh, + struct nl_info *nlinfo) { - __remove_nexthop_fib(net, nh); + bool need_flush = __remove_nexthop_fib(net, nh); if (nh->is_group) { remove_nexthop_group(nh, nlinfo); @@ -2189,13 +2193,17 @@ static void __remove_nexthop(struct net *net, struct nexthop *nh, if (nhi->fib_nhc.nhc_dev) hlist_del(&nhi->dev_hash); - remove_nexthop_from_groups(net, nh, nlinfo); + need_flush |= remove_nexthop_from_groups(net, nh, nlinfo); } + + return need_flush; } -static void remove_nexthop(struct net *net, struct nexthop *nh, - struct nl_info *nlinfo) +static bool __must_check remove_nexthop(struct net *net, struct nexthop *nh, + struct nl_info *nlinfo) { + bool need_flush; + call_nexthop_notifiers(net, NEXTHOP_EVENT_DEL, nh, NULL); /* remove from the tree */ @@ -2204,10 +2212,19 @@ static void remove_nexthop(struct net *net, struct nexthop *nh, if (nlinfo) nexthop_notify(RTM_DELNEXTHOP, nh, nlinfo); - __remove_nexthop(net, nh, nlinfo); + need_flush = __remove_nexthop(net, nh, nlinfo); nh_base_seq_inc(net); nexthop_put(nh); + + return need_flush; +} + +static void remove_one_nexthop(struct net *net, struct nexthop *nh, + struct nl_info *nlinfo) +{ + if (remove_nexthop(net, nh, nlinfo)) + fib_flush(net); } /* if any FIB entries reference this nexthop, any dst entries @@ -2589,7 +2606,7 @@ static int replace_nexthop(struct net *net, struct nexthop *old, if (!err) { nh_rt_cache_flush(net, old, new); - __remove_nexthop(net, new, NULL); + WARN_ON_ONCE(__remove_nexthop(net, new, NULL)); nexthop_put(new); } @@ -2696,6 +2713,7 @@ static void nexthop_flush_dev(struct net_device *dev, unsigned long event) unsigned int hash = nh_dev_hashfn(dev->ifindex); struct net *net = dev_net(dev); struct hlist_head *head = &net->nexthop.devhash[hash]; + bool need_flush = false; struct hlist_node *n; struct nh_info *nhi; @@ -2707,22 +2725,28 @@ static void nexthop_flush_dev(struct net_device *dev, unsigned long event) (event == NETDEV_DOWN || event == NETDEV_CHANGE)) continue; - remove_nexthop(net, nhi->nh_parent, NULL); + need_flush |= remove_nexthop(net, nhi->nh_parent, NULL); } + + if (need_flush) + fib_flush(net); } /* rtnl; called when net namespace is deleted */ static void flush_all_nexthops(struct net *net) { struct rb_root *root = &net->nexthop.rb_root; + bool need_flush = false; struct rb_node *node; struct nexthop *nh; while ((node = rb_first(root))) { nh = rb_entry(node, struct nexthop, rb_node); - remove_nexthop(net, nh, NULL); + need_flush |= remove_nexthop(net, nh, NULL); cond_resched(); } + if (need_flush) + fib_flush(net); } static struct nexthop *nexthop_create_group(struct net *net, @@ -2992,7 +3016,7 @@ static struct nexthop *nexthop_add(struct net *net, struct nh_config *cfg, err = insert_nexthop(net, nh, cfg, extack); if (err) { - __remove_nexthop(net, nh, NULL); + WARN_ON_ONCE(__remove_nexthop(net, nh, NULL)); nexthop_put(nh); nh = ERR_PTR(err); } @@ -3361,7 +3385,7 @@ static int rtm_del_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh, nh = nexthop_find_by_id(net, id); if (nh) - remove_nexthop(net, nh, &nlinfo); + remove_one_nexthop(net, nh, &nlinfo); else err = -ENOENT; diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c index 57a4566904067..3912128d9b866 100644 --- a/net/mptcp/pm.c +++ b/net/mptcp/pm.c @@ -337,8 +337,12 @@ static void mptcp_pm_add_timer(struct timer_list *timer) if (inet_sk_state_load(sk) == TCP_CLOSE) return; - if (!entry->addr.id) - return; + bh_lock_sock(sk); + if (sock_owned_by_user(sk)) { + /* Try again later. */ + sk_reset_timer(sk, timer, jiffies + HZ / 20); + goto out; + } if (mptcp_pm_should_add_signal_addr(msk)) { sk_reset_timer(sk, timer, jiffies + TCP_RTO_MAX / 8); @@ -368,6 +372,7 @@ static void mptcp_pm_add_timer(struct timer_list *timer) mptcp_pm_subflow_established(msk); out: + bh_unlock_sock(sk); __sock_put(sk); } diff --git a/net/netfilter/nft_inner.c b/net/netfilter/nft_inner.c index c4569d4b92285..1b3e7a976f560 100644 --- a/net/netfilter/nft_inner.c +++ b/net/netfilter/nft_inner.c @@ -163,7 +163,6 @@ static int nft_inner_parse_l2l3(const struct nft_inner *priv, return -1; if (fragoff == 0) { - thoff = nhoff + sizeof(_ip6h); ctx->flags |= NFT_PAYLOAD_CTX_INNER_TH; ctx->inner_thoff = thoff; ctx->l4proto = l4proto; diff --git a/sound/usb/mixer.c b/sound/usb/mixer.c index 21191da4c0496..bf6b6b8582ef9 100644 --- a/sound/usb/mixer.c +++ b/sound/usb/mixer.c @@ -1714,7 +1714,7 @@ static void __build_feature_ctl(struct usb_mixer_interface *mixer, cval->master_readonly = readonly_mask; } else { int i, c = 0; - for (i = 0; i < 16; i++) + for (i = 0; i < MAX_CHANNELS; i++) if (ctl_mask & BIT(i)) c++; cval->channels = c; diff --git a/tools/workqueue/wq_dump.py b/tools/workqueue/wq_dump.py index d29b918306b48..06948ffcfc4b6 100644 --- a/tools/workqueue/wq_dump.py +++ b/tools/workqueue/wq_dump.py @@ -107,6 +107,7 @@ def wq_type_str(wq): WQ_AFFN_CPU = prog['WQ_AFFN_CPU'] WQ_AFFN_SMT = prog['WQ_AFFN_SMT'] WQ_AFFN_CACHE = prog['WQ_AFFN_CACHE'] +WQ_AFFN_CACHE_SHARD = prog['WQ_AFFN_CACHE_SHARD'] WQ_AFFN_NUMA = prog['WQ_AFFN_NUMA'] WQ_AFFN_SYSTEM = prog['WQ_AFFN_SYSTEM'] @@ -138,7 +139,7 @@ def print_pod_type(pt): print(f' [{cpu}]={pt.cpu_pod[cpu].value_()}', end='') print('') -for affn in [WQ_AFFN_CPU, WQ_AFFN_SMT, WQ_AFFN_CACHE, WQ_AFFN_NUMA, WQ_AFFN_SYSTEM]: +for affn in [WQ_AFFN_CPU, WQ_AFFN_SMT, WQ_AFFN_CACHE, WQ_AFFN_CACHE_SHARD, WQ_AFFN_NUMA, WQ_AFFN_SYSTEM]: print('') print(f'{wq_affn_names[affn].string_().decode().upper()}{" (default)" if affn == wq_affn_dfl else ""}') print_pod_type(wq_pod_types[affn])