From 831eea229853d610c82dc2f4777da1612d672dce Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Thu, 18 Dec 2025 16:42:12 +0100 Subject: [PATCH 08/11] fixes Signed-off-by: Peter Jung --- arch/arm/include/asm/mmu_context.h | 2 +- arch/riscv/include/asm/sync_core.h | 2 +- arch/s390/include/asm/mmu_context.h | 2 +- arch/sparc/include/asm/mmu_context_64.h | 2 +- arch/x86/include/asm/mmu_context.h | 23 +++++- arch/x86/include/asm/sync_core.h | 2 +- arch/x86/kernel/cpu/amd.c | 6 ++ arch/x86/mm/tlb.c | 21 ------ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 3 +- drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c | 58 +++++---------- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 11 +-- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 4 - .../drm/amd/amdkfd/kfd_device_queue_manager.c | 73 +++++++++++++------ .../amd/display/amdgpu_dm/amdgpu_dm_helpers.c | 1 + .../amd/display/include/ddc_service_types.h | 1 + drivers/gpu/drm/i915/gt/intel_rc6.c | 13 +++- include/linux/perf_event.h | 2 +- include/linux/sched/mm.h | 10 +-- include/linux/tick.h | 4 +- include/linux/vtime.h | 8 +- kernel/sched/core.c | 19 ++--- kernel/sched/sched.h | 24 +++--- net/ipv4/esp4_offload.c | 4 +- net/ipv6/esp6_offload.c | 4 +- sound/soc/codecs/cs35l41.c | 7 +- 25 files changed, 161 insertions(+), 145 deletions(-) diff --git a/arch/arm/include/asm/mmu_context.h b/arch/arm/include/asm/mmu_context.h index db2cb06aa8cf..bebde469f81a 100644 --- a/arch/arm/include/asm/mmu_context.h +++ b/arch/arm/include/asm/mmu_context.h @@ -80,7 +80,7 @@ static inline void check_and_switch_context(struct mm_struct *mm, #ifndef MODULE #define finish_arch_post_lock_switch \ finish_arch_post_lock_switch -static inline void finish_arch_post_lock_switch(void) +static __always_inline void finish_arch_post_lock_switch(void) { struct mm_struct *mm = current->mm; diff --git a/arch/riscv/include/asm/sync_core.h b/arch/riscv/include/asm/sync_core.h index 9153016da8f1..2fe6b7fe6b12 100644 --- a/arch/riscv/include/asm/sync_core.h +++ b/arch/riscv/include/asm/sync_core.h @@ -6,7 +6,7 @@ * RISC-V implements return to user-space through an xRET instruction, * which is not core serializing. */ -static inline void sync_core_before_usermode(void) +static __always_inline void sync_core_before_usermode(void) { asm volatile ("fence.i" ::: "memory"); } diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index d9b8501bc93d..c124ef6a01b3 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -97,7 +97,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, } #define finish_arch_post_lock_switch finish_arch_post_lock_switch -static inline void finish_arch_post_lock_switch(void) +static __always_inline void finish_arch_post_lock_switch(void) { struct task_struct *tsk = current; struct mm_struct *mm = tsk->mm; diff --git a/arch/sparc/include/asm/mmu_context_64.h b/arch/sparc/include/asm/mmu_context_64.h index 78bbacc14d2d..d1967214ef25 100644 --- a/arch/sparc/include/asm/mmu_context_64.h +++ b/arch/sparc/include/asm/mmu_context_64.h @@ -160,7 +160,7 @@ static inline void arch_start_context_switch(struct task_struct *prev) } #define finish_arch_post_lock_switch finish_arch_post_lock_switch -static inline void finish_arch_post_lock_switch(void) +static __always_inline void finish_arch_post_lock_switch(void) { /* Restore the state of MCDPER register for the new process * just switched to. diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h index 73bf3b1b44e8..ecd134dcfb34 100644 --- a/arch/x86/include/asm/mmu_context.h +++ b/arch/x86/include/asm/mmu_context.h @@ -136,8 +136,29 @@ static inline void mm_reset_untag_mask(struct mm_struct *mm) } #endif +/* + * Please ignore the name of this function. It should be called + * switch_to_kernel_thread(). + * + * enter_lazy_tlb() is a hint from the scheduler that we are entering a + * kernel thread or other context without an mm. Acceptable implementations + * include doing nothing whatsoever, switching to init_mm, or various clever + * lazy tricks to try to minimize TLB flushes. + * + * The scheduler reserves the right to call enter_lazy_tlb() several times + * in a row. It will notify us that we're going back to a real mm by + * calling switch_mm_irqs_off(). + */ #define enter_lazy_tlb enter_lazy_tlb -extern void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk); +#ifndef MODULE +static __always_inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) +{ + if (this_cpu_read(cpu_tlbstate.loaded_mm) == &init_mm) + return; + + this_cpu_write(cpu_tlbstate_shared.is_lazy, true); +} +#endif #define mm_init_global_asid mm_init_global_asid extern void mm_init_global_asid(struct mm_struct *mm); diff --git a/arch/x86/include/asm/sync_core.h b/arch/x86/include/asm/sync_core.h index 96bda43538ee..4b55fa353bb5 100644 --- a/arch/x86/include/asm/sync_core.h +++ b/arch/x86/include/asm/sync_core.h @@ -93,7 +93,7 @@ static __always_inline void sync_core(void) * to user-mode. x86 implements return to user-space through sysexit, * sysrel, and sysretq, which are not core serializing. */ -static inline void sync_core_before_usermode(void) +static __always_inline void sync_core_before_usermode(void) { /* With PTI, we unconditionally serialize before running user code. */ if (static_cpu_has(X86_FEATURE_PTI)) diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 5d46709c58d0..1bf7f64e321f 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -1035,6 +1035,7 @@ static void init_amd_zen4(struct cpuinfo_x86 *c) } } +#ifndef CONFIG_CACHY static const struct x86_cpu_id zen5_rdseed_microcode[] = { ZEN_MODEL_STEP_UCODE(0x1a, 0x02, 0x1, 0x0b00215a), ZEN_MODEL_STEP_UCODE(0x1a, 0x08, 0x1, 0x0b008121), @@ -1056,6 +1057,11 @@ static void init_amd_zen5(struct cpuinfo_x86 *c) pr_emerg_once("RDSEED32 is broken. Disabling the corresponding CPUID bit.\n"); } } +#else +static void init_amd_zen5(struct cpuinfo_x86 *c) +{ +} +#endif /* !CONFIG_CACHY */ static void init_amd(struct cpuinfo_x86 *c) { diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 5d221709353e..cb715e8e75e4 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -970,27 +970,6 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next, } } -/* - * Please ignore the name of this function. It should be called - * switch_to_kernel_thread(). - * - * enter_lazy_tlb() is a hint from the scheduler that we are entering a - * kernel thread or other context without an mm. Acceptable implementations - * include doing nothing whatsoever, switching to init_mm, or various clever - * lazy tricks to try to minimize TLB flushes. - * - * The scheduler reserves the right to call enter_lazy_tlb() several times - * in a row. It will notify us that we're going back to a real mm by - * calling switch_mm_irqs_off(). - */ -void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) -{ - if (this_cpu_read(cpu_tlbstate.loaded_mm) == &init_mm) - return; - - this_cpu_write(cpu_tlbstate_shared.is_lazy, true); -} - /* * Using a temporary mm allows to set temporary mappings that are not accessible * by other CPUs. Such mappings are needed to perform sensitive memory writes diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 96b6738e6252..9640818d21a0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -3416,11 +3416,10 @@ int amdgpu_device_set_pg_state(struct amdgpu_device *adev, (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA)) continue; - /* skip CG for VCE/UVD/VPE, it's handled specially */ + /* skip CG for VCE/UVD, it's handled specially */ if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && - adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VPE && adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && adev->ip_blocks[i].version->funcs->set_powergating_state) { /* enable powergating to save power */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c index 3ef5bc95642c..9cab36322c16 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c @@ -201,58 +201,34 @@ static int amdgpu_vmid_grab_idle(struct amdgpu_ring *ring, struct amdgpu_device *adev = ring->adev; unsigned vmhub = ring->vm_hub; struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; - struct dma_fence **fences; - unsigned i; + /* If anybody is waiting for a VMID let everybody wait for fairness */ if (!dma_fence_is_signaled(ring->vmid_wait)) { *fence = dma_fence_get(ring->vmid_wait); return 0; } - fences = kmalloc_array(id_mgr->num_ids, sizeof(void *), GFP_NOWAIT); - if (!fences) - return -ENOMEM; - /* Check if we have an idle VMID */ - i = 0; - list_for_each_entry((*idle), &id_mgr->ids_lru, list) { + list_for_each_entry_reverse((*idle), &id_mgr->ids_lru, list) { /* Don't use per engine and per process VMID at the same time */ struct amdgpu_ring *r = adev->vm_manager.concurrent_flush ? NULL : ring; - fences[i] = amdgpu_sync_peek_fence(&(*idle)->active, r); - if (!fences[i]) - break; - ++i; + *fence = amdgpu_sync_peek_fence(&(*idle)->active, r); + if (!(*fence)) + return 0; } - /* If we can't find a idle VMID to use, wait till one becomes available */ - if (&(*idle)->list == &id_mgr->ids_lru) { - u64 fence_context = adev->vm_manager.fence_context + ring->idx; - unsigned seqno = ++adev->vm_manager.seqno[ring->idx]; - struct dma_fence_array *array; - unsigned j; - - *idle = NULL; - for (j = 0; j < i; ++j) - dma_fence_get(fences[j]); - - array = dma_fence_array_create(i, fences, fence_context, - seqno, true); - if (!array) { - for (j = 0; j < i; ++j) - dma_fence_put(fences[j]); - kfree(fences); - return -ENOMEM; - } - - *fence = dma_fence_get(&array->base); - dma_fence_put(ring->vmid_wait); - ring->vmid_wait = &array->base; - return 0; - } - kfree(fences); + /* + * If we can't find a idle VMID to use, wait on a fence from the least + * recently used in the hope that it will be available soon. + */ + *idle = NULL; + dma_fence_put(ring->vmid_wait); + ring->vmid_wait = dma_fence_get(*fence); + /* This is the reference we return */ + dma_fence_get(*fence); return 0; } @@ -313,7 +289,7 @@ static int amdgpu_vmid_grab_reserved(struct amdgpu_vm *vm, * user of the VMID. */ r = amdgpu_sync_fence(&(*id)->active, &job->base.s_fence->finished, - GFP_NOWAIT); + GFP_ATOMIC); if (r) return r; @@ -373,7 +349,7 @@ static int amdgpu_vmid_grab_used(struct amdgpu_vm *vm, */ r = amdgpu_sync_fence(&(*id)->active, &job->base.s_fence->finished, - GFP_NOWAIT); + GFP_ATOMIC); if (r) return r; @@ -426,7 +402,7 @@ int amdgpu_vmid_grab(struct amdgpu_vm *vm, struct amdgpu_ring *ring, /* Remember this submission as user of the VMID */ r = amdgpu_sync_fence(&id->active, &job->base.s_fence->finished, - GFP_NOWAIT); + GFP_ATOMIC); if (r) goto error; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 676e24fb8864..c8f2a7a0d1cb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -1066,7 +1066,9 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, } /* Prepare a TLB flush fence to be attached to PTs */ - if (!params->unlocked) { + if (!params->unlocked && + /* SI doesn't support pasid or KIQ/MES */ + params->adev->family > AMDGPU_FAMILY_SI) { amdgpu_vm_tlb_fence_create(params->adev, vm, fence); /* Makes sure no PD/PT is freed before the flush */ @@ -2828,8 +2830,6 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) */ void amdgpu_vm_manager_init(struct amdgpu_device *adev) { - unsigned i; - /* Concurrent flushes are only possible starting with Vega10 and * are broken on Navi10 and Navi14. */ @@ -2838,11 +2838,6 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev) adev->asic_type == CHIP_NAVI14); amdgpu_vmid_mgr_init(adev); - adev->vm_manager.fence_context = - dma_fence_context_alloc(AMDGPU_MAX_RINGS); - for (i = 0; i < AMDGPU_MAX_RINGS; ++i) - adev->vm_manager.seqno[i] = 0; - spin_lock_init(&adev->vm_manager.prt_lock); atomic_set(&adev->vm_manager.num_prt_users, 0); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index cf0ec94e8a07..15d757c016cb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -453,10 +453,6 @@ struct amdgpu_vm_manager { unsigned int first_kfd_vmid; bool concurrent_flush; - /* Handling of VM fences */ - u64 fence_context; - unsigned seqno[AMDGPU_MAX_RINGS]; - uint64_t max_pfn; uint32_t num_level; uint32_t block_size; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 6e7bc983fc0b..6c5c7c1bf5ed 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -1209,15 +1209,6 @@ static int evict_process_queues_cpsch(struct device_queue_manager *dqm, pr_debug_ratelimited("Evicting process pid %d queues\n", pdd->process->lead_thread->pid); - if (dqm->dev->kfd->shared_resources.enable_mes) { - pdd->last_evict_timestamp = get_jiffies_64(); - retval = suspend_all_queues_mes(dqm); - if (retval) { - dev_err(dev, "Suspending all queues failed"); - goto out; - } - } - /* Mark all queues as evicted. Deactivate all active queues on * the qpd. */ @@ -1230,27 +1221,23 @@ static int evict_process_queues_cpsch(struct device_queue_manager *dqm, decrement_queue_count(dqm, qpd, q); if (dqm->dev->kfd->shared_resources.enable_mes) { - retval = remove_queue_mes(dqm, q, qpd); - if (retval) { + int err; + + err = remove_queue_mes(dqm, q, qpd); + if (err) { dev_err(dev, "Failed to evict queue %d\n", q->properties.queue_id); - goto out; + retval = err; } } } - - if (!dqm->dev->kfd->shared_resources.enable_mes) { - pdd->last_evict_timestamp = get_jiffies_64(); + pdd->last_evict_timestamp = get_jiffies_64(); + if (!dqm->dev->kfd->shared_resources.enable_mes) retval = execute_queues_cpsch(dqm, qpd->is_debug ? KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES : KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD); - } else { - retval = resume_all_queues_mes(dqm); - if (retval) - dev_err(dev, "Resuming all queues failed"); - } out: dqm_unlock(dqm); @@ -3111,17 +3098,61 @@ int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbel return ret; } +static int kfd_dqm_evict_pasid_mes(struct device_queue_manager *dqm, + struct qcm_process_device *qpd) +{ + struct device *dev = dqm->dev->adev->dev; + int ret = 0; + + /* Check if process is already evicted */ + dqm_lock(dqm); + if (qpd->evicted) { + /* Increment the evicted count to make sure the + * process stays evicted before its terminated. + */ + qpd->evicted++; + dqm_unlock(dqm); + goto out; + } + dqm_unlock(dqm); + + ret = suspend_all_queues_mes(dqm); + if (ret) { + dev_err(dev, "Suspending all queues failed"); + goto out; + } + + ret = dqm->ops.evict_process_queues(dqm, qpd); + if (ret) { + dev_err(dev, "Evicting process queues failed"); + goto out; + } + + ret = resume_all_queues_mes(dqm); + if (ret) + dev_err(dev, "Resuming all queues failed"); + +out: + return ret; +} + int kfd_evict_process_device(struct kfd_process_device *pdd) { struct device_queue_manager *dqm; struct kfd_process *p; + int ret = 0; p = pdd->process; dqm = pdd->dev->dqm; WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid); - return dqm->ops.evict_process_queues(dqm, &pdd->qpd); + if (dqm->dev->kfd->shared_resources.enable_mes) + ret = kfd_dqm_evict_pasid_mes(dqm, &pdd->qpd); + else + ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd); + + return ret; } int reserve_debug_trap_vmid(struct device_queue_manager *dqm, diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c index d0f770dd0a95..f01f30a245ad 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c @@ -1374,6 +1374,7 @@ static bool dm_is_freesync_pcon_whitelist(const uint32_t branch_dev_id) case DP_BRANCH_DEVICE_ID_0060AD: case DP_BRANCH_DEVICE_ID_00E04C: case DP_BRANCH_DEVICE_ID_90CC24: + case DP_BRANCH_DEVICE_ID_2B02F0: ret_val = true; break; default: diff --git a/drivers/gpu/drm/amd/display/include/ddc_service_types.h b/drivers/gpu/drm/amd/display/include/ddc_service_types.h index 1c603b12957f..e838f7c1269c 100644 --- a/drivers/gpu/drm/amd/display/include/ddc_service_types.h +++ b/drivers/gpu/drm/amd/display/include/ddc_service_types.h @@ -36,6 +36,7 @@ #define DP_BRANCH_DEVICE_ID_006037 0x006037 #define DP_BRANCH_DEVICE_ID_001CF8 0x001CF8 #define DP_BRANCH_DEVICE_ID_0060AD 0x0060AD +#define DP_BRANCH_DEVICE_ID_2B02F0 0x2B02F0 /* Chrontel CH7218 */ #define DP_BRANCH_HW_REV_10 0x10 #define DP_BRANCH_HW_REV_20 0x20 diff --git a/drivers/gpu/drm/i915/gt/intel_rc6.c b/drivers/gpu/drm/i915/gt/intel_rc6.c index bf38cc5fe872..787b3fd2925c 100644 --- a/drivers/gpu/drm/i915/gt/intel_rc6.c +++ b/drivers/gpu/drm/i915/gt/intel_rc6.c @@ -2,9 +2,9 @@ /* * Copyright © 2019 Intel Corporation */ - #include #include +#include #include "gem/i915_gem_region.h" #include "i915_drv.h" @@ -494,6 +494,17 @@ static bool rc6_supported(struct intel_rc6 *rc6) if (!HAS_RC6(i915)) return false; + if (dmi_match(DMI_PRODUCT_SKU, "IBP1XI08MK1") || + dmi_match(DMI_PRODUCT_SKU, "IBP14I08MK2") || + dmi_match(DMI_PRODUCT_SKU, "IBP1XI08MK2") || + dmi_match(DMI_PRODUCT_SKU, "IBP16I08MK2") || + dmi_match(DMI_PRODUCT_SKU, "OMNIA08IMK1") || + dmi_match(DMI_PRODUCT_SKU, "OMNIA08IMK2")) { + drm_notice(&i915->drm, + "RC6 disabled by quirk\n"); + return false; + } + if (intel_vgpu_active(i915)) return false; diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index fd1d91017b99..2b1c752af207 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1624,7 +1624,7 @@ static inline void perf_event_task_migrate(struct task_struct *task) task->sched_migrated = 1; } -static inline void perf_event_task_sched_in(struct task_struct *prev, +static __always_inline void perf_event_task_sched_in(struct task_struct *prev, struct task_struct *task) { if (static_branch_unlikely(&perf_sched_events)) diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 0232d983b715..650164baddae 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -44,7 +44,7 @@ static inline void smp_mb__after_mmgrab(void) extern void __mmdrop(struct mm_struct *mm); -static inline void mmdrop(struct mm_struct *mm) +static __always_inline void mmdrop(struct mm_struct *mm) { /* * The implicit full barrier implied by atomic_dec_and_test() is @@ -71,14 +71,14 @@ static inline void __mmdrop_delayed(struct rcu_head *rhp) * Invoked from finish_task_switch(). Delegates the heavy lifting on RT * kernels via RCU. */ -static inline void mmdrop_sched(struct mm_struct *mm) +static __always_inline void mmdrop_sched(struct mm_struct *mm) { /* Provides a full memory barrier. See mmdrop() */ if (atomic_dec_and_test(&mm->mm_count)) call_rcu(&mm->delayed_drop, __mmdrop_delayed); } #else -static inline void mmdrop_sched(struct mm_struct *mm) +static __always_inline void mmdrop_sched(struct mm_struct *mm) { mmdrop(mm); } @@ -104,7 +104,7 @@ static inline void mmdrop_lazy_tlb(struct mm_struct *mm) } } -static inline void mmdrop_lazy_tlb_sched(struct mm_struct *mm) +static __always_inline void mmdrop_lazy_tlb_sched(struct mm_struct *mm) { if (IS_ENABLED(CONFIG_MMU_LAZY_TLB_REFCOUNT)) mmdrop_sched(mm); @@ -529,7 +529,7 @@ enum { #include #endif -static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) +static __always_inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) { /* * The atomic_read() below prevents CSE. The following should diff --git a/include/linux/tick.h b/include/linux/tick.h index ac76ae9fa36d..fce16aa10ba2 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h @@ -175,7 +175,7 @@ extern cpumask_var_t tick_nohz_full_mask; #ifdef CONFIG_NO_HZ_FULL extern bool tick_nohz_full_running; -static inline bool tick_nohz_full_enabled(void) +static __always_inline bool tick_nohz_full_enabled(void) { if (!context_tracking_enabled()) return false; @@ -299,7 +299,7 @@ static inline void __tick_nohz_task_switch(void) { } static inline void tick_nohz_full_setup(cpumask_var_t cpumask) { } #endif -static inline void tick_nohz_task_switch(void) +static __always_inline void tick_nohz_task_switch(void) { if (tick_nohz_full_enabled()) __tick_nohz_task_switch(); diff --git a/include/linux/vtime.h b/include/linux/vtime.h index 29dd5b91dd7d..428464bb81b3 100644 --- a/include/linux/vtime.h +++ b/include/linux/vtime.h @@ -67,24 +67,24 @@ static __always_inline void vtime_account_guest_exit(void) * For now vtime state is tied to context tracking. We might want to decouple * those later if necessary. */ -static inline bool vtime_accounting_enabled(void) +static __always_inline bool vtime_accounting_enabled(void) { return context_tracking_enabled(); } -static inline bool vtime_accounting_enabled_cpu(int cpu) +static __always_inline bool vtime_accounting_enabled_cpu(int cpu) { return context_tracking_enabled_cpu(cpu); } -static inline bool vtime_accounting_enabled_this_cpu(void) +static __always_inline bool vtime_accounting_enabled_this_cpu(void) { return context_tracking_enabled_this_cpu(); } extern void vtime_task_switch_generic(struct task_struct *prev); -static inline void vtime_task_switch(struct task_struct *prev) +static __always_inline void vtime_task_switch(struct task_struct *prev) { if (vtime_accounting_enabled_this_cpu()) vtime_task_switch_generic(prev); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index f754a60de848..d90b290aad66 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -677,11 +677,6 @@ bool raw_spin_rq_trylock(struct rq *rq) } } -void raw_spin_rq_unlock(struct rq *rq) -{ - raw_spin_unlock(rq_lockp(rq)); -} - /* * double_rq_lock - safely lock two runqueues */ @@ -4947,7 +4942,7 @@ static inline void prepare_task(struct task_struct *next) WRITE_ONCE(next->on_cpu, 1); } -static inline void finish_task(struct task_struct *prev) +static __always_inline void finish_task(struct task_struct *prev) { /* * This must be the very last reference to @prev from this CPU. After @@ -4963,7 +4958,7 @@ static inline void finish_task(struct task_struct *prev) smp_store_release(&prev->on_cpu, 0); } -static void do_balance_callbacks(struct rq *rq, struct balance_callback *head) +static __always_inline void do_balance_callbacks(struct rq *rq, struct balance_callback *head) { void (*func)(struct rq *rq); struct balance_callback *next; @@ -4998,7 +4993,7 @@ struct balance_callback balance_push_callback = { .func = balance_push, }; -static inline struct balance_callback * +static __always_inline struct balance_callback * __splice_balance_callbacks(struct rq *rq, bool split) { struct balance_callback *head = rq->balance_callback; @@ -5028,7 +5023,7 @@ struct balance_callback *splice_balance_callbacks(struct rq *rq) return __splice_balance_callbacks(rq, true); } -static void __balance_callbacks(struct rq *rq) +static __always_inline void __balance_callbacks(struct rq *rq) { do_balance_callbacks(rq, __splice_balance_callbacks(rq, false)); } @@ -5061,7 +5056,7 @@ prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf #endif } -static inline void finish_lock_switch(struct rq *rq) +static __always_inline void finish_lock_switch(struct rq *rq) { /* * If we are tracking spinlock dependencies then we have to @@ -5093,7 +5088,7 @@ static inline void kmap_local_sched_out(void) #endif } -static inline void kmap_local_sched_in(void) +static __always_inline void kmap_local_sched_in(void) { #ifdef CONFIG_KMAP_LOCAL if (unlikely(current->kmap_ctrl.idx)) @@ -5147,7 +5142,7 @@ prepare_task_switch(struct rq *rq, struct task_struct *prev, * past. 'prev == current' is still correct but we need to recalculate this_rq * because prev may have moved to another CPU. */ -static struct rq *finish_task_switch(struct task_struct *prev) +static __always_inline struct rq *finish_task_switch(struct task_struct *prev) __releases(rq->lock) { struct rq *rq = this_rq(); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 51e432eb9c48..73d20396d6f4 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1373,12 +1373,12 @@ static inline struct cpumask *sched_group_span(struct sched_group *sg); DECLARE_STATIC_KEY_FALSE(__sched_core_enabled); -static inline bool sched_core_enabled(struct rq *rq) +static __always_inline bool sched_core_enabled(struct rq *rq) { return static_branch_unlikely(&__sched_core_enabled) && rq->core_enabled; } -static inline bool sched_core_disabled(void) +static __always_inline bool sched_core_disabled(void) { return !static_branch_unlikely(&__sched_core_enabled); } @@ -1387,7 +1387,7 @@ static inline bool sched_core_disabled(void) * Be careful with this function; not for general use. The return value isn't * stable unless you actually hold a relevant rq->__lock. */ -static inline raw_spinlock_t *rq_lockp(struct rq *rq) +static __always_inline raw_spinlock_t *rq_lockp(struct rq *rq) { if (sched_core_enabled(rq)) return &rq->core->__lock; @@ -1395,7 +1395,7 @@ static inline raw_spinlock_t *rq_lockp(struct rq *rq) return &rq->__lock; } -static inline raw_spinlock_t *__rq_lockp(struct rq *rq) +static __always_inline raw_spinlock_t *__rq_lockp(struct rq *rq) { if (rq->core_enabled) return &rq->core->__lock; @@ -1486,12 +1486,12 @@ static inline bool sched_core_disabled(void) return true; } -static inline raw_spinlock_t *rq_lockp(struct rq *rq) +static __always_inline raw_spinlock_t *rq_lockp(struct rq *rq) { return &rq->__lock; } -static inline raw_spinlock_t *__rq_lockp(struct rq *rq) +static __always_inline raw_spinlock_t *__rq_lockp(struct rq *rq) { return &rq->__lock; } @@ -1540,20 +1540,24 @@ static inline void lockdep_assert_rq_held(struct rq *rq) extern void raw_spin_rq_lock_nested(struct rq *rq, int subclass); extern bool raw_spin_rq_trylock(struct rq *rq); -extern void raw_spin_rq_unlock(struct rq *rq); -static inline void raw_spin_rq_lock(struct rq *rq) +static __always_inline void raw_spin_rq_lock(struct rq *rq) { raw_spin_rq_lock_nested(rq, 0); } -static inline void raw_spin_rq_lock_irq(struct rq *rq) +static __always_inline void raw_spin_rq_unlock(struct rq *rq) +{ + raw_spin_unlock(rq_lockp(rq)); +} + +static __always_inline void raw_spin_rq_lock_irq(struct rq *rq) { local_irq_disable(); raw_spin_rq_lock(rq); } -static inline void raw_spin_rq_unlock_irq(struct rq *rq) +static __always_inline void raw_spin_rq_unlock_irq(struct rq *rq) { raw_spin_rq_unlock(rq); local_irq_enable(); diff --git a/net/ipv4/esp4_offload.c b/net/ipv4/esp4_offload.c index 05828d4cb6cd..abd77162f5e7 100644 --- a/net/ipv4/esp4_offload.c +++ b/net/ipv4/esp4_offload.c @@ -122,8 +122,8 @@ static struct sk_buff *xfrm4_tunnel_gso_segment(struct xfrm_state *x, struct sk_buff *skb, netdev_features_t features) { - const struct xfrm_mode *inner_mode = xfrm_ip2inner_mode(x, - XFRM_MODE_SKB_CB(skb)->protocol); + struct xfrm_offload *xo = xfrm_offload(skb); + const struct xfrm_mode *inner_mode = xfrm_ip2inner_mode(x, xo->proto); __be16 type = inner_mode->family == AF_INET6 ? htons(ETH_P_IPV6) : htons(ETH_P_IP); diff --git a/net/ipv6/esp6_offload.c b/net/ipv6/esp6_offload.c index 22410243ebe8..22895521a57d 100644 --- a/net/ipv6/esp6_offload.c +++ b/net/ipv6/esp6_offload.c @@ -158,8 +158,8 @@ static struct sk_buff *xfrm6_tunnel_gso_segment(struct xfrm_state *x, struct sk_buff *skb, netdev_features_t features) { - const struct xfrm_mode *inner_mode = xfrm_ip2inner_mode(x, - XFRM_MODE_SKB_CB(skb)->protocol); + struct xfrm_offload *xo = xfrm_offload(skb); + const struct xfrm_mode *inner_mode = xfrm_ip2inner_mode(x, xo->proto); __be16 type = inner_mode->family == AF_INET ? htons(ETH_P_IP) : htons(ETH_P_IPV6); diff --git a/sound/soc/codecs/cs35l41.c b/sound/soc/codecs/cs35l41.c index 173d7c59b725..5001a546a3e7 100644 --- a/sound/soc/codecs/cs35l41.c +++ b/sound/soc/codecs/cs35l41.c @@ -1188,13 +1188,14 @@ static int cs35l41_get_system_name(struct cs35l41_private *cs35l41) } } -err: if (sub) { cs35l41->dsp.system_name = sub; dev_info(cs35l41->dev, "Subsystem ID: %s\n", cs35l41->dsp.system_name); - } else - dev_warn(cs35l41->dev, "Subsystem ID not found\n"); + return 0; + } +err: + dev_warn(cs35l41->dev, "Subsystem ID not found\n"); return ret; } -- 2.52.0