Skip to content

Commit 54b44ad

Browse files
committed
Merge branch kvm-arm64/sgi-injection into kvmarm/next
* kvm-arm64/sgi-injection: : vSGI injection improvements + fixes, courtesy Marc Zyngier : : Avoid linearly searching for vSGI targets using a compressed MPIDR to : index a cache. While at it, fix some egregious bugs in KVM's mishandling : of vcpuid (user-controlled value) and vcpu_idx. KVM: arm64: Clarify the ordering requirements for vcpu/RD creation KVM: arm64: vgic-v3: Optimize affinity-based SGI injection KVM: arm64: Fast-track kvm_mpidr_to_vcpu() when mpidr_data is available KVM: arm64: Build MPIDR to vcpu index cache at runtime KVM: arm64: Simplify kvm_vcpu_get_mpidr_aff() KVM: arm64: Use vcpu_idx for invalidation tracking KVM: arm64: vgic: Use vcpu_idx for the debug information KVM: arm64: vgic-v2: Use cpuid from userspace as vcpu_id KVM: arm64: vgic-v3: Refactor GICv3 SGI generation KVM: arm64: vgic-its: Treat the collection target address as a vcpu_id KVM: arm64: vgic: Make kvm_vgic_inject_irq() take a vcpu pointer Signed-off-by: Oliver Upton <oliver.upton@linux.dev>
2 parents df26b77 + f994041 commit 54b44ad

File tree

13 files changed

+212
-153
lines changed

13 files changed

+212
-153
lines changed

Documentation/virt/kvm/devices/arm-vgic-v3.rst

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,13 @@ Groups:
5959
It is invalid to mix calls with KVM_VGIC_V3_ADDR_TYPE_REDIST and
6060
KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION attributes.
6161

62+
Note that to obtain reproducible results (the same VCPU being associated
63+
with the same redistributor across a save/restore operation), VCPU creation
64+
order, redistributor region creation order as well as the respective
65+
interleaves of VCPU and region creation MUST be preserved. Any change in
66+
either ordering may result in a different vcpu_id/redistributor association,
67+
resulting in a VM that will fail to run at restore time.
68+
6269
Errors:
6370

6471
======= =============================================================

arch/arm64/include/asm/kvm_emulate.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -470,7 +470,7 @@ static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
470470

471471
static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu)
472472
{
473-
return vcpu_read_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK;
473+
return __vcpu_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK;
474474
}
475475

476476
static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu)

arch/arm64/include/asm/kvm_host.h

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,31 @@ struct kvm_protected_vm {
212212
struct kvm_hyp_memcache teardown_mc;
213213
};
214214

215+
struct kvm_mpidr_data {
216+
u64 mpidr_mask;
217+
DECLARE_FLEX_ARRAY(u16, cmpidr_to_idx);
218+
};
219+
220+
static inline u16 kvm_mpidr_index(struct kvm_mpidr_data *data, u64 mpidr)
221+
{
222+
unsigned long mask = data->mpidr_mask;
223+
u64 aff = mpidr & MPIDR_HWID_BITMASK;
224+
int nbits, bit, bit_idx = 0;
225+
u16 index = 0;
226+
227+
/*
228+
* If this looks like RISC-V's BEXT or x86's PEXT
229+
* instructions, it isn't by accident.
230+
*/
231+
nbits = fls(mask);
232+
for_each_set_bit(bit, &mask, nbits) {
233+
index |= (aff & BIT(bit)) >> (bit - bit_idx);
234+
bit_idx++;
235+
}
236+
237+
return index;
238+
}
239+
215240
struct kvm_arch {
216241
struct kvm_s2_mmu mmu;
217242

@@ -253,6 +278,9 @@ struct kvm_arch {
253278
/* VM-wide vCPU feature set */
254279
DECLARE_BITMAP(vcpu_features, KVM_VCPU_MAX_FEATURES);
255280

281+
/* MPIDR to vcpu index mapping, optional */
282+
struct kvm_mpidr_data *mpidr_data;
283+
256284
/*
257285
* VM-wide PMU filter, implemented as a bitmap and big enough for
258286
* up to 2^10 events (ARMv8.0) or 2^16 events (ARMv8.1+).

arch/arm64/kvm/arch_timer.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -458,7 +458,7 @@ static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
458458
timer_ctx->irq.level);
459459

460460
if (!userspace_irqchip(vcpu->kvm)) {
461-
ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
461+
ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu,
462462
timer_irq(timer_ctx),
463463
timer_ctx->irq.level,
464464
timer_ctx);

arch/arm64/kvm/arm.c

Lines changed: 76 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
205205
if (is_protected_kvm_enabled())
206206
pkvm_destroy_hyp_vm(kvm);
207207

208+
kfree(kvm->arch.mpidr_data);
208209
kvm_destroy_vcpus(kvm);
209210

210211
kvm_unshare_hyp(kvm, kvm + 1);
@@ -437,9 +438,9 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
437438
* We might get preempted before the vCPU actually runs, but
438439
* over-invalidation doesn't affect correctness.
439440
*/
440-
if (*last_ran != vcpu->vcpu_id) {
441+
if (*last_ran != vcpu->vcpu_idx) {
441442
kvm_call_hyp(__kvm_flush_cpu_context, mmu);
442-
*last_ran = vcpu->vcpu_id;
443+
*last_ran = vcpu->vcpu_idx;
443444
}
444445

445446
vcpu->cpu = cpu;
@@ -577,6 +578,57 @@ static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)
577578
return vcpu_get_flag(vcpu, VCPU_INITIALIZED);
578579
}
579580

581+
static void kvm_init_mpidr_data(struct kvm *kvm)
582+
{
583+
struct kvm_mpidr_data *data = NULL;
584+
unsigned long c, mask, nr_entries;
585+
u64 aff_set = 0, aff_clr = ~0UL;
586+
struct kvm_vcpu *vcpu;
587+
588+
mutex_lock(&kvm->arch.config_lock);
589+
590+
if (kvm->arch.mpidr_data || atomic_read(&kvm->online_vcpus) == 1)
591+
goto out;
592+
593+
kvm_for_each_vcpu(c, vcpu, kvm) {
594+
u64 aff = kvm_vcpu_get_mpidr_aff(vcpu);
595+
aff_set |= aff;
596+
aff_clr &= aff;
597+
}
598+
599+
/*
600+
* A significant bit can be either 0 or 1, and will only appear in
601+
* aff_set. Use aff_clr to weed out the useless stuff.
602+
*/
603+
mask = aff_set ^ aff_clr;
604+
nr_entries = BIT_ULL(hweight_long(mask));
605+
606+
/*
607+
* Don't let userspace fool us. If we need more than a single page
608+
* to describe the compressed MPIDR array, just fall back to the
609+
* iterative method. Single vcpu VMs do not need this either.
610+
*/
611+
if (struct_size(data, cmpidr_to_idx, nr_entries) <= PAGE_SIZE)
612+
data = kzalloc(struct_size(data, cmpidr_to_idx, nr_entries),
613+
GFP_KERNEL_ACCOUNT);
614+
615+
if (!data)
616+
goto out;
617+
618+
data->mpidr_mask = mask;
619+
620+
kvm_for_each_vcpu(c, vcpu, kvm) {
621+
u64 aff = kvm_vcpu_get_mpidr_aff(vcpu);
622+
u16 index = kvm_mpidr_index(data, aff);
623+
624+
data->cmpidr_to_idx[index] = c;
625+
}
626+
627+
kvm->arch.mpidr_data = data;
628+
out:
629+
mutex_unlock(&kvm->arch.config_lock);
630+
}
631+
580632
/*
581633
* Handle both the initialisation that is being done when the vcpu is
582634
* run for the first time, as well as the updates that must be
@@ -600,6 +652,8 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
600652
if (likely(vcpu_has_run_once(vcpu)))
601653
return 0;
602654

655+
kvm_init_mpidr_data(kvm);
656+
603657
kvm_arm_vcpu_init_debug(vcpu);
604658

605659
if (likely(irqchip_in_kernel(kvm))) {
@@ -1136,27 +1190,23 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
11361190
bool line_status)
11371191
{
11381192
u32 irq = irq_level->irq;
1139-
unsigned int irq_type, vcpu_idx, irq_num;
1140-
int nrcpus = atomic_read(&kvm->online_vcpus);
1193+
unsigned int irq_type, vcpu_id, irq_num;
11411194
struct kvm_vcpu *vcpu = NULL;
11421195
bool level = irq_level->level;
11431196

11441197
irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
1145-
vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
1146-
vcpu_idx += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1);
1198+
vcpu_id = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
1199+
vcpu_id += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1);
11471200
irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
11481201

1149-
trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level);
1202+
trace_kvm_irq_line(irq_type, vcpu_id, irq_num, irq_level->level);
11501203

11511204
switch (irq_type) {
11521205
case KVM_ARM_IRQ_TYPE_CPU:
11531206
if (irqchip_in_kernel(kvm))
11541207
return -ENXIO;
11551208

1156-
if (vcpu_idx >= nrcpus)
1157-
return -EINVAL;
1158-
1159-
vcpu = kvm_get_vcpu(kvm, vcpu_idx);
1209+
vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
11601210
if (!vcpu)
11611211
return -EINVAL;
11621212

@@ -1168,25 +1218,22 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
11681218
if (!irqchip_in_kernel(kvm))
11691219
return -ENXIO;
11701220

1171-
if (vcpu_idx >= nrcpus)
1172-
return -EINVAL;
1173-
1174-
vcpu = kvm_get_vcpu(kvm, vcpu_idx);
1221+
vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
11751222
if (!vcpu)
11761223
return -EINVAL;
11771224

11781225
if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
11791226
return -EINVAL;
11801227

1181-
return kvm_vgic_inject_irq(kvm, vcpu->vcpu_id, irq_num, level, NULL);
1228+
return kvm_vgic_inject_irq(kvm, vcpu, irq_num, level, NULL);
11821229
case KVM_ARM_IRQ_TYPE_SPI:
11831230
if (!irqchip_in_kernel(kvm))
11841231
return -ENXIO;
11851232

11861233
if (irq_num < VGIC_NR_PRIVATE_IRQS)
11871234
return -EINVAL;
11881235

1189-
return kvm_vgic_inject_irq(kvm, 0, irq_num, level, NULL);
1236+
return kvm_vgic_inject_irq(kvm, NULL, irq_num, level, NULL);
11901237
}
11911238

11921239
return -EINVAL;
@@ -2378,6 +2425,18 @@ struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
23782425
unsigned long i;
23792426

23802427
mpidr &= MPIDR_HWID_BITMASK;
2428+
2429+
if (kvm->arch.mpidr_data) {
2430+
u16 idx = kvm_mpidr_index(kvm->arch.mpidr_data, mpidr);
2431+
2432+
vcpu = kvm_get_vcpu(kvm,
2433+
kvm->arch.mpidr_data->cmpidr_to_idx[idx]);
2434+
if (mpidr != kvm_vcpu_get_mpidr_aff(vcpu))
2435+
vcpu = NULL;
2436+
2437+
return vcpu;
2438+
}
2439+
23812440
kvm_for_each_vcpu(i, vcpu, kvm) {
23822441
if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu))
23832442
return vcpu;

arch/arm64/kvm/pmu-emul.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,7 @@ static void kvm_pmu_update_state(struct kvm_vcpu *vcpu)
365365
pmu->irq_level = overflow;
366366

367367
if (likely(irqchip_in_kernel(vcpu->kvm))) {
368-
int ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
368+
int ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu,
369369
pmu->irq_num, overflow, pmu);
370370
WARN_ON(ret);
371371
}

arch/arm64/kvm/vgic/vgic-debug.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ static void print_header(struct seq_file *s, struct vgic_irq *irq,
166166

167167
if (vcpu) {
168168
hdr = "VCPU";
169-
id = vcpu->vcpu_id;
169+
id = vcpu->vcpu_idx;
170170
}
171171

172172
seq_printf(s, "\n");
@@ -212,7 +212,7 @@ static void print_irq_state(struct seq_file *s, struct vgic_irq *irq,
212212
" %2d "
213213
"\n",
214214
type, irq->intid,
215-
(irq->target_vcpu) ? irq->target_vcpu->vcpu_id : -1,
215+
(irq->target_vcpu) ? irq->target_vcpu->vcpu_idx : -1,
216216
pending,
217217
irq->line_level,
218218
irq->active,
@@ -224,7 +224,7 @@ static void print_irq_state(struct seq_file *s, struct vgic_irq *irq,
224224
irq->mpidr,
225225
irq->source,
226226
irq->priority,
227-
(irq->vcpu) ? irq->vcpu->vcpu_id : -1);
227+
(irq->vcpu) ? irq->vcpu->vcpu_idx : -1);
228228
}
229229

230230
static int vgic_debug_show(struct seq_file *s, void *v)

arch/arm64/kvm/vgic/vgic-irqfd.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ static int vgic_irqfd_set_irq(struct kvm_kernel_irq_routing_entry *e,
2323

2424
if (!vgic_valid_spi(kvm, spi_id))
2525
return -EINVAL;
26-
return kvm_vgic_inject_irq(kvm, 0, spi_id, level, NULL);
26+
return kvm_vgic_inject_irq(kvm, NULL, spi_id, level, NULL);
2727
}
2828

2929
/**

0 commit comments

Comments
 (0)