~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/virt/kvm/arm/vgic/vgic-its.c

Version: ~ [ linux-5.9.1 ] ~ [ linux-5.8.16 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.72 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.152 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.202 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.240 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.240 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * GICv3 ITS emulation
  3  *
  4  * Copyright (C) 2015,2016 ARM Ltd.
  5  * Author: Andre Przywara <andre.przywara@arm.com>
  6  *
  7  * This program is free software; you can redistribute it and/or modify
  8  * it under the terms of the GNU General Public License version 2 as
  9  * published by the Free Software Foundation.
 10  *
 11  * This program is distributed in the hope that it will be useful,
 12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 14  * GNU General Public License for more details.
 15  *
 16  * You should have received a copy of the GNU General Public License
 17  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 18  */
 19 
 20 #include <linux/cpu.h>
 21 #include <linux/kvm.h>
 22 #include <linux/kvm_host.h>
 23 #include <linux/interrupt.h>
 24 #include <linux/list.h>
 25 #include <linux/uaccess.h>
 26 #include <linux/list_sort.h>
 27 
 28 #include <linux/irqchip/arm-gic-v3.h>
 29 
 30 #include <asm/kvm_emulate.h>
 31 #include <asm/kvm_arm.h>
 32 #include <asm/kvm_mmu.h>
 33 
 34 #include "vgic.h"
 35 #include "vgic-mmio.h"
 36 
 37 static int vgic_its_save_tables_v0(struct vgic_its *its);
 38 static int vgic_its_restore_tables_v0(struct vgic_its *its);
 39 static int vgic_its_commit_v0(struct vgic_its *its);
 40 static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
 41                              struct kvm_vcpu *filter_vcpu, bool needs_inv);
 42 
 43 /*
 44  * Creates a new (reference to a) struct vgic_irq for a given LPI.
 45  * If this LPI is already mapped on another ITS, we increase its refcount
 46  * and return a pointer to the existing structure.
 47  * If this is a "new" LPI, we allocate and initialize a new struct vgic_irq.
 48  * This function returns a pointer to the _unlocked_ structure.
 49  */
 50 static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid,
 51                                      struct kvm_vcpu *vcpu)
 52 {
 53         struct vgic_dist *dist = &kvm->arch.vgic;
 54         struct vgic_irq *irq = vgic_get_irq(kvm, NULL, intid), *oldirq;
 55         int ret;
 56 
 57         /* In this case there is no put, since we keep the reference. */
 58         if (irq)
 59                 return irq;
 60 
 61         irq = kzalloc(sizeof(struct vgic_irq), GFP_KERNEL);
 62         if (!irq)
 63                 return ERR_PTR(-ENOMEM);
 64 
 65         INIT_LIST_HEAD(&irq->lpi_list);
 66         INIT_LIST_HEAD(&irq->ap_list);
 67         spin_lock_init(&irq->irq_lock);
 68 
 69         irq->config = VGIC_CONFIG_EDGE;
 70         kref_init(&irq->refcount);
 71         irq->intid = intid;
 72         irq->target_vcpu = vcpu;
 73 
 74         spin_lock(&dist->lpi_list_lock);
 75 
 76         /*
 77          * There could be a race with another vgic_add_lpi(), so we need to
 78          * check that we don't add a second list entry with the same LPI.
 79          */
 80         list_for_each_entry(oldirq, &dist->lpi_list_head, lpi_list) {
 81                 if (oldirq->intid != intid)
 82                         continue;
 83 
 84                 /* Someone was faster with adding this LPI, lets use that. */
 85                 kfree(irq);
 86                 irq = oldirq;
 87 
 88                 /*
 89                  * This increases the refcount, the caller is expected to
 90                  * call vgic_put_irq() on the returned pointer once it's
 91                  * finished with the IRQ.
 92                  */
 93                 vgic_get_irq_kref(irq);
 94 
 95                 goto out_unlock;
 96         }
 97 
 98         list_add_tail(&irq->lpi_list, &dist->lpi_list_head);
 99         dist->lpi_list_count++;
100 
101 out_unlock:
102         spin_unlock(&dist->lpi_list_lock);
103 
104         /*
105          * We "cache" the configuration table entries in our struct vgic_irq's.
106          * However we only have those structs for mapped IRQs, so we read in
107          * the respective config data from memory here upon mapping the LPI.
108          */
109         ret = update_lpi_config(kvm, irq, NULL, false);
110         if (ret)
111                 return ERR_PTR(ret);
112 
113         ret = vgic_v3_lpi_sync_pending_status(kvm, irq);
114         if (ret)
115                 return ERR_PTR(ret);
116 
117         return irq;
118 }
119 
120 struct its_device {
121         struct list_head dev_list;
122 
123         /* the head for the list of ITTEs */
124         struct list_head itt_head;
125         u32 num_eventid_bits;
126         gpa_t itt_addr;
127         u32 device_id;
128 };
129 
130 #define COLLECTION_NOT_MAPPED ((u32)~0)
131 
132 struct its_collection {
133         struct list_head coll_list;
134 
135         u32 collection_id;
136         u32 target_addr;
137 };
138 
139 #define its_is_collection_mapped(coll) ((coll) && \
140                                 ((coll)->target_addr != COLLECTION_NOT_MAPPED))
141 
142 struct its_ite {
143         struct list_head ite_list;
144 
145         struct vgic_irq *irq;
146         struct its_collection *collection;
147         u32 event_id;
148 };
149 
150 /**
151  * struct vgic_its_abi - ITS abi ops and settings
152  * @cte_esz: collection table entry size
153  * @dte_esz: device table entry size
154  * @ite_esz: interrupt translation table entry size
155  * @save tables: save the ITS tables into guest RAM
156  * @restore_tables: restore the ITS internal structs from tables
157  *  stored in guest RAM
158  * @commit: initialize the registers which expose the ABI settings,
159  *  especially the entry sizes
160  */
161 struct vgic_its_abi {
162         int cte_esz;
163         int dte_esz;
164         int ite_esz;
165         int (*save_tables)(struct vgic_its *its);
166         int (*restore_tables)(struct vgic_its *its);
167         int (*commit)(struct vgic_its *its);
168 };
169 
170 static const struct vgic_its_abi its_table_abi_versions[] = {
171         [0] = {.cte_esz = 8, .dte_esz = 8, .ite_esz = 8,
172          .save_tables = vgic_its_save_tables_v0,
173          .restore_tables = vgic_its_restore_tables_v0,
174          .commit = vgic_its_commit_v0,
175         },
176 };
177 
178 #define NR_ITS_ABIS     ARRAY_SIZE(its_table_abi_versions)
179 
180 inline const struct vgic_its_abi *vgic_its_get_abi(struct vgic_its *its)
181 {
182         return &its_table_abi_versions[its->abi_rev];
183 }
184 
185 int vgic_its_set_abi(struct vgic_its *its, int rev)
186 {
187         const struct vgic_its_abi *abi;
188 
189         its->abi_rev = rev;
190         abi = vgic_its_get_abi(its);
191         return abi->commit(its);
192 }
193 
194 /*
195  * Find and returns a device in the device table for an ITS.
196  * Must be called with the its_lock mutex held.
197  */
198 static struct its_device *find_its_device(struct vgic_its *its, u32 device_id)
199 {
200         struct its_device *device;
201 
202         list_for_each_entry(device, &its->device_list, dev_list)
203                 if (device_id == device->device_id)
204                         return device;
205 
206         return NULL;
207 }
208 
209 /*
210  * Find and returns an interrupt translation table entry (ITTE) for a given
211  * Device ID/Event ID pair on an ITS.
212  * Must be called with the its_lock mutex held.
213  */
214 static struct its_ite *find_ite(struct vgic_its *its, u32 device_id,
215                                   u32 event_id)
216 {
217         struct its_device *device;
218         struct its_ite *ite;
219 
220         device = find_its_device(its, device_id);
221         if (device == NULL)
222                 return NULL;
223 
224         list_for_each_entry(ite, &device->itt_head, ite_list)
225                 if (ite->event_id == event_id)
226                         return ite;
227 
228         return NULL;
229 }
230 
231 /* To be used as an iterator this macro misses the enclosing parentheses */
232 #define for_each_lpi_its(dev, ite, its) \
233         list_for_each_entry(dev, &(its)->device_list, dev_list) \
234                 list_for_each_entry(ite, &(dev)->itt_head, ite_list)
235 
236 /*
237  * We only implement 48 bits of PA at the moment, although the ITS
238  * supports more. Let's be restrictive here.
239  */
240 #define BASER_ADDRESS(x)        ((x) & GENMASK_ULL(47, 16))
241 #define CBASER_ADDRESS(x)       ((x) & GENMASK_ULL(47, 12))
242 
243 #define GIC_LPI_OFFSET 8192
244 
245 #define VITS_TYPER_IDBITS 16
246 #define VITS_TYPER_DEVBITS 16
247 #define VITS_DTE_MAX_DEVID_OFFSET       (BIT(14) - 1)
248 #define VITS_ITE_MAX_EVENTID_OFFSET     (BIT(16) - 1)
249 
250 /*
251  * Finds and returns a collection in the ITS collection table.
252  * Must be called with the its_lock mutex held.
253  */
254 static struct its_collection *find_collection(struct vgic_its *its, int coll_id)
255 {
256         struct its_collection *collection;
257 
258         list_for_each_entry(collection, &its->collection_list, coll_list) {
259                 if (coll_id == collection->collection_id)
260                         return collection;
261         }
262 
263         return NULL;
264 }
265 
266 #define LPI_PROP_ENABLE_BIT(p)  ((p) & LPI_PROP_ENABLED)
267 #define LPI_PROP_PRIORITY(p)    ((p) & 0xfc)
268 
269 /*
270  * Reads the configuration data for a given LPI from guest memory and
271  * updates the fields in struct vgic_irq.
272  * If filter_vcpu is not NULL, applies only if the IRQ is targeting this
273  * VCPU. Unconditionally applies if filter_vcpu is NULL.
274  */
275 static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
276                              struct kvm_vcpu *filter_vcpu, bool needs_inv)
277 {
278         u64 propbase = GICR_PROPBASER_ADDRESS(kvm->arch.vgic.propbaser);
279         u8 prop;
280         int ret;
281         unsigned long flags;
282 
283         ret = kvm_read_guest(kvm, propbase + irq->intid - GIC_LPI_OFFSET,
284                              &prop, 1);
285 
286         if (ret)
287                 return ret;
288 
289         spin_lock_irqsave(&irq->irq_lock, flags);
290 
291         if (!filter_vcpu || filter_vcpu == irq->target_vcpu) {
292                 irq->priority = LPI_PROP_PRIORITY(prop);
293                 irq->enabled = LPI_PROP_ENABLE_BIT(prop);
294 
295                 if (!irq->hw) {
296                         vgic_queue_irq_unlock(kvm, irq, flags);
297                         return 0;
298                 }
299         }
300 
301         spin_unlock_irqrestore(&irq->irq_lock, flags);
302 
303         if (irq->hw)
304                 return its_prop_update_vlpi(irq->host_irq, prop, needs_inv);
305 
306         return 0;
307 }
308 
309 /*
310  * Create a snapshot of the current LPIs targeting @vcpu, so that we can
311  * enumerate those LPIs without holding any lock.
312  * Returns their number and puts the kmalloc'ed array into intid_ptr.
313  */
314 static int vgic_copy_lpi_list(struct kvm_vcpu *vcpu, u32 **intid_ptr)
315 {
316         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
317         struct vgic_irq *irq;
318         u32 *intids;
319         int irq_count = dist->lpi_list_count, i = 0;
320 
321         /*
322          * We use the current value of the list length, which may change
323          * after the kmalloc. We don't care, because the guest shouldn't
324          * change anything while the command handling is still running,
325          * and in the worst case we would miss a new IRQ, which one wouldn't
326          * expect to be covered by this command anyway.
327          */
328         intids = kmalloc_array(irq_count, sizeof(intids[0]), GFP_KERNEL);
329         if (!intids)
330                 return -ENOMEM;
331 
332         spin_lock(&dist->lpi_list_lock);
333         list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
334                 /* We don't need to "get" the IRQ, as we hold the list lock. */
335                 if (irq->target_vcpu != vcpu)
336                         continue;
337                 intids[i++] = irq->intid;
338         }
339         spin_unlock(&dist->lpi_list_lock);
340 
341         *intid_ptr = intids;
342         return i;
343 }
344 
345 static int update_affinity(struct vgic_irq *irq, struct kvm_vcpu *vcpu)
346 {
347         int ret = 0;
348 
349         spin_lock(&irq->irq_lock);
350         irq->target_vcpu = vcpu;
351         spin_unlock(&irq->irq_lock);
352 
353         if (irq->hw) {
354                 struct its_vlpi_map map;
355 
356                 ret = its_get_vlpi(irq->host_irq, &map);
357                 if (ret)
358                         return ret;
359 
360                 map.vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
361 
362                 ret = its_map_vlpi(irq->host_irq, &map);
363         }
364 
365         return ret;
366 }
367 
368 /*
369  * Promotes the ITS view of affinity of an ITTE (which redistributor this LPI
370  * is targeting) to the VGIC's view, which deals with target VCPUs.
371  * Needs to be called whenever either the collection for a LPIs has
372  * changed or the collection itself got retargeted.
373  */
374 static void update_affinity_ite(struct kvm *kvm, struct its_ite *ite)
375 {
376         struct kvm_vcpu *vcpu;
377 
378         if (!its_is_collection_mapped(ite->collection))
379                 return;
380 
381         vcpu = kvm_get_vcpu(kvm, ite->collection->target_addr);
382         update_affinity(ite->irq, vcpu);
383 }
384 
385 /*
386  * Updates the target VCPU for every LPI targeting this collection.
387  * Must be called with the its_lock mutex held.
388  */
389 static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its,
390                                        struct its_collection *coll)
391 {
392         struct its_device *device;
393         struct its_ite *ite;
394 
395         for_each_lpi_its(device, ite, its) {
396                 if (!ite->collection || coll != ite->collection)
397                         continue;
398 
399                 update_affinity_ite(kvm, ite);
400         }
401 }
402 
403 static u32 max_lpis_propbaser(u64 propbaser)
404 {
405         int nr_idbits = (propbaser & 0x1f) + 1;
406 
407         return 1U << min(nr_idbits, INTERRUPT_ID_BITS_ITS);
408 }
409 
410 /*
411  * Sync the pending table pending bit of LPIs targeting @vcpu
412  * with our own data structures. This relies on the LPI being
413  * mapped before.
414  */
415 static int its_sync_lpi_pending_table(struct kvm_vcpu *vcpu)
416 {
417         gpa_t pendbase = GICR_PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser);
418         struct vgic_irq *irq;
419         int last_byte_offset = -1;
420         int ret = 0;
421         u32 *intids;
422         int nr_irqs, i;
423         unsigned long flags;
424         u8 pendmask;
425 
426         nr_irqs = vgic_copy_lpi_list(vcpu, &intids);
427         if (nr_irqs < 0)
428                 return nr_irqs;
429 
430         for (i = 0; i < nr_irqs; i++) {
431                 int byte_offset, bit_nr;
432 
433                 byte_offset = intids[i] / BITS_PER_BYTE;
434                 bit_nr = intids[i] % BITS_PER_BYTE;
435 
436                 /*
437                  * For contiguously allocated LPIs chances are we just read
438                  * this very same byte in the last iteration. Reuse that.
439                  */
440                 if (byte_offset != last_byte_offset) {
441                         ret = kvm_read_guest(vcpu->kvm, pendbase + byte_offset,
442                                              &pendmask, 1);
443                         if (ret) {
444                                 kfree(intids);
445                                 return ret;
446                         }
447                         last_byte_offset = byte_offset;
448                 }
449 
450                 irq = vgic_get_irq(vcpu->kvm, NULL, intids[i]);
451                 spin_lock_irqsave(&irq->irq_lock, flags);
452                 irq->pending_latch = pendmask & (1U << bit_nr);
453                 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
454                 vgic_put_irq(vcpu->kvm, irq);
455         }
456 
457         kfree(intids);
458 
459         return ret;
460 }
461 
462 static unsigned long vgic_mmio_read_its_typer(struct kvm *kvm,
463                                               struct vgic_its *its,
464                                               gpa_t addr, unsigned int len)
465 {
466         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
467         u64 reg = GITS_TYPER_PLPIS;
468 
469         /*
470          * We use linear CPU numbers for redistributor addressing,
471          * so GITS_TYPER.PTA is 0.
472          * Also we force all PROPBASER registers to be the same, so
473          * CommonLPIAff is 0 as well.
474          * To avoid memory waste in the guest, we keep the number of IDBits and
475          * DevBits low - as least for the time being.
476          */
477         reg |= GIC_ENCODE_SZ(VITS_TYPER_DEVBITS, 5) << GITS_TYPER_DEVBITS_SHIFT;
478         reg |= GIC_ENCODE_SZ(VITS_TYPER_IDBITS, 5) << GITS_TYPER_IDBITS_SHIFT;
479         reg |= GIC_ENCODE_SZ(abi->ite_esz, 4) << GITS_TYPER_ITT_ENTRY_SIZE_SHIFT;
480 
481         return extract_bytes(reg, addr & 7, len);
482 }
483 
484 static unsigned long vgic_mmio_read_its_iidr(struct kvm *kvm,
485                                              struct vgic_its *its,
486                                              gpa_t addr, unsigned int len)
487 {
488         u32 val;
489 
490         val = (its->abi_rev << GITS_IIDR_REV_SHIFT) & GITS_IIDR_REV_MASK;
491         val |= (PRODUCT_ID_KVM << GITS_IIDR_PRODUCTID_SHIFT) | IMPLEMENTER_ARM;
492         return val;
493 }
494 
495 static int vgic_mmio_uaccess_write_its_iidr(struct kvm *kvm,
496                                             struct vgic_its *its,
497                                             gpa_t addr, unsigned int len,
498                                             unsigned long val)
499 {
500         u32 rev = GITS_IIDR_REV(val);
501 
502         if (rev >= NR_ITS_ABIS)
503                 return -EINVAL;
504         return vgic_its_set_abi(its, rev);
505 }
506 
507 static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm,
508                                                struct vgic_its *its,
509                                                gpa_t addr, unsigned int len)
510 {
511         switch (addr & 0xffff) {
512         case GITS_PIDR0:
513                 return 0x92;    /* part number, bits[7:0] */
514         case GITS_PIDR1:
515                 return 0xb4;    /* part number, bits[11:8] */
516         case GITS_PIDR2:
517                 return GIC_PIDR2_ARCH_GICv3 | 0x0b;
518         case GITS_PIDR4:
519                 return 0x40;    /* This is a 64K software visible page */
520         /* The following are the ID registers for (any) GIC. */
521         case GITS_CIDR0:
522                 return 0x0d;
523         case GITS_CIDR1:
524                 return 0xf0;
525         case GITS_CIDR2:
526                 return 0x05;
527         case GITS_CIDR3:
528                 return 0xb1;
529         }
530 
531         return 0;
532 }
533 
534 int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
535                          u32 devid, u32 eventid, struct vgic_irq **irq)
536 {
537         struct kvm_vcpu *vcpu;
538         struct its_ite *ite;
539 
540         if (!its->enabled)
541                 return -EBUSY;
542 
543         ite = find_ite(its, devid, eventid);
544         if (!ite || !its_is_collection_mapped(ite->collection))
545                 return E_ITS_INT_UNMAPPED_INTERRUPT;
546 
547         vcpu = kvm_get_vcpu(kvm, ite->collection->target_addr);
548         if (!vcpu)
549                 return E_ITS_INT_UNMAPPED_INTERRUPT;
550 
551         if (!vcpu->arch.vgic_cpu.lpis_enabled)
552                 return -EBUSY;
553 
554         *irq = ite->irq;
555         return 0;
556 }
557 
558 struct vgic_its *vgic_msi_to_its(struct kvm *kvm, struct kvm_msi *msi)
559 {
560         u64 address;
561         struct kvm_io_device *kvm_io_dev;
562         struct vgic_io_device *iodev;
563 
564         if (!vgic_has_its(kvm))
565                 return ERR_PTR(-ENODEV);
566 
567         if (!(msi->flags & KVM_MSI_VALID_DEVID))
568                 return ERR_PTR(-EINVAL);
569 
570         address = (u64)msi->address_hi << 32 | msi->address_lo;
571 
572         kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, address);
573         if (!kvm_io_dev)
574                 return ERR_PTR(-EINVAL);
575 
576         if (kvm_io_dev->ops != &kvm_io_gic_ops)
577                 return ERR_PTR(-EINVAL);
578 
579         iodev = container_of(kvm_io_dev, struct vgic_io_device, dev);
580         if (iodev->iodev_type != IODEV_ITS)
581                 return ERR_PTR(-EINVAL);
582 
583         return iodev->its;
584 }
585 
586 /*
587  * Find the target VCPU and the LPI number for a given devid/eventid pair
588  * and make this IRQ pending, possibly injecting it.
589  * Must be called with the its_lock mutex held.
590  * Returns 0 on success, a positive error value for any ITS mapping
591  * related errors and negative error values for generic errors.
592  */
593 static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,
594                                 u32 devid, u32 eventid)
595 {
596         struct vgic_irq *irq = NULL;
597         unsigned long flags;
598         int err;
599 
600         err = vgic_its_resolve_lpi(kvm, its, devid, eventid, &irq);
601         if (err)
602                 return err;
603 
604         if (irq->hw)
605                 return irq_set_irqchip_state(irq->host_irq,
606                                              IRQCHIP_STATE_PENDING, true);
607 
608         spin_lock_irqsave(&irq->irq_lock, flags);
609         irq->pending_latch = true;
610         vgic_queue_irq_unlock(kvm, irq, flags);
611 
612         return 0;
613 }
614 
615 /*
616  * Queries the KVM IO bus framework to get the ITS pointer from the given
617  * doorbell address.
618  * We then call vgic_its_trigger_msi() with the decoded data.
619  * According to the KVM_SIGNAL_MSI API description returns 1 on success.
620  */
621 int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
622 {
623         struct vgic_its *its;
624         int ret;
625 
626         its = vgic_msi_to_its(kvm, msi);
627         if (IS_ERR(its))
628                 return PTR_ERR(its);
629 
630         mutex_lock(&its->its_lock);
631         ret = vgic_its_trigger_msi(kvm, its, msi->devid, msi->data);
632         mutex_unlock(&its->its_lock);
633 
634         if (ret < 0)
635                 return ret;
636 
637         /*
638          * KVM_SIGNAL_MSI demands a return value > 0 for success and 0
639          * if the guest has blocked the MSI. So we map any LPI mapping
640          * related error to that.
641          */
642         if (ret)
643                 return 0;
644         else
645                 return 1;
646 }
647 
648 /* Requires the its_lock to be held. */
649 static void its_free_ite(struct kvm *kvm, struct its_ite *ite)
650 {
651         list_del(&ite->ite_list);
652 
653         /* This put matches the get in vgic_add_lpi. */
654         if (ite->irq) {
655                 if (ite->irq->hw)
656                         WARN_ON(its_unmap_vlpi(ite->irq->host_irq));
657 
658                 vgic_put_irq(kvm, ite->irq);
659         }
660 
661         kfree(ite);
662 }
663 
664 static u64 its_cmd_mask_field(u64 *its_cmd, int word, int shift, int size)
665 {
666         return (le64_to_cpu(its_cmd[word]) >> shift) & (BIT_ULL(size) - 1);
667 }
668 
669 #define its_cmd_get_command(cmd)        its_cmd_mask_field(cmd, 0,  0,  8)
670 #define its_cmd_get_deviceid(cmd)       its_cmd_mask_field(cmd, 0, 32, 32)
671 #define its_cmd_get_size(cmd)           (its_cmd_mask_field(cmd, 1,  0,  5) + 1)
672 #define its_cmd_get_id(cmd)             its_cmd_mask_field(cmd, 1,  0, 32)
673 #define its_cmd_get_physical_id(cmd)    its_cmd_mask_field(cmd, 1, 32, 32)
674 #define its_cmd_get_collection(cmd)     its_cmd_mask_field(cmd, 2,  0, 16)
675 #define its_cmd_get_ittaddr(cmd)        (its_cmd_mask_field(cmd, 2,  8, 44) << 8)
676 #define its_cmd_get_target_addr(cmd)    its_cmd_mask_field(cmd, 2, 16, 32)
677 #define its_cmd_get_validbit(cmd)       its_cmd_mask_field(cmd, 2, 63,  1)
678 
679 /*
680  * The DISCARD command frees an Interrupt Translation Table Entry (ITTE).
681  * Must be called with the its_lock mutex held.
682  */
683 static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,
684                                        u64 *its_cmd)
685 {
686         u32 device_id = its_cmd_get_deviceid(its_cmd);
687         u32 event_id = its_cmd_get_id(its_cmd);
688         struct its_ite *ite;
689 
690 
691         ite = find_ite(its, device_id, event_id);
692         if (ite && ite->collection) {
693                 /*
694                  * Though the spec talks about removing the pending state, we
695                  * don't bother here since we clear the ITTE anyway and the
696                  * pending state is a property of the ITTE struct.
697                  */
698                 its_free_ite(kvm, ite);
699                 return 0;
700         }
701 
702         return E_ITS_DISCARD_UNMAPPED_INTERRUPT;
703 }
704 
705 /*
706  * The MOVI command moves an ITTE to a different collection.
707  * Must be called with the its_lock mutex held.
708  */
709 static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,
710                                     u64 *its_cmd)
711 {
712         u32 device_id = its_cmd_get_deviceid(its_cmd);
713         u32 event_id = its_cmd_get_id(its_cmd);
714         u32 coll_id = its_cmd_get_collection(its_cmd);
715         struct kvm_vcpu *vcpu;
716         struct its_ite *ite;
717         struct its_collection *collection;
718 
719         ite = find_ite(its, device_id, event_id);
720         if (!ite)
721                 return E_ITS_MOVI_UNMAPPED_INTERRUPT;
722 
723         if (!its_is_collection_mapped(ite->collection))
724                 return E_ITS_MOVI_UNMAPPED_COLLECTION;
725 
726         collection = find_collection(its, coll_id);
727         if (!its_is_collection_mapped(collection))
728                 return E_ITS_MOVI_UNMAPPED_COLLECTION;
729 
730         ite->collection = collection;
731         vcpu = kvm_get_vcpu(kvm, collection->target_addr);
732 
733         return update_affinity(ite->irq, vcpu);
734 }
735 
736 /*
737  * Check whether an ID can be stored into the corresponding guest table.
738  * For a direct table this is pretty easy, but gets a bit nasty for
739  * indirect tables. We check whether the resulting guest physical address
740  * is actually valid (covered by a memslot and guest accessible).
741  * For this we have to read the respective first level entry.
742  */
743 static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id,
744                               gpa_t *eaddr)
745 {
746         int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
747         u64 indirect_ptr, type = GITS_BASER_TYPE(baser);
748         int esz = GITS_BASER_ENTRY_SIZE(baser);
749         int index;
750         gfn_t gfn;
751 
752         switch (type) {
753         case GITS_BASER_TYPE_DEVICE:
754                 if (id >= BIT_ULL(VITS_TYPER_DEVBITS))
755                         return false;
756                 break;
757         case GITS_BASER_TYPE_COLLECTION:
758                 /* as GITS_TYPER.CIL == 0, ITS supports 16-bit collection ID */
759                 if (id >= BIT_ULL(16))
760                         return false;
761                 break;
762         default:
763                 return false;
764         }
765 
766         if (!(baser & GITS_BASER_INDIRECT)) {
767                 phys_addr_t addr;
768 
769                 if (id >= (l1_tbl_size / esz))
770                         return false;
771 
772                 addr = BASER_ADDRESS(baser) + id * esz;
773                 gfn = addr >> PAGE_SHIFT;
774 
775                 if (eaddr)
776                         *eaddr = addr;
777                 return kvm_is_visible_gfn(its->dev->kvm, gfn);
778         }
779 
780         /* calculate and check the index into the 1st level */
781         index = id / (SZ_64K / esz);
782         if (index >= (l1_tbl_size / sizeof(u64)))
783                 return false;
784 
785         /* Each 1st level entry is represented by a 64-bit value. */
786         if (kvm_read_guest(its->dev->kvm,
787                            BASER_ADDRESS(baser) + index * sizeof(indirect_ptr),
788                            &indirect_ptr, sizeof(indirect_ptr)))
789                 return false;
790 
791         indirect_ptr = le64_to_cpu(indirect_ptr);
792 
793         /* check the valid bit of the first level entry */
794         if (!(indirect_ptr & BIT_ULL(63)))
795                 return false;
796 
797         /*
798          * Mask the guest physical address and calculate the frame number.
799          * Any address beyond our supported 48 bits of PA will be caught
800          * by the actual check in the final step.
801          */
802         indirect_ptr &= GENMASK_ULL(51, 16);
803 
804         /* Find the address of the actual entry */
805         index = id % (SZ_64K / esz);
806         indirect_ptr += index * esz;
807         gfn = indirect_ptr >> PAGE_SHIFT;
808 
809         if (eaddr)
810                 *eaddr = indirect_ptr;
811         return kvm_is_visible_gfn(its->dev->kvm, gfn);
812 }
813 
814 static int vgic_its_alloc_collection(struct vgic_its *its,
815                                      struct its_collection **colp,
816                                      u32 coll_id)
817 {
818         struct its_collection *collection;
819 
820         if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))
821                 return E_ITS_MAPC_COLLECTION_OOR;
822 
823         collection = kzalloc(sizeof(*collection), GFP_KERNEL);
824         if (!collection)
825                 return -ENOMEM;
826 
827         collection->collection_id = coll_id;
828         collection->target_addr = COLLECTION_NOT_MAPPED;
829 
830         list_add_tail(&collection->coll_list, &its->collection_list);
831         *colp = collection;
832 
833         return 0;
834 }
835 
836 static void vgic_its_free_collection(struct vgic_its *its, u32 coll_id)
837 {
838         struct its_collection *collection;
839         struct its_device *device;
840         struct its_ite *ite;
841 
842         /*
843          * Clearing the mapping for that collection ID removes the
844          * entry from the list. If there wasn't any before, we can
845          * go home early.
846          */
847         collection = find_collection(its, coll_id);
848         if (!collection)
849                 return;
850 
851         for_each_lpi_its(device, ite, its)
852                 if (ite->collection &&
853                     ite->collection->collection_id == coll_id)
854                         ite->collection = NULL;
855 
856         list_del(&collection->coll_list);
857         kfree(collection);
858 }
859 
860 /* Must be called with its_lock mutex held */
861 static struct its_ite *vgic_its_alloc_ite(struct its_device *device,
862                                           struct its_collection *collection,
863                                           u32 event_id)
864 {
865         struct its_ite *ite;
866 
867         ite = kzalloc(sizeof(*ite), GFP_KERNEL);
868         if (!ite)
869                 return ERR_PTR(-ENOMEM);
870 
871         ite->event_id   = event_id;
872         ite->collection = collection;
873 
874         list_add_tail(&ite->ite_list, &device->itt_head);
875         return ite;
876 }
877 
878 /*
879  * The MAPTI and MAPI commands map LPIs to ITTEs.
880  * Must be called with its_lock mutex held.
881  */
882 static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,
883                                     u64 *its_cmd)
884 {
885         u32 device_id = its_cmd_get_deviceid(its_cmd);
886         u32 event_id = its_cmd_get_id(its_cmd);
887         u32 coll_id = its_cmd_get_collection(its_cmd);
888         struct its_ite *ite;
889         struct kvm_vcpu *vcpu = NULL;
890         struct its_device *device;
891         struct its_collection *collection, *new_coll = NULL;
892         struct vgic_irq *irq;
893         int lpi_nr;
894 
895         device = find_its_device(its, device_id);
896         if (!device)
897                 return E_ITS_MAPTI_UNMAPPED_DEVICE;
898 
899         if (event_id >= BIT_ULL(device->num_eventid_bits))
900                 return E_ITS_MAPTI_ID_OOR;
901 
902         if (its_cmd_get_command(its_cmd) == GITS_CMD_MAPTI)
903                 lpi_nr = its_cmd_get_physical_id(its_cmd);
904         else
905                 lpi_nr = event_id;
906         if (lpi_nr < GIC_LPI_OFFSET ||
907             lpi_nr >= max_lpis_propbaser(kvm->arch.vgic.propbaser))
908                 return E_ITS_MAPTI_PHYSICALID_OOR;
909 
910         /* If there is an existing mapping, behavior is UNPREDICTABLE. */
911         if (find_ite(its, device_id, event_id))
912                 return 0;
913 
914         collection = find_collection(its, coll_id);
915         if (!collection) {
916                 int ret = vgic_its_alloc_collection(its, &collection, coll_id);
917                 if (ret)
918                         return ret;
919                 new_coll = collection;
920         }
921 
922         ite = vgic_its_alloc_ite(device, collection, event_id);
923         if (IS_ERR(ite)) {
924                 if (new_coll)
925                         vgic_its_free_collection(its, coll_id);
926                 return PTR_ERR(ite);
927         }
928 
929         if (its_is_collection_mapped(collection))
930                 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
931 
932         irq = vgic_add_lpi(kvm, lpi_nr, vcpu);
933         if (IS_ERR(irq)) {
934                 if (new_coll)
935                         vgic_its_free_collection(its, coll_id);
936                 its_free_ite(kvm, ite);
937                 return PTR_ERR(irq);
938         }
939         ite->irq = irq;
940 
941         return 0;
942 }
943 
944 /* Requires the its_lock to be held. */
945 static void vgic_its_free_device(struct kvm *kvm, struct its_device *device)
946 {
947         struct its_ite *ite, *temp;
948 
949         /*
950          * The spec says that unmapping a device with still valid
951          * ITTEs associated is UNPREDICTABLE. We remove all ITTEs,
952          * since we cannot leave the memory unreferenced.
953          */
954         list_for_each_entry_safe(ite, temp, &device->itt_head, ite_list)
955                 its_free_ite(kvm, ite);
956 
957         list_del(&device->dev_list);
958         kfree(device);
959 }
960 
961 /* its lock must be held */
962 static void vgic_its_free_device_list(struct kvm *kvm, struct vgic_its *its)
963 {
964         struct its_device *cur, *temp;
965 
966         list_for_each_entry_safe(cur, temp, &its->device_list, dev_list)
967                 vgic_its_free_device(kvm, cur);
968 }
969 
970 /* its lock must be held */
971 static void vgic_its_free_collection_list(struct kvm *kvm, struct vgic_its *its)
972 {
973         struct its_collection *cur, *temp;
974 
975         list_for_each_entry_safe(cur, temp, &its->collection_list, coll_list)
976                 vgic_its_free_collection(its, cur->collection_id);
977 }
978 
979 /* Must be called with its_lock mutex held */
980 static struct its_device *vgic_its_alloc_device(struct vgic_its *its,
981                                                 u32 device_id, gpa_t itt_addr,
982                                                 u8 num_eventid_bits)
983 {
984         struct its_device *device;
985 
986         device = kzalloc(sizeof(*device), GFP_KERNEL);
987         if (!device)
988                 return ERR_PTR(-ENOMEM);
989 
990         device->device_id = device_id;
991         device->itt_addr = itt_addr;
992         device->num_eventid_bits = num_eventid_bits;
993         INIT_LIST_HEAD(&device->itt_head);
994 
995         list_add_tail(&device->dev_list, &its->device_list);
996         return device;
997 }
998 
999 /*
1000  * MAPD maps or unmaps a device ID to Interrupt Translation Tables (ITTs).
1001  * Must be called with the its_lock mutex held.
1002  */
1003 static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its,
1004                                     u64 *its_cmd)
1005 {
1006         u32 device_id = its_cmd_get_deviceid(its_cmd);
1007         bool valid = its_cmd_get_validbit(its_cmd);
1008         u8 num_eventid_bits = its_cmd_get_size(its_cmd);
1009         gpa_t itt_addr = its_cmd_get_ittaddr(its_cmd);
1010         struct its_device *device;
1011 
1012         if (!vgic_its_check_id(its, its->baser_device_table, device_id, NULL))
1013                 return E_ITS_MAPD_DEVICE_OOR;
1014 
1015         if (valid && num_eventid_bits > VITS_TYPER_IDBITS)
1016                 return E_ITS_MAPD_ITTSIZE_OOR;
1017 
1018         device = find_its_device(its, device_id);
1019 
1020         /*
1021          * The spec says that calling MAPD on an already mapped device
1022          * invalidates all cached data for this device. We implement this
1023          * by removing the mapping and re-establishing it.
1024          */
1025         if (device)
1026                 vgic_its_free_device(kvm, device);
1027 
1028         /*
1029          * The spec does not say whether unmapping a not-mapped device
1030          * is an error, so we are done in any case.
1031          */
1032         if (!valid)
1033                 return 0;
1034 
1035         device = vgic_its_alloc_device(its, device_id, itt_addr,
1036                                        num_eventid_bits);
1037         if (IS_ERR(device))
1038                 return PTR_ERR(device);
1039 
1040         return 0;
1041 }
1042 
1043 /*
1044  * The MAPC command maps collection IDs to redistributors.
1045  * Must be called with the its_lock mutex held.
1046  */
1047 static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,
1048                                     u64 *its_cmd)
1049 {
1050         u16 coll_id;
1051         u32 target_addr;
1052         struct its_collection *collection;
1053         bool valid;
1054 
1055         valid = its_cmd_get_validbit(its_cmd);
1056         coll_id = its_cmd_get_collection(its_cmd);
1057         target_addr = its_cmd_get_target_addr(its_cmd);
1058 
1059         if (target_addr >= atomic_read(&kvm->online_vcpus))
1060                 return E_ITS_MAPC_PROCNUM_OOR;
1061 
1062         if (!valid) {
1063                 vgic_its_free_collection(its, coll_id);
1064         } else {
1065                 collection = find_collection(its, coll_id);
1066 
1067                 if (!collection) {
1068                         int ret;
1069 
1070                         ret = vgic_its_alloc_collection(its, &collection,
1071                                                         coll_id);
1072                         if (ret)
1073                                 return ret;
1074                         collection->target_addr = target_addr;
1075                 } else {
1076                         collection->target_addr = target_addr;
1077                         update_affinity_collection(kvm, its, collection);
1078                 }
1079         }
1080 
1081         return 0;
1082 }
1083 
1084 /*
1085  * The CLEAR command removes the pending state for a particular LPI.
1086  * Must be called with the its_lock mutex held.
1087  */
1088 static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its,
1089                                      u64 *its_cmd)
1090 {
1091         u32 device_id = its_cmd_get_deviceid(its_cmd);
1092         u32 event_id = its_cmd_get_id(its_cmd);
1093         struct its_ite *ite;
1094 
1095 
1096         ite = find_ite(its, device_id, event_id);
1097         if (!ite)
1098                 return E_ITS_CLEAR_UNMAPPED_INTERRUPT;
1099 
1100         ite->irq->pending_latch = false;
1101 
1102         if (ite->irq->hw)
1103                 return irq_set_irqchip_state(ite->irq->host_irq,
1104                                              IRQCHIP_STATE_PENDING, false);
1105 
1106         return 0;
1107 }
1108 
1109 /*
1110  * The INV command syncs the configuration bits from the memory table.
1111  * Must be called with the its_lock mutex held.
1112  */
1113 static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its,
1114                                    u64 *its_cmd)
1115 {
1116         u32 device_id = its_cmd_get_deviceid(its_cmd);
1117         u32 event_id = its_cmd_get_id(its_cmd);
1118         struct its_ite *ite;
1119 
1120 
1121         ite = find_ite(its, device_id, event_id);
1122         if (!ite)
1123                 return E_ITS_INV_UNMAPPED_INTERRUPT;
1124 
1125         return update_lpi_config(kvm, ite->irq, NULL, true);
1126 }
1127 
1128 /*
1129  * The INVALL command requests flushing of all IRQ data in this collection.
1130  * Find the VCPU mapped to that collection, then iterate over the VM's list
1131  * of mapped LPIs and update the configuration for each IRQ which targets
1132  * the specified vcpu. The configuration will be read from the in-memory
1133  * configuration table.
1134  * Must be called with the its_lock mutex held.
1135  */
1136 static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,
1137                                       u64 *its_cmd)
1138 {
1139         u32 coll_id = its_cmd_get_collection(its_cmd);
1140         struct its_collection *collection;
1141         struct kvm_vcpu *vcpu;
1142         struct vgic_irq *irq;
1143         u32 *intids;
1144         int irq_count, i;
1145 
1146         collection = find_collection(its, coll_id);
1147         if (!its_is_collection_mapped(collection))
1148                 return E_ITS_INVALL_UNMAPPED_COLLECTION;
1149 
1150         vcpu = kvm_get_vcpu(kvm, collection->target_addr);
1151 
1152         irq_count = vgic_copy_lpi_list(vcpu, &intids);
1153         if (irq_count < 0)
1154                 return irq_count;
1155 
1156         for (i = 0; i < irq_count; i++) {
1157                 irq = vgic_get_irq(kvm, NULL, intids[i]);
1158                 if (!irq)
1159                         continue;
1160                 update_lpi_config(kvm, irq, vcpu, false);
1161                 vgic_put_irq(kvm, irq);
1162         }
1163 
1164         kfree(intids);
1165 
1166         if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.its_vm)
1167                 its_invall_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe);
1168 
1169         return 0;
1170 }
1171 
1172 /*
1173  * The MOVALL command moves the pending state of all IRQs targeting one
1174  * redistributor to another. We don't hold the pending state in the VCPUs,
1175  * but in the IRQs instead, so there is really not much to do for us here.
1176  * However the spec says that no IRQ must target the old redistributor
1177  * afterwards, so we make sure that no LPI is using the associated target_vcpu.
1178  * This command affects all LPIs in the system that target that redistributor.
1179  */
1180 static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
1181                                       u64 *its_cmd)
1182 {
1183         u32 target1_addr = its_cmd_get_target_addr(its_cmd);
1184         u32 target2_addr = its_cmd_mask_field(its_cmd, 3, 16, 32);
1185         struct kvm_vcpu *vcpu1, *vcpu2;
1186         struct vgic_irq *irq;
1187         u32 *intids;
1188         int irq_count, i;
1189 
1190         if (target1_addr >= atomic_read(&kvm->online_vcpus) ||
1191             target2_addr >= atomic_read(&kvm->online_vcpus))
1192                 return E_ITS_MOVALL_PROCNUM_OOR;
1193 
1194         if (target1_addr == target2_addr)
1195                 return 0;
1196 
1197         vcpu1 = kvm_get_vcpu(kvm, target1_addr);
1198         vcpu2 = kvm_get_vcpu(kvm, target2_addr);
1199 
1200         irq_count = vgic_copy_lpi_list(vcpu1, &intids);
1201         if (irq_count < 0)
1202                 return irq_count;
1203 
1204         for (i = 0; i < irq_count; i++) {
1205                 irq = vgic_get_irq(kvm, NULL, intids[i]);
1206 
1207                 update_affinity(irq, vcpu2);
1208 
1209                 vgic_put_irq(kvm, irq);
1210         }
1211 
1212         kfree(intids);
1213         return 0;
1214 }
1215 
1216 /*
1217  * The INT command injects the LPI associated with that DevID/EvID pair.
1218  * Must be called with the its_lock mutex held.
1219  */
1220 static int vgic_its_cmd_handle_int(struct kvm *kvm, struct vgic_its *its,
1221                                    u64 *its_cmd)
1222 {
1223         u32 msi_data = its_cmd_get_id(its_cmd);
1224         u64 msi_devid = its_cmd_get_deviceid(its_cmd);
1225 
1226         return vgic_its_trigger_msi(kvm, its, msi_devid, msi_data);
1227 }
1228 
1229 /*
1230  * This function is called with the its_cmd lock held, but the ITS data
1231  * structure lock dropped.
1232  */
1233 static int vgic_its_handle_command(struct kvm *kvm, struct vgic_its *its,
1234                                    u64 *its_cmd)
1235 {
1236         int ret = -ENODEV;
1237 
1238         mutex_lock(&its->its_lock);
1239         switch (its_cmd_get_command(its_cmd)) {
1240         case GITS_CMD_MAPD:
1241                 ret = vgic_its_cmd_handle_mapd(kvm, its, its_cmd);
1242                 break;
1243         case GITS_CMD_MAPC:
1244                 ret = vgic_its_cmd_handle_mapc(kvm, its, its_cmd);
1245                 break;
1246         case GITS_CMD_MAPI:
1247                 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1248                 break;
1249         case GITS_CMD_MAPTI:
1250                 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1251                 break;
1252         case GITS_CMD_MOVI:
1253                 ret = vgic_its_cmd_handle_movi(kvm, its, its_cmd);
1254                 break;
1255         case GITS_CMD_DISCARD:
1256                 ret = vgic_its_cmd_handle_discard(kvm, its, its_cmd);
1257                 break;
1258         case GITS_CMD_CLEAR:
1259                 ret = vgic_its_cmd_handle_clear(kvm, its, its_cmd);
1260                 break;
1261         case GITS_CMD_MOVALL:
1262                 ret = vgic_its_cmd_handle_movall(kvm, its, its_cmd);
1263                 break;
1264         case GITS_CMD_INT:
1265                 ret = vgic_its_cmd_handle_int(kvm, its, its_cmd);
1266                 break;
1267         case GITS_CMD_INV:
1268                 ret = vgic_its_cmd_handle_inv(kvm, its, its_cmd);
1269                 break;
1270         case GITS_CMD_INVALL:
1271                 ret = vgic_its_cmd_handle_invall(kvm, its, its_cmd);
1272                 break;
1273         case GITS_CMD_SYNC:
1274                 /* we ignore this command: we are in sync all of the time */
1275                 ret = 0;
1276                 break;
1277         }
1278         mutex_unlock(&its->its_lock);
1279 
1280         return ret;
1281 }
1282 
1283 static u64 vgic_sanitise_its_baser(u64 reg)
1284 {
1285         reg = vgic_sanitise_field(reg, GITS_BASER_SHAREABILITY_MASK,
1286                                   GITS_BASER_SHAREABILITY_SHIFT,
1287                                   vgic_sanitise_shareability);
1288         reg = vgic_sanitise_field(reg, GITS_BASER_INNER_CACHEABILITY_MASK,
1289                                   GITS_BASER_INNER_CACHEABILITY_SHIFT,
1290                                   vgic_sanitise_inner_cacheability);
1291         reg = vgic_sanitise_field(reg, GITS_BASER_OUTER_CACHEABILITY_MASK,
1292                                   GITS_BASER_OUTER_CACHEABILITY_SHIFT,
1293                                   vgic_sanitise_outer_cacheability);
1294 
1295         /* Bits 15:12 contain bits 51:48 of the PA, which we don't support. */
1296         reg &= ~GENMASK_ULL(15, 12);
1297 
1298         /* We support only one (ITS) page size: 64K */
1299         reg = (reg & ~GITS_BASER_PAGE_SIZE_MASK) | GITS_BASER_PAGE_SIZE_64K;
1300 
1301         return reg;
1302 }
1303 
1304 static u64 vgic_sanitise_its_cbaser(u64 reg)
1305 {
1306         reg = vgic_sanitise_field(reg, GITS_CBASER_SHAREABILITY_MASK,
1307                                   GITS_CBASER_SHAREABILITY_SHIFT,
1308                                   vgic_sanitise_shareability);
1309         reg = vgic_sanitise_field(reg, GITS_CBASER_INNER_CACHEABILITY_MASK,
1310                                   GITS_CBASER_INNER_CACHEABILITY_SHIFT,
1311                                   vgic_sanitise_inner_cacheability);
1312         reg = vgic_sanitise_field(reg, GITS_CBASER_OUTER_CACHEABILITY_MASK,
1313                                   GITS_CBASER_OUTER_CACHEABILITY_SHIFT,
1314                                   vgic_sanitise_outer_cacheability);
1315 
1316         /*
1317          * Sanitise the physical address to be 64k aligned.
1318          * Also limit the physical addresses to 48 bits.
1319          */
1320         reg &= ~(GENMASK_ULL(51, 48) | GENMASK_ULL(15, 12));
1321 
1322         return reg;
1323 }
1324 
1325 static unsigned long vgic_mmio_read_its_cbaser(struct kvm *kvm,
1326                                                struct vgic_its *its,
1327                                                gpa_t addr, unsigned int len)
1328 {
1329         return extract_bytes(its->cbaser, addr & 7, len);
1330 }
1331 
1332 static void vgic_mmio_write_its_cbaser(struct kvm *kvm, struct vgic_its *its,
1333                                        gpa_t addr, unsigned int len,
1334                                        unsigned long val)
1335 {
1336         /* When GITS_CTLR.Enable is 1, this register is RO. */
1337         if (its->enabled)
1338                 return;
1339 
1340         mutex_lock(&its->cmd_lock);
1341         its->cbaser = update_64bit_reg(its->cbaser, addr & 7, len, val);
1342         its->cbaser = vgic_sanitise_its_cbaser(its->cbaser);
1343         its->creadr = 0;
1344         /*
1345          * CWRITER is architecturally UNKNOWN on reset, but we need to reset
1346          * it to CREADR to make sure we start with an empty command buffer.
1347          */
1348         its->cwriter = its->creadr;
1349         mutex_unlock(&its->cmd_lock);
1350 }
1351 
1352 #define ITS_CMD_BUFFER_SIZE(baser)      ((((baser) & 0xff) + 1) << 12)
1353 #define ITS_CMD_SIZE                    32
1354 #define ITS_CMD_OFFSET(reg)             ((reg) & GENMASK(19, 5))
1355 
1356 /* Must be called with the cmd_lock held. */
1357 static void vgic_its_process_commands(struct kvm *kvm, struct vgic_its *its)
1358 {
1359         gpa_t cbaser;
1360         u64 cmd_buf[4];
1361 
1362         /* Commands are only processed when the ITS is enabled. */
1363         if (!its->enabled)
1364                 return;
1365 
1366         cbaser = CBASER_ADDRESS(its->cbaser);
1367 
1368         while (its->cwriter != its->creadr) {
1369                 int ret = kvm_read_guest(kvm, cbaser + its->creadr,
1370                                          cmd_buf, ITS_CMD_SIZE);
1371                 /*
1372                  * If kvm_read_guest() fails, this could be due to the guest
1373                  * programming a bogus value in CBASER or something else going
1374                  * wrong from which we cannot easily recover.
1375                  * According to section 6.3.2 in the GICv3 spec we can just
1376                  * ignore that command then.
1377                  */
1378                 if (!ret)
1379                         vgic_its_handle_command(kvm, its, cmd_buf);
1380 
1381                 its->creadr += ITS_CMD_SIZE;
1382                 if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser))
1383                         its->creadr = 0;
1384         }
1385 }
1386 
1387 /*
1388  * By writing to CWRITER the guest announces new commands to be processed.
1389  * To avoid any races in the first place, we take the its_cmd lock, which
1390  * protects our ring buffer variables, so that there is only one user
1391  * per ITS handling commands at a given time.
1392  */
1393 static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,
1394                                         gpa_t addr, unsigned int len,
1395                                         unsigned long val)
1396 {
1397         u64 reg;
1398 
1399         if (!its)
1400                 return;
1401 
1402         mutex_lock(&its->cmd_lock);
1403 
1404         reg = update_64bit_reg(its->cwriter, addr & 7, len, val);
1405         reg = ITS_CMD_OFFSET(reg);
1406         if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1407                 mutex_unlock(&its->cmd_lock);
1408                 return;
1409         }
1410         its->cwriter = reg;
1411 
1412         vgic_its_process_commands(kvm, its);
1413 
1414         mutex_unlock(&its->cmd_lock);
1415 }
1416 
1417 static unsigned long vgic_mmio_read_its_cwriter(struct kvm *kvm,
1418                                                 struct vgic_its *its,
1419                                                 gpa_t addr, unsigned int len)
1420 {
1421         return extract_bytes(its->cwriter, addr & 0x7, len);
1422 }
1423 
1424 static unsigned long vgic_mmio_read_its_creadr(struct kvm *kvm,
1425                                                struct vgic_its *its,
1426                                                gpa_t addr, unsigned int len)
1427 {
1428         return extract_bytes(its->creadr, addr & 0x7, len);
1429 }
1430 
1431 static int vgic_mmio_uaccess_write_its_creadr(struct kvm *kvm,
1432                                               struct vgic_its *its,
1433                                               gpa_t addr, unsigned int len,
1434                                               unsigned long val)
1435 {
1436         u32 cmd_offset;
1437         int ret = 0;
1438 
1439         mutex_lock(&its->cmd_lock);
1440 
1441         if (its->enabled) {
1442                 ret = -EBUSY;
1443                 goto out;
1444         }
1445 
1446         cmd_offset = ITS_CMD_OFFSET(val);
1447         if (cmd_offset >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1448                 ret = -EINVAL;
1449                 goto out;
1450         }
1451 
1452         its->creadr = cmd_offset;
1453 out:
1454         mutex_unlock(&its->cmd_lock);
1455         return ret;
1456 }
1457 
1458 #define BASER_INDEX(addr) (((addr) / sizeof(u64)) & 0x7)
1459 static unsigned long vgic_mmio_read_its_baser(struct kvm *kvm,
1460                                               struct vgic_its *its,
1461                                               gpa_t addr, unsigned int len)
1462 {
1463         u64 reg;
1464 
1465         switch (BASER_INDEX(addr)) {
1466         case 0:
1467                 reg = its->baser_device_table;
1468                 break;
1469         case 1:
1470                 reg = its->baser_coll_table;
1471                 break;
1472         default:
1473                 reg = 0;
1474                 break;
1475         }
1476 
1477         return extract_bytes(reg, addr & 7, len);
1478 }
1479 
1480 #define GITS_BASER_RO_MASK      (GENMASK_ULL(52, 48) | GENMASK_ULL(58, 56))
1481 static void vgic_mmio_write_its_baser(struct kvm *kvm,
1482                                       struct vgic_its *its,
1483                                       gpa_t addr, unsigned int len,
1484                                       unsigned long val)
1485 {
1486         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1487         u64 entry_size, table_type;
1488         u64 reg, *regptr, clearbits = 0;
1489 
1490         /* When GITS_CTLR.Enable is 1, we ignore write accesses. */
1491         if (its->enabled)
1492                 return;
1493 
1494         switch (BASER_INDEX(addr)) {
1495         case 0:
1496                 regptr = &its->baser_device_table;
1497                 entry_size = abi->dte_esz;
1498                 table_type = GITS_BASER_TYPE_DEVICE;
1499                 break;
1500         case 1:
1501                 regptr = &its->baser_coll_table;
1502                 entry_size = abi->cte_esz;
1503                 table_type = GITS_BASER_TYPE_COLLECTION;
1504                 clearbits = GITS_BASER_INDIRECT;
1505                 break;
1506         default:
1507                 return;
1508         }
1509 
1510         reg = update_64bit_reg(*regptr, addr & 7, len, val);
1511         reg &= ~GITS_BASER_RO_MASK;
1512         reg &= ~clearbits;
1513 
1514         reg |= (entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT;
1515         reg |= table_type << GITS_BASER_TYPE_SHIFT;
1516         reg = vgic_sanitise_its_baser(reg);
1517 
1518         *regptr = reg;
1519 
1520         if (!(reg & GITS_BASER_VALID)) {
1521                 /* Take the its_lock to prevent a race with a save/restore */
1522                 mutex_lock(&its->its_lock);
1523                 switch (table_type) {
1524                 case GITS_BASER_TYPE_DEVICE:
1525                         vgic_its_free_device_list(kvm, its);
1526                         break;
1527                 case GITS_BASER_TYPE_COLLECTION:
1528                         vgic_its_free_collection_list(kvm, its);
1529                         break;
1530                 }
1531                 mutex_unlock(&its->its_lock);
1532         }
1533 }
1534 
1535 static unsigned long vgic_mmio_read_its_ctlr(struct kvm *vcpu,
1536                                              struct vgic_its *its,
1537                                              gpa_t addr, unsigned int len)
1538 {
1539         u32 reg = 0;
1540 
1541         mutex_lock(&its->cmd_lock);
1542         if (its->creadr == its->cwriter)
1543                 reg |= GITS_CTLR_QUIESCENT;
1544         if (its->enabled)
1545                 reg |= GITS_CTLR_ENABLE;
1546         mutex_unlock(&its->cmd_lock);
1547 
1548         return reg;
1549 }
1550 
1551 static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,
1552                                      gpa_t addr, unsigned int len,
1553                                      unsigned long val)
1554 {
1555         mutex_lock(&its->cmd_lock);
1556 
1557         /*
1558          * It is UNPREDICTABLE to enable the ITS if any of the CBASER or
1559          * device/collection BASER are invalid
1560          */
1561         if (!its->enabled && (val & GITS_CTLR_ENABLE) &&
1562                 (!(its->baser_device_table & GITS_BASER_VALID) ||
1563                  !(its->baser_coll_table & GITS_BASER_VALID) ||
1564                  !(its->cbaser & GITS_CBASER_VALID)))
1565                 goto out;
1566 
1567         its->enabled = !!(val & GITS_CTLR_ENABLE);
1568 
1569         /*
1570          * Try to process any pending commands. This function bails out early
1571          * if the ITS is disabled or no commands have been queued.
1572          */
1573         vgic_its_process_commands(kvm, its);
1574 
1575 out:
1576         mutex_unlock(&its->cmd_lock);
1577 }
1578 
1579 #define REGISTER_ITS_DESC(off, rd, wr, length, acc)             \
1580 {                                                               \
1581         .reg_offset = off,                                      \
1582         .len = length,                                          \
1583         .access_flags = acc,                                    \
1584         .its_read = rd,                                         \
1585         .its_write = wr,                                        \
1586 }
1587 
1588 #define REGISTER_ITS_DESC_UACCESS(off, rd, wr, uwr, length, acc)\
1589 {                                                               \
1590         .reg_offset = off,                                      \
1591         .len = length,                                          \
1592         .access_flags = acc,                                    \
1593         .its_read = rd,                                         \
1594         .its_write = wr,                                        \
1595         .uaccess_its_write = uwr,                               \
1596 }
1597 
1598 static void its_mmio_write_wi(struct kvm *kvm, struct vgic_its *its,
1599                               gpa_t addr, unsigned int len, unsigned long val)
1600 {
1601         /* Ignore */
1602 }
1603 
1604 static struct vgic_register_region its_registers[] = {
1605         REGISTER_ITS_DESC(GITS_CTLR,
1606                 vgic_mmio_read_its_ctlr, vgic_mmio_write_its_ctlr, 4,
1607                 VGIC_ACCESS_32bit),
1608         REGISTER_ITS_DESC_UACCESS(GITS_IIDR,
1609                 vgic_mmio_read_its_iidr, its_mmio_write_wi,
1610                 vgic_mmio_uaccess_write_its_iidr, 4,
1611                 VGIC_ACCESS_32bit),
1612         REGISTER_ITS_DESC(GITS_TYPER,
1613                 vgic_mmio_read_its_typer, its_mmio_write_wi, 8,
1614                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1615         REGISTER_ITS_DESC(GITS_CBASER,
1616                 vgic_mmio_read_its_cbaser, vgic_mmio_write_its_cbaser, 8,
1617                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1618         REGISTER_ITS_DESC(GITS_CWRITER,
1619                 vgic_mmio_read_its_cwriter, vgic_mmio_write_its_cwriter, 8,
1620                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1621         REGISTER_ITS_DESC_UACCESS(GITS_CREADR,
1622                 vgic_mmio_read_its_creadr, its_mmio_write_wi,
1623                 vgic_mmio_uaccess_write_its_creadr, 8,
1624                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1625         REGISTER_ITS_DESC(GITS_BASER,
1626                 vgic_mmio_read_its_baser, vgic_mmio_write_its_baser, 0x40,
1627                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1628         REGISTER_ITS_DESC(GITS_IDREGS_BASE,
1629                 vgic_mmio_read_its_idregs, its_mmio_write_wi, 0x30,
1630                 VGIC_ACCESS_32bit),
1631 };
1632 
1633 /* This is called on setting the LPI enable bit in the redistributor. */
1634 void vgic_enable_lpis(struct kvm_vcpu *vcpu)
1635 {
1636         if (!(vcpu->arch.vgic_cpu.pendbaser & GICR_PENDBASER_PTZ))
1637                 its_sync_lpi_pending_table(vcpu);
1638 }
1639 
1640 static int vgic_register_its_iodev(struct kvm *kvm, struct vgic_its *its,
1641                                    u64 addr)
1642 {
1643         struct vgic_io_device *iodev = &its->iodev;
1644         int ret;
1645 
1646         mutex_lock(&kvm->slots_lock);
1647         if (!IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1648                 ret = -EBUSY;
1649                 goto out;
1650         }
1651 
1652         its->vgic_its_base = addr;
1653         iodev->regions = its_registers;
1654         iodev->nr_regions = ARRAY_SIZE(its_registers);
1655         kvm_iodevice_init(&iodev->dev, &kvm_io_gic_ops);
1656 
1657         iodev->base_addr = its->vgic_its_base;
1658         iodev->iodev_type = IODEV_ITS;
1659         iodev->its = its;
1660         ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, iodev->base_addr,
1661                                       KVM_VGIC_V3_ITS_SIZE, &iodev->dev);
1662 out:
1663         mutex_unlock(&kvm->slots_lock);
1664 
1665         return ret;
1666 }
1667 
1668 #define INITIAL_BASER_VALUE                                               \
1669         (GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb)                | \
1670          GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner)         | \
1671          GIC_BASER_SHAREABILITY(GITS_BASER, InnerShareable)             | \
1672          GITS_BASER_PAGE_SIZE_64K)
1673 
1674 #define INITIAL_PROPBASER_VALUE                                           \
1675         (GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, RaWb)            | \
1676          GIC_BASER_CACHEABILITY(GICR_PROPBASER, OUTER, SameAsInner)     | \
1677          GIC_BASER_SHAREABILITY(GICR_PROPBASER, InnerShareable))
1678 
1679 static int vgic_its_create(struct kvm_device *dev, u32 type)
1680 {
1681         struct vgic_its *its;
1682 
1683         if (type != KVM_DEV_TYPE_ARM_VGIC_ITS)
1684                 return -ENODEV;
1685 
1686         its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL);
1687         if (!its)
1688                 return -ENOMEM;
1689 
1690         if (vgic_initialized(dev->kvm)) {
1691                 int ret = vgic_v4_init(dev->kvm);
1692                 if (ret < 0) {
1693                         kfree(its);
1694                         return ret;
1695                 }
1696         }
1697 
1698         mutex_init(&its->its_lock);
1699         mutex_init(&its->cmd_lock);
1700 
1701         its->vgic_its_base = VGIC_ADDR_UNDEF;
1702 
1703         INIT_LIST_HEAD(&its->device_list);
1704         INIT_LIST_HEAD(&its->collection_list);
1705 
1706         dev->kvm->arch.vgic.msis_require_devid = true;
1707         dev->kvm->arch.vgic.has_its = true;
1708         its->enabled = false;
1709         its->dev = dev;
1710 
1711         its->baser_device_table = INITIAL_BASER_VALUE                   |
1712                 ((u64)GITS_BASER_TYPE_DEVICE << GITS_BASER_TYPE_SHIFT);
1713         its->baser_coll_table = INITIAL_BASER_VALUE |
1714                 ((u64)GITS_BASER_TYPE_COLLECTION << GITS_BASER_TYPE_SHIFT);
1715         dev->kvm->arch.vgic.propbaser = INITIAL_PROPBASER_VALUE;
1716 
1717         dev->private = its;
1718 
1719         return vgic_its_set_abi(its, NR_ITS_ABIS - 1);
1720 }
1721 
1722 static void vgic_its_destroy(struct kvm_device *kvm_dev)
1723 {
1724         struct kvm *kvm = kvm_dev->kvm;
1725         struct vgic_its *its = kvm_dev->private;
1726 
1727         mutex_lock(&its->its_lock);
1728 
1729         vgic_its_free_device_list(kvm, its);
1730         vgic_its_free_collection_list(kvm, its);
1731 
1732         mutex_unlock(&its->its_lock);
1733         kfree(its);
1734 }
1735 
1736 int vgic_its_has_attr_regs(struct kvm_device *dev,
1737                            struct kvm_device_attr *attr)
1738 {
1739         const struct vgic_register_region *region;
1740         gpa_t offset = attr->attr;
1741         int align;
1742 
1743         align = (offset < GITS_TYPER) || (offset >= GITS_PIDR4) ? 0x3 : 0x7;
1744 
1745         if (offset & align)
1746                 return -EINVAL;
1747 
1748         region = vgic_find_mmio_region(its_registers,
1749                                        ARRAY_SIZE(its_registers),
1750                                        offset);
1751         if (!region)
1752                 return -ENXIO;
1753 
1754         return 0;
1755 }
1756 
1757 int vgic_its_attr_regs_access(struct kvm_device *dev,
1758                               struct kvm_device_attr *attr,
1759                               u64 *reg, bool is_write)
1760 {
1761         const struct vgic_register_region *region;
1762         struct vgic_its *its;
1763         gpa_t addr, offset;
1764         unsigned int len;
1765         int align, ret = 0;
1766 
1767         its = dev->private;
1768         offset = attr->attr;
1769 
1770         /*
1771          * Although the spec supports upper/lower 32-bit accesses to
1772          * 64-bit ITS registers, the userspace ABI requires 64-bit
1773          * accesses to all 64-bit wide registers. We therefore only
1774          * support 32-bit accesses to GITS_CTLR, GITS_IIDR and GITS ID
1775          * registers
1776          */
1777         if ((offset < GITS_TYPER) || (offset >= GITS_PIDR4))
1778                 align = 0x3;
1779         else
1780                 align = 0x7;
1781 
1782         if (offset & align)
1783                 return -EINVAL;
1784 
1785         mutex_lock(&dev->kvm->lock);
1786 
1787         if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1788                 ret = -ENXIO;
1789                 goto out;
1790         }
1791 
1792         region = vgic_find_mmio_region(its_registers,
1793                                        ARRAY_SIZE(its_registers),
1794                                        offset);
1795         if (!region) {
1796                 ret = -ENXIO;
1797                 goto out;
1798         }
1799 
1800         if (!lock_all_vcpus(dev->kvm)) {
1801                 ret = -EBUSY;
1802                 goto out;
1803         }
1804 
1805         addr = its->vgic_its_base + offset;
1806 
1807         len = region->access_flags & VGIC_ACCESS_64bit ? 8 : 4;
1808 
1809         if (is_write) {
1810                 if (region->uaccess_its_write)
1811                         ret = region->uaccess_its_write(dev->kvm, its, addr,
1812                                                         len, *reg);
1813                 else
1814                         region->its_write(dev->kvm, its, addr, len, *reg);
1815         } else {
1816                 *reg = region->its_read(dev->kvm, its, addr, len);
1817         }
1818         unlock_all_vcpus(dev->kvm);
1819 out:
1820         mutex_unlock(&dev->kvm->lock);
1821         return ret;
1822 }
1823 
1824 static u32 compute_next_devid_offset(struct list_head *h,
1825                                      struct its_device *dev)
1826 {
1827         struct its_device *next;
1828         u32 next_offset;
1829 
1830         if (list_is_last(&dev->dev_list, h))
1831                 return 0;
1832         next = list_next_entry(dev, dev_list);
1833         next_offset = next->device_id - dev->device_id;
1834 
1835         return min_t(u32, next_offset, VITS_DTE_MAX_DEVID_OFFSET);
1836 }
1837 
1838 static u32 compute_next_eventid_offset(struct list_head *h, struct its_ite *ite)
1839 {
1840         struct its_ite *next;
1841         u32 next_offset;
1842 
1843         if (list_is_last(&ite->ite_list, h))
1844                 return 0;
1845         next = list_next_entry(ite, ite_list);
1846         next_offset = next->event_id - ite->event_id;
1847 
1848         return min_t(u32, next_offset, VITS_ITE_MAX_EVENTID_OFFSET);
1849 }
1850 
1851 /**
1852  * entry_fn_t - Callback called on a table entry restore path
1853  * @its: its handle
1854  * @id: id of the entry
1855  * @entry: pointer to the entry
1856  * @opaque: pointer to an opaque data
1857  *
1858  * Return: < 0 on error, 0 if last element was identified, id offset to next
1859  * element otherwise
1860  */
1861 typedef int (*entry_fn_t)(struct vgic_its *its, u32 id, void *entry,
1862                           void *opaque);
1863 
1864 /**
1865  * scan_its_table - Scan a contiguous table in guest RAM and applies a function
1866  * to each entry
1867  *
1868  * @its: its handle
1869  * @base: base gpa of the table
1870  * @size: size of the table in bytes
1871  * @esz: entry size in bytes
1872  * @start_id: the ID of the first entry in the table
1873  * (non zero for 2d level tables)
1874  * @fn: function to apply on each entry
1875  *
1876  * Return: < 0 on error, 0 if last element was identified, 1 otherwise
1877  * (the last element may not be found on second level tables)
1878  */
1879 static int scan_its_table(struct vgic_its *its, gpa_t base, int size, int esz,
1880                           int start_id, entry_fn_t fn, void *opaque)
1881 {
1882         struct kvm *kvm = its->dev->kvm;
1883         unsigned long len = size;
1884         int id = start_id;
1885         gpa_t gpa = base;
1886         char entry[esz];
1887         int ret;
1888 
1889         memset(entry, 0, esz);
1890 
1891         while (len > 0) {
1892                 int next_offset;
1893                 size_t byte_offset;
1894 
1895                 ret = kvm_read_guest(kvm, gpa, entry, esz);
1896                 if (ret)
1897                         return ret;
1898 
1899                 next_offset = fn(its, id, entry, opaque);
1900                 if (next_offset <= 0)
1901                         return next_offset;
1902 
1903                 byte_offset = next_offset * esz;
1904                 id += next_offset;
1905                 gpa += byte_offset;
1906                 len -= byte_offset;
1907         }
1908         return 1;
1909 }
1910 
1911 /**
1912  * vgic_its_save_ite - Save an interrupt translation entry at @gpa
1913  */
1914 static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev,
1915                               struct its_ite *ite, gpa_t gpa, int ite_esz)
1916 {
1917         struct kvm *kvm = its->dev->kvm;
1918         u32 next_offset;
1919         u64 val;
1920 
1921         next_offset = compute_next_eventid_offset(&dev->itt_head, ite);
1922         val = ((u64)next_offset << KVM_ITS_ITE_NEXT_SHIFT) |
1923                ((u64)ite->irq->intid << KVM_ITS_ITE_PINTID_SHIFT) |
1924                 ite->collection->collection_id;
1925         val = cpu_to_le64(val);
1926         return kvm_write_guest(kvm, gpa, &val, ite_esz);
1927 }
1928 
1929 /**
1930  * vgic_its_restore_ite - restore an interrupt translation entry
1931  * @event_id: id used for indexing
1932  * @ptr: pointer to the ITE entry
1933  * @opaque: pointer to the its_device
1934  */
1935 static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id,
1936                                 void *ptr, void *opaque)
1937 {
1938         struct its_device *dev = (struct its_device *)opaque;
1939         struct its_collection *collection;
1940         struct kvm *kvm = its->dev->kvm;
1941         struct kvm_vcpu *vcpu = NULL;
1942         u64 val;
1943         u64 *p = (u64 *)ptr;
1944         struct vgic_irq *irq;
1945         u32 coll_id, lpi_id;
1946         struct its_ite *ite;
1947         u32 offset;
1948 
1949         val = *p;
1950 
1951         val = le64_to_cpu(val);
1952 
1953         coll_id = val & KVM_ITS_ITE_ICID_MASK;
1954         lpi_id = (val & KVM_ITS_ITE_PINTID_MASK) >> KVM_ITS_ITE_PINTID_SHIFT;
1955 
1956         if (!lpi_id)
1957                 return 1; /* invalid entry, no choice but to scan next entry */
1958 
1959         if (lpi_id < VGIC_MIN_LPI)
1960                 return -EINVAL;
1961 
1962         offset = val >> KVM_ITS_ITE_NEXT_SHIFT;
1963         if (event_id + offset >= BIT_ULL(dev->num_eventid_bits))
1964                 return -EINVAL;
1965 
1966         collection = find_collection(its, coll_id);
1967         if (!collection)
1968                 return -EINVAL;
1969 
1970         ite = vgic_its_alloc_ite(dev, collection, event_id);
1971         if (IS_ERR(ite))
1972                 return PTR_ERR(ite);
1973 
1974         if (its_is_collection_mapped(collection))
1975                 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
1976 
1977         irq = vgic_add_lpi(kvm, lpi_id, vcpu);
1978         if (IS_ERR(irq))
1979                 return PTR_ERR(irq);
1980         ite->irq = irq;
1981 
1982         return offset;
1983 }
1984 
1985 static int vgic_its_ite_cmp(void *priv, struct list_head *a,
1986                             struct list_head *b)
1987 {
1988         struct its_ite *itea = container_of(a, struct its_ite, ite_list);
1989         struct its_ite *iteb = container_of(b, struct its_ite, ite_list);
1990 
1991         if (itea->event_id < iteb->event_id)
1992                 return -1;
1993         else
1994                 return 1;
1995 }
1996 
1997 static int vgic_its_save_itt(struct vgic_its *its, struct its_device *device)
1998 {
1999         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2000         gpa_t base = device->itt_addr;
2001         struct its_ite *ite;
2002         int ret;
2003         int ite_esz = abi->ite_esz;
2004 
2005         list_sort(NULL, &device->itt_head, vgic_its_ite_cmp);
2006 
2007         list_for_each_entry(ite, &device->itt_head, ite_list) {
2008                 gpa_t gpa = base + ite->event_id * ite_esz;
2009 
2010                 /*
2011                  * If an LPI carries the HW bit, this means that this
2012                  * interrupt is controlled by GICv4, and we do not
2013                  * have direct access to that state. Let's simply fail
2014                  * the save operation...
2015                  */
2016                 if (ite->irq->hw)
2017                         return -EACCES;
2018 
2019                 ret = vgic_its_save_ite(its, device, ite, gpa, ite_esz);
2020                 if (ret)
2021                         return ret;
2022         }
2023         return 0;
2024 }
2025 
2026 /**
2027  * vgic_its_restore_itt - restore the ITT of a device
2028  *
2029  * @its: its handle
2030  * @dev: device handle
2031  *
2032  * Return 0 on success, < 0 on error
2033  */
2034 static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev)
2035 {
2036         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2037         gpa_t base = dev->itt_addr;
2038         int ret;
2039         int ite_esz = abi->ite_esz;
2040         size_t max_size = BIT_ULL(dev->num_eventid_bits) * ite_esz;
2041 
2042         ret = scan_its_table(its, base, max_size, ite_esz, 0,
2043                              vgic_its_restore_ite, dev);
2044 
2045         /* scan_its_table returns +1 if all ITEs are invalid */
2046         if (ret > 0)
2047                 ret = 0;
2048 
2049         return ret;
2050 }
2051 
2052 /**
2053  * vgic_its_save_dte - Save a device table entry at a given GPA
2054  *
2055  * @its: ITS handle
2056  * @dev: ITS device
2057  * @ptr: GPA
2058  */
2059 static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev,
2060                              gpa_t ptr, int dte_esz)
2061 {
2062         struct kvm *kvm = its->dev->kvm;
2063         u64 val, itt_addr_field;
2064         u32 next_offset;
2065 
2066         itt_addr_field = dev->itt_addr >> 8;
2067         next_offset = compute_next_devid_offset(&its->device_list, dev);
2068         val = (1ULL << KVM_ITS_DTE_VALID_SHIFT |
2069                ((u64)next_offset << KVM_ITS_DTE_NEXT_SHIFT) |
2070                (itt_addr_field << KVM_ITS_DTE_ITTADDR_SHIFT) |
2071                 (dev->num_eventid_bits - 1));
2072         val = cpu_to_le64(val);
2073         return kvm_write_guest(kvm, ptr, &val, dte_esz);
2074 }
2075 
2076 /**
2077  * vgic_its_restore_dte - restore a device table entry
2078  *
2079  * @its: its handle
2080  * @id: device id the DTE corresponds to
2081  * @ptr: kernel VA where the 8 byte DTE is located
2082  * @opaque: unused
2083  *
2084  * Return: < 0 on error, 0 if the dte is the last one, id offset to the
2085  * next dte otherwise
2086  */
2087 static int vgic_its_restore_dte(struct vgic_its *its, u32 id,
2088                                 void *ptr, void *opaque)
2089 {
2090         struct its_device *dev;
2091         gpa_t itt_addr;
2092         u8 num_eventid_bits;
2093         u64 entry = *(u64 *)ptr;
2094         bool valid;
2095         u32 offset;
2096         int ret;
2097 
2098         entry = le64_to_cpu(entry);
2099 
2100         valid = entry >> KVM_ITS_DTE_VALID_SHIFT;
2101         num_eventid_bits = (entry & KVM_ITS_DTE_SIZE_MASK) + 1;
2102         itt_addr = ((entry & KVM_ITS_DTE_ITTADDR_MASK)
2103                         >> KVM_ITS_DTE_ITTADDR_SHIFT) << 8;
2104 
2105         if (!valid)
2106                 return 1;
2107 
2108         /* dte entry is valid */
2109         offset = (entry & KVM_ITS_DTE_NEXT_MASK) >> KVM_ITS_DTE_NEXT_SHIFT;
2110 
2111         dev = vgic_its_alloc_device(its, id, itt_addr, num_eventid_bits);
2112         if (IS_ERR(dev))
2113                 return PTR_ERR(dev);
2114 
2115         ret = vgic_its_restore_itt(its, dev);
2116         if (ret) {
2117                 vgic_its_free_device(its->dev->kvm, dev);
2118                 return ret;
2119         }
2120 
2121         return offset;
2122 }
2123 
2124 static int vgic_its_device_cmp(void *priv, struct list_head *a,
2125                                struct list_head *b)
2126 {
2127         struct its_device *deva = container_of(a, struct its_device, dev_list);
2128         struct its_device *devb = container_of(b, struct its_device, dev_list);
2129 
2130         if (deva->device_id < devb->device_id)
2131                 return -1;
2132         else
2133                 return 1;
2134 }
2135 
2136 /**
2137  * vgic_its_save_device_tables - Save the device table and all ITT
2138  * into guest RAM
2139  *
2140  * L1/L2 handling is hidden by vgic_its_check_id() helper which directly
2141  * returns the GPA of the device entry
2142  */
2143 static int vgic_its_save_device_tables(struct vgic_its *its)
2144 {
2145         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2146         u64 baser = its->baser_device_table;
2147         struct its_device *dev;
2148         int dte_esz = abi->dte_esz;
2149 
2150         if (!(baser & GITS_BASER_VALID))
2151                 return 0;
2152 
2153         list_sort(NULL, &its->device_list, vgic_its_device_cmp);
2154 
2155         list_for_each_entry(dev, &its->device_list, dev_list) {
2156                 int ret;
2157                 gpa_t eaddr;
2158 
2159                 if (!vgic_its_check_id(its, baser,
2160                                        dev->device_id, &eaddr))
2161                         return -EINVAL;
2162 
2163                 ret = vgic_its_save_itt(its, dev);
2164                 if (ret)
2165                         return ret;
2166 
2167                 ret = vgic_its_save_dte(its, dev, eaddr, dte_esz);
2168                 if (ret)
2169                         return ret;
2170         }
2171         return 0;
2172 }
2173 
2174 /**
2175  * handle_l1_dte - callback used for L1 device table entries (2 stage case)
2176  *
2177  * @its: its handle
2178  * @id: index of the entry in the L1 table
2179  * @addr: kernel VA
2180  * @opaque: unused
2181  *
2182  * L1 table entries are scanned by steps of 1 entry
2183  * Return < 0 if error, 0 if last dte was found when scanning the L2
2184  * table, +1 otherwise (meaning next L1 entry must be scanned)
2185  */
2186 static int handle_l1_dte(struct vgic_its *its, u32 id, void *addr,
2187                          void *opaque)
2188 {
2189         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2190         int l2_start_id = id * (SZ_64K / abi->dte_esz);
2191         u64 entry = *(u64 *)addr;
2192         int dte_esz = abi->dte_esz;
2193         gpa_t gpa;
2194         int ret;
2195 
2196         entry = le64_to_cpu(entry);
2197 
2198         if (!(entry & KVM_ITS_L1E_VALID_MASK))
2199                 return 1;
2200 
2201         gpa = entry & KVM_ITS_L1E_ADDR_MASK;
2202 
2203         ret = scan_its_table(its, gpa, SZ_64K, dte_esz,
2204                              l2_start_id, vgic_its_restore_dte, NULL);
2205 
2206         return ret;
2207 }
2208 
2209 /**
2210  * vgic_its_restore_device_tables - Restore the device table and all ITT
2211  * from guest RAM to internal data structs
2212  */
2213 static int vgic_its_restore_device_tables(struct vgic_its *its)
2214 {
2215         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2216         u64 baser = its->baser_device_table;
2217         int l1_esz, ret;
2218         int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2219         gpa_t l1_gpa;
2220 
2221         if (!(baser & GITS_BASER_VALID))
2222                 return 0;
2223 
2224         l1_gpa = BASER_ADDRESS(baser);
2225 
2226         if (baser & GITS_BASER_INDIRECT) {
2227                 l1_esz = GITS_LVL1_ENTRY_SIZE;
2228                 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2229                                      handle_l1_dte, NULL);
2230         } else {
2231                 l1_esz = abi->dte_esz;
2232                 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2233                                      vgic_its_restore_dte, NULL);
2234         }
2235 
2236         /* scan_its_table returns +1 if all entries are invalid */
2237         if (ret > 0)
2238                 ret = 0;
2239 
2240         return ret;
2241 }
2242 
2243 static int vgic_its_save_cte(struct vgic_its *its,
2244                              struct its_collection *collection,
2245                              gpa_t gpa, int esz)
2246 {
2247         u64 val;
2248 
2249         val = (1ULL << KVM_ITS_CTE_VALID_SHIFT |
2250                ((u64)collection->target_addr << KVM_ITS_CTE_RDBASE_SHIFT) |
2251                collection->collection_id);
2252         val = cpu_to_le64(val);
2253         return kvm_write_guest(its->dev->kvm, gpa, &val, esz);
2254 }
2255 
2256 static int vgic_its_restore_cte(struct vgic_its *its, gpa_t gpa, int esz)
2257 {
2258         struct its_collection *collection;
2259         struct kvm *kvm = its->dev->kvm;
2260         u32 target_addr, coll_id;
2261         u64 val;
2262         int ret;
2263 
2264         BUG_ON(esz > sizeof(val));
2265         ret = kvm_read_guest(kvm, gpa, &val, esz);
2266         if (ret)
2267                 return ret;
2268         val = le64_to_cpu(val);
2269         if (!(val & KVM_ITS_CTE_VALID_MASK))
2270                 return 0;
2271 
2272         target_addr = (u32)(val >> KVM_ITS_CTE_RDBASE_SHIFT);
2273         coll_id = val & KVM_ITS_CTE_ICID_MASK;
2274 
2275         if (target_addr >= atomic_read(&kvm->online_vcpus))
2276                 return -EINVAL;
2277 
2278         collection = find_collection(its, coll_id);
2279         if (collection)
2280                 return -EEXIST;
2281         ret = vgic_its_alloc_collection(its, &collection, coll_id);
2282         if (ret)
2283                 return ret;
2284         collection->target_addr = target_addr;
2285         return 1;
2286 }
2287 
2288 /**
2289  * vgic_its_save_collection_table - Save the collection table into
2290  * guest RAM
2291  */
2292 static int vgic_its_save_collection_table(struct vgic_its *its)
2293 {
2294         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2295         u64 baser = its->baser_coll_table;
2296         gpa_t gpa = BASER_ADDRESS(baser);
2297         struct its_collection *collection;
2298         u64 val;
2299         size_t max_size, filled = 0;
2300         int ret, cte_esz = abi->cte_esz;
2301 
2302         if (!(baser & GITS_BASER_VALID))
2303                 return 0;
2304 
2305         max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2306 
2307         list_for_each_entry(collection, &its->collection_list, coll_list) {
2308                 ret = vgic_its_save_cte(its, collection, gpa, cte_esz);
2309                 if (ret)
2310                         return ret;
2311                 gpa += cte_esz;
2312                 filled += cte_esz;
2313         }
2314 
2315         if (filled == max_size)
2316                 return 0;
2317 
2318         /*
2319          * table is not fully filled, add a last dummy element
2320          * with valid bit unset
2321          */
2322         val = 0;
2323         BUG_ON(cte_esz > sizeof(val));
2324         ret = kvm_write_guest(its->dev->kvm, gpa, &val, cte_esz);
2325         return ret;
2326 }
2327 
2328 /**
2329  * vgic_its_restore_collection_table - reads the collection table
2330  * in guest memory and restores the ITS internal state. Requires the
2331  * BASER registers to be restored before.
2332  */
2333 static int vgic_its_restore_collection_table(struct vgic_its *its)
2334 {
2335         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2336         u64 baser = its->baser_coll_table;
2337         int cte_esz = abi->cte_esz;
2338         size_t max_size, read = 0;
2339         gpa_t gpa;
2340         int ret;
2341 
2342         if (!(baser & GITS_BASER_VALID))
2343                 return 0;
2344 
2345         gpa = BASER_ADDRESS(baser);
2346 
2347         max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2348 
2349         while (read < max_size) {
2350                 ret = vgic_its_restore_cte(its, gpa, cte_esz);
2351                 if (ret <= 0)
2352                         break;
2353                 gpa += cte_esz;
2354                 read += cte_esz;
2355         }
2356 
2357         if (ret > 0)
2358                 return 0;
2359 
2360         return ret;
2361 }
2362 
2363 /**
2364  * vgic_its_save_tables_v0 - Save the ITS tables into guest ARM
2365  * according to v0 ABI
2366  */
2367 static int vgic_its_save_tables_v0(struct vgic_its *its)
2368 {
2369         int ret;
2370 
2371         ret = vgic_its_save_device_tables(its);
2372         if (ret)
2373                 return ret;
2374 
2375         return vgic_its_save_collection_table(its);
2376 }
2377 
2378 /**
2379  * vgic_its_restore_tables_v0 - Restore the ITS tables from guest RAM
2380  * to internal data structs according to V0 ABI
2381  *
2382  */
2383 static int vgic_its_restore_tables_v0(struct vgic_its *its)
2384 {
2385         int ret;
2386 
2387         ret = vgic_its_restore_collection_table(its);
2388         if (ret)
2389                 return ret;
2390 
2391         return vgic_its_restore_device_tables(its);
2392 }
2393 
2394 static int vgic_its_commit_v0(struct vgic_its *its)
2395 {
2396         const struct vgic_its_abi *abi;
2397 
2398         abi = vgic_its_get_abi(its);
2399         its->baser_coll_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2400         its->baser_device_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2401 
2402         its->baser_coll_table |= (GIC_ENCODE_SZ(abi->cte_esz, 5)
2403                                         << GITS_BASER_ENTRY_SIZE_SHIFT);
2404 
2405         its->baser_device_table |= (GIC_ENCODE_SZ(abi->dte_esz, 5)
2406                                         << GITS_BASER_ENTRY_SIZE_SHIFT);
2407         return 0;
2408 }
2409 
2410 static void vgic_its_reset(struct kvm *kvm, struct vgic_its *its)
2411 {
2412         /* We need to keep the ABI specific field values */
2413         its->baser_coll_table &= ~GITS_BASER_VALID;
2414         its->baser_device_table &= ~GITS_BASER_VALID;
2415         its->cbaser = 0;
2416         its->creadr = 0;
2417         its->cwriter = 0;
2418         its->enabled = 0;
2419         vgic_its_free_device_list(kvm, its);
2420         vgic_its_free_collection_list(kvm, its);
2421 }
2422 
2423 static int vgic_its_has_attr(struct kvm_device *dev,
2424                              struct kvm_device_attr *attr)
2425 {
2426         switch (attr->group) {
2427         case KVM_DEV_ARM_VGIC_GRP_ADDR:
2428                 switch (attr->attr) {
2429                 case KVM_VGIC_ITS_ADDR_TYPE:
2430                         return 0;
2431                 }
2432                 break;
2433         case KVM_DEV_ARM_VGIC_GRP_CTRL:
2434                 switch (attr->attr) {
2435                 case KVM_DEV_ARM_VGIC_CTRL_INIT:
2436                         return 0;
2437                 case KVM_DEV_ARM_ITS_CTRL_RESET:
2438                         return 0;
2439                 case KVM_DEV_ARM_ITS_SAVE_TABLES:
2440                         return 0;
2441                 case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2442                         return 0;
2443                 }
2444                 break;
2445         case KVM_DEV_ARM_VGIC_GRP_ITS_REGS:
2446                 return vgic_its_has_attr_regs(dev, attr);
2447         }
2448         return -ENXIO;
2449 }
2450 
2451 static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)
2452 {
2453         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2454         int ret = 0;
2455 
2456         if (attr == KVM_DEV_ARM_VGIC_CTRL_INIT) /* Nothing to do */
2457                 return 0;
2458 
2459         mutex_lock(&kvm->lock);
2460         mutex_lock(&its->its_lock);
2461 
2462         if (!lock_all_vcpus(kvm)) {
2463                 mutex_unlock(&its->its_lock);
2464                 mutex_unlock(&kvm->lock);
2465                 return -EBUSY;
2466         }
2467 
2468         switch (attr) {
2469         case KVM_DEV_ARM_ITS_CTRL_RESET:
2470                 vgic_its_reset(kvm, its);
2471                 break;
2472         case KVM_DEV_ARM_ITS_SAVE_TABLES:
2473                 ret = abi->save_tables(its);
2474                 break;
2475         case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2476                 ret = abi->restore_tables(its);
2477                 break;
2478         }
2479 
2480         unlock_all_vcpus(kvm);
2481         mutex_unlock(&its->its_lock);
2482         mutex_unlock(&kvm->lock);
2483         return ret;
2484 }
2485 
2486 static int vgic_its_set_attr(struct kvm_device *dev,
2487                              struct kvm_device_attr *attr)
2488 {
2489         struct vgic_its *its = dev->private;
2490         int ret;
2491 
2492         switch (attr->group) {
2493         case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2494                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2495                 unsigned long type = (unsigned long)attr->attr;
2496                 u64 addr;
2497 
2498                 if (type != KVM_VGIC_ITS_ADDR_TYPE)
2499                         return -ENODEV;
2500 
2501                 if (copy_from_user(&addr, uaddr, sizeof(addr)))
2502                         return -EFAULT;
2503 
2504                 ret = vgic_check_ioaddr(dev->kvm, &its->vgic_its_base,
2505                                         addr, SZ_64K);
2506                 if (ret)
2507                         return ret;
2508 
2509                 return vgic_register_its_iodev(dev->kvm, its, addr);
2510         }
2511         case KVM_DEV_ARM_VGIC_GRP_CTRL:
2512                 return vgic_its_ctrl(dev->kvm, its, attr->attr);
2513         case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
2514                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2515                 u64 reg;
2516 
2517                 if (get_user(reg, uaddr))
2518                         return -EFAULT;
2519 
2520                 return vgic_its_attr_regs_access(dev, attr, &reg, true);
2521         }
2522         }
2523         return -ENXIO;
2524 }
2525 
2526 static int vgic_its_get_attr(struct kvm_device *dev,
2527                              struct kvm_device_attr *attr)
2528 {
2529         switch (attr->group) {
2530         case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2531                 struct vgic_its *its = dev->private;
2532                 u64 addr = its->vgic_its_base;
2533                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2534                 unsigned long type = (unsigned long)attr->attr;
2535 
2536                 if (type != KVM_VGIC_ITS_ADDR_TYPE)
2537                         return -ENODEV;
2538 
2539                 if (copy_to_user(uaddr, &addr, sizeof(addr)))
2540                         return -EFAULT;
2541                 break;
2542         }
2543         case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
2544                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2545                 u64 reg;
2546                 int ret;
2547 
2548                 ret = vgic_its_attr_regs_access(dev, attr, &reg, false);
2549                 if (ret)
2550                         return ret;
2551                 return put_user(reg, uaddr);
2552         }
2553         default:
2554                 return -ENXIO;
2555         }
2556 
2557         return 0;
2558 }
2559 
2560 static struct kvm_device_ops kvm_arm_vgic_its_ops = {
2561         .name = "kvm-arm-vgic-its",
2562         .create = vgic_its_create,
2563         .destroy = vgic_its_destroy,
2564         .set_attr = vgic_its_set_attr,
2565         .get_attr = vgic_its_get_attr,
2566         .has_attr = vgic_its_has_attr,
2567 };
2568 
2569 int kvm_vgic_register_its_device(void)
2570 {
2571         return kvm_register_device_ops(&kvm_arm_vgic_its_ops,
2572                                        KVM_DEV_TYPE_ARM_VGIC_ITS);
2573 }
2574 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp