~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/powerpc/perf/hv-24x7.c

Version: ~ [ linux-5.0-rc6 ] ~ [ linux-4.20.10 ] ~ [ linux-4.19.23 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.101 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.158 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.174 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.134 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.63 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Hypervisor supplied "24x7" performance counter support
  3  *
  4  * Author: Cody P Schafer <cody@linux.vnet.ibm.com>
  5  * Copyright 2014 IBM Corporation.
  6  *
  7  * This program is free software; you can redistribute it and/or
  8  * modify it under the terms of the GNU General Public License
  9  * as published by the Free Software Foundation; either version
 10  * 2 of the License, or (at your option) any later version.
 11  */
 12 
 13 #define pr_fmt(fmt) "hv-24x7: " fmt
 14 
 15 #include <linux/perf_event.h>
 16 #include <linux/rbtree.h>
 17 #include <linux/module.h>
 18 #include <linux/slab.h>
 19 #include <linux/vmalloc.h>
 20 
 21 #include <asm/cputhreads.h>
 22 #include <asm/firmware.h>
 23 #include <asm/hvcall.h>
 24 #include <asm/io.h>
 25 #include <linux/byteorder/generic.h>
 26 
 27 #include "hv-24x7.h"
 28 #include "hv-24x7-catalog.h"
 29 #include "hv-common.h"
 30 
 31 /* Version of the 24x7 hypervisor API that we should use in this machine. */
 32 static int interface_version;
 33 
 34 /* Whether we have to aggregate result data for some domains. */
 35 static bool aggregate_result_elements;
 36 
 37 static bool domain_is_valid(unsigned domain)
 38 {
 39         switch (domain) {
 40 #define DOMAIN(n, v, x, c)              \
 41         case HV_PERF_DOMAIN_##n:        \
 42                 /* fall through */
 43 #include "hv-24x7-domains.h"
 44 #undef DOMAIN
 45                 return true;
 46         default:
 47                 return false;
 48         }
 49 }
 50 
 51 static bool is_physical_domain(unsigned domain)
 52 {
 53         switch (domain) {
 54 #define DOMAIN(n, v, x, c)              \
 55         case HV_PERF_DOMAIN_##n:        \
 56                 return c;
 57 #include "hv-24x7-domains.h"
 58 #undef DOMAIN
 59         default:
 60                 return false;
 61         }
 62 }
 63 
 64 /* Domains for which more than one result element are returned for each event. */
 65 static bool domain_needs_aggregation(unsigned int domain)
 66 {
 67         return aggregate_result_elements &&
 68                         (domain == HV_PERF_DOMAIN_PHYS_CORE ||
 69                          (domain >= HV_PERF_DOMAIN_VCPU_HOME_CORE &&
 70                           domain <= HV_PERF_DOMAIN_VCPU_REMOTE_NODE));
 71 }
 72 
 73 static const char *domain_name(unsigned domain)
 74 {
 75         if (!domain_is_valid(domain))
 76                 return NULL;
 77 
 78         switch (domain) {
 79         case HV_PERF_DOMAIN_PHYS_CHIP:          return "Physical Chip";
 80         case HV_PERF_DOMAIN_PHYS_CORE:          return "Physical Core";
 81         case HV_PERF_DOMAIN_VCPU_HOME_CORE:     return "VCPU Home Core";
 82         case HV_PERF_DOMAIN_VCPU_HOME_CHIP:     return "VCPU Home Chip";
 83         case HV_PERF_DOMAIN_VCPU_HOME_NODE:     return "VCPU Home Node";
 84         case HV_PERF_DOMAIN_VCPU_REMOTE_NODE:   return "VCPU Remote Node";
 85         }
 86 
 87         WARN_ON_ONCE(domain);
 88         return NULL;
 89 }
 90 
 91 static bool catalog_entry_domain_is_valid(unsigned domain)
 92 {
 93         /* POWER8 doesn't support virtual domains. */
 94         if (interface_version == 1)
 95                 return is_physical_domain(domain);
 96         else
 97                 return domain_is_valid(domain);
 98 }
 99 
100 /*
101  * TODO: Merging events:
102  * - Think of the hcall as an interface to a 4d array of counters:
103  *   - x = domains
104  *   - y = indexes in the domain (core, chip, vcpu, node, etc)
105  *   - z = offset into the counter space
106  *   - w = lpars (guest vms, "logical partitions")
107  * - A single request is: x,y,y_last,z,z_last,w,w_last
108  *   - this means we can retrieve a rectangle of counters in y,z for a single x.
109  *
110  * - Things to consider (ignoring w):
111  *   - input  cost_per_request = 16
112  *   - output cost_per_result(ys,zs)  = 8 + 8 * ys + ys * zs
113  *   - limited number of requests per hcall (must fit into 4K bytes)
114  *     - 4k = 16 [buffer header] - 16 [request size] * request_count
115  *     - 255 requests per hcall
116  *   - sometimes it will be more efficient to read extra data and discard
117  */
118 
119 /*
120  * Example usage:
121  *  perf stat -e 'hv_24x7/domain=2,offset=8,vcpu=0,lpar=0xffffffff/'
122  */
123 
124 /* u3 0-6, one of HV_24X7_PERF_DOMAIN */
125 EVENT_DEFINE_RANGE_FORMAT(domain, config, 0, 3);
126 /* u16 */
127 EVENT_DEFINE_RANGE_FORMAT(core, config, 16, 31);
128 EVENT_DEFINE_RANGE_FORMAT(chip, config, 16, 31);
129 EVENT_DEFINE_RANGE_FORMAT(vcpu, config, 16, 31);
130 /* u32, see "data_offset" */
131 EVENT_DEFINE_RANGE_FORMAT(offset, config, 32, 63);
132 /* u16 */
133 EVENT_DEFINE_RANGE_FORMAT(lpar, config1, 0, 15);
134 
135 EVENT_DEFINE_RANGE(reserved1, config,   4, 15);
136 EVENT_DEFINE_RANGE(reserved2, config1, 16, 63);
137 EVENT_DEFINE_RANGE(reserved3, config2,  0, 63);
138 
139 static struct attribute *format_attrs[] = {
140         &format_attr_domain.attr,
141         &format_attr_offset.attr,
142         &format_attr_core.attr,
143         &format_attr_chip.attr,
144         &format_attr_vcpu.attr,
145         &format_attr_lpar.attr,
146         NULL,
147 };
148 
149 static struct attribute_group format_group = {
150         .name = "format",
151         .attrs = format_attrs,
152 };
153 
154 static struct attribute_group event_group = {
155         .name = "events",
156         /* .attrs is set in init */
157 };
158 
159 static struct attribute_group event_desc_group = {
160         .name = "event_descs",
161         /* .attrs is set in init */
162 };
163 
164 static struct attribute_group event_long_desc_group = {
165         .name = "event_long_descs",
166         /* .attrs is set in init */
167 };
168 
169 static struct kmem_cache *hv_page_cache;
170 
171 DEFINE_PER_CPU(int, hv_24x7_txn_flags);
172 DEFINE_PER_CPU(int, hv_24x7_txn_err);
173 
174 struct hv_24x7_hw {
175         struct perf_event *events[255];
176 };
177 
178 DEFINE_PER_CPU(struct hv_24x7_hw, hv_24x7_hw);
179 
180 /*
181  * request_buffer and result_buffer are not required to be 4k aligned,
182  * but are not allowed to cross any 4k boundary. Aligning them to 4k is
183  * the simplest way to ensure that.
184  */
185 #define H24x7_DATA_BUFFER_SIZE  4096
186 DEFINE_PER_CPU(char, hv_24x7_reqb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096);
187 DEFINE_PER_CPU(char, hv_24x7_resb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096);
188 
189 static unsigned int max_num_requests(int interface_version)
190 {
191         return (H24x7_DATA_BUFFER_SIZE - sizeof(struct hv_24x7_request_buffer))
192                 / H24x7_REQUEST_SIZE(interface_version);
193 }
194 
195 static char *event_name(struct hv_24x7_event_data *ev, int *len)
196 {
197         *len = be16_to_cpu(ev->event_name_len) - 2;
198         return (char *)ev->remainder;
199 }
200 
201 static char *event_desc(struct hv_24x7_event_data *ev, int *len)
202 {
203         unsigned nl = be16_to_cpu(ev->event_name_len);
204         __be16 *desc_len = (__be16 *)(ev->remainder + nl - 2);
205 
206         *len = be16_to_cpu(*desc_len) - 2;
207         return (char *)ev->remainder + nl;
208 }
209 
210 static char *event_long_desc(struct hv_24x7_event_data *ev, int *len)
211 {
212         unsigned nl = be16_to_cpu(ev->event_name_len);
213         __be16 *desc_len_ = (__be16 *)(ev->remainder + nl - 2);
214         unsigned desc_len = be16_to_cpu(*desc_len_);
215         __be16 *long_desc_len = (__be16 *)(ev->remainder + nl + desc_len - 2);
216 
217         *len = be16_to_cpu(*long_desc_len) - 2;
218         return (char *)ev->remainder + nl + desc_len;
219 }
220 
221 static bool event_fixed_portion_is_within(struct hv_24x7_event_data *ev,
222                                           void *end)
223 {
224         void *start = ev;
225 
226         return (start + offsetof(struct hv_24x7_event_data, remainder)) < end;
227 }
228 
229 /*
230  * Things we don't check:
231  *  - padding for desc, name, and long/detailed desc is required to be '\0'
232  *    bytes.
233  *
234  *  Return NULL if we pass end,
235  *  Otherwise return the address of the byte just following the event.
236  */
237 static void *event_end(struct hv_24x7_event_data *ev, void *end)
238 {
239         void *start = ev;
240         __be16 *dl_, *ldl_;
241         unsigned dl, ldl;
242         unsigned nl = be16_to_cpu(ev->event_name_len);
243 
244         if (nl < 2) {
245                 pr_debug("%s: name length too short: %d", __func__, nl);
246                 return NULL;
247         }
248 
249         if (start + nl > end) {
250                 pr_debug("%s: start=%p + nl=%u > end=%p",
251                                 __func__, start, nl, end);
252                 return NULL;
253         }
254 
255         dl_ = (__be16 *)(ev->remainder + nl - 2);
256         if (!IS_ALIGNED((uintptr_t)dl_, 2))
257                 pr_warn("desc len not aligned %p", dl_);
258         dl = be16_to_cpu(*dl_);
259         if (dl < 2) {
260                 pr_debug("%s: desc len too short: %d", __func__, dl);
261                 return NULL;
262         }
263 
264         if (start + nl + dl > end) {
265                 pr_debug("%s: (start=%p + nl=%u + dl=%u)=%p > end=%p",
266                                 __func__, start, nl, dl, start + nl + dl, end);
267                 return NULL;
268         }
269 
270         ldl_ = (__be16 *)(ev->remainder + nl + dl - 2);
271         if (!IS_ALIGNED((uintptr_t)ldl_, 2))
272                 pr_warn("long desc len not aligned %p", ldl_);
273         ldl = be16_to_cpu(*ldl_);
274         if (ldl < 2) {
275                 pr_debug("%s: long desc len too short (ldl=%u)",
276                                 __func__, ldl);
277                 return NULL;
278         }
279 
280         if (start + nl + dl + ldl > end) {
281                 pr_debug("%s: start=%p + nl=%u + dl=%u + ldl=%u > end=%p",
282                                 __func__, start, nl, dl, ldl, end);
283                 return NULL;
284         }
285 
286         return start + nl + dl + ldl;
287 }
288 
289 static long h_get_24x7_catalog_page_(unsigned long phys_4096,
290                                      unsigned long version, unsigned long index)
291 {
292         pr_devel("h_get_24x7_catalog_page(0x%lx, %lu, %lu)",
293                         phys_4096, version, index);
294 
295         WARN_ON(!IS_ALIGNED(phys_4096, 4096));
296 
297         return plpar_hcall_norets(H_GET_24X7_CATALOG_PAGE,
298                         phys_4096, version, index);
299 }
300 
301 static long h_get_24x7_catalog_page(char page[], u64 version, u32 index)
302 {
303         return h_get_24x7_catalog_page_(virt_to_phys(page),
304                                         version, index);
305 }
306 
307 /*
308  * Each event we find in the catalog, will have a sysfs entry. Format the
309  * data for this sysfs entry based on the event's domain.
310  *
311  * Events belonging to the Chip domain can only be monitored in that domain.
312  * i.e the domain for these events is a fixed/knwon value.
313  *
314  * Events belonging to the Core domain can be monitored either in the physical
315  * core or in one of the virtual CPU domains. So the domain value for these
316  * events must be specified by the user (i.e is a required parameter). Format
317  * the Core events with 'domain=?' so the perf-tool can error check required
318  * parameters.
319  *
320  * NOTE: For the Core domain events, rather than making domain a required
321  *       parameter we could default it to PHYS_CORE and allowe users to
322  *       override the domain to one of the VCPU domains.
323  *
324  *       However, this can make the interface a little inconsistent.
325  *
326  *       If we set domain=2 (PHYS_CHIP) and allow user to override this field
327  *       the user may be tempted to also modify the "offset=x" field in which
328  *       can lead to confusing usage. Consider the HPM_PCYC (offset=0x18) and
329  *       HPM_INST (offset=0x20) events. With:
330  *
331  *              perf stat -e hv_24x7/HPM_PCYC,offset=0x20/
332  *
333  *      we end up monitoring HPM_INST, while the command line has HPM_PCYC.
334  *
335  *      By not assigning a default value to the domain for the Core events,
336  *      we can have simple guidelines:
337  *
338  *              - Specifying values for parameters with "=?" is required.
339  *
340  *              - Specifying (i.e overriding) values for other parameters
341  *                is undefined.
342  */
343 static char *event_fmt(struct hv_24x7_event_data *event, unsigned domain)
344 {
345         const char *sindex;
346         const char *lpar;
347         const char *domain_str;
348         char buf[8];
349 
350         switch (domain) {
351         case HV_PERF_DOMAIN_PHYS_CHIP:
352                 snprintf(buf, sizeof(buf), "%d", domain);
353                 domain_str = buf;
354                 lpar = "0x0";
355                 sindex = "chip";
356                 break;
357         case HV_PERF_DOMAIN_PHYS_CORE:
358                 domain_str = "?";
359                 lpar = "0x0";
360                 sindex = "core";
361                 break;
362         default:
363                 domain_str = "?";
364                 lpar = "?";
365                 sindex = "vcpu";
366         }
367 
368         return kasprintf(GFP_KERNEL,
369                         "domain=%s,offset=0x%x,%s=?,lpar=%s",
370                         domain_str,
371                         be16_to_cpu(event->event_counter_offs) +
372                                 be16_to_cpu(event->event_group_record_offs),
373                         sindex,
374                         lpar);
375 }
376 
377 /* Avoid trusting fw to NUL terminate strings */
378 static char *memdup_to_str(char *maybe_str, int max_len, gfp_t gfp)
379 {
380         return kasprintf(gfp, "%.*s", max_len, maybe_str);
381 }
382 
383 static ssize_t device_show_string(struct device *dev,
384                 struct device_attribute *attr, char *buf)
385 {
386         struct dev_ext_attribute *d;
387 
388         d = container_of(attr, struct dev_ext_attribute, attr);
389 
390         return sprintf(buf, "%s\n", (char *)d->var);
391 }
392 
393 static struct attribute *device_str_attr_create_(char *name, char *str)
394 {
395         struct dev_ext_attribute *attr = kzalloc(sizeof(*attr), GFP_KERNEL);
396 
397         if (!attr)
398                 return NULL;
399 
400         sysfs_attr_init(&attr->attr.attr);
401 
402         attr->var = str;
403         attr->attr.attr.name = name;
404         attr->attr.attr.mode = 0444;
405         attr->attr.show = device_show_string;
406 
407         return &attr->attr.attr;
408 }
409 
410 /*
411  * Allocate and initialize strings representing event attributes.
412  *
413  * NOTE: The strings allocated here are never destroyed and continue to
414  *       exist till shutdown. This is to allow us to create as many events
415  *       from the catalog as possible, even if we encounter errors with some.
416  *       In case of changes to error paths in future, these may need to be
417  *       freed by the caller.
418  */
419 static struct attribute *device_str_attr_create(char *name, int name_max,
420                                                 int name_nonce,
421                                                 char *str, size_t str_max)
422 {
423         char *n;
424         char *s = memdup_to_str(str, str_max, GFP_KERNEL);
425         struct attribute *a;
426 
427         if (!s)
428                 return NULL;
429 
430         if (!name_nonce)
431                 n = kasprintf(GFP_KERNEL, "%.*s", name_max, name);
432         else
433                 n = kasprintf(GFP_KERNEL, "%.*s__%d", name_max, name,
434                                         name_nonce);
435         if (!n)
436                 goto out_s;
437 
438         a = device_str_attr_create_(n, s);
439         if (!a)
440                 goto out_n;
441 
442         return a;
443 out_n:
444         kfree(n);
445 out_s:
446         kfree(s);
447         return NULL;
448 }
449 
450 static struct attribute *event_to_attr(unsigned ix,
451                                        struct hv_24x7_event_data *event,
452                                        unsigned domain,
453                                        int nonce)
454 {
455         int event_name_len;
456         char *ev_name, *a_ev_name, *val;
457         struct attribute *attr;
458 
459         if (!domain_is_valid(domain)) {
460                 pr_warn("catalog event %u has invalid domain %u\n",
461                                 ix, domain);
462                 return NULL;
463         }
464 
465         val = event_fmt(event, domain);
466         if (!val)
467                 return NULL;
468 
469         ev_name = event_name(event, &event_name_len);
470         if (!nonce)
471                 a_ev_name = kasprintf(GFP_KERNEL, "%.*s",
472                                 (int)event_name_len, ev_name);
473         else
474                 a_ev_name = kasprintf(GFP_KERNEL, "%.*s__%d",
475                                 (int)event_name_len, ev_name, nonce);
476 
477         if (!a_ev_name)
478                 goto out_val;
479 
480         attr = device_str_attr_create_(a_ev_name, val);
481         if (!attr)
482                 goto out_name;
483 
484         return attr;
485 out_name:
486         kfree(a_ev_name);
487 out_val:
488         kfree(val);
489         return NULL;
490 }
491 
492 static struct attribute *event_to_desc_attr(struct hv_24x7_event_data *event,
493                                             int nonce)
494 {
495         int nl, dl;
496         char *name = event_name(event, &nl);
497         char *desc = event_desc(event, &dl);
498 
499         /* If there isn't a description, don't create the sysfs file */
500         if (!dl)
501                 return NULL;
502 
503         return device_str_attr_create(name, nl, nonce, desc, dl);
504 }
505 
506 static struct attribute *
507 event_to_long_desc_attr(struct hv_24x7_event_data *event, int nonce)
508 {
509         int nl, dl;
510         char *name = event_name(event, &nl);
511         char *desc = event_long_desc(event, &dl);
512 
513         /* If there isn't a description, don't create the sysfs file */
514         if (!dl)
515                 return NULL;
516 
517         return device_str_attr_create(name, nl, nonce, desc, dl);
518 }
519 
520 static int event_data_to_attrs(unsigned ix, struct attribute **attrs,
521                                    struct hv_24x7_event_data *event, int nonce)
522 {
523         *attrs = event_to_attr(ix, event, event->domain, nonce);
524         if (!*attrs)
525                 return -1;
526 
527         return 0;
528 }
529 
530 /* */
531 struct event_uniq {
532         struct rb_node node;
533         const char *name;
534         int nl;
535         unsigned ct;
536         unsigned domain;
537 };
538 
539 static int memord(const void *d1, size_t s1, const void *d2, size_t s2)
540 {
541         if (s1 < s2)
542                 return 1;
543         if (s1 > s2)
544                 return -1;
545 
546         return memcmp(d1, d2, s1);
547 }
548 
549 static int ev_uniq_ord(const void *v1, size_t s1, unsigned d1, const void *v2,
550                        size_t s2, unsigned d2)
551 {
552         int r = memord(v1, s1, v2, s2);
553 
554         if (r)
555                 return r;
556         if (d1 > d2)
557                 return 1;
558         if (d2 > d1)
559                 return -1;
560         return 0;
561 }
562 
563 static int event_uniq_add(struct rb_root *root, const char *name, int nl,
564                           unsigned domain)
565 {
566         struct rb_node **new = &(root->rb_node), *parent = NULL;
567         struct event_uniq *data;
568 
569         /* Figure out where to put new node */
570         while (*new) {
571                 struct event_uniq *it;
572                 int result;
573 
574                 it = container_of(*new, struct event_uniq, node);
575                 result = ev_uniq_ord(name, nl, domain, it->name, it->nl,
576                                         it->domain);
577 
578                 parent = *new;
579                 if (result < 0)
580                         new = &((*new)->rb_left);
581                 else if (result > 0)
582                         new = &((*new)->rb_right);
583                 else {
584                         it->ct++;
585                         pr_info("found a duplicate event %.*s, ct=%u\n", nl,
586                                                 name, it->ct);
587                         return it->ct;
588                 }
589         }
590 
591         data = kmalloc(sizeof(*data), GFP_KERNEL);
592         if (!data)
593                 return -ENOMEM;
594 
595         *data = (struct event_uniq) {
596                 .name = name,
597                 .nl = nl,
598                 .ct = 0,
599                 .domain = domain,
600         };
601 
602         /* Add new node and rebalance tree. */
603         rb_link_node(&data->node, parent, new);
604         rb_insert_color(&data->node, root);
605 
606         /* data->ct */
607         return 0;
608 }
609 
610 static void event_uniq_destroy(struct rb_root *root)
611 {
612         /*
613          * the strings we point to are in the giant block of memory filled by
614          * the catalog, and are freed separately.
615          */
616         struct event_uniq *pos, *n;
617 
618         rbtree_postorder_for_each_entry_safe(pos, n, root, node)
619                 kfree(pos);
620 }
621 
622 
623 /*
624  * ensure the event structure's sizes are self consistent and don't cause us to
625  * read outside of the event
626  *
627  * On success, return the event length in bytes.
628  * Otherwise, return -1 (and print as appropriate).
629  */
630 static ssize_t catalog_event_len_validate(struct hv_24x7_event_data *event,
631                                           size_t event_idx,
632                                           size_t event_data_bytes,
633                                           size_t event_entry_count,
634                                           size_t offset, void *end)
635 {
636         ssize_t ev_len;
637         void *ev_end, *calc_ev_end;
638 
639         if (offset >= event_data_bytes)
640                 return -1;
641 
642         if (event_idx >= event_entry_count) {
643                 pr_devel("catalog event data has %zu bytes of padding after last event\n",
644                                 event_data_bytes - offset);
645                 return -1;
646         }
647 
648         if (!event_fixed_portion_is_within(event, end)) {
649                 pr_warn("event %zu fixed portion is not within range\n",
650                                 event_idx);
651                 return -1;
652         }
653 
654         ev_len = be16_to_cpu(event->length);
655 
656         if (ev_len % 16)
657                 pr_info("event %zu has length %zu not divisible by 16: event=%pK\n",
658                                 event_idx, ev_len, event);
659 
660         ev_end = (__u8 *)event + ev_len;
661         if (ev_end > end) {
662                 pr_warn("event %zu has .length=%zu, ends after buffer end: ev_end=%pK > end=%pK, offset=%zu\n",
663                                 event_idx, ev_len, ev_end, end,
664                                 offset);
665                 return -1;
666         }
667 
668         calc_ev_end = event_end(event, end);
669         if (!calc_ev_end) {
670                 pr_warn("event %zu has a calculated length which exceeds buffer length %zu: event=%pK end=%pK, offset=%zu\n",
671                         event_idx, event_data_bytes, event, end,
672                         offset);
673                 return -1;
674         }
675 
676         if (calc_ev_end > ev_end) {
677                 pr_warn("event %zu exceeds it's own length: event=%pK, end=%pK, offset=%zu, calc_ev_end=%pK\n",
678                         event_idx, event, ev_end, offset, calc_ev_end);
679                 return -1;
680         }
681 
682         return ev_len;
683 }
684 
685 #define MAX_4K (SIZE_MAX / 4096)
686 
687 static int create_events_from_catalog(struct attribute ***events_,
688                                       struct attribute ***event_descs_,
689                                       struct attribute ***event_long_descs_)
690 {
691         long hret;
692         size_t catalog_len, catalog_page_len, event_entry_count,
693                event_data_len, event_data_offs,
694                event_data_bytes, junk_events, event_idx, event_attr_ct, i,
695                attr_max, event_idx_last, desc_ct, long_desc_ct;
696         ssize_t ct, ev_len;
697         uint64_t catalog_version_num;
698         struct attribute **events, **event_descs, **event_long_descs;
699         struct hv_24x7_catalog_page_0 *page_0 =
700                 kmem_cache_alloc(hv_page_cache, GFP_KERNEL);
701         void *page = page_0;
702         void *event_data, *end;
703         struct hv_24x7_event_data *event;
704         struct rb_root ev_uniq = RB_ROOT;
705         int ret = 0;
706 
707         if (!page) {
708                 ret = -ENOMEM;
709                 goto e_out;
710         }
711 
712         hret = h_get_24x7_catalog_page(page, 0, 0);
713         if (hret) {
714                 ret = -EIO;
715                 goto e_free;
716         }
717 
718         catalog_version_num = be64_to_cpu(page_0->version);
719         catalog_page_len = be32_to_cpu(page_0->length);
720 
721         if (MAX_4K < catalog_page_len) {
722                 pr_err("invalid page count: %zu\n", catalog_page_len);
723                 ret = -EIO;
724                 goto e_free;
725         }
726 
727         catalog_len = catalog_page_len * 4096;
728 
729         event_entry_count = be16_to_cpu(page_0->event_entry_count);
730         event_data_offs   = be16_to_cpu(page_0->event_data_offs);
731         event_data_len    = be16_to_cpu(page_0->event_data_len);
732 
733         pr_devel("cv %llu cl %zu eec %zu edo %zu edl %zu\n",
734                         catalog_version_num, catalog_len,
735                         event_entry_count, event_data_offs, event_data_len);
736 
737         if ((MAX_4K < event_data_len)
738                         || (MAX_4K < event_data_offs)
739                         || (MAX_4K - event_data_offs < event_data_len)) {
740                 pr_err("invalid event data offs %zu and/or len %zu\n",
741                                 event_data_offs, event_data_len);
742                 ret = -EIO;
743                 goto e_free;
744         }
745 
746         if ((event_data_offs + event_data_len) > catalog_page_len) {
747                 pr_err("event data %zu-%zu does not fit inside catalog 0-%zu\n",
748                                 event_data_offs,
749                                 event_data_offs + event_data_len,
750                                 catalog_page_len);
751                 ret = -EIO;
752                 goto e_free;
753         }
754 
755         if (SIZE_MAX - 1 < event_entry_count) {
756                 pr_err("event_entry_count %zu is invalid\n", event_entry_count);
757                 ret = -EIO;
758                 goto e_free;
759         }
760 
761         event_data_bytes = event_data_len * 4096;
762 
763         /*
764          * event data can span several pages, events can cross between these
765          * pages. Use vmalloc to make this easier.
766          */
767         event_data = vmalloc(event_data_bytes);
768         if (!event_data) {
769                 pr_err("could not allocate event data\n");
770                 ret = -ENOMEM;
771                 goto e_free;
772         }
773 
774         end = event_data + event_data_bytes;
775 
776         /*
777          * using vmalloc_to_phys() like this only works if PAGE_SIZE is
778          * divisible by 4096
779          */
780         BUILD_BUG_ON(PAGE_SIZE % 4096);
781 
782         for (i = 0; i < event_data_len; i++) {
783                 hret = h_get_24x7_catalog_page_(
784                                 vmalloc_to_phys(event_data + i * 4096),
785                                 catalog_version_num,
786                                 i + event_data_offs);
787                 if (hret) {
788                         pr_err("Failed to get event data in page %zu: rc=%ld\n",
789                                i + event_data_offs, hret);
790                         ret = -EIO;
791                         goto e_event_data;
792                 }
793         }
794 
795         /*
796          * scan the catalog to determine the number of attributes we need, and
797          * verify it at the same time.
798          */
799         for (junk_events = 0, event = event_data, event_idx = 0, attr_max = 0;
800              ;
801              event_idx++, event = (void *)event + ev_len) {
802                 size_t offset = (void *)event - (void *)event_data;
803                 char *name;
804                 int nl;
805 
806                 ev_len = catalog_event_len_validate(event, event_idx,
807                                                     event_data_bytes,
808                                                     event_entry_count,
809                                                     offset, end);
810                 if (ev_len < 0)
811                         break;
812 
813                 name = event_name(event, &nl);
814 
815                 if (event->event_group_record_len == 0) {
816                         pr_devel("invalid event %zu (%.*s): group_record_len == 0, skipping\n",
817                                         event_idx, nl, name);
818                         junk_events++;
819                         continue;
820                 }
821 
822                 if (!catalog_entry_domain_is_valid(event->domain)) {
823                         pr_info("event %zu (%.*s) has invalid domain %d\n",
824                                         event_idx, nl, name, event->domain);
825                         junk_events++;
826                         continue;
827                 }
828 
829                 attr_max++;
830         }
831 
832         event_idx_last = event_idx;
833         if (event_idx_last != event_entry_count)
834                 pr_warn("event buffer ended before listed # of events were parsed (got %zu, wanted %zu, junk %zu)\n",
835                                 event_idx_last, event_entry_count, junk_events);
836 
837         events = kmalloc_array(attr_max + 1, sizeof(*events), GFP_KERNEL);
838         if (!events) {
839                 ret = -ENOMEM;
840                 goto e_event_data;
841         }
842 
843         event_descs = kmalloc_array(event_idx + 1, sizeof(*event_descs),
844                                 GFP_KERNEL);
845         if (!event_descs) {
846                 ret = -ENOMEM;
847                 goto e_event_attrs;
848         }
849 
850         event_long_descs = kmalloc_array(event_idx + 1,
851                         sizeof(*event_long_descs), GFP_KERNEL);
852         if (!event_long_descs) {
853                 ret = -ENOMEM;
854                 goto e_event_descs;
855         }
856 
857         /* Iterate over the catalog filling in the attribute vector */
858         for (junk_events = 0, event_attr_ct = 0, desc_ct = 0, long_desc_ct = 0,
859                                 event = event_data, event_idx = 0;
860                         event_idx < event_idx_last;
861                         event_idx++, ev_len = be16_to_cpu(event->length),
862                                 event = (void *)event + ev_len) {
863                 char *name;
864                 int nl;
865                 int nonce;
866                 /*
867                  * these are the only "bad" events that are intermixed and that
868                  * we can ignore without issue. make sure to skip them here
869                  */
870                 if (event->event_group_record_len == 0)
871                         continue;
872                 if (!catalog_entry_domain_is_valid(event->domain))
873                         continue;
874 
875                 name  = event_name(event, &nl);
876                 nonce = event_uniq_add(&ev_uniq, name, nl, event->domain);
877                 ct    = event_data_to_attrs(event_idx, events + event_attr_ct,
878                                             event, nonce);
879                 if (ct < 0) {
880                         pr_warn("event %zu (%.*s) creation failure, skipping\n",
881                                 event_idx, nl, name);
882                         junk_events++;
883                 } else {
884                         event_attr_ct++;
885                         event_descs[desc_ct] = event_to_desc_attr(event, nonce);
886                         if (event_descs[desc_ct])
887                                 desc_ct++;
888                         event_long_descs[long_desc_ct] =
889                                         event_to_long_desc_attr(event, nonce);
890                         if (event_long_descs[long_desc_ct])
891                                 long_desc_ct++;
892                 }
893         }
894 
895         pr_info("read %zu catalog entries, created %zu event attrs (%zu failures), %zu descs\n",
896                         event_idx, event_attr_ct, junk_events, desc_ct);
897 
898         events[event_attr_ct] = NULL;
899         event_descs[desc_ct] = NULL;
900         event_long_descs[long_desc_ct] = NULL;
901 
902         event_uniq_destroy(&ev_uniq);
903         vfree(event_data);
904         kmem_cache_free(hv_page_cache, page);
905 
906         *events_ = events;
907         *event_descs_ = event_descs;
908         *event_long_descs_ = event_long_descs;
909         return 0;
910 
911 e_event_descs:
912         kfree(event_descs);
913 e_event_attrs:
914         kfree(events);
915 e_event_data:
916         vfree(event_data);
917 e_free:
918         kmem_cache_free(hv_page_cache, page);
919 e_out:
920         *events_ = NULL;
921         *event_descs_ = NULL;
922         *event_long_descs_ = NULL;
923         return ret;
924 }
925 
926 static ssize_t catalog_read(struct file *filp, struct kobject *kobj,
927                             struct bin_attribute *bin_attr, char *buf,
928                             loff_t offset, size_t count)
929 {
930         long hret;
931         ssize_t ret = 0;
932         size_t catalog_len = 0, catalog_page_len = 0;
933         loff_t page_offset = 0;
934         loff_t offset_in_page;
935         size_t copy_len;
936         uint64_t catalog_version_num = 0;
937         void *page = kmem_cache_alloc(hv_page_cache, GFP_USER);
938         struct hv_24x7_catalog_page_0 *page_0 = page;
939 
940         if (!page)
941                 return -ENOMEM;
942 
943         hret = h_get_24x7_catalog_page(page, 0, 0);
944         if (hret) {
945                 ret = -EIO;
946                 goto e_free;
947         }
948 
949         catalog_version_num = be64_to_cpu(page_0->version);
950         catalog_page_len = be32_to_cpu(page_0->length);
951         catalog_len = catalog_page_len * 4096;
952 
953         page_offset = offset / 4096;
954         offset_in_page = offset % 4096;
955 
956         if (page_offset >= catalog_page_len)
957                 goto e_free;
958 
959         if (page_offset != 0) {
960                 hret = h_get_24x7_catalog_page(page, catalog_version_num,
961                                                page_offset);
962                 if (hret) {
963                         ret = -EIO;
964                         goto e_free;
965                 }
966         }
967 
968         copy_len = 4096 - offset_in_page;
969         if (copy_len > count)
970                 copy_len = count;
971 
972         memcpy(buf, page+offset_in_page, copy_len);
973         ret = copy_len;
974 
975 e_free:
976         if (hret)
977                 pr_err("h_get_24x7_catalog_page(ver=%lld, page=%lld) failed:"
978                        " rc=%ld\n",
979                        catalog_version_num, page_offset, hret);
980         kmem_cache_free(hv_page_cache, page);
981 
982         pr_devel("catalog_read: offset=%lld(%lld) count=%zu "
983                         "catalog_len=%zu(%zu) => %zd\n", offset, page_offset,
984                         count, catalog_len, catalog_page_len, ret);
985 
986         return ret;
987 }
988 
989 static ssize_t domains_show(struct device *dev, struct device_attribute *attr,
990                             char *page)
991 {
992         int d, n, count = 0;
993         const char *str;
994 
995         for (d = 0; d < HV_PERF_DOMAIN_MAX; d++) {
996                 str = domain_name(d);
997                 if (!str)
998                         continue;
999 
1000                 n = sprintf(page, "%d: %s\n", d, str);
1001                 if (n < 0)
1002                         break;
1003 
1004                 count += n;
1005                 page += n;
1006         }
1007         return count;
1008 }
1009 
1010 #define PAGE_0_ATTR(_name, _fmt, _expr)                         \
1011 static ssize_t _name##_show(struct device *dev,                 \
1012                             struct device_attribute *dev_attr,  \
1013                             char *buf)                          \
1014 {                                                               \
1015         long hret;                                              \
1016         ssize_t ret = 0;                                        \
1017         void *page = kmem_cache_alloc(hv_page_cache, GFP_USER); \
1018         struct hv_24x7_catalog_page_0 *page_0 = page;           \
1019         if (!page)                                              \
1020                 return -ENOMEM;                                 \
1021         hret = h_get_24x7_catalog_page(page, 0, 0);             \
1022         if (hret) {                                             \
1023                 ret = -EIO;                                     \
1024                 goto e_free;                                    \
1025         }                                                       \
1026         ret = sprintf(buf, _fmt, _expr);                        \
1027 e_free:                                                         \
1028         kmem_cache_free(hv_page_cache, page);                   \
1029         return ret;                                             \
1030 }                                                               \
1031 static DEVICE_ATTR_RO(_name)
1032 
1033 PAGE_0_ATTR(catalog_version, "%lld\n",
1034                 (unsigned long long)be64_to_cpu(page_0->version));
1035 PAGE_0_ATTR(catalog_len, "%lld\n",
1036                 (unsigned long long)be32_to_cpu(page_0->length) * 4096);
1037 static BIN_ATTR_RO(catalog, 0/* real length varies */);
1038 static DEVICE_ATTR_RO(domains);
1039 
1040 static struct bin_attribute *if_bin_attrs[] = {
1041         &bin_attr_catalog,
1042         NULL,
1043 };
1044 
1045 static struct attribute *if_attrs[] = {
1046         &dev_attr_catalog_len.attr,
1047         &dev_attr_catalog_version.attr,
1048         &dev_attr_domains.attr,
1049         NULL,
1050 };
1051 
1052 static struct attribute_group if_group = {
1053         .name = "interface",
1054         .bin_attrs = if_bin_attrs,
1055         .attrs = if_attrs,
1056 };
1057 
1058 static const struct attribute_group *attr_groups[] = {
1059         &format_group,
1060         &event_group,
1061         &event_desc_group,
1062         &event_long_desc_group,
1063         &if_group,
1064         NULL,
1065 };
1066 
1067 /*
1068  * Start the process for a new H_GET_24x7_DATA hcall.
1069  */
1070 static void init_24x7_request(struct hv_24x7_request_buffer *request_buffer,
1071                               struct hv_24x7_data_result_buffer *result_buffer)
1072 {
1073 
1074         memset(request_buffer, 0, H24x7_DATA_BUFFER_SIZE);
1075         memset(result_buffer, 0, H24x7_DATA_BUFFER_SIZE);
1076 
1077         request_buffer->interface_version = interface_version;
1078         /* memset above set request_buffer->num_requests to 0 */
1079 }
1080 
1081 /*
1082  * Commit (i.e perform) the H_GET_24x7_DATA hcall using the data collected
1083  * by 'init_24x7_request()' and 'add_event_to_24x7_request()'.
1084  */
1085 static int make_24x7_request(struct hv_24x7_request_buffer *request_buffer,
1086                              struct hv_24x7_data_result_buffer *result_buffer)
1087 {
1088         long ret;
1089 
1090         /*
1091          * NOTE: Due to variable number of array elements in request and
1092          *       result buffer(s), sizeof() is not reliable. Use the actual
1093          *       allocated buffer size, H24x7_DATA_BUFFER_SIZE.
1094          */
1095         ret = plpar_hcall_norets(H_GET_24X7_DATA,
1096                         virt_to_phys(request_buffer), H24x7_DATA_BUFFER_SIZE,
1097                         virt_to_phys(result_buffer),  H24x7_DATA_BUFFER_SIZE);
1098 
1099         if (ret) {
1100                 struct hv_24x7_request *req;
1101 
1102                 req = request_buffer->requests;
1103                 pr_notice_ratelimited("hcall failed: [%d %#x %#x %d] => ret 0x%lx (%ld) detail=0x%x failing ix=%x\n",
1104                                       req->performance_domain, req->data_offset,
1105                                       req->starting_ix, req->starting_lpar_ix,
1106                                       ret, ret, result_buffer->detailed_rc,
1107                                       result_buffer->failing_request_ix);
1108                 return -EIO;
1109         }
1110 
1111         return 0;
1112 }
1113 
1114 /*
1115  * Add the given @event to the next slot in the 24x7 request_buffer.
1116  *
1117  * Note that H_GET_24X7_DATA hcall allows reading several counters'
1118  * values in a single HCALL. We expect the caller to add events to the
1119  * request buffer one by one, make the HCALL and process the results.
1120  */
1121 static int add_event_to_24x7_request(struct perf_event *event,
1122                                 struct hv_24x7_request_buffer *request_buffer)
1123 {
1124         u16 idx;
1125         int i;
1126         size_t req_size;
1127         struct hv_24x7_request *req;
1128 
1129         if (request_buffer->num_requests >=
1130             max_num_requests(request_buffer->interface_version)) {
1131                 pr_devel("Too many requests for 24x7 HCALL %d\n",
1132                                 request_buffer->num_requests);
1133                 return -EINVAL;
1134         }
1135 
1136         switch (event_get_domain(event)) {
1137         case HV_PERF_DOMAIN_PHYS_CHIP:
1138                 idx = event_get_chip(event);
1139                 break;
1140         case HV_PERF_DOMAIN_PHYS_CORE:
1141                 idx = event_get_core(event);
1142                 break;
1143         default:
1144                 idx = event_get_vcpu(event);
1145         }
1146 
1147         req_size = H24x7_REQUEST_SIZE(request_buffer->interface_version);
1148 
1149         i = request_buffer->num_requests++;
1150         req = (void *) request_buffer->requests + i * req_size;
1151 
1152         req->performance_domain = event_get_domain(event);
1153         req->data_size = cpu_to_be16(8);
1154         req->data_offset = cpu_to_be32(event_get_offset(event));
1155         req->starting_lpar_ix = cpu_to_be16(event_get_lpar(event));
1156         req->max_num_lpars = cpu_to_be16(1);
1157         req->starting_ix = cpu_to_be16(idx);
1158         req->max_ix = cpu_to_be16(1);
1159 
1160         if (request_buffer->interface_version > 1) {
1161                 if (domain_needs_aggregation(req->performance_domain))
1162                         req->max_num_thread_groups = -1;
1163                 else if (req->performance_domain != HV_PERF_DOMAIN_PHYS_CHIP) {
1164                         req->starting_thread_group_ix = idx % 2;
1165                         req->max_num_thread_groups = 1;
1166                 }
1167         }
1168 
1169         return 0;
1170 }
1171 
1172 /**
1173  * get_count_from_result - get event count from all result elements in result
1174  *
1175  * If the event corresponding to this result needs aggregation of the result
1176  * element values, then this function does that.
1177  *
1178  * @event:      Event associated with @res.
1179  * @resb:       Result buffer containing @res.
1180  * @res:        Result to work on.
1181  * @countp:     Output variable containing the event count.
1182  * @next:       Optional output variable pointing to the next result in @resb.
1183  */
1184 static int get_count_from_result(struct perf_event *event,
1185                                  struct hv_24x7_data_result_buffer *resb,
1186                                  struct hv_24x7_result *res, u64 *countp,
1187                                  struct hv_24x7_result **next)
1188 {
1189         u16 num_elements = be16_to_cpu(res->num_elements_returned);
1190         u16 data_size = be16_to_cpu(res->result_element_data_size);
1191         unsigned int data_offset;
1192         void *element_data;
1193         int i;
1194         u64 count;
1195 
1196         /*
1197          * We can bail out early if the result is empty.
1198          */
1199         if (!num_elements) {
1200                 pr_debug("Result of request %hhu is empty, nothing to do\n",
1201                          res->result_ix);
1202 
1203                 if (next)
1204                         *next = (struct hv_24x7_result *) res->elements;
1205 
1206                 return -ENODATA;
1207         }
1208 
1209         /*
1210          * Since we always specify 1 as the maximum for the smallest resource
1211          * we're requesting, there should to be only one element per result.
1212          * Except when an event needs aggregation, in which case there are more.
1213          */
1214         if (num_elements != 1 &&
1215             !domain_needs_aggregation(event_get_domain(event))) {
1216                 pr_err("Error: result of request %hhu has %hu elements\n",
1217                        res->result_ix, num_elements);
1218 
1219                 return -EIO;
1220         }
1221 
1222         if (data_size != sizeof(u64)) {
1223                 pr_debug("Error: result of request %hhu has data of %hu bytes\n",
1224                          res->result_ix, data_size);
1225 
1226                 return -ENOTSUPP;
1227         }
1228 
1229         if (resb->interface_version == 1)
1230                 data_offset = offsetof(struct hv_24x7_result_element_v1,
1231                                        element_data);
1232         else
1233                 data_offset = offsetof(struct hv_24x7_result_element_v2,
1234                                        element_data);
1235 
1236         /* Go through the result elements in the result. */
1237         for (i = count = 0, element_data = res->elements + data_offset;
1238              i < num_elements;
1239              i++, element_data += data_size + data_offset)
1240                 count += be64_to_cpu(*((u64 *) element_data));
1241 
1242         *countp = count;
1243 
1244         /* The next result is after the last result element. */
1245         if (next)
1246                 *next = element_data - data_offset;
1247 
1248         return 0;
1249 }
1250 
1251 static int single_24x7_request(struct perf_event *event, u64 *count)
1252 {
1253         int ret;
1254         struct hv_24x7_request_buffer *request_buffer;
1255         struct hv_24x7_data_result_buffer *result_buffer;
1256 
1257         BUILD_BUG_ON(sizeof(*request_buffer) > 4096);
1258         BUILD_BUG_ON(sizeof(*result_buffer) > 4096);
1259 
1260         request_buffer = (void *)get_cpu_var(hv_24x7_reqb);
1261         result_buffer = (void *)get_cpu_var(hv_24x7_resb);
1262 
1263         init_24x7_request(request_buffer, result_buffer);
1264 
1265         ret = add_event_to_24x7_request(event, request_buffer);
1266         if (ret)
1267                 goto out;
1268 
1269         ret = make_24x7_request(request_buffer, result_buffer);
1270         if (ret)
1271                 goto out;
1272 
1273         /* process result from hcall */
1274         ret = get_count_from_result(event, result_buffer,
1275                                     result_buffer->results, count, NULL);
1276 
1277 out:
1278         put_cpu_var(hv_24x7_reqb);
1279         put_cpu_var(hv_24x7_resb);
1280         return ret;
1281 }
1282 
1283 
1284 static int h_24x7_event_init(struct perf_event *event)
1285 {
1286         struct hv_perf_caps caps;
1287         unsigned domain;
1288         unsigned long hret;
1289         u64 ct;
1290 
1291         /* Not our event */
1292         if (event->attr.type != event->pmu->type)
1293                 return -ENOENT;
1294 
1295         /* Unused areas must be 0 */
1296         if (event_get_reserved1(event) ||
1297             event_get_reserved2(event) ||
1298             event_get_reserved3(event)) {
1299                 pr_devel("reserved set when forbidden 0x%llx(0x%llx) 0x%llx(0x%llx) 0x%llx(0x%llx)\n",
1300                                 event->attr.config,
1301                                 event_get_reserved1(event),
1302                                 event->attr.config1,
1303                                 event_get_reserved2(event),
1304                                 event->attr.config2,
1305                                 event_get_reserved3(event));
1306                 return -EINVAL;
1307         }
1308 
1309         /* unsupported modes and filters */
1310         if (event->attr.exclude_user   ||
1311             event->attr.exclude_kernel ||
1312             event->attr.exclude_hv     ||
1313             event->attr.exclude_idle   ||
1314             event->attr.exclude_host   ||
1315             event->attr.exclude_guest)
1316                 return -EINVAL;
1317 
1318         /* no branch sampling */
1319         if (has_branch_stack(event))
1320                 return -EOPNOTSUPP;
1321 
1322         /* offset must be 8 byte aligned */
1323         if (event_get_offset(event) % 8) {
1324                 pr_devel("bad alignment\n");
1325                 return -EINVAL;
1326         }
1327 
1328         domain = event_get_domain(event);
1329         if (domain >= HV_PERF_DOMAIN_MAX) {
1330                 pr_devel("invalid domain %d\n", domain);
1331                 return -EINVAL;
1332         }
1333 
1334         hret = hv_perf_caps_get(&caps);
1335         if (hret) {
1336                 pr_devel("could not get capabilities: rc=%ld\n", hret);
1337                 return -EIO;
1338         }
1339 
1340         /* Physical domains & other lpars require extra capabilities */
1341         if (!caps.collect_privileged && (is_physical_domain(domain) ||
1342                 (event_get_lpar(event) != event_get_lpar_max()))) {
1343                 pr_devel("hv permissions disallow: is_physical_domain:%d, lpar=0x%llx\n",
1344                                 is_physical_domain(domain),
1345                                 event_get_lpar(event));
1346                 return -EACCES;
1347         }
1348 
1349         /* Get the initial value of the counter for this event */
1350         if (single_24x7_request(event, &ct)) {
1351                 pr_devel("test hcall failed\n");
1352                 return -EIO;
1353         }
1354         (void)local64_xchg(&event->hw.prev_count, ct);
1355 
1356         return 0;
1357 }
1358 
1359 static u64 h_24x7_get_value(struct perf_event *event)
1360 {
1361         u64 ct;
1362 
1363         if (single_24x7_request(event, &ct))
1364                 /* We checked this in event init, shouldn't fail here... */
1365                 return 0;
1366 
1367         return ct;
1368 }
1369 
1370 static void update_event_count(struct perf_event *event, u64 now)
1371 {
1372         s64 prev;
1373 
1374         prev = local64_xchg(&event->hw.prev_count, now);
1375         local64_add(now - prev, &event->count);
1376 }
1377 
1378 static void h_24x7_event_read(struct perf_event *event)
1379 {
1380         u64 now;
1381         struct hv_24x7_request_buffer *request_buffer;
1382         struct hv_24x7_hw *h24x7hw;
1383         int txn_flags;
1384 
1385         txn_flags = __this_cpu_read(hv_24x7_txn_flags);
1386 
1387         /*
1388          * If in a READ transaction, add this counter to the list of
1389          * counters to read during the next HCALL (i.e commit_txn()).
1390          * If not in a READ transaction, go ahead and make the HCALL
1391          * to read this counter by itself.
1392          */
1393 
1394         if (txn_flags & PERF_PMU_TXN_READ) {
1395                 int i;
1396                 int ret;
1397 
1398                 if (__this_cpu_read(hv_24x7_txn_err))
1399                         return;
1400 
1401                 request_buffer = (void *)get_cpu_var(hv_24x7_reqb);
1402 
1403                 ret = add_event_to_24x7_request(event, request_buffer);
1404                 if (ret) {
1405                         __this_cpu_write(hv_24x7_txn_err, ret);
1406                 } else {
1407                         /*
1408                          * Associate the event with the HCALL request index,
1409                          * so ->commit_txn() can quickly find/update count.
1410                          */
1411                         i = request_buffer->num_requests - 1;
1412 
1413                         h24x7hw = &get_cpu_var(hv_24x7_hw);
1414                         h24x7hw->events[i] = event;
1415                         put_cpu_var(h24x7hw);
1416                         /*
1417                          * Clear the event count so we can compute the _change_
1418                          * in the 24x7 raw counter value at the end of the txn.
1419                          *
1420                          * Note that we could alternatively read the 24x7 value
1421                          * now and save its value in event->hw.prev_count. But
1422                          * that would require issuing a hcall, which would then
1423                          * defeat the purpose of using the txn interface.
1424                          */
1425                         local64_set(&event->count, 0);
1426                 }
1427 
1428                 put_cpu_var(hv_24x7_reqb);
1429         } else {
1430                 now = h_24x7_get_value(event);
1431                 update_event_count(event, now);
1432         }
1433 }
1434 
1435 static void h_24x7_event_start(struct perf_event *event, int flags)
1436 {
1437         if (flags & PERF_EF_RELOAD)
1438                 local64_set(&event->hw.prev_count, h_24x7_get_value(event));
1439 }
1440 
1441 static void h_24x7_event_stop(struct perf_event *event, int flags)
1442 {
1443         h_24x7_event_read(event);
1444 }
1445 
1446 static int h_24x7_event_add(struct perf_event *event, int flags)
1447 {
1448         if (flags & PERF_EF_START)
1449                 h_24x7_event_start(event, flags);
1450 
1451         return 0;
1452 }
1453 
1454 /*
1455  * 24x7 counters only support READ transactions. They are
1456  * always counting and dont need/support ADD transactions.
1457  * Cache the flags, but otherwise ignore transactions that
1458  * are not PERF_PMU_TXN_READ.
1459  */
1460 static void h_24x7_event_start_txn(struct pmu *pmu, unsigned int flags)
1461 {
1462         struct hv_24x7_request_buffer *request_buffer;
1463         struct hv_24x7_data_result_buffer *result_buffer;
1464 
1465         /* We should not be called if we are already in a txn */
1466         WARN_ON_ONCE(__this_cpu_read(hv_24x7_txn_flags));
1467 
1468         __this_cpu_write(hv_24x7_txn_flags, flags);
1469         if (flags & ~PERF_PMU_TXN_READ)
1470                 return;
1471 
1472         request_buffer = (void *)get_cpu_var(hv_24x7_reqb);
1473         result_buffer = (void *)get_cpu_var(hv_24x7_resb);
1474 
1475         init_24x7_request(request_buffer, result_buffer);
1476 
1477         put_cpu_var(hv_24x7_resb);
1478         put_cpu_var(hv_24x7_reqb);
1479 }
1480 
1481 /*
1482  * Clean up transaction state.
1483  *
1484  * NOTE: Ignore state of request and result buffers for now.
1485  *       We will initialize them during the next read/txn.
1486  */
1487 static void reset_txn(void)
1488 {
1489         __this_cpu_write(hv_24x7_txn_flags, 0);
1490         __this_cpu_write(hv_24x7_txn_err, 0);
1491 }
1492 
1493 /*
1494  * 24x7 counters only support READ transactions. They are always counting
1495  * and dont need/support ADD transactions. Clear ->txn_flags but otherwise
1496  * ignore transactions that are not of type PERF_PMU_TXN_READ.
1497  *
1498  * For READ transactions, submit all pending 24x7 requests (i.e requests
1499  * that were queued by h_24x7_event_read()), to the hypervisor and update
1500  * the event counts.
1501  */
1502 static int h_24x7_event_commit_txn(struct pmu *pmu)
1503 {
1504         struct hv_24x7_request_buffer *request_buffer;
1505         struct hv_24x7_data_result_buffer *result_buffer;
1506         struct hv_24x7_result *res, *next_res;
1507         u64 count;
1508         int i, ret, txn_flags;
1509         struct hv_24x7_hw *h24x7hw;
1510 
1511         txn_flags = __this_cpu_read(hv_24x7_txn_flags);
1512         WARN_ON_ONCE(!txn_flags);
1513 
1514         ret = 0;
1515         if (txn_flags & ~PERF_PMU_TXN_READ)
1516                 goto out;
1517 
1518         ret = __this_cpu_read(hv_24x7_txn_err);
1519         if (ret)
1520                 goto out;
1521 
1522         request_buffer = (void *)get_cpu_var(hv_24x7_reqb);
1523         result_buffer = (void *)get_cpu_var(hv_24x7_resb);
1524 
1525         ret = make_24x7_request(request_buffer, result_buffer);
1526         if (ret)
1527                 goto put_reqb;
1528 
1529         h24x7hw = &get_cpu_var(hv_24x7_hw);
1530 
1531         /* Go through results in the result buffer to update event counts. */
1532         for (i = 0, res = result_buffer->results;
1533              i < result_buffer->num_results; i++, res = next_res) {
1534                 struct perf_event *event = h24x7hw->events[res->result_ix];
1535 
1536                 ret = get_count_from_result(event, result_buffer, res, &count,
1537                                             &next_res);
1538                 if (ret)
1539                         break;
1540 
1541                 update_event_count(event, count);
1542         }
1543 
1544         put_cpu_var(hv_24x7_hw);
1545 
1546 put_reqb:
1547         put_cpu_var(hv_24x7_resb);
1548         put_cpu_var(hv_24x7_reqb);
1549 out:
1550         reset_txn();
1551         return ret;
1552 }
1553 
1554 /*
1555  * 24x7 counters only support READ transactions. They are always counting
1556  * and dont need/support ADD transactions. However, regardless of type
1557  * of transaction, all we need to do is cleanup, so we don't have to check
1558  * the type of transaction.
1559  */
1560 static void h_24x7_event_cancel_txn(struct pmu *pmu)
1561 {
1562         WARN_ON_ONCE(!__this_cpu_read(hv_24x7_txn_flags));
1563         reset_txn();
1564 }
1565 
1566 static struct pmu h_24x7_pmu = {
1567         .task_ctx_nr = perf_invalid_context,
1568 
1569         .name = "hv_24x7",
1570         .attr_groups = attr_groups,
1571         .event_init  = h_24x7_event_init,
1572         .add         = h_24x7_event_add,
1573         .del         = h_24x7_event_stop,
1574         .start       = h_24x7_event_start,
1575         .stop        = h_24x7_event_stop,
1576         .read        = h_24x7_event_read,
1577         .start_txn   = h_24x7_event_start_txn,
1578         .commit_txn  = h_24x7_event_commit_txn,
1579         .cancel_txn  = h_24x7_event_cancel_txn,
1580 };
1581 
1582 static int hv_24x7_init(void)
1583 {
1584         int r;
1585         unsigned long hret;
1586         struct hv_perf_caps caps;
1587 
1588         if (!firmware_has_feature(FW_FEATURE_LPAR)) {
1589                 pr_debug("not a virtualized system, not enabling\n");
1590                 return -ENODEV;
1591         } else if (!cur_cpu_spec->oprofile_cpu_type)
1592                 return -ENODEV;
1593 
1594         /* POWER8 only supports v1, while POWER9 only supports v2. */
1595         if (!strcmp(cur_cpu_spec->oprofile_cpu_type, "ppc64/power8"))
1596                 interface_version = 1;
1597         else {
1598                 interface_version = 2;
1599 
1600                 /* SMT8 in POWER9 needs to aggregate result elements. */
1601                 if (threads_per_core == 8)
1602                         aggregate_result_elements = true;
1603         }
1604 
1605         hret = hv_perf_caps_get(&caps);
1606         if (hret) {
1607                 pr_debug("could not obtain capabilities, not enabling, rc=%ld\n",
1608                                 hret);
1609                 return -ENODEV;
1610         }
1611 
1612         hv_page_cache = kmem_cache_create("hv-page-4096", 4096, 4096, 0, NULL);
1613         if (!hv_page_cache)
1614                 return -ENOMEM;
1615 
1616         /* sampling not supported */
1617         h_24x7_pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1618 
1619         r = create_events_from_catalog(&event_group.attrs,
1620                                    &event_desc_group.attrs,
1621                                    &event_long_desc_group.attrs);
1622 
1623         if (r)
1624                 return r;
1625 
1626         r = perf_pmu_register(&h_24x7_pmu, h_24x7_pmu.name, -1);
1627         if (r)
1628                 return r;
1629 
1630         return 0;
1631 }
1632 
1633 device_initcall(hv_24x7_init);
1634 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp