1/* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License version 2 as 4 * published by the Free Software Foundation. 5 * 6 * This program is distributed in the hope that it will be useful, 7 * but WITHOUT ANY WARRANTY; without even the implied warranty of 8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 * GNU General Public License for more details. 10 * 11 * Copyright (C) 2014 ARM Limited 12 */ 13 14#include <linux/ctype.h> 15#include <linux/hrtimer.h> 16#include <linux/idr.h> 17#include <linux/interrupt.h> 18#include <linux/io.h> 19#include <linux/module.h> 20#include <linux/perf_event.h> 21#include <linux/platform_device.h> 22#include <linux/slab.h> 23 24#define CCN_NUM_XP_PORTS 2 25#define CCN_NUM_VCS 4 26#define CCN_NUM_REGIONS 256 27#define CCN_REGION_SIZE 0x10000 28 29#define CCN_ALL_OLY_ID 0xff00 30#define CCN_ALL_OLY_ID__OLY_ID__SHIFT 0 31#define CCN_ALL_OLY_ID__OLY_ID__MASK 0x1f 32#define CCN_ALL_OLY_ID__NODE_ID__SHIFT 8 33#define CCN_ALL_OLY_ID__NODE_ID__MASK 0x3f 34 35#define CCN_MN_ERRINT_STATUS 0x0008 36#define CCN_MN_ERRINT_STATUS__INTREQ__DESSERT 0x11 37#define CCN_MN_ERRINT_STATUS__ALL_ERRORS__ENABLE 0x02 38#define CCN_MN_ERRINT_STATUS__ALL_ERRORS__DISABLED 0x20 39#define CCN_MN_ERRINT_STATUS__ALL_ERRORS__DISABLE 0x22 40#define CCN_MN_ERRINT_STATUS__CORRECTED_ERRORS_ENABLE 0x04 41#define CCN_MN_ERRINT_STATUS__CORRECTED_ERRORS_DISABLED 0x40 42#define CCN_MN_ERRINT_STATUS__CORRECTED_ERRORS_DISABLE 0x44 43#define CCN_MN_ERRINT_STATUS__PMU_EVENTS__ENABLE 0x08 44#define CCN_MN_ERRINT_STATUS__PMU_EVENTS__DISABLED 0x80 45#define CCN_MN_ERRINT_STATUS__PMU_EVENTS__DISABLE 0x88 46#define CCN_MN_OLY_COMP_LIST_63_0 0x01e0 47#define CCN_MN_ERR_SIG_VAL_63_0 0x0300 48#define CCN_MN_ERR_SIG_VAL_63_0__DT (1 << 1) 49 50#define CCN_DT_ACTIVE_DSM 0x0000 51#define CCN_DT_ACTIVE_DSM__DSM_ID__SHIFT(n) ((n) * 8) 52#define CCN_DT_ACTIVE_DSM__DSM_ID__MASK 0xff 53#define CCN_DT_CTL 0x0028 54#define CCN_DT_CTL__DT_EN (1 << 0) 55#define CCN_DT_PMEVCNT(n) (0x0100 + (n) * 0x8) 56#define CCN_DT_PMCCNTR 0x0140 57#define CCN_DT_PMCCNTRSR 0x0190 58#define CCN_DT_PMOVSR 0x0198 59#define CCN_DT_PMOVSR_CLR 0x01a0 60#define CCN_DT_PMOVSR_CLR__MASK 0x1f 61#define CCN_DT_PMCR 0x01a8 62#define CCN_DT_PMCR__OVFL_INTR_EN (1 << 6) 63#define CCN_DT_PMCR__PMU_EN (1 << 0) 64#define CCN_DT_PMSR 0x01b0 65#define CCN_DT_PMSR_REQ 0x01b8 66#define CCN_DT_PMSR_CLR 0x01c0 67 68#define CCN_HNF_PMU_EVENT_SEL 0x0600 69#define CCN_HNF_PMU_EVENT_SEL__ID__SHIFT(n) ((n) * 4) 70#define CCN_HNF_PMU_EVENT_SEL__ID__MASK 0xf 71 72#define CCN_XP_DT_CONFIG 0x0300 73#define CCN_XP_DT_CONFIG__DT_CFG__SHIFT(n) ((n) * 4) 74#define CCN_XP_DT_CONFIG__DT_CFG__MASK 0xf 75#define CCN_XP_DT_CONFIG__DT_CFG__PASS_THROUGH 0x0 76#define CCN_XP_DT_CONFIG__DT_CFG__WATCHPOINT_0_OR_1 0x1 77#define CCN_XP_DT_CONFIG__DT_CFG__WATCHPOINT(n) (0x2 + (n)) 78#define CCN_XP_DT_CONFIG__DT_CFG__XP_PMU_EVENT(n) (0x4 + (n)) 79#define CCN_XP_DT_CONFIG__DT_CFG__DEVICE_PMU_EVENT(d, n) (0x8 + (d) * 4 + (n)) 80#define CCN_XP_DT_INTERFACE_SEL 0x0308 81#define CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__SHIFT(n) (0 + (n) * 8) 82#define CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__MASK 0x1 83#define CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__SHIFT(n) (1 + (n) * 8) 84#define CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__MASK 0x1 85#define CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__SHIFT(n) (2 + (n) * 8) 86#define CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__MASK 0x3 87#define CCN_XP_DT_CMP_VAL_L(n) (0x0310 + (n) * 0x40) 88#define CCN_XP_DT_CMP_VAL_H(n) (0x0318 + (n) * 0x40) 89#define CCN_XP_DT_CMP_MASK_L(n) (0x0320 + (n) * 0x40) 90#define CCN_XP_DT_CMP_MASK_H(n) (0x0328 + (n) * 0x40) 91#define CCN_XP_DT_CONTROL 0x0370 92#define CCN_XP_DT_CONTROL__DT_ENABLE (1 << 0) 93#define CCN_XP_DT_CONTROL__WP_ARM_SEL__SHIFT(n) (12 + (n) * 4) 94#define CCN_XP_DT_CONTROL__WP_ARM_SEL__MASK 0xf 95#define CCN_XP_DT_CONTROL__WP_ARM_SEL__ALWAYS 0xf 96#define CCN_XP_PMU_EVENT_SEL 0x0600 97#define CCN_XP_PMU_EVENT_SEL__ID__SHIFT(n) ((n) * 7) 98#define CCN_XP_PMU_EVENT_SEL__ID__MASK 0x3f 99 100#define CCN_SBAS_PMU_EVENT_SEL 0x0600 101#define CCN_SBAS_PMU_EVENT_SEL__ID__SHIFT(n) ((n) * 4) 102#define CCN_SBAS_PMU_EVENT_SEL__ID__MASK 0xf 103 104#define CCN_RNI_PMU_EVENT_SEL 0x0600 105#define CCN_RNI_PMU_EVENT_SEL__ID__SHIFT(n) ((n) * 4) 106#define CCN_RNI_PMU_EVENT_SEL__ID__MASK 0xf 107 108#define CCN_TYPE_MN 0x01 109#define CCN_TYPE_DT 0x02 110#define CCN_TYPE_HNF 0x04 111#define CCN_TYPE_HNI 0x05 112#define CCN_TYPE_XP 0x08 113#define CCN_TYPE_SBSX 0x0c 114#define CCN_TYPE_SBAS 0x10 115#define CCN_TYPE_RNI_1P 0x14 116#define CCN_TYPE_RNI_2P 0x15 117#define CCN_TYPE_RNI_3P 0x16 118#define CCN_TYPE_RND_1P 0x18 /* RN-D = RN-I + DVM */ 119#define CCN_TYPE_RND_2P 0x19 120#define CCN_TYPE_RND_3P 0x1a 121#define CCN_TYPE_CYCLES 0xff /* Pseudotype */ 122 123#define CCN_EVENT_WATCHPOINT 0xfe /* Pseudoevent */ 124 125#define CCN_NUM_PMU_EVENTS 4 126#define CCN_NUM_XP_WATCHPOINTS 2 /* See DT.dbg_id.num_watchpoints */ 127#define CCN_NUM_PMU_EVENT_COUNTERS 8 /* See DT.dbg_id.num_pmucntr */ 128#define CCN_IDX_PMU_CYCLE_COUNTER CCN_NUM_PMU_EVENT_COUNTERS 129 130#define CCN_NUM_PREDEFINED_MASKS 4 131#define CCN_IDX_MASK_ANY (CCN_NUM_PMU_EVENT_COUNTERS + 0) 132#define CCN_IDX_MASK_EXACT (CCN_NUM_PMU_EVENT_COUNTERS + 1) 133#define CCN_IDX_MASK_ORDER (CCN_NUM_PMU_EVENT_COUNTERS + 2) 134#define CCN_IDX_MASK_OPCODE (CCN_NUM_PMU_EVENT_COUNTERS + 3) 135 136struct arm_ccn_component { 137 void __iomem *base; 138 u32 type; 139 140 DECLARE_BITMAP(pmu_events_mask, CCN_NUM_PMU_EVENTS); 141 union { 142 struct { 143 DECLARE_BITMAP(dt_cmp_mask, CCN_NUM_XP_WATCHPOINTS); 144 } xp; 145 }; 146}; 147 148#define pmu_to_arm_ccn(_pmu) container_of(container_of(_pmu, \ 149 struct arm_ccn_dt, pmu), struct arm_ccn, dt) 150 151struct arm_ccn_dt { 152 int id; 153 void __iomem *base; 154 155 spinlock_t config_lock; 156 157 DECLARE_BITMAP(pmu_counters_mask, CCN_NUM_PMU_EVENT_COUNTERS + 1); 158 struct { 159 struct arm_ccn_component *source; 160 struct perf_event *event; 161 } pmu_counters[CCN_NUM_PMU_EVENT_COUNTERS + 1]; 162 163 struct { 164 u64 l, h; 165 } cmp_mask[CCN_NUM_PMU_EVENT_COUNTERS + CCN_NUM_PREDEFINED_MASKS]; 166 167 struct hrtimer hrtimer; 168 169 struct pmu pmu; 170}; 171 172struct arm_ccn { 173 struct device *dev; 174 void __iomem *base; 175 unsigned irq_used:1; 176 unsigned sbas_present:1; 177 unsigned sbsx_present:1; 178 179 int num_nodes; 180 struct arm_ccn_component *node; 181 182 int num_xps; 183 struct arm_ccn_component *xp; 184 185 struct arm_ccn_dt dt; 186}; 187 188 189static int arm_ccn_node_to_xp(int node) 190{ 191 return node / CCN_NUM_XP_PORTS; 192} 193 194static int arm_ccn_node_to_xp_port(int node) 195{ 196 return node % CCN_NUM_XP_PORTS; 197} 198 199 200/* 201 * Bit shifts and masks in these defines must be kept in sync with 202 * arm_ccn_pmu_config_set() and CCN_FORMAT_ATTRs below! 203 */ 204#define CCN_CONFIG_NODE(_config) (((_config) >> 0) & 0xff) 205#define CCN_CONFIG_XP(_config) (((_config) >> 0) & 0xff) 206#define CCN_CONFIG_TYPE(_config) (((_config) >> 8) & 0xff) 207#define CCN_CONFIG_EVENT(_config) (((_config) >> 16) & 0xff) 208#define CCN_CONFIG_PORT(_config) (((_config) >> 24) & 0x3) 209#define CCN_CONFIG_VC(_config) (((_config) >> 26) & 0x7) 210#define CCN_CONFIG_DIR(_config) (((_config) >> 29) & 0x1) 211#define CCN_CONFIG_MASK(_config) (((_config) >> 30) & 0xf) 212 213static void arm_ccn_pmu_config_set(u64 *config, u32 node_xp, u32 type, u32 port) 214{ 215 *config &= ~((0xff << 0) | (0xff << 8) | (0xff << 24)); 216 *config |= (node_xp << 0) | (type << 8) | (port << 24); 217} 218 219static ssize_t arm_ccn_pmu_format_show(struct device *dev, 220 struct device_attribute *attr, char *buf) 221{ 222 struct dev_ext_attribute *ea = container_of(attr, 223 struct dev_ext_attribute, attr); 224 225 return snprintf(buf, PAGE_SIZE, "%s\n", (char *)ea->var); 226} 227 228#define CCN_FORMAT_ATTR(_name, _config) \ 229 struct dev_ext_attribute arm_ccn_pmu_format_attr_##_name = \ 230 { __ATTR(_name, S_IRUGO, arm_ccn_pmu_format_show, \ 231 NULL), _config } 232 233static CCN_FORMAT_ATTR(node, "config:0-7"); 234static CCN_FORMAT_ATTR(xp, "config:0-7"); 235static CCN_FORMAT_ATTR(type, "config:8-15"); 236static CCN_FORMAT_ATTR(event, "config:16-23"); 237static CCN_FORMAT_ATTR(port, "config:24-25"); 238static CCN_FORMAT_ATTR(vc, "config:26-28"); 239static CCN_FORMAT_ATTR(dir, "config:29-29"); 240static CCN_FORMAT_ATTR(mask, "config:30-33"); 241static CCN_FORMAT_ATTR(cmp_l, "config1:0-62"); 242static CCN_FORMAT_ATTR(cmp_h, "config2:0-59"); 243 244static struct attribute *arm_ccn_pmu_format_attrs[] = { 245 &arm_ccn_pmu_format_attr_node.attr.attr, 246 &arm_ccn_pmu_format_attr_xp.attr.attr, 247 &arm_ccn_pmu_format_attr_type.attr.attr, 248 &arm_ccn_pmu_format_attr_event.attr.attr, 249 &arm_ccn_pmu_format_attr_port.attr.attr, 250 &arm_ccn_pmu_format_attr_vc.attr.attr, 251 &arm_ccn_pmu_format_attr_dir.attr.attr, 252 &arm_ccn_pmu_format_attr_mask.attr.attr, 253 &arm_ccn_pmu_format_attr_cmp_l.attr.attr, 254 &arm_ccn_pmu_format_attr_cmp_h.attr.attr, 255 NULL 256}; 257 258static struct attribute_group arm_ccn_pmu_format_attr_group = { 259 .name = "format", 260 .attrs = arm_ccn_pmu_format_attrs, 261}; 262 263 264struct arm_ccn_pmu_event { 265 struct device_attribute attr; 266 u32 type; 267 u32 event; 268 int num_ports; 269 int num_vcs; 270 const char *def; 271 int mask; 272}; 273 274#define CCN_EVENT_ATTR(_name) \ 275 __ATTR(_name, S_IRUGO, arm_ccn_pmu_event_show, NULL) 276 277/* 278 * Events defined in TRM for MN, HN-I and SBSX are actually watchpoints set on 279 * their ports in XP they are connected to. For the sake of usability they are 280 * explicitly defined here (and translated into a relevant watchpoint in 281 * arm_ccn_pmu_event_init()) so the user can easily request them without deep 282 * knowledge of the flit format. 283 */ 284 285#define CCN_EVENT_MN(_name, _def, _mask) { .attr = CCN_EVENT_ATTR(mn_##_name), \ 286 .type = CCN_TYPE_MN, .event = CCN_EVENT_WATCHPOINT, \ 287 .num_ports = CCN_NUM_XP_PORTS, .num_vcs = CCN_NUM_VCS, \ 288 .def = _def, .mask = _mask, } 289 290#define CCN_EVENT_HNI(_name, _def, _mask) { \ 291 .attr = CCN_EVENT_ATTR(hni_##_name), .type = CCN_TYPE_HNI, \ 292 .event = CCN_EVENT_WATCHPOINT, .num_ports = CCN_NUM_XP_PORTS, \ 293 .num_vcs = CCN_NUM_VCS, .def = _def, .mask = _mask, } 294 295#define CCN_EVENT_SBSX(_name, _def, _mask) { \ 296 .attr = CCN_EVENT_ATTR(sbsx_##_name), .type = CCN_TYPE_SBSX, \ 297 .event = CCN_EVENT_WATCHPOINT, .num_ports = CCN_NUM_XP_PORTS, \ 298 .num_vcs = CCN_NUM_VCS, .def = _def, .mask = _mask, } 299 300#define CCN_EVENT_HNF(_name, _event) { .attr = CCN_EVENT_ATTR(hnf_##_name), \ 301 .type = CCN_TYPE_HNF, .event = _event, } 302 303#define CCN_EVENT_XP(_name, _event) { .attr = CCN_EVENT_ATTR(xp_##_name), \ 304 .type = CCN_TYPE_XP, .event = _event, \ 305 .num_ports = CCN_NUM_XP_PORTS, .num_vcs = CCN_NUM_VCS, } 306 307/* 308 * RN-I & RN-D (RN-D = RN-I + DVM) nodes have different type ID depending 309 * on configuration. One of them is picked to represent the whole group, 310 * as they all share the same event types. 311 */ 312#define CCN_EVENT_RNI(_name, _event) { .attr = CCN_EVENT_ATTR(rni_##_name), \ 313 .type = CCN_TYPE_RNI_3P, .event = _event, } 314 315#define CCN_EVENT_SBAS(_name, _event) { .attr = CCN_EVENT_ATTR(sbas_##_name), \ 316 .type = CCN_TYPE_SBAS, .event = _event, } 317 318#define CCN_EVENT_CYCLES(_name) { .attr = CCN_EVENT_ATTR(_name), \ 319 .type = CCN_TYPE_CYCLES } 320 321 322static ssize_t arm_ccn_pmu_event_show(struct device *dev, 323 struct device_attribute *attr, char *buf) 324{ 325 struct arm_ccn_pmu_event *event = container_of(attr, 326 struct arm_ccn_pmu_event, attr); 327 ssize_t res; 328 329 res = snprintf(buf, PAGE_SIZE, "type=0x%x", event->type); 330 if (event->event) 331 res += snprintf(buf + res, PAGE_SIZE - res, ",event=0x%x", 332 event->event); 333 if (event->def) 334 res += snprintf(buf + res, PAGE_SIZE - res, ",%s", 335 event->def); 336 if (event->mask) 337 res += snprintf(buf + res, PAGE_SIZE - res, ",mask=0x%x", 338 event->mask); 339 res += snprintf(buf + res, PAGE_SIZE - res, "\n"); 340 341 return res; 342} 343 344static umode_t arm_ccn_pmu_events_is_visible(struct kobject *kobj, 345 struct attribute *attr, int index) 346{ 347 struct device *dev = kobj_to_dev(kobj); 348 struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev)); 349 struct device_attribute *dev_attr = container_of(attr, 350 struct device_attribute, attr); 351 struct arm_ccn_pmu_event *event = container_of(dev_attr, 352 struct arm_ccn_pmu_event, attr); 353 354 if (event->type == CCN_TYPE_SBAS && !ccn->sbas_present) 355 return 0; 356 if (event->type == CCN_TYPE_SBSX && !ccn->sbsx_present) 357 return 0; 358 359 return attr->mode; 360} 361 362static struct arm_ccn_pmu_event arm_ccn_pmu_events[] = { 363 CCN_EVENT_MN(eobarrier, "dir=0,vc=0,cmp_h=0x1c00", CCN_IDX_MASK_OPCODE), 364 CCN_EVENT_MN(ecbarrier, "dir=0,vc=0,cmp_h=0x1e00", CCN_IDX_MASK_OPCODE), 365 CCN_EVENT_MN(dvmop, "dir=0,vc=0,cmp_h=0x2800", CCN_IDX_MASK_OPCODE), 366 CCN_EVENT_HNI(txdatflits, "dir=1,vc=3", CCN_IDX_MASK_ANY), 367 CCN_EVENT_HNI(rxdatflits, "dir=0,vc=3", CCN_IDX_MASK_ANY), 368 CCN_EVENT_HNI(txreqflits, "dir=1,vc=0", CCN_IDX_MASK_ANY), 369 CCN_EVENT_HNI(rxreqflits, "dir=0,vc=0", CCN_IDX_MASK_ANY), 370 CCN_EVENT_HNI(rxreqflits_order, "dir=0,vc=0,cmp_h=0x8000", 371 CCN_IDX_MASK_ORDER), 372 CCN_EVENT_SBSX(txdatflits, "dir=1,vc=3", CCN_IDX_MASK_ANY), 373 CCN_EVENT_SBSX(rxdatflits, "dir=0,vc=3", CCN_IDX_MASK_ANY), 374 CCN_EVENT_SBSX(txreqflits, "dir=1,vc=0", CCN_IDX_MASK_ANY), 375 CCN_EVENT_SBSX(rxreqflits, "dir=0,vc=0", CCN_IDX_MASK_ANY), 376 CCN_EVENT_SBSX(rxreqflits_order, "dir=0,vc=0,cmp_h=0x8000", 377 CCN_IDX_MASK_ORDER), 378 CCN_EVENT_HNF(cache_miss, 0x1), 379 CCN_EVENT_HNF(l3_sf_cache_access, 0x02), 380 CCN_EVENT_HNF(cache_fill, 0x3), 381 CCN_EVENT_HNF(pocq_retry, 0x4), 382 CCN_EVENT_HNF(pocq_reqs_recvd, 0x5), 383 CCN_EVENT_HNF(sf_hit, 0x6), 384 CCN_EVENT_HNF(sf_evictions, 0x7), 385 CCN_EVENT_HNF(snoops_sent, 0x8), 386 CCN_EVENT_HNF(snoops_broadcast, 0x9), 387 CCN_EVENT_HNF(l3_eviction, 0xa), 388 CCN_EVENT_HNF(l3_fill_invalid_way, 0xb), 389 CCN_EVENT_HNF(mc_retries, 0xc), 390 CCN_EVENT_HNF(mc_reqs, 0xd), 391 CCN_EVENT_HNF(qos_hh_retry, 0xe), 392 CCN_EVENT_RNI(rdata_beats_p0, 0x1), 393 CCN_EVENT_RNI(rdata_beats_p1, 0x2), 394 CCN_EVENT_RNI(rdata_beats_p2, 0x3), 395 CCN_EVENT_RNI(rxdat_flits, 0x4), 396 CCN_EVENT_RNI(txdat_flits, 0x5), 397 CCN_EVENT_RNI(txreq_flits, 0x6), 398 CCN_EVENT_RNI(txreq_flits_retried, 0x7), 399 CCN_EVENT_RNI(rrt_full, 0x8), 400 CCN_EVENT_RNI(wrt_full, 0x9), 401 CCN_EVENT_RNI(txreq_flits_replayed, 0xa), 402 CCN_EVENT_XP(upload_starvation, 0x1), 403 CCN_EVENT_XP(download_starvation, 0x2), 404 CCN_EVENT_XP(respin, 0x3), 405 CCN_EVENT_XP(valid_flit, 0x4), 406 CCN_EVENT_XP(watchpoint, CCN_EVENT_WATCHPOINT), 407 CCN_EVENT_SBAS(rdata_beats_p0, 0x1), 408 CCN_EVENT_SBAS(rxdat_flits, 0x4), 409 CCN_EVENT_SBAS(txdat_flits, 0x5), 410 CCN_EVENT_SBAS(txreq_flits, 0x6), 411 CCN_EVENT_SBAS(txreq_flits_retried, 0x7), 412 CCN_EVENT_SBAS(rrt_full, 0x8), 413 CCN_EVENT_SBAS(wrt_full, 0x9), 414 CCN_EVENT_SBAS(txreq_flits_replayed, 0xa), 415 CCN_EVENT_CYCLES(cycles), 416}; 417 418/* Populated in arm_ccn_init() */ 419static struct attribute 420 *arm_ccn_pmu_events_attrs[ARRAY_SIZE(arm_ccn_pmu_events) + 1]; 421 422static struct attribute_group arm_ccn_pmu_events_attr_group = { 423 .name = "events", 424 .is_visible = arm_ccn_pmu_events_is_visible, 425 .attrs = arm_ccn_pmu_events_attrs, 426}; 427 428 429static u64 *arm_ccn_pmu_get_cmp_mask(struct arm_ccn *ccn, const char *name) 430{ 431 unsigned long i; 432 433 if (WARN_ON(!name || !name[0] || !isxdigit(name[0]) || !name[1])) 434 return NULL; 435 i = isdigit(name[0]) ? name[0] - '0' : 0xa + tolower(name[0]) - 'a'; 436 437 switch (name[1]) { 438 case 'l': 439 return &ccn->dt.cmp_mask[i].l; 440 case 'h': 441 return &ccn->dt.cmp_mask[i].h; 442 default: 443 return NULL; 444 } 445} 446 447static ssize_t arm_ccn_pmu_cmp_mask_show(struct device *dev, 448 struct device_attribute *attr, char *buf) 449{ 450 struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev)); 451 u64 *mask = arm_ccn_pmu_get_cmp_mask(ccn, attr->attr.name); 452 453 return mask ? snprintf(buf, PAGE_SIZE, "0x%016llx\n", *mask) : -EINVAL; 454} 455 456static ssize_t arm_ccn_pmu_cmp_mask_store(struct device *dev, 457 struct device_attribute *attr, const char *buf, size_t count) 458{ 459 struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev)); 460 u64 *mask = arm_ccn_pmu_get_cmp_mask(ccn, attr->attr.name); 461 int err = -EINVAL; 462 463 if (mask) 464 err = kstrtoull(buf, 0, mask); 465 466 return err ? err : count; 467} 468 469#define CCN_CMP_MASK_ATTR(_name) \ 470 struct device_attribute arm_ccn_pmu_cmp_mask_attr_##_name = \ 471 __ATTR(_name, S_IRUGO | S_IWUSR, \ 472 arm_ccn_pmu_cmp_mask_show, arm_ccn_pmu_cmp_mask_store) 473 474#define CCN_CMP_MASK_ATTR_RO(_name) \ 475 struct device_attribute arm_ccn_pmu_cmp_mask_attr_##_name = \ 476 __ATTR(_name, S_IRUGO, arm_ccn_pmu_cmp_mask_show, NULL) 477 478static CCN_CMP_MASK_ATTR(0l); 479static CCN_CMP_MASK_ATTR(0h); 480static CCN_CMP_MASK_ATTR(1l); 481static CCN_CMP_MASK_ATTR(1h); 482static CCN_CMP_MASK_ATTR(2l); 483static CCN_CMP_MASK_ATTR(2h); 484static CCN_CMP_MASK_ATTR(3l); 485static CCN_CMP_MASK_ATTR(3h); 486static CCN_CMP_MASK_ATTR(4l); 487static CCN_CMP_MASK_ATTR(4h); 488static CCN_CMP_MASK_ATTR(5l); 489static CCN_CMP_MASK_ATTR(5h); 490static CCN_CMP_MASK_ATTR(6l); 491static CCN_CMP_MASK_ATTR(6h); 492static CCN_CMP_MASK_ATTR(7l); 493static CCN_CMP_MASK_ATTR(7h); 494static CCN_CMP_MASK_ATTR_RO(8l); 495static CCN_CMP_MASK_ATTR_RO(8h); 496static CCN_CMP_MASK_ATTR_RO(9l); 497static CCN_CMP_MASK_ATTR_RO(9h); 498static CCN_CMP_MASK_ATTR_RO(al); 499static CCN_CMP_MASK_ATTR_RO(ah); 500static CCN_CMP_MASK_ATTR_RO(bl); 501static CCN_CMP_MASK_ATTR_RO(bh); 502 503static struct attribute *arm_ccn_pmu_cmp_mask_attrs[] = { 504 &arm_ccn_pmu_cmp_mask_attr_0l.attr, &arm_ccn_pmu_cmp_mask_attr_0h.attr, 505 &arm_ccn_pmu_cmp_mask_attr_1l.attr, &arm_ccn_pmu_cmp_mask_attr_1h.attr, 506 &arm_ccn_pmu_cmp_mask_attr_2l.attr, &arm_ccn_pmu_cmp_mask_attr_2h.attr, 507 &arm_ccn_pmu_cmp_mask_attr_3l.attr, &arm_ccn_pmu_cmp_mask_attr_3h.attr, 508 &arm_ccn_pmu_cmp_mask_attr_4l.attr, &arm_ccn_pmu_cmp_mask_attr_4h.attr, 509 &arm_ccn_pmu_cmp_mask_attr_5l.attr, &arm_ccn_pmu_cmp_mask_attr_5h.attr, 510 &arm_ccn_pmu_cmp_mask_attr_6l.attr, &arm_ccn_pmu_cmp_mask_attr_6h.attr, 511 &arm_ccn_pmu_cmp_mask_attr_7l.attr, &arm_ccn_pmu_cmp_mask_attr_7h.attr, 512 &arm_ccn_pmu_cmp_mask_attr_8l.attr, &arm_ccn_pmu_cmp_mask_attr_8h.attr, 513 &arm_ccn_pmu_cmp_mask_attr_9l.attr, &arm_ccn_pmu_cmp_mask_attr_9h.attr, 514 &arm_ccn_pmu_cmp_mask_attr_al.attr, &arm_ccn_pmu_cmp_mask_attr_ah.attr, 515 &arm_ccn_pmu_cmp_mask_attr_bl.attr, &arm_ccn_pmu_cmp_mask_attr_bh.attr, 516 NULL 517}; 518 519static struct attribute_group arm_ccn_pmu_cmp_mask_attr_group = { 520 .name = "cmp_mask", 521 .attrs = arm_ccn_pmu_cmp_mask_attrs, 522}; 523 524 525/* 526 * Default poll period is 10ms, which is way over the top anyway, 527 * as in the worst case scenario (an event every cycle), with 1GHz 528 * clocked bus, the smallest, 32 bit counter will overflow in 529 * more than 4s. 530 */ 531static unsigned int arm_ccn_pmu_poll_period_us = 10000; 532module_param_named(pmu_poll_period_us, arm_ccn_pmu_poll_period_us, uint, 533 S_IRUGO | S_IWUSR); 534 535static ktime_t arm_ccn_pmu_timer_period(void) 536{ 537 return ns_to_ktime((u64)arm_ccn_pmu_poll_period_us * 1000); 538} 539 540 541static const struct attribute_group *arm_ccn_pmu_attr_groups[] = { 542 &arm_ccn_pmu_events_attr_group, 543 &arm_ccn_pmu_format_attr_group, 544 &arm_ccn_pmu_cmp_mask_attr_group, 545 NULL 546}; 547 548 549static int arm_ccn_pmu_alloc_bit(unsigned long *bitmap, unsigned long size) 550{ 551 int bit; 552 553 do { 554 bit = find_first_zero_bit(bitmap, size); 555 if (bit >= size) 556 return -EAGAIN; 557 } while (test_and_set_bit(bit, bitmap)); 558 559 return bit; 560} 561 562/* All RN-I and RN-D nodes have identical PMUs */ 563static int arm_ccn_pmu_type_eq(u32 a, u32 b) 564{ 565 if (a == b) 566 return 1; 567 568 switch (a) { 569 case CCN_TYPE_RNI_1P: 570 case CCN_TYPE_RNI_2P: 571 case CCN_TYPE_RNI_3P: 572 case CCN_TYPE_RND_1P: 573 case CCN_TYPE_RND_2P: 574 case CCN_TYPE_RND_3P: 575 switch (b) { 576 case CCN_TYPE_RNI_1P: 577 case CCN_TYPE_RNI_2P: 578 case CCN_TYPE_RNI_3P: 579 case CCN_TYPE_RND_1P: 580 case CCN_TYPE_RND_2P: 581 case CCN_TYPE_RND_3P: 582 return 1; 583 } 584 break; 585 } 586 587 return 0; 588} 589 590static void arm_ccn_pmu_event_destroy(struct perf_event *event) 591{ 592 struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); 593 struct hw_perf_event *hw = &event->hw; 594 595 if (hw->idx == CCN_IDX_PMU_CYCLE_COUNTER) { 596 clear_bit(CCN_IDX_PMU_CYCLE_COUNTER, ccn->dt.pmu_counters_mask); 597 } else { 598 struct arm_ccn_component *source = 599 ccn->dt.pmu_counters[hw->idx].source; 600 601 if (CCN_CONFIG_TYPE(event->attr.config) == CCN_TYPE_XP && 602 CCN_CONFIG_EVENT(event->attr.config) == 603 CCN_EVENT_WATCHPOINT) 604 clear_bit(hw->config_base, source->xp.dt_cmp_mask); 605 else 606 clear_bit(hw->config_base, source->pmu_events_mask); 607 clear_bit(hw->idx, ccn->dt.pmu_counters_mask); 608 } 609 610 ccn->dt.pmu_counters[hw->idx].source = NULL; 611 ccn->dt.pmu_counters[hw->idx].event = NULL; 612} 613 614static int arm_ccn_pmu_event_init(struct perf_event *event) 615{ 616 struct arm_ccn *ccn; 617 struct hw_perf_event *hw = &event->hw; 618 u32 node_xp, type, event_id; 619 int valid, bit; 620 struct arm_ccn_component *source; 621 int i; 622 623 if (event->attr.type != event->pmu->type) 624 return -ENOENT; 625 626 ccn = pmu_to_arm_ccn(event->pmu); 627 event->destroy = arm_ccn_pmu_event_destroy; 628 629 if (hw->sample_period) { 630 dev_warn(ccn->dev, "Sampling not supported!\n"); 631 return -EOPNOTSUPP; 632 } 633 634 if (has_branch_stack(event) || event->attr.exclude_user || 635 event->attr.exclude_kernel || event->attr.exclude_hv || 636 event->attr.exclude_idle) { 637 dev_warn(ccn->dev, "Can't exclude execution levels!\n"); 638 return -EOPNOTSUPP; 639 } 640 641 if (event->cpu < 0) { 642 dev_warn(ccn->dev, "Can't provide per-task data!\n"); 643 return -EOPNOTSUPP; 644 } 645 646 node_xp = CCN_CONFIG_NODE(event->attr.config); 647 type = CCN_CONFIG_TYPE(event->attr.config); 648 event_id = CCN_CONFIG_EVENT(event->attr.config); 649 650 /* Validate node/xp vs topology */ 651 switch (type) { 652 case CCN_TYPE_XP: 653 if (node_xp >= ccn->num_xps) { 654 dev_warn(ccn->dev, "Invalid XP ID %d!\n", node_xp); 655 return -EINVAL; 656 } 657 break; 658 case CCN_TYPE_CYCLES: 659 break; 660 default: 661 if (node_xp >= ccn->num_nodes) { 662 dev_warn(ccn->dev, "Invalid node ID %d!\n", node_xp); 663 return -EINVAL; 664 } 665 if (!arm_ccn_pmu_type_eq(type, ccn->node[node_xp].type)) { 666 dev_warn(ccn->dev, "Invalid type 0x%x for node %d!\n", 667 type, node_xp); 668 return -EINVAL; 669 } 670 break; 671 } 672 673 /* Validate event ID vs available for the type */ 674 for (i = 0, valid = 0; i < ARRAY_SIZE(arm_ccn_pmu_events) && !valid; 675 i++) { 676 struct arm_ccn_pmu_event *e = &arm_ccn_pmu_events[i]; 677 u32 port = CCN_CONFIG_PORT(event->attr.config); 678 u32 vc = CCN_CONFIG_VC(event->attr.config); 679 680 if (!arm_ccn_pmu_type_eq(type, e->type)) 681 continue; 682 if (event_id != e->event) 683 continue; 684 if (e->num_ports && port >= e->num_ports) { 685 dev_warn(ccn->dev, "Invalid port %d for node/XP %d!\n", 686 port, node_xp); 687 return -EINVAL; 688 } 689 if (e->num_vcs && vc >= e->num_vcs) { 690 dev_warn(ccn->dev, "Invalid vc %d for node/XP %d!\n", 691 vc, node_xp); 692 return -EINVAL; 693 } 694 valid = 1; 695 } 696 if (!valid) { 697 dev_warn(ccn->dev, "Invalid event 0x%x for node/XP %d!\n", 698 event_id, node_xp); 699 return -EINVAL; 700 } 701 702 /* Watchpoint-based event for a node is actually set on XP */ 703 if (event_id == CCN_EVENT_WATCHPOINT && type != CCN_TYPE_XP) { 704 u32 port; 705 706 type = CCN_TYPE_XP; 707 port = arm_ccn_node_to_xp_port(node_xp); 708 node_xp = arm_ccn_node_to_xp(node_xp); 709 710 arm_ccn_pmu_config_set(&event->attr.config, 711 node_xp, type, port); 712 } 713 714 /* Allocate the cycle counter */ 715 if (type == CCN_TYPE_CYCLES) { 716 if (test_and_set_bit(CCN_IDX_PMU_CYCLE_COUNTER, 717 ccn->dt.pmu_counters_mask)) 718 return -EAGAIN; 719 720 hw->idx = CCN_IDX_PMU_CYCLE_COUNTER; 721 ccn->dt.pmu_counters[CCN_IDX_PMU_CYCLE_COUNTER].event = event; 722 723 return 0; 724 } 725 726 /* Allocate an event counter */ 727 hw->idx = arm_ccn_pmu_alloc_bit(ccn->dt.pmu_counters_mask, 728 CCN_NUM_PMU_EVENT_COUNTERS); 729 if (hw->idx < 0) { 730 dev_warn(ccn->dev, "No more counters available!\n"); 731 return -EAGAIN; 732 } 733 734 if (type == CCN_TYPE_XP) 735 source = &ccn->xp[node_xp]; 736 else 737 source = &ccn->node[node_xp]; 738 ccn->dt.pmu_counters[hw->idx].source = source; 739 740 /* Allocate an event source or a watchpoint */ 741 if (type == CCN_TYPE_XP && event_id == CCN_EVENT_WATCHPOINT) 742 bit = arm_ccn_pmu_alloc_bit(source->xp.dt_cmp_mask, 743 CCN_NUM_XP_WATCHPOINTS); 744 else 745 bit = arm_ccn_pmu_alloc_bit(source->pmu_events_mask, 746 CCN_NUM_PMU_EVENTS); 747 if (bit < 0) { 748 dev_warn(ccn->dev, "No more event sources/watchpoints on node/XP %d!\n", 749 node_xp); 750 clear_bit(hw->idx, ccn->dt.pmu_counters_mask); 751 return -EAGAIN; 752 } 753 hw->config_base = bit; 754 755 ccn->dt.pmu_counters[hw->idx].event = event; 756 757 return 0; 758} 759 760static u64 arm_ccn_pmu_read_counter(struct arm_ccn *ccn, int idx) 761{ 762 u64 res; 763 764 if (idx == CCN_IDX_PMU_CYCLE_COUNTER) { 765#ifdef readq 766 res = readq(ccn->dt.base + CCN_DT_PMCCNTR); 767#else 768 /* 40 bit counter, can do snapshot and read in two parts */ 769 writel(0x1, ccn->dt.base + CCN_DT_PMSR_REQ); 770 while (!(readl(ccn->dt.base + CCN_DT_PMSR) & 0x1)) 771 ; 772 writel(0x1, ccn->dt.base + CCN_DT_PMSR_CLR); 773 res = readl(ccn->dt.base + CCN_DT_PMCCNTRSR + 4) & 0xff; 774 res <<= 32; 775 res |= readl(ccn->dt.base + CCN_DT_PMCCNTRSR); 776#endif 777 } else { 778 res = readl(ccn->dt.base + CCN_DT_PMEVCNT(idx)); 779 } 780 781 return res; 782} 783 784static void arm_ccn_pmu_event_update(struct perf_event *event) 785{ 786 struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); 787 struct hw_perf_event *hw = &event->hw; 788 u64 prev_count, new_count, mask; 789 790 do { 791 prev_count = local64_read(&hw->prev_count); 792 new_count = arm_ccn_pmu_read_counter(ccn, hw->idx); 793 } while (local64_xchg(&hw->prev_count, new_count) != prev_count); 794 795 mask = (1LLU << (hw->idx == CCN_IDX_PMU_CYCLE_COUNTER ? 40 : 32)) - 1; 796 797 local64_add((new_count - prev_count) & mask, &event->count); 798} 799 800static void arm_ccn_pmu_xp_dt_config(struct perf_event *event, int enable) 801{ 802 struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); 803 struct hw_perf_event *hw = &event->hw; 804 struct arm_ccn_component *xp; 805 u32 val, dt_cfg; 806 807 if (CCN_CONFIG_TYPE(event->attr.config) == CCN_TYPE_XP) 808 xp = &ccn->xp[CCN_CONFIG_XP(event->attr.config)]; 809 else 810 xp = &ccn->xp[arm_ccn_node_to_xp( 811 CCN_CONFIG_NODE(event->attr.config))]; 812 813 if (enable) 814 dt_cfg = hw->event_base; 815 else 816 dt_cfg = CCN_XP_DT_CONFIG__DT_CFG__PASS_THROUGH; 817 818 spin_lock(&ccn->dt.config_lock); 819 820 val = readl(xp->base + CCN_XP_DT_CONFIG); 821 val &= ~(CCN_XP_DT_CONFIG__DT_CFG__MASK << 822 CCN_XP_DT_CONFIG__DT_CFG__SHIFT(hw->idx)); 823 val |= dt_cfg << CCN_XP_DT_CONFIG__DT_CFG__SHIFT(hw->idx); 824 writel(val, xp->base + CCN_XP_DT_CONFIG); 825 826 spin_unlock(&ccn->dt.config_lock); 827} 828 829static void arm_ccn_pmu_event_start(struct perf_event *event, int flags) 830{ 831 struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); 832 struct hw_perf_event *hw = &event->hw; 833 834 local64_set(&event->hw.prev_count, 835 arm_ccn_pmu_read_counter(ccn, hw->idx)); 836 hw->state = 0; 837 838 if (!ccn->irq_used) 839 hrtimer_start(&ccn->dt.hrtimer, arm_ccn_pmu_timer_period(), 840 HRTIMER_MODE_REL); 841 842 /* Set the DT bus input, engaging the counter */ 843 arm_ccn_pmu_xp_dt_config(event, 1); 844} 845 846static void arm_ccn_pmu_event_stop(struct perf_event *event, int flags) 847{ 848 struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); 849 struct hw_perf_event *hw = &event->hw; 850 u64 timeout; 851 852 /* Disable counting, setting the DT bus to pass-through mode */ 853 arm_ccn_pmu_xp_dt_config(event, 0); 854 855 if (!ccn->irq_used) 856 hrtimer_cancel(&ccn->dt.hrtimer); 857 858 /* Let the DT bus drain */ 859 timeout = arm_ccn_pmu_read_counter(ccn, CCN_IDX_PMU_CYCLE_COUNTER) + 860 ccn->num_xps; 861 while (arm_ccn_pmu_read_counter(ccn, CCN_IDX_PMU_CYCLE_COUNTER) < 862 timeout) 863 cpu_relax(); 864 865 if (flags & PERF_EF_UPDATE) 866 arm_ccn_pmu_event_update(event); 867 868 hw->state |= PERF_HES_STOPPED; 869} 870 871static void arm_ccn_pmu_xp_watchpoint_config(struct perf_event *event) 872{ 873 struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); 874 struct hw_perf_event *hw = &event->hw; 875 struct arm_ccn_component *source = 876 ccn->dt.pmu_counters[hw->idx].source; 877 unsigned long wp = hw->config_base; 878 u32 val; 879 u64 cmp_l = event->attr.config1; 880 u64 cmp_h = event->attr.config2; 881 u64 mask_l = ccn->dt.cmp_mask[CCN_CONFIG_MASK(event->attr.config)].l; 882 u64 mask_h = ccn->dt.cmp_mask[CCN_CONFIG_MASK(event->attr.config)].h; 883 884 hw->event_base = CCN_XP_DT_CONFIG__DT_CFG__WATCHPOINT(wp); 885 886 /* Direction (RX/TX), device (port) & virtual channel */ 887 val = readl(source->base + CCN_XP_DT_INTERFACE_SEL); 888 val &= ~(CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__MASK << 889 CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__SHIFT(wp)); 890 val |= CCN_CONFIG_DIR(event->attr.config) << 891 CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__SHIFT(wp); 892 val &= ~(CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__MASK << 893 CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__SHIFT(wp)); 894 val |= CCN_CONFIG_PORT(event->attr.config) << 895 CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__SHIFT(wp); 896 val &= ~(CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__MASK << 897 CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__SHIFT(wp)); 898 val |= CCN_CONFIG_VC(event->attr.config) << 899 CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__SHIFT(wp); 900 writel(val, source->base + CCN_XP_DT_INTERFACE_SEL); 901 902 /* Comparison values */ 903 writel(cmp_l & 0xffffffff, source->base + CCN_XP_DT_CMP_VAL_L(wp)); 904 writel((cmp_l >> 32) & 0xefffffff, 905 source->base + CCN_XP_DT_CMP_VAL_L(wp) + 4); 906 writel(cmp_h & 0xffffffff, source->base + CCN_XP_DT_CMP_VAL_H(wp)); 907 writel((cmp_h >> 32) & 0x0fffffff, 908 source->base + CCN_XP_DT_CMP_VAL_H(wp) + 4); 909 910 /* Mask */ 911 writel(mask_l & 0xffffffff, source->base + CCN_XP_DT_CMP_MASK_L(wp)); 912 writel((mask_l >> 32) & 0xefffffff, 913 source->base + CCN_XP_DT_CMP_MASK_L(wp) + 4); 914 writel(mask_h & 0xffffffff, source->base + CCN_XP_DT_CMP_MASK_H(wp)); 915 writel((mask_h >> 32) & 0x0fffffff, 916 source->base + CCN_XP_DT_CMP_MASK_H(wp) + 4); 917} 918 919static void arm_ccn_pmu_xp_event_config(struct perf_event *event) 920{ 921 struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); 922 struct hw_perf_event *hw = &event->hw; 923 struct arm_ccn_component *source = 924 ccn->dt.pmu_counters[hw->idx].source; 925 u32 val, id; 926 927 hw->event_base = CCN_XP_DT_CONFIG__DT_CFG__XP_PMU_EVENT(hw->config_base); 928 929 id = (CCN_CONFIG_VC(event->attr.config) << 4) | 930 (CCN_CONFIG_PORT(event->attr.config) << 3) | 931 (CCN_CONFIG_EVENT(event->attr.config) << 0); 932 933 val = readl(source->base + CCN_XP_PMU_EVENT_SEL); 934 val &= ~(CCN_XP_PMU_EVENT_SEL__ID__MASK << 935 CCN_XP_PMU_EVENT_SEL__ID__SHIFT(hw->config_base)); 936 val |= id << CCN_XP_PMU_EVENT_SEL__ID__SHIFT(hw->config_base); 937 writel(val, source->base + CCN_XP_PMU_EVENT_SEL); 938} 939 940static void arm_ccn_pmu_node_event_config(struct perf_event *event) 941{ 942 struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); 943 struct hw_perf_event *hw = &event->hw; 944 struct arm_ccn_component *source = 945 ccn->dt.pmu_counters[hw->idx].source; 946 u32 type = CCN_CONFIG_TYPE(event->attr.config); 947 u32 val, port; 948 949 port = arm_ccn_node_to_xp_port(CCN_CONFIG_NODE(event->attr.config)); 950 hw->event_base = CCN_XP_DT_CONFIG__DT_CFG__DEVICE_PMU_EVENT(port, 951 hw->config_base); 952 953 /* These *_event_sel regs should be identical, but let's make sure... */ 954 BUILD_BUG_ON(CCN_HNF_PMU_EVENT_SEL != CCN_SBAS_PMU_EVENT_SEL); 955 BUILD_BUG_ON(CCN_SBAS_PMU_EVENT_SEL != CCN_RNI_PMU_EVENT_SEL); 956 BUILD_BUG_ON(CCN_HNF_PMU_EVENT_SEL__ID__SHIFT(1) != 957 CCN_SBAS_PMU_EVENT_SEL__ID__SHIFT(1)); 958 BUILD_BUG_ON(CCN_SBAS_PMU_EVENT_SEL__ID__SHIFT(1) != 959 CCN_RNI_PMU_EVENT_SEL__ID__SHIFT(1)); 960 BUILD_BUG_ON(CCN_HNF_PMU_EVENT_SEL__ID__MASK != 961 CCN_SBAS_PMU_EVENT_SEL__ID__MASK); 962 BUILD_BUG_ON(CCN_SBAS_PMU_EVENT_SEL__ID__MASK != 963 CCN_RNI_PMU_EVENT_SEL__ID__MASK); 964 if (WARN_ON(type != CCN_TYPE_HNF && type != CCN_TYPE_SBAS && 965 !arm_ccn_pmu_type_eq(type, CCN_TYPE_RNI_3P))) 966 return; 967 968 /* Set the event id for the pre-allocated counter */ 969 val = readl(source->base + CCN_HNF_PMU_EVENT_SEL); 970 val &= ~(CCN_HNF_PMU_EVENT_SEL__ID__MASK << 971 CCN_HNF_PMU_EVENT_SEL__ID__SHIFT(hw->config_base)); 972 val |= CCN_CONFIG_EVENT(event->attr.config) << 973 CCN_HNF_PMU_EVENT_SEL__ID__SHIFT(hw->config_base); 974 writel(val, source->base + CCN_HNF_PMU_EVENT_SEL); 975} 976 977static void arm_ccn_pmu_event_config(struct perf_event *event) 978{ 979 struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); 980 struct hw_perf_event *hw = &event->hw; 981 u32 xp, offset, val; 982 983 /* Cycle counter requires no setup */ 984 if (hw->idx == CCN_IDX_PMU_CYCLE_COUNTER) 985 return; 986 987 if (CCN_CONFIG_TYPE(event->attr.config) == CCN_TYPE_XP) 988 xp = CCN_CONFIG_XP(event->attr.config); 989 else 990 xp = arm_ccn_node_to_xp(CCN_CONFIG_NODE(event->attr.config)); 991 992 spin_lock(&ccn->dt.config_lock); 993 994 /* Set the DT bus "distance" register */ 995 offset = (hw->idx / 4) * 4; 996 val = readl(ccn->dt.base + CCN_DT_ACTIVE_DSM + offset); 997 val &= ~(CCN_DT_ACTIVE_DSM__DSM_ID__MASK << 998 CCN_DT_ACTIVE_DSM__DSM_ID__SHIFT(hw->idx % 4)); 999 val |= xp << CCN_DT_ACTIVE_DSM__DSM_ID__SHIFT(hw->idx % 4); 1000 writel(val, ccn->dt.base + CCN_DT_ACTIVE_DSM + offset); 1001 1002 if (CCN_CONFIG_TYPE(event->attr.config) == CCN_TYPE_XP) { 1003 if (CCN_CONFIG_EVENT(event->attr.config) == 1004 CCN_EVENT_WATCHPOINT) 1005 arm_ccn_pmu_xp_watchpoint_config(event); 1006 else 1007 arm_ccn_pmu_xp_event_config(event); 1008 } else { 1009 arm_ccn_pmu_node_event_config(event); 1010 } 1011 1012 spin_unlock(&ccn->dt.config_lock); 1013} 1014 1015static int arm_ccn_pmu_event_add(struct perf_event *event, int flags) 1016{ 1017 struct hw_perf_event *hw = &event->hw; 1018 1019 arm_ccn_pmu_event_config(event); 1020 1021 hw->state = PERF_HES_STOPPED; 1022 1023 if (flags & PERF_EF_START) 1024 arm_ccn_pmu_event_start(event, PERF_EF_UPDATE); 1025 1026 return 0; 1027} 1028 1029static void arm_ccn_pmu_event_del(struct perf_event *event, int flags) 1030{ 1031 arm_ccn_pmu_event_stop(event, PERF_EF_UPDATE); 1032} 1033 1034static void arm_ccn_pmu_event_read(struct perf_event *event) 1035{ 1036 arm_ccn_pmu_event_update(event); 1037} 1038 1039static irqreturn_t arm_ccn_pmu_overflow_handler(struct arm_ccn_dt *dt) 1040{ 1041 u32 pmovsr = readl(dt->base + CCN_DT_PMOVSR); 1042 int idx; 1043 1044 if (!pmovsr) 1045 return IRQ_NONE; 1046 1047 writel(pmovsr, dt->base + CCN_DT_PMOVSR_CLR); 1048 1049 BUILD_BUG_ON(CCN_IDX_PMU_CYCLE_COUNTER != CCN_NUM_PMU_EVENT_COUNTERS); 1050 1051 for (idx = 0; idx < CCN_NUM_PMU_EVENT_COUNTERS + 1; idx++) { 1052 struct perf_event *event = dt->pmu_counters[idx].event; 1053 int overflowed = pmovsr & BIT(idx); 1054 1055 WARN_ON_ONCE(overflowed && !event && 1056 idx != CCN_IDX_PMU_CYCLE_COUNTER); 1057 1058 if (!event || !overflowed) 1059 continue; 1060 1061 arm_ccn_pmu_event_update(event); 1062 } 1063 1064 return IRQ_HANDLED; 1065} 1066 1067static enum hrtimer_restart arm_ccn_pmu_timer_handler(struct hrtimer *hrtimer) 1068{ 1069 struct arm_ccn_dt *dt = container_of(hrtimer, struct arm_ccn_dt, 1070 hrtimer); 1071 unsigned long flags; 1072 1073 local_irq_save(flags); 1074 arm_ccn_pmu_overflow_handler(dt); 1075 local_irq_restore(flags); 1076 1077 hrtimer_forward_now(hrtimer, arm_ccn_pmu_timer_period()); 1078 return HRTIMER_RESTART; 1079} 1080 1081 1082static DEFINE_IDA(arm_ccn_pmu_ida); 1083 1084static int arm_ccn_pmu_init(struct arm_ccn *ccn) 1085{ 1086 int i; 1087 char *name; 1088 1089 /* Initialize DT subsystem */ 1090 ccn->dt.base = ccn->base + CCN_REGION_SIZE; 1091 spin_lock_init(&ccn->dt.config_lock); 1092 writel(CCN_DT_PMOVSR_CLR__MASK, ccn->dt.base + CCN_DT_PMOVSR_CLR); 1093 writel(CCN_DT_CTL__DT_EN, ccn->dt.base + CCN_DT_CTL); 1094 writel(CCN_DT_PMCR__OVFL_INTR_EN | CCN_DT_PMCR__PMU_EN, 1095 ccn->dt.base + CCN_DT_PMCR); 1096 writel(0x1, ccn->dt.base + CCN_DT_PMSR_CLR); 1097 for (i = 0; i < ccn->num_xps; i++) { 1098 writel(0, ccn->xp[i].base + CCN_XP_DT_CONFIG); 1099 writel((CCN_XP_DT_CONTROL__WP_ARM_SEL__ALWAYS << 1100 CCN_XP_DT_CONTROL__WP_ARM_SEL__SHIFT(0)) | 1101 (CCN_XP_DT_CONTROL__WP_ARM_SEL__ALWAYS << 1102 CCN_XP_DT_CONTROL__WP_ARM_SEL__SHIFT(1)) | 1103 CCN_XP_DT_CONTROL__DT_ENABLE, 1104 ccn->xp[i].base + CCN_XP_DT_CONTROL); 1105 } 1106 ccn->dt.cmp_mask[CCN_IDX_MASK_ANY].l = ~0; 1107 ccn->dt.cmp_mask[CCN_IDX_MASK_ANY].h = ~0; 1108 ccn->dt.cmp_mask[CCN_IDX_MASK_EXACT].l = 0; 1109 ccn->dt.cmp_mask[CCN_IDX_MASK_EXACT].h = 0; 1110 ccn->dt.cmp_mask[CCN_IDX_MASK_ORDER].l = ~0; 1111 ccn->dt.cmp_mask[CCN_IDX_MASK_ORDER].h = ~(0x1 << 15); 1112 ccn->dt.cmp_mask[CCN_IDX_MASK_OPCODE].l = ~0; 1113 ccn->dt.cmp_mask[CCN_IDX_MASK_OPCODE].h = ~(0x1f << 9); 1114 1115 /* Get a convenient /sys/event_source/devices/ name */ 1116 ccn->dt.id = ida_simple_get(&arm_ccn_pmu_ida, 0, 0, GFP_KERNEL); 1117 if (ccn->dt.id == 0) { 1118 name = "ccn"; 1119 } else { 1120 int len = snprintf(NULL, 0, "ccn_%d", ccn->dt.id); 1121 1122 name = devm_kzalloc(ccn->dev, len + 1, GFP_KERNEL); 1123 snprintf(name, len + 1, "ccn_%d", ccn->dt.id); 1124 } 1125 1126 /* Perf driver registration */ 1127 ccn->dt.pmu = (struct pmu) { 1128 .attr_groups = arm_ccn_pmu_attr_groups, 1129 .task_ctx_nr = perf_invalid_context, 1130 .event_init = arm_ccn_pmu_event_init, 1131 .add = arm_ccn_pmu_event_add, 1132 .del = arm_ccn_pmu_event_del, 1133 .start = arm_ccn_pmu_event_start, 1134 .stop = arm_ccn_pmu_event_stop, 1135 .read = arm_ccn_pmu_event_read, 1136 }; 1137 1138 /* No overflow interrupt? Have to use a timer instead. */ 1139 if (!ccn->irq_used) { 1140 dev_info(ccn->dev, "No access to interrupts, using timer.\n"); 1141 hrtimer_init(&ccn->dt.hrtimer, CLOCK_MONOTONIC, 1142 HRTIMER_MODE_REL); 1143 ccn->dt.hrtimer.function = arm_ccn_pmu_timer_handler; 1144 } 1145 1146 return perf_pmu_register(&ccn->dt.pmu, name, -1); 1147} 1148 1149static void arm_ccn_pmu_cleanup(struct arm_ccn *ccn) 1150{ 1151 int i; 1152 1153 for (i = 0; i < ccn->num_xps; i++) 1154 writel(0, ccn->xp[i].base + CCN_XP_DT_CONTROL); 1155 writel(0, ccn->dt.base + CCN_DT_PMCR); 1156 perf_pmu_unregister(&ccn->dt.pmu); 1157 ida_simple_remove(&arm_ccn_pmu_ida, ccn->dt.id); 1158} 1159 1160 1161static int arm_ccn_for_each_valid_region(struct arm_ccn *ccn, 1162 int (*callback)(struct arm_ccn *ccn, int region, 1163 void __iomem *base, u32 type, u32 id)) 1164{ 1165 int region; 1166 1167 for (region = 0; region < CCN_NUM_REGIONS; region++) { 1168 u32 val, type, id; 1169 void __iomem *base; 1170 int err; 1171 1172 val = readl(ccn->base + CCN_MN_OLY_COMP_LIST_63_0 + 1173 4 * (region / 32)); 1174 if (!(val & (1 << (region % 32)))) 1175 continue; 1176 1177 base = ccn->base + region * CCN_REGION_SIZE; 1178 val = readl(base + CCN_ALL_OLY_ID); 1179 type = (val >> CCN_ALL_OLY_ID__OLY_ID__SHIFT) & 1180 CCN_ALL_OLY_ID__OLY_ID__MASK; 1181 id = (val >> CCN_ALL_OLY_ID__NODE_ID__SHIFT) & 1182 CCN_ALL_OLY_ID__NODE_ID__MASK; 1183 1184 err = callback(ccn, region, base, type, id); 1185 if (err) 1186 return err; 1187 } 1188 1189 return 0; 1190} 1191 1192static int arm_ccn_get_nodes_num(struct arm_ccn *ccn, int region, 1193 void __iomem *base, u32 type, u32 id) 1194{ 1195 1196 if (type == CCN_TYPE_XP && id >= ccn->num_xps) 1197 ccn->num_xps = id + 1; 1198 else if (id >= ccn->num_nodes) 1199 ccn->num_nodes = id + 1; 1200 1201 return 0; 1202} 1203 1204static int arm_ccn_init_nodes(struct arm_ccn *ccn, int region, 1205 void __iomem *base, u32 type, u32 id) 1206{ 1207 struct arm_ccn_component *component; 1208 1209 dev_dbg(ccn->dev, "Region %d: id=%u, type=0x%02x\n", region, id, type); 1210 1211 switch (type) { 1212 case CCN_TYPE_MN: 1213 case CCN_TYPE_DT: 1214 return 0; 1215 case CCN_TYPE_XP: 1216 component = &ccn->xp[id]; 1217 break; 1218 case CCN_TYPE_SBSX: 1219 ccn->sbsx_present = 1; 1220 component = &ccn->node[id]; 1221 break; 1222 case CCN_TYPE_SBAS: 1223 ccn->sbas_present = 1; 1224 /* Fall-through */ 1225 default: 1226 component = &ccn->node[id]; 1227 break; 1228 } 1229 1230 component->base = base; 1231 component->type = type; 1232 1233 return 0; 1234} 1235 1236 1237static irqreturn_t arm_ccn_error_handler(struct arm_ccn *ccn, 1238 const u32 *err_sig_val) 1239{ 1240 /* This should be really handled by firmware... */ 1241 dev_err(ccn->dev, "Error reported in %08x%08x%08x%08x%08x%08x.\n", 1242 err_sig_val[5], err_sig_val[4], err_sig_val[3], 1243 err_sig_val[2], err_sig_val[1], err_sig_val[0]); 1244 dev_err(ccn->dev, "Disabling interrupt generation for all errors.\n"); 1245 writel(CCN_MN_ERRINT_STATUS__ALL_ERRORS__DISABLE, 1246 ccn->base + CCN_MN_ERRINT_STATUS); 1247 1248 return IRQ_HANDLED; 1249} 1250 1251 1252static irqreturn_t arm_ccn_irq_handler(int irq, void *dev_id) 1253{ 1254 irqreturn_t res = IRQ_NONE; 1255 struct arm_ccn *ccn = dev_id; 1256 u32 err_sig_val[6]; 1257 u32 err_or; 1258 int i; 1259 1260 /* PMU overflow is a special case */ 1261 err_or = err_sig_val[0] = readl(ccn->base + CCN_MN_ERR_SIG_VAL_63_0); 1262 if (err_or & CCN_MN_ERR_SIG_VAL_63_0__DT) { 1263 err_or &= ~CCN_MN_ERR_SIG_VAL_63_0__DT; 1264 res = arm_ccn_pmu_overflow_handler(&ccn->dt); 1265 } 1266 1267 /* Have to read all err_sig_vals to clear them */ 1268 for (i = 1; i < ARRAY_SIZE(err_sig_val); i++) { 1269 err_sig_val[i] = readl(ccn->base + 1270 CCN_MN_ERR_SIG_VAL_63_0 + i * 4); 1271 err_or |= err_sig_val[i]; 1272 } 1273 if (err_or) 1274 res |= arm_ccn_error_handler(ccn, err_sig_val); 1275 1276 if (res != IRQ_NONE) 1277 writel(CCN_MN_ERRINT_STATUS__INTREQ__DESSERT, 1278 ccn->base + CCN_MN_ERRINT_STATUS); 1279 1280 return res; 1281} 1282 1283 1284static int arm_ccn_probe(struct platform_device *pdev) 1285{ 1286 struct arm_ccn *ccn; 1287 struct resource *res; 1288 int err; 1289 1290 ccn = devm_kzalloc(&pdev->dev, sizeof(*ccn), GFP_KERNEL); 1291 if (!ccn) 1292 return -ENOMEM; 1293 ccn->dev = &pdev->dev; 1294 platform_set_drvdata(pdev, ccn); 1295 1296 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1297 if (!res) 1298 return -EINVAL; 1299 1300 if (!devm_request_mem_region(ccn->dev, res->start, 1301 resource_size(res), pdev->name)) 1302 return -EBUSY; 1303 1304 ccn->base = devm_ioremap(ccn->dev, res->start, 1305 resource_size(res)); 1306 if (!ccn->base) 1307 return -EFAULT; 1308 1309 res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); 1310 if (!res) 1311 return -EINVAL; 1312 1313 /* Check if we can use the interrupt */ 1314 writel(CCN_MN_ERRINT_STATUS__PMU_EVENTS__DISABLE, 1315 ccn->base + CCN_MN_ERRINT_STATUS); 1316 if (readl(ccn->base + CCN_MN_ERRINT_STATUS) & 1317 CCN_MN_ERRINT_STATUS__PMU_EVENTS__DISABLED) { 1318 /* Can set 'disable' bits, so can acknowledge interrupts */ 1319 writel(CCN_MN_ERRINT_STATUS__PMU_EVENTS__ENABLE, 1320 ccn->base + CCN_MN_ERRINT_STATUS); 1321 err = devm_request_irq(ccn->dev, res->start, 1322 arm_ccn_irq_handler, 0, dev_name(ccn->dev), 1323 ccn); 1324 if (err) 1325 return err; 1326 1327 ccn->irq_used = 1; 1328 } 1329 1330 1331 /* Build topology */ 1332 1333 err = arm_ccn_for_each_valid_region(ccn, arm_ccn_get_nodes_num); 1334 if (err) 1335 return err; 1336 1337 ccn->node = devm_kzalloc(ccn->dev, sizeof(*ccn->node) * ccn->num_nodes, 1338 GFP_KERNEL); 1339 ccn->xp = devm_kzalloc(ccn->dev, sizeof(*ccn->node) * ccn->num_xps, 1340 GFP_KERNEL); 1341 if (!ccn->node || !ccn->xp) 1342 return -ENOMEM; 1343 1344 err = arm_ccn_for_each_valid_region(ccn, arm_ccn_init_nodes); 1345 if (err) 1346 return err; 1347 1348 return arm_ccn_pmu_init(ccn); 1349} 1350 1351static int arm_ccn_remove(struct platform_device *pdev) 1352{ 1353 struct arm_ccn *ccn = platform_get_drvdata(pdev); 1354 1355 arm_ccn_pmu_cleanup(ccn); 1356 1357 return 0; 1358} 1359 1360static const struct of_device_id arm_ccn_match[] = { 1361 { .compatible = "arm,ccn-504", }, 1362 {}, 1363}; 1364 1365static struct platform_driver arm_ccn_driver = { 1366 .driver = { 1367 .name = "arm-ccn", 1368 .of_match_table = arm_ccn_match, 1369 }, 1370 .probe = arm_ccn_probe, 1371 .remove = arm_ccn_remove, 1372}; 1373 1374static int __init arm_ccn_init(void) 1375{ 1376 int i; 1377 1378 for (i = 0; i < ARRAY_SIZE(arm_ccn_pmu_events); i++) 1379 arm_ccn_pmu_events_attrs[i] = &arm_ccn_pmu_events[i].attr.attr; 1380 1381 return platform_driver_register(&arm_ccn_driver); 1382} 1383 1384static void __exit arm_ccn_exit(void) 1385{ 1386 platform_driver_unregister(&arm_ccn_driver); 1387} 1388 1389module_init(arm_ccn_init); 1390module_exit(arm_ccn_exit); 1391 1392MODULE_AUTHOR("Pawel Moll <pawel.moll@arm.com>"); 1393MODULE_LICENSE("GPL"); 1394