~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm/mm/cache-uniphier.c

Version: ~ [ linux-5.11-rc3 ] ~ [ linux-5.10.7 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.89 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.167 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.215 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.251 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.251 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Copyright (C) 2015-2016 Socionext Inc.
  3  *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
  4  *
  5  * This program is free software; you can redistribute it and/or modify
  6  * it under the terms of the GNU General Public License as published by
  7  * the Free Software Foundation; either version 2 of the License, or
  8  * (at your option) any later version.
  9  *
 10  * This program is distributed in the hope that it will be useful,
 11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13  * GNU General Public License for more details.
 14  */
 15 
 16 #define pr_fmt(fmt)             "uniphier: " fmt
 17 
 18 #include <linux/bitops.h>
 19 #include <linux/init.h>
 20 #include <linux/io.h>
 21 #include <linux/log2.h>
 22 #include <linux/of_address.h>
 23 #include <linux/slab.h>
 24 #include <asm/hardware/cache-uniphier.h>
 25 #include <asm/outercache.h>
 26 
 27 /* control registers */
 28 #define UNIPHIER_SSCC           0x0     /* Control Register */
 29 #define    UNIPHIER_SSCC_BST                    BIT(20) /* UCWG burst read */
 30 #define    UNIPHIER_SSCC_ACT                    BIT(19) /* Inst-Data separate */
 31 #define    UNIPHIER_SSCC_WTG                    BIT(18) /* WT gathering on */
 32 #define    UNIPHIER_SSCC_PRD                    BIT(17) /* enable pre-fetch */
 33 #define    UNIPHIER_SSCC_ON                     BIT(0)  /* enable cache */
 34 #define UNIPHIER_SSCLPDAWCR     0x30    /* Unified/Data Active Way Control */
 35 #define UNIPHIER_SSCLPIAWCR     0x34    /* Instruction Active Way Control */
 36 
 37 /* revision registers */
 38 #define UNIPHIER_SSCID          0x0     /* ID Register */
 39 
 40 /* operation registers */
 41 #define UNIPHIER_SSCOPE         0x244   /* Cache Operation Primitive Entry */
 42 #define    UNIPHIER_SSCOPE_CM_INV               0x0     /* invalidate */
 43 #define    UNIPHIER_SSCOPE_CM_CLEAN             0x1     /* clean */
 44 #define    UNIPHIER_SSCOPE_CM_FLUSH             0x2     /* flush */
 45 #define    UNIPHIER_SSCOPE_CM_SYNC              0x8     /* sync (drain bufs) */
 46 #define    UNIPHIER_SSCOPE_CM_FLUSH_PREFETCH    0x9     /* flush p-fetch buf */
 47 #define UNIPHIER_SSCOQM         0x248   /* Cache Operation Queue Mode */
 48 #define    UNIPHIER_SSCOQM_S_MASK               (0x3 << 17)
 49 #define    UNIPHIER_SSCOQM_S_RANGE              (0x0 << 17)
 50 #define    UNIPHIER_SSCOQM_S_ALL                (0x1 << 17)
 51 #define    UNIPHIER_SSCOQM_CE                   BIT(15) /* notify completion */
 52 #define    UNIPHIER_SSCOQM_CM_INV               0x0     /* invalidate */
 53 #define    UNIPHIER_SSCOQM_CM_CLEAN             0x1     /* clean */
 54 #define    UNIPHIER_SSCOQM_CM_FLUSH             0x2     /* flush */
 55 #define UNIPHIER_SSCOQAD        0x24c   /* Cache Operation Queue Address */
 56 #define UNIPHIER_SSCOQSZ        0x250   /* Cache Operation Queue Size */
 57 #define UNIPHIER_SSCOPPQSEF     0x25c   /* Cache Operation Queue Set Complete*/
 58 #define    UNIPHIER_SSCOPPQSEF_FE               BIT(1)
 59 #define    UNIPHIER_SSCOPPQSEF_OE               BIT(0)
 60 #define UNIPHIER_SSCOLPQS       0x260   /* Cache Operation Queue Status */
 61 #define    UNIPHIER_SSCOLPQS_EF                 BIT(2)
 62 #define    UNIPHIER_SSCOLPQS_EST                BIT(1)
 63 #define    UNIPHIER_SSCOLPQS_QST                BIT(0)
 64 
 65 /* Is the operation region specified by address range? */
 66 #define UNIPHIER_SSCOQM_S_IS_RANGE(op) \
 67                 ((op & UNIPHIER_SSCOQM_S_MASK) == UNIPHIER_SSCOQM_S_RANGE)
 68 
 69 /**
 70  * uniphier_cache_data - UniPhier outer cache specific data
 71  *
 72  * @ctrl_base: virtual base address of control registers
 73  * @rev_base: virtual base address of revision registers
 74  * @op_base: virtual base address of operation registers
 75  * @way_mask: each bit specifies if the way is present
 76  * @nsets: number of associativity sets
 77  * @line_size: line size in bytes
 78  * @range_op_max_size: max size that can be handled by a single range operation
 79  * @list: list node to include this level in the whole cache hierarchy
 80  */
 81 struct uniphier_cache_data {
 82         void __iomem *ctrl_base;
 83         void __iomem *rev_base;
 84         void __iomem *op_base;
 85         void __iomem *way_ctrl_base;
 86         u32 way_mask;
 87         u32 nsets;
 88         u32 line_size;
 89         u32 range_op_max_size;
 90         struct list_head list;
 91 };
 92 
 93 /*
 94  * List of the whole outer cache hierarchy.  This list is only modified during
 95  * the early boot stage, so no mutex is taken for the access to the list.
 96  */
 97 static LIST_HEAD(uniphier_cache_list);
 98 
 99 /**
100  * __uniphier_cache_sync - perform a sync point for a particular cache level
101  *
102  * @data: cache controller specific data
103  */
104 static void __uniphier_cache_sync(struct uniphier_cache_data *data)
105 {
106         /* This sequence need not be atomic.  Do not disable IRQ. */
107         writel_relaxed(UNIPHIER_SSCOPE_CM_SYNC,
108                        data->op_base + UNIPHIER_SSCOPE);
109         /* need a read back to confirm */
110         readl_relaxed(data->op_base + UNIPHIER_SSCOPE);
111 }
112 
113 /**
114  * __uniphier_cache_maint_common - run a queue operation for a particular level
115  *
116  * @data: cache controller specific data
117  * @start: start address of range operation (don't care for "all" operation)
118  * @size: data size of range operation (don't care for "all" operation)
119  * @operation: flags to specify the desired cache operation
120  */
121 static void __uniphier_cache_maint_common(struct uniphier_cache_data *data,
122                                           unsigned long start,
123                                           unsigned long size,
124                                           u32 operation)
125 {
126         unsigned long flags;
127 
128         /*
129          * No spin lock is necessary here because:
130          *
131          * [1] This outer cache controller is able to accept maintenance
132          * operations from multiple CPUs at a time in an SMP system; if a
133          * maintenance operation is under way and another operation is issued,
134          * the new one is stored in the queue.  The controller performs one
135          * operation after another.  If the queue is full, the status register,
136          * UNIPHIER_SSCOPPQSEF, indicates that the queue registration has
137          * failed.  The status registers, UNIPHIER_{SSCOPPQSEF, SSCOLPQS}, have
138          * different instances for each CPU, i.e. each CPU can track the status
139          * of the maintenance operations triggered by itself.
140          *
141          * [2] The cache command registers, UNIPHIER_{SSCOQM, SSCOQAD, SSCOQSZ,
142          * SSCOQWN}, are shared between multiple CPUs, but the hardware still
143          * guarantees the registration sequence is atomic; the write access to
144          * them are arbitrated by the hardware.  The first accessor to the
145          * register, UNIPHIER_SSCOQM, holds the access right and it is released
146          * by reading the status register, UNIPHIER_SSCOPPQSEF.  While one CPU
147          * is holding the access right, other CPUs fail to register operations.
148          * One CPU should not hold the access right for a long time, so local
149          * IRQs should be disabled while the following sequence.
150          */
151         local_irq_save(flags);
152 
153         /* clear the complete notification flag */
154         writel_relaxed(UNIPHIER_SSCOLPQS_EF, data->op_base + UNIPHIER_SSCOLPQS);
155 
156         do {
157                 /* set cache operation */
158                 writel_relaxed(UNIPHIER_SSCOQM_CE | operation,
159                                data->op_base + UNIPHIER_SSCOQM);
160 
161                 /* set address range if needed */
162                 if (likely(UNIPHIER_SSCOQM_S_IS_RANGE(operation))) {
163                         writel_relaxed(start, data->op_base + UNIPHIER_SSCOQAD);
164                         writel_relaxed(size, data->op_base + UNIPHIER_SSCOQSZ);
165                 }
166         } while (unlikely(readl_relaxed(data->op_base + UNIPHIER_SSCOPPQSEF) &
167                           (UNIPHIER_SSCOPPQSEF_FE | UNIPHIER_SSCOPPQSEF_OE)));
168 
169         /* wait until the operation is completed */
170         while (likely(readl_relaxed(data->op_base + UNIPHIER_SSCOLPQS) !=
171                       UNIPHIER_SSCOLPQS_EF))
172                 cpu_relax();
173 
174         local_irq_restore(flags);
175 }
176 
177 static void __uniphier_cache_maint_all(struct uniphier_cache_data *data,
178                                        u32 operation)
179 {
180         __uniphier_cache_maint_common(data, 0, 0,
181                                       UNIPHIER_SSCOQM_S_ALL | operation);
182 
183         __uniphier_cache_sync(data);
184 }
185 
186 static void __uniphier_cache_maint_range(struct uniphier_cache_data *data,
187                                          unsigned long start, unsigned long end,
188                                          u32 operation)
189 {
190         unsigned long size;
191 
192         /*
193          * If the start address is not aligned,
194          * perform a cache operation for the first cache-line
195          */
196         start = start & ~(data->line_size - 1);
197 
198         size = end - start;
199 
200         if (unlikely(size >= (unsigned long)(-data->line_size))) {
201                 /* this means cache operation for all range */
202                 __uniphier_cache_maint_all(data, operation);
203                 return;
204         }
205 
206         /*
207          * If the end address is not aligned,
208          * perform a cache operation for the last cache-line
209          */
210         size = ALIGN(size, data->line_size);
211 
212         while (size) {
213                 unsigned long chunk_size = min_t(unsigned long, size,
214                                                  data->range_op_max_size);
215 
216                 __uniphier_cache_maint_common(data, start, chunk_size,
217                                         UNIPHIER_SSCOQM_S_RANGE | operation);
218 
219                 start += chunk_size;
220                 size -= chunk_size;
221         }
222 
223         __uniphier_cache_sync(data);
224 }
225 
226 static void __uniphier_cache_enable(struct uniphier_cache_data *data, bool on)
227 {
228         u32 val = 0;
229 
230         if (on)
231                 val = UNIPHIER_SSCC_WTG | UNIPHIER_SSCC_PRD | UNIPHIER_SSCC_ON;
232 
233         writel_relaxed(val, data->ctrl_base + UNIPHIER_SSCC);
234 }
235 
236 static void __init __uniphier_cache_set_active_ways(
237                                         struct uniphier_cache_data *data)
238 {
239         unsigned int cpu;
240 
241         for_each_possible_cpu(cpu)
242                 writel_relaxed(data->way_mask, data->way_ctrl_base + 4 * cpu);
243 }
244 
245 static void uniphier_cache_maint_range(unsigned long start, unsigned long end,
246                                        u32 operation)
247 {
248         struct uniphier_cache_data *data;
249 
250         list_for_each_entry(data, &uniphier_cache_list, list)
251                 __uniphier_cache_maint_range(data, start, end, operation);
252 }
253 
254 static void uniphier_cache_maint_all(u32 operation)
255 {
256         struct uniphier_cache_data *data;
257 
258         list_for_each_entry(data, &uniphier_cache_list, list)
259                 __uniphier_cache_maint_all(data, operation);
260 }
261 
262 static void uniphier_cache_inv_range(unsigned long start, unsigned long end)
263 {
264         uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_INV);
265 }
266 
267 static void uniphier_cache_clean_range(unsigned long start, unsigned long end)
268 {
269         uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_CLEAN);
270 }
271 
272 static void uniphier_cache_flush_range(unsigned long start, unsigned long end)
273 {
274         uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_FLUSH);
275 }
276 
277 static void __init uniphier_cache_inv_all(void)
278 {
279         uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_INV);
280 }
281 
282 static void uniphier_cache_flush_all(void)
283 {
284         uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_FLUSH);
285 }
286 
287 static void uniphier_cache_disable(void)
288 {
289         struct uniphier_cache_data *data;
290 
291         list_for_each_entry_reverse(data, &uniphier_cache_list, list)
292                 __uniphier_cache_enable(data, false);
293 
294         uniphier_cache_flush_all();
295 }
296 
297 static void __init uniphier_cache_enable(void)
298 {
299         struct uniphier_cache_data *data;
300 
301         uniphier_cache_inv_all();
302 
303         list_for_each_entry(data, &uniphier_cache_list, list) {
304                 __uniphier_cache_enable(data, true);
305                 __uniphier_cache_set_active_ways(data);
306         }
307 }
308 
309 static void uniphier_cache_sync(void)
310 {
311         struct uniphier_cache_data *data;
312 
313         list_for_each_entry(data, &uniphier_cache_list, list)
314                 __uniphier_cache_sync(data);
315 }
316 
317 static const struct of_device_id uniphier_cache_match[] __initconst = {
318         { .compatible = "socionext,uniphier-system-cache" },
319         { /* sentinel */ }
320 };
321 
322 static int __init __uniphier_cache_init(struct device_node *np,
323                                         unsigned int *cache_level)
324 {
325         struct uniphier_cache_data *data;
326         u32 level, cache_size;
327         struct device_node *next_np;
328         int ret = 0;
329 
330         if (!of_match_node(uniphier_cache_match, np)) {
331                 pr_err("L%d: not compatible with uniphier cache\n",
332                        *cache_level);
333                 return -EINVAL;
334         }
335 
336         if (of_property_read_u32(np, "cache-level", &level)) {
337                 pr_err("L%d: cache-level is not specified\n", *cache_level);
338                 return -EINVAL;
339         }
340 
341         if (level != *cache_level) {
342                 pr_err("L%d: cache-level is unexpected value %d\n",
343                        *cache_level, level);
344                 return -EINVAL;
345         }
346 
347         if (!of_property_read_bool(np, "cache-unified")) {
348                 pr_err("L%d: cache-unified is not specified\n", *cache_level);
349                 return -EINVAL;
350         }
351 
352         data = kzalloc(sizeof(*data), GFP_KERNEL);
353         if (!data)
354                 return -ENOMEM;
355 
356         if (of_property_read_u32(np, "cache-line-size", &data->line_size) ||
357             !is_power_of_2(data->line_size)) {
358                 pr_err("L%d: cache-line-size is unspecified or invalid\n",
359                        *cache_level);
360                 ret = -EINVAL;
361                 goto err;
362         }
363 
364         if (of_property_read_u32(np, "cache-sets", &data->nsets) ||
365             !is_power_of_2(data->nsets)) {
366                 pr_err("L%d: cache-sets is unspecified or invalid\n",
367                        *cache_level);
368                 ret = -EINVAL;
369                 goto err;
370         }
371 
372         if (of_property_read_u32(np, "cache-size", &cache_size) ||
373             cache_size == 0 || cache_size % (data->nsets * data->line_size)) {
374                 pr_err("L%d: cache-size is unspecified or invalid\n",
375                        *cache_level);
376                 ret = -EINVAL;
377                 goto err;
378         }
379 
380         data->way_mask = GENMASK(cache_size / data->nsets / data->line_size - 1,
381                                  0);
382 
383         data->ctrl_base = of_iomap(np, 0);
384         if (!data->ctrl_base) {
385                 pr_err("L%d: failed to map control register\n", *cache_level);
386                 ret = -ENOMEM;
387                 goto err;
388         }
389 
390         data->rev_base = of_iomap(np, 1);
391         if (!data->rev_base) {
392                 pr_err("L%d: failed to map revision register\n", *cache_level);
393                 ret = -ENOMEM;
394                 goto err;
395         }
396 
397         data->op_base = of_iomap(np, 2);
398         if (!data->op_base) {
399                 pr_err("L%d: failed to map operation register\n", *cache_level);
400                 ret = -ENOMEM;
401                 goto err;
402         }
403 
404         data->way_ctrl_base = data->ctrl_base + 0xc00;
405 
406         if (*cache_level == 2) {
407                 u32 revision = readl(data->rev_base + UNIPHIER_SSCID);
408                 /*
409                  * The size of range operation is limited to (1 << 22) or less
410                  * for PH-sLD8 or older SoCs.
411                  */
412                 if (revision <= 0x16)
413                         data->range_op_max_size = (u32)1 << 22;
414 
415                 /*
416                  * Unfortunatly, the offset address of active way control base
417                  * varies from SoC to SoC.
418                  */
419                 switch (revision) {
420                 case 0x11:      /* sLD3 */
421                         data->way_ctrl_base = data->ctrl_base + 0x870;
422                         break;
423                 case 0x12:      /* LD4 */
424                 case 0x16:      /* sld8 */
425                         data->way_ctrl_base = data->ctrl_base + 0x840;
426                         break;
427                 default:
428                         break;
429                 }
430         }
431 
432         data->range_op_max_size -= data->line_size;
433 
434         INIT_LIST_HEAD(&data->list);
435         list_add_tail(&data->list, &uniphier_cache_list); /* no mutex */
436 
437         /*
438          * OK, this level has been successfully initialized.  Look for the next
439          * level cache.  Do not roll back even if the initialization of the
440          * next level cache fails because we want to continue with available
441          * cache levels.
442          */
443         next_np = of_find_next_cache_node(np);
444         if (next_np) {
445                 (*cache_level)++;
446                 ret = __uniphier_cache_init(next_np, cache_level);
447         }
448         of_node_put(next_np);
449 
450         return ret;
451 err:
452         iounmap(data->op_base);
453         iounmap(data->rev_base);
454         iounmap(data->ctrl_base);
455         kfree(data);
456 
457         return ret;
458 }
459 
460 int __init uniphier_cache_init(void)
461 {
462         struct device_node *np = NULL;
463         unsigned int cache_level;
464         int ret = 0;
465 
466         /* look for level 2 cache */
467         while ((np = of_find_matching_node(np, uniphier_cache_match)))
468                 if (!of_property_read_u32(np, "cache-level", &cache_level) &&
469                     cache_level == 2)
470                         break;
471 
472         if (!np)
473                 return -ENODEV;
474 
475         ret = __uniphier_cache_init(np, &cache_level);
476         of_node_put(np);
477 
478         if (ret) {
479                 /*
480                  * Error out iif L2 initialization fails.  Continue with any
481                  * error on L3 or outer because they are optional.
482                  */
483                 if (cache_level == 2) {
484                         pr_err("failed to initialize L2 cache\n");
485                         return ret;
486                 }
487 
488                 cache_level--;
489                 ret = 0;
490         }
491 
492         outer_cache.inv_range = uniphier_cache_inv_range;
493         outer_cache.clean_range = uniphier_cache_clean_range;
494         outer_cache.flush_range = uniphier_cache_flush_range;
495         outer_cache.flush_all = uniphier_cache_flush_all;
496         outer_cache.disable = uniphier_cache_disable;
497         outer_cache.sync = uniphier_cache_sync;
498 
499         uniphier_cache_enable();
500 
501         pr_info("enabled outer cache (cache level: %d)\n", cache_level);
502 
503         return ret;
504 }
505 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp