~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm/mm/cache-uniphier.c

Version: ~ [ linux-5.8 ] ~ [ linux-5.7.12 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.55 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.136 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.191 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.232 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.232 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-or-later
  2 /*
  3  * Copyright (C) 2015-2016 Socionext Inc.
  4  *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
  5  */
  6 
  7 #define pr_fmt(fmt)             "uniphier: " fmt
  8 
  9 #include <linux/bitops.h>
 10 #include <linux/init.h>
 11 #include <linux/io.h>
 12 #include <linux/log2.h>
 13 #include <linux/of_address.h>
 14 #include <linux/slab.h>
 15 #include <asm/hardware/cache-uniphier.h>
 16 #include <asm/outercache.h>
 17 
 18 /* control registers */
 19 #define UNIPHIER_SSCC           0x0     /* Control Register */
 20 #define    UNIPHIER_SSCC_BST                    BIT(20) /* UCWG burst read */
 21 #define    UNIPHIER_SSCC_ACT                    BIT(19) /* Inst-Data separate */
 22 #define    UNIPHIER_SSCC_WTG                    BIT(18) /* WT gathering on */
 23 #define    UNIPHIER_SSCC_PRD                    BIT(17) /* enable pre-fetch */
 24 #define    UNIPHIER_SSCC_ON                     BIT(0)  /* enable cache */
 25 #define UNIPHIER_SSCLPDAWCR     0x30    /* Unified/Data Active Way Control */
 26 #define UNIPHIER_SSCLPIAWCR     0x34    /* Instruction Active Way Control */
 27 
 28 /* revision registers */
 29 #define UNIPHIER_SSCID          0x0     /* ID Register */
 30 
 31 /* operation registers */
 32 #define UNIPHIER_SSCOPE         0x244   /* Cache Operation Primitive Entry */
 33 #define    UNIPHIER_SSCOPE_CM_INV               0x0     /* invalidate */
 34 #define    UNIPHIER_SSCOPE_CM_CLEAN             0x1     /* clean */
 35 #define    UNIPHIER_SSCOPE_CM_FLUSH             0x2     /* flush */
 36 #define    UNIPHIER_SSCOPE_CM_SYNC              0x8     /* sync (drain bufs) */
 37 #define    UNIPHIER_SSCOPE_CM_FLUSH_PREFETCH    0x9     /* flush p-fetch buf */
 38 #define UNIPHIER_SSCOQM         0x248   /* Cache Operation Queue Mode */
 39 #define    UNIPHIER_SSCOQM_S_MASK               (0x3 << 17)
 40 #define    UNIPHIER_SSCOQM_S_RANGE              (0x0 << 17)
 41 #define    UNIPHIER_SSCOQM_S_ALL                (0x1 << 17)
 42 #define    UNIPHIER_SSCOQM_CE                   BIT(15) /* notify completion */
 43 #define    UNIPHIER_SSCOQM_CM_INV               0x0     /* invalidate */
 44 #define    UNIPHIER_SSCOQM_CM_CLEAN             0x1     /* clean */
 45 #define    UNIPHIER_SSCOQM_CM_FLUSH             0x2     /* flush */
 46 #define UNIPHIER_SSCOQAD        0x24c   /* Cache Operation Queue Address */
 47 #define UNIPHIER_SSCOQSZ        0x250   /* Cache Operation Queue Size */
 48 #define UNIPHIER_SSCOPPQSEF     0x25c   /* Cache Operation Queue Set Complete*/
 49 #define    UNIPHIER_SSCOPPQSEF_FE               BIT(1)
 50 #define    UNIPHIER_SSCOPPQSEF_OE               BIT(0)
 51 #define UNIPHIER_SSCOLPQS       0x260   /* Cache Operation Queue Status */
 52 #define    UNIPHIER_SSCOLPQS_EF                 BIT(2)
 53 #define    UNIPHIER_SSCOLPQS_EST                BIT(1)
 54 #define    UNIPHIER_SSCOLPQS_QST                BIT(0)
 55 
 56 /* Is the operation region specified by address range? */
 57 #define UNIPHIER_SSCOQM_S_IS_RANGE(op) \
 58                 ((op & UNIPHIER_SSCOQM_S_MASK) == UNIPHIER_SSCOQM_S_RANGE)
 59 
 60 /**
 61  * uniphier_cache_data - UniPhier outer cache specific data
 62  *
 63  * @ctrl_base: virtual base address of control registers
 64  * @rev_base: virtual base address of revision registers
 65  * @op_base: virtual base address of operation registers
 66  * @way_mask: each bit specifies if the way is present
 67  * @nsets: number of associativity sets
 68  * @line_size: line size in bytes
 69  * @range_op_max_size: max size that can be handled by a single range operation
 70  * @list: list node to include this level in the whole cache hierarchy
 71  */
 72 struct uniphier_cache_data {
 73         void __iomem *ctrl_base;
 74         void __iomem *rev_base;
 75         void __iomem *op_base;
 76         void __iomem *way_ctrl_base;
 77         u32 way_mask;
 78         u32 nsets;
 79         u32 line_size;
 80         u32 range_op_max_size;
 81         struct list_head list;
 82 };
 83 
 84 /*
 85  * List of the whole outer cache hierarchy.  This list is only modified during
 86  * the early boot stage, so no mutex is taken for the access to the list.
 87  */
 88 static LIST_HEAD(uniphier_cache_list);
 89 
 90 /**
 91  * __uniphier_cache_sync - perform a sync point for a particular cache level
 92  *
 93  * @data: cache controller specific data
 94  */
 95 static void __uniphier_cache_sync(struct uniphier_cache_data *data)
 96 {
 97         /* This sequence need not be atomic.  Do not disable IRQ. */
 98         writel_relaxed(UNIPHIER_SSCOPE_CM_SYNC,
 99                        data->op_base + UNIPHIER_SSCOPE);
100         /* need a read back to confirm */
101         readl_relaxed(data->op_base + UNIPHIER_SSCOPE);
102 }
103 
104 /**
105  * __uniphier_cache_maint_common - run a queue operation for a particular level
106  *
107  * @data: cache controller specific data
108  * @start: start address of range operation (don't care for "all" operation)
109  * @size: data size of range operation (don't care for "all" operation)
110  * @operation: flags to specify the desired cache operation
111  */
112 static void __uniphier_cache_maint_common(struct uniphier_cache_data *data,
113                                           unsigned long start,
114                                           unsigned long size,
115                                           u32 operation)
116 {
117         unsigned long flags;
118 
119         /*
120          * No spin lock is necessary here because:
121          *
122          * [1] This outer cache controller is able to accept maintenance
123          * operations from multiple CPUs at a time in an SMP system; if a
124          * maintenance operation is under way and another operation is issued,
125          * the new one is stored in the queue.  The controller performs one
126          * operation after another.  If the queue is full, the status register,
127          * UNIPHIER_SSCOPPQSEF, indicates that the queue registration has
128          * failed.  The status registers, UNIPHIER_{SSCOPPQSEF, SSCOLPQS}, have
129          * different instances for each CPU, i.e. each CPU can track the status
130          * of the maintenance operations triggered by itself.
131          *
132          * [2] The cache command registers, UNIPHIER_{SSCOQM, SSCOQAD, SSCOQSZ,
133          * SSCOQWN}, are shared between multiple CPUs, but the hardware still
134          * guarantees the registration sequence is atomic; the write access to
135          * them are arbitrated by the hardware.  The first accessor to the
136          * register, UNIPHIER_SSCOQM, holds the access right and it is released
137          * by reading the status register, UNIPHIER_SSCOPPQSEF.  While one CPU
138          * is holding the access right, other CPUs fail to register operations.
139          * One CPU should not hold the access right for a long time, so local
140          * IRQs should be disabled while the following sequence.
141          */
142         local_irq_save(flags);
143 
144         /* clear the complete notification flag */
145         writel_relaxed(UNIPHIER_SSCOLPQS_EF, data->op_base + UNIPHIER_SSCOLPQS);
146 
147         do {
148                 /* set cache operation */
149                 writel_relaxed(UNIPHIER_SSCOQM_CE | operation,
150                                data->op_base + UNIPHIER_SSCOQM);
151 
152                 /* set address range if needed */
153                 if (likely(UNIPHIER_SSCOQM_S_IS_RANGE(operation))) {
154                         writel_relaxed(start, data->op_base + UNIPHIER_SSCOQAD);
155                         writel_relaxed(size, data->op_base + UNIPHIER_SSCOQSZ);
156                 }
157         } while (unlikely(readl_relaxed(data->op_base + UNIPHIER_SSCOPPQSEF) &
158                           (UNIPHIER_SSCOPPQSEF_FE | UNIPHIER_SSCOPPQSEF_OE)));
159 
160         /* wait until the operation is completed */
161         while (likely(readl_relaxed(data->op_base + UNIPHIER_SSCOLPQS) !=
162                       UNIPHIER_SSCOLPQS_EF))
163                 cpu_relax();
164 
165         local_irq_restore(flags);
166 }
167 
168 static void __uniphier_cache_maint_all(struct uniphier_cache_data *data,
169                                        u32 operation)
170 {
171         __uniphier_cache_maint_common(data, 0, 0,
172                                       UNIPHIER_SSCOQM_S_ALL | operation);
173 
174         __uniphier_cache_sync(data);
175 }
176 
177 static void __uniphier_cache_maint_range(struct uniphier_cache_data *data,
178                                          unsigned long start, unsigned long end,
179                                          u32 operation)
180 {
181         unsigned long size;
182 
183         /*
184          * If the start address is not aligned,
185          * perform a cache operation for the first cache-line
186          */
187         start = start & ~(data->line_size - 1);
188 
189         size = end - start;
190 
191         if (unlikely(size >= (unsigned long)(-data->line_size))) {
192                 /* this means cache operation for all range */
193                 __uniphier_cache_maint_all(data, operation);
194                 return;
195         }
196 
197         /*
198          * If the end address is not aligned,
199          * perform a cache operation for the last cache-line
200          */
201         size = ALIGN(size, data->line_size);
202 
203         while (size) {
204                 unsigned long chunk_size = min_t(unsigned long, size,
205                                                  data->range_op_max_size);
206 
207                 __uniphier_cache_maint_common(data, start, chunk_size,
208                                         UNIPHIER_SSCOQM_S_RANGE | operation);
209 
210                 start += chunk_size;
211                 size -= chunk_size;
212         }
213 
214         __uniphier_cache_sync(data);
215 }
216 
217 static void __uniphier_cache_enable(struct uniphier_cache_data *data, bool on)
218 {
219         u32 val = 0;
220 
221         if (on)
222                 val = UNIPHIER_SSCC_WTG | UNIPHIER_SSCC_PRD | UNIPHIER_SSCC_ON;
223 
224         writel_relaxed(val, data->ctrl_base + UNIPHIER_SSCC);
225 }
226 
227 static void __init __uniphier_cache_set_active_ways(
228                                         struct uniphier_cache_data *data)
229 {
230         unsigned int cpu;
231 
232         for_each_possible_cpu(cpu)
233                 writel_relaxed(data->way_mask, data->way_ctrl_base + 4 * cpu);
234 }
235 
236 static void uniphier_cache_maint_range(unsigned long start, unsigned long end,
237                                        u32 operation)
238 {
239         struct uniphier_cache_data *data;
240 
241         list_for_each_entry(data, &uniphier_cache_list, list)
242                 __uniphier_cache_maint_range(data, start, end, operation);
243 }
244 
245 static void uniphier_cache_maint_all(u32 operation)
246 {
247         struct uniphier_cache_data *data;
248 
249         list_for_each_entry(data, &uniphier_cache_list, list)
250                 __uniphier_cache_maint_all(data, operation);
251 }
252 
253 static void uniphier_cache_inv_range(unsigned long start, unsigned long end)
254 {
255         uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_INV);
256 }
257 
258 static void uniphier_cache_clean_range(unsigned long start, unsigned long end)
259 {
260         uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_CLEAN);
261 }
262 
263 static void uniphier_cache_flush_range(unsigned long start, unsigned long end)
264 {
265         uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_FLUSH);
266 }
267 
268 static void __init uniphier_cache_inv_all(void)
269 {
270         uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_INV);
271 }
272 
273 static void uniphier_cache_flush_all(void)
274 {
275         uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_FLUSH);
276 }
277 
278 static void uniphier_cache_disable(void)
279 {
280         struct uniphier_cache_data *data;
281 
282         list_for_each_entry_reverse(data, &uniphier_cache_list, list)
283                 __uniphier_cache_enable(data, false);
284 
285         uniphier_cache_flush_all();
286 }
287 
288 static void __init uniphier_cache_enable(void)
289 {
290         struct uniphier_cache_data *data;
291 
292         uniphier_cache_inv_all();
293 
294         list_for_each_entry(data, &uniphier_cache_list, list) {
295                 __uniphier_cache_enable(data, true);
296                 __uniphier_cache_set_active_ways(data);
297         }
298 }
299 
300 static void uniphier_cache_sync(void)
301 {
302         struct uniphier_cache_data *data;
303 
304         list_for_each_entry(data, &uniphier_cache_list, list)
305                 __uniphier_cache_sync(data);
306 }
307 
308 static const struct of_device_id uniphier_cache_match[] __initconst = {
309         { .compatible = "socionext,uniphier-system-cache" },
310         { /* sentinel */ }
311 };
312 
313 static int __init __uniphier_cache_init(struct device_node *np,
314                                         unsigned int *cache_level)
315 {
316         struct uniphier_cache_data *data;
317         u32 level, cache_size;
318         struct device_node *next_np;
319         int ret = 0;
320 
321         if (!of_match_node(uniphier_cache_match, np)) {
322                 pr_err("L%d: not compatible with uniphier cache\n",
323                        *cache_level);
324                 return -EINVAL;
325         }
326 
327         if (of_property_read_u32(np, "cache-level", &level)) {
328                 pr_err("L%d: cache-level is not specified\n", *cache_level);
329                 return -EINVAL;
330         }
331 
332         if (level != *cache_level) {
333                 pr_err("L%d: cache-level is unexpected value %d\n",
334                        *cache_level, level);
335                 return -EINVAL;
336         }
337 
338         if (!of_property_read_bool(np, "cache-unified")) {
339                 pr_err("L%d: cache-unified is not specified\n", *cache_level);
340                 return -EINVAL;
341         }
342 
343         data = kzalloc(sizeof(*data), GFP_KERNEL);
344         if (!data)
345                 return -ENOMEM;
346 
347         if (of_property_read_u32(np, "cache-line-size", &data->line_size) ||
348             !is_power_of_2(data->line_size)) {
349                 pr_err("L%d: cache-line-size is unspecified or invalid\n",
350                        *cache_level);
351                 ret = -EINVAL;
352                 goto err;
353         }
354 
355         if (of_property_read_u32(np, "cache-sets", &data->nsets) ||
356             !is_power_of_2(data->nsets)) {
357                 pr_err("L%d: cache-sets is unspecified or invalid\n",
358                        *cache_level);
359                 ret = -EINVAL;
360                 goto err;
361         }
362 
363         if (of_property_read_u32(np, "cache-size", &cache_size) ||
364             cache_size == 0 || cache_size % (data->nsets * data->line_size)) {
365                 pr_err("L%d: cache-size is unspecified or invalid\n",
366                        *cache_level);
367                 ret = -EINVAL;
368                 goto err;
369         }
370 
371         data->way_mask = GENMASK(cache_size / data->nsets / data->line_size - 1,
372                                  0);
373 
374         data->ctrl_base = of_iomap(np, 0);
375         if (!data->ctrl_base) {
376                 pr_err("L%d: failed to map control register\n", *cache_level);
377                 ret = -ENOMEM;
378                 goto err;
379         }
380 
381         data->rev_base = of_iomap(np, 1);
382         if (!data->rev_base) {
383                 pr_err("L%d: failed to map revision register\n", *cache_level);
384                 ret = -ENOMEM;
385                 goto err;
386         }
387 
388         data->op_base = of_iomap(np, 2);
389         if (!data->op_base) {
390                 pr_err("L%d: failed to map operation register\n", *cache_level);
391                 ret = -ENOMEM;
392                 goto err;
393         }
394 
395         data->way_ctrl_base = data->ctrl_base + 0xc00;
396 
397         if (*cache_level == 2) {
398                 u32 revision = readl(data->rev_base + UNIPHIER_SSCID);
399                 /*
400                  * The size of range operation is limited to (1 << 22) or less
401                  * for PH-sLD8 or older SoCs.
402                  */
403                 if (revision <= 0x16)
404                         data->range_op_max_size = (u32)1 << 22;
405 
406                 /*
407                  * Unfortunatly, the offset address of active way control base
408                  * varies from SoC to SoC.
409                  */
410                 switch (revision) {
411                 case 0x11:      /* sLD3 */
412                         data->way_ctrl_base = data->ctrl_base + 0x870;
413                         break;
414                 case 0x12:      /* LD4 */
415                 case 0x16:      /* sld8 */
416                         data->way_ctrl_base = data->ctrl_base + 0x840;
417                         break;
418                 default:
419                         break;
420                 }
421         }
422 
423         data->range_op_max_size -= data->line_size;
424 
425         INIT_LIST_HEAD(&data->list);
426         list_add_tail(&data->list, &uniphier_cache_list); /* no mutex */
427 
428         /*
429          * OK, this level has been successfully initialized.  Look for the next
430          * level cache.  Do not roll back even if the initialization of the
431          * next level cache fails because we want to continue with available
432          * cache levels.
433          */
434         next_np = of_find_next_cache_node(np);
435         if (next_np) {
436                 (*cache_level)++;
437                 ret = __uniphier_cache_init(next_np, cache_level);
438         }
439         of_node_put(next_np);
440 
441         return ret;
442 err:
443         iounmap(data->op_base);
444         iounmap(data->rev_base);
445         iounmap(data->ctrl_base);
446         kfree(data);
447 
448         return ret;
449 }
450 
451 int __init uniphier_cache_init(void)
452 {
453         struct device_node *np = NULL;
454         unsigned int cache_level;
455         int ret = 0;
456 
457         /* look for level 2 cache */
458         while ((np = of_find_matching_node(np, uniphier_cache_match)))
459                 if (!of_property_read_u32(np, "cache-level", &cache_level) &&
460                     cache_level == 2)
461                         break;
462 
463         if (!np)
464                 return -ENODEV;
465 
466         ret = __uniphier_cache_init(np, &cache_level);
467         of_node_put(np);
468 
469         if (ret) {
470                 /*
471                  * Error out iif L2 initialization fails.  Continue with any
472                  * error on L3 or outer because they are optional.
473                  */
474                 if (cache_level == 2) {
475                         pr_err("failed to initialize L2 cache\n");
476                         return ret;
477                 }
478 
479                 cache_level--;
480                 ret = 0;
481         }
482 
483         outer_cache.inv_range = uniphier_cache_inv_range;
484         outer_cache.clean_range = uniphier_cache_clean_range;
485         outer_cache.flush_range = uniphier_cache_flush_range;
486         outer_cache.flush_all = uniphier_cache_flush_all;
487         outer_cache.disable = uniphier_cache_disable;
488         outer_cache.sync = uniphier_cache_sync;
489 
490         uniphier_cache_enable();
491 
492         pr_info("enabled outer cache (cache level: %d)\n", cache_level);
493 
494         return ret;
495 }
496 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp