~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arc/mm/cache.c

Version: ~ [ linux-5.16-rc3 ] ~ [ linux-5.15.5 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.82 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.162 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.218 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.256 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.291 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.293 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-only
  2 /*
  3  * ARC Cache Management
  4  *
  5  * Copyright (C) 2014-15 Synopsys, Inc. (www.synopsys.com)
  6  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
  7  */
  8 
  9 #include <linux/module.h>
 10 #include <linux/mm.h>
 11 #include <linux/sched.h>
 12 #include <linux/cache.h>
 13 #include <linux/mmu_context.h>
 14 #include <linux/syscalls.h>
 15 #include <linux/uaccess.h>
 16 #include <linux/pagemap.h>
 17 #include <asm/cacheflush.h>
 18 #include <asm/cachectl.h>
 19 #include <asm/setup.h>
 20 
 21 #ifdef CONFIG_ISA_ARCV2
 22 #define USE_RGN_FLSH    1
 23 #endif
 24 
 25 static int l2_line_sz;
 26 static int ioc_exists;
 27 int slc_enable = 1, ioc_enable = 1;
 28 unsigned long perip_base = ARC_UNCACHED_ADDR_SPACE; /* legacy value for boot */
 29 unsigned long perip_end = 0xFFFFFFFF; /* legacy value */
 30 
 31 void (*_cache_line_loop_ic_fn)(phys_addr_t paddr, unsigned long vaddr,
 32                                unsigned long sz, const int op, const int full_page);
 33 
 34 void (*__dma_cache_wback_inv)(phys_addr_t start, unsigned long sz);
 35 void (*__dma_cache_inv)(phys_addr_t start, unsigned long sz);
 36 void (*__dma_cache_wback)(phys_addr_t start, unsigned long sz);
 37 
 38 char *arc_cache_mumbojumbo(int c, char *buf, int len)
 39 {
 40         int n = 0;
 41         struct cpuinfo_arc_cache *p;
 42 
 43 #define PR_CACHE(p, cfg, str)                                           \
 44         if (!(p)->line_len)                                             \
 45                 n += scnprintf(buf + n, len - n, str"\t\t: N/A\n");     \
 46         else                                                            \
 47                 n += scnprintf(buf + n, len - n,                        \
 48                         str"\t\t: %uK, %dway/set, %uB Line, %s%s%s\n",  \
 49                         (p)->sz_k, (p)->assoc, (p)->line_len,           \
 50                         (p)->vipt ? "VIPT" : "PIPT",                    \
 51                         (p)->alias ? " aliasing" : "",                  \
 52                         IS_USED_CFG(cfg));
 53 
 54         PR_CACHE(&cpuinfo_arc700[c].icache, CONFIG_ARC_HAS_ICACHE, "I-Cache");
 55         PR_CACHE(&cpuinfo_arc700[c].dcache, CONFIG_ARC_HAS_DCACHE, "D-Cache");
 56 
 57         p = &cpuinfo_arc700[c].slc;
 58         if (p->line_len)
 59                 n += scnprintf(buf + n, len - n,
 60                                "SLC\t\t: %uK, %uB Line%s\n",
 61                                p->sz_k, p->line_len, IS_USED_RUN(slc_enable));
 62 
 63         n += scnprintf(buf + n, len - n, "Peripherals\t: %#lx%s%s\n",
 64                        perip_base,
 65                        IS_AVAIL3(ioc_exists, ioc_enable, ", IO-Coherency (per-device) "));
 66 
 67         return buf;
 68 }
 69 
 70 /*
 71  * Read the Cache Build Confuration Registers, Decode them and save into
 72  * the cpuinfo structure for later use.
 73  * No Validation done here, simply read/convert the BCRs
 74  */
 75 static void read_decode_cache_bcr_arcv2(int cpu)
 76 {
 77         struct cpuinfo_arc_cache *p_slc = &cpuinfo_arc700[cpu].slc;
 78         struct bcr_generic sbcr;
 79 
 80         struct bcr_slc_cfg {
 81 #ifdef CONFIG_CPU_BIG_ENDIAN
 82                 unsigned int pad:24, way:2, lsz:2, sz:4;
 83 #else
 84                 unsigned int sz:4, lsz:2, way:2, pad:24;
 85 #endif
 86         } slc_cfg;
 87 
 88         struct bcr_clust_cfg {
 89 #ifdef CONFIG_CPU_BIG_ENDIAN
 90                 unsigned int pad:7, c:1, num_entries:8, num_cores:8, ver:8;
 91 #else
 92                 unsigned int ver:8, num_cores:8, num_entries:8, c:1, pad:7;
 93 #endif
 94         } cbcr;
 95 
 96         struct bcr_volatile {
 97 #ifdef CONFIG_CPU_BIG_ENDIAN
 98                 unsigned int start:4, limit:4, pad:22, order:1, disable:1;
 99 #else
100                 unsigned int disable:1, order:1, pad:22, limit:4, start:4;
101 #endif
102         } vol;
103 
104 
105         READ_BCR(ARC_REG_SLC_BCR, sbcr);
106         if (sbcr.ver) {
107                 READ_BCR(ARC_REG_SLC_CFG, slc_cfg);
108                 p_slc->sz_k = 128 << slc_cfg.sz;
109                 l2_line_sz = p_slc->line_len = (slc_cfg.lsz == 0) ? 128 : 64;
110         }
111 
112         READ_BCR(ARC_REG_CLUSTER_BCR, cbcr);
113         if (cbcr.c) {
114                 ioc_exists = 1;
115 
116                 /*
117                  * As for today we don't support both IOC and ZONE_HIGHMEM enabled
118                  * simultaneously. This happens because as of today IOC aperture covers
119                  * only ZONE_NORMAL (low mem) and any dma transactions outside this
120                  * region won't be HW coherent.
121                  * If we want to use both IOC and ZONE_HIGHMEM we can use
122                  * bounce_buffer to handle dma transactions to HIGHMEM.
123                  * Also it is possible to modify dma_direct cache ops or increase IOC
124                  * aperture size if we are planning to use HIGHMEM without PAE.
125                  */
126                 if (IS_ENABLED(CONFIG_HIGHMEM) || is_pae40_enabled())
127                         ioc_enable = 0;
128         } else {
129                 ioc_enable = 0;
130         }
131 
132         /* HS 2.0 didn't have AUX_VOL */
133         if (cpuinfo_arc700[cpu].core.family > 0x51) {
134                 READ_BCR(AUX_VOL, vol);
135                 perip_base = vol.start << 28;
136                 /* HS 3.0 has limit and strict-ordering fields */
137                 if (cpuinfo_arc700[cpu].core.family > 0x52)
138                         perip_end = (vol.limit << 28) - 1;
139         }
140 }
141 
142 void read_decode_cache_bcr(void)
143 {
144         struct cpuinfo_arc_cache *p_ic, *p_dc;
145         unsigned int cpu = smp_processor_id();
146         struct bcr_cache {
147 #ifdef CONFIG_CPU_BIG_ENDIAN
148                 unsigned int pad:12, line_len:4, sz:4, config:4, ver:8;
149 #else
150                 unsigned int ver:8, config:4, sz:4, line_len:4, pad:12;
151 #endif
152         } ibcr, dbcr;
153 
154         p_ic = &cpuinfo_arc700[cpu].icache;
155         READ_BCR(ARC_REG_IC_BCR, ibcr);
156 
157         if (!ibcr.ver)
158                 goto dc_chk;
159 
160         if (ibcr.ver <= 3) {
161                 BUG_ON(ibcr.config != 3);
162                 p_ic->assoc = 2;                /* Fixed to 2w set assoc */
163         } else if (ibcr.ver >= 4) {
164                 p_ic->assoc = 1 << ibcr.config; /* 1,2,4,8 */
165         }
166 
167         p_ic->line_len = 8 << ibcr.line_len;
168         p_ic->sz_k = 1 << (ibcr.sz - 1);
169         p_ic->vipt = 1;
170         p_ic->alias = p_ic->sz_k/p_ic->assoc/TO_KB(PAGE_SIZE) > 1;
171 
172 dc_chk:
173         p_dc = &cpuinfo_arc700[cpu].dcache;
174         READ_BCR(ARC_REG_DC_BCR, dbcr);
175 
176         if (!dbcr.ver)
177                 goto slc_chk;
178 
179         if (dbcr.ver <= 3) {
180                 BUG_ON(dbcr.config != 2);
181                 p_dc->assoc = 4;                /* Fixed to 4w set assoc */
182                 p_dc->vipt = 1;
183                 p_dc->alias = p_dc->sz_k/p_dc->assoc/TO_KB(PAGE_SIZE) > 1;
184         } else if (dbcr.ver >= 4) {
185                 p_dc->assoc = 1 << dbcr.config; /* 1,2,4,8 */
186                 p_dc->vipt = 0;
187                 p_dc->alias = 0;                /* PIPT so can't VIPT alias */
188         }
189 
190         p_dc->line_len = 16 << dbcr.line_len;
191         p_dc->sz_k = 1 << (dbcr.sz - 1);
192 
193 slc_chk:
194         if (is_isa_arcv2())
195                 read_decode_cache_bcr_arcv2(cpu);
196 }
197 
198 /*
199  * Line Operation on {I,D}-Cache
200  */
201 
202 #define OP_INV          0x1
203 #define OP_FLUSH        0x2
204 #define OP_FLUSH_N_INV  0x3
205 #define OP_INV_IC       0x4
206 
207 /*
208  *              I-Cache Aliasing in ARC700 VIPT caches (MMU v1-v3)
209  *
210  * ARC VIPT I-cache uses vaddr to index into cache and paddr to match the tag.
211  * The orig Cache Management Module "CDU" only required paddr to invalidate a
212  * certain line since it sufficed as index in Non-Aliasing VIPT cache-geometry.
213  * Infact for distinct V1,V2,P: all of {V1-P},{V2-P},{P-P} would end up fetching
214  * the exact same line.
215  *
216  * However for larger Caches (way-size > page-size) - i.e. in Aliasing config,
217  * paddr alone could not be used to correctly index the cache.
218  *
219  * ------------------
220  * MMU v1/v2 (Fixed Page Size 8k)
221  * ------------------
222  * The solution was to provide CDU with these additonal vaddr bits. These
223  * would be bits [x:13], x would depend on cache-geometry, 13 comes from
224  * standard page size of 8k.
225  * H/w folks chose [17:13] to be a future safe range, and moreso these 5 bits
226  * of vaddr could easily be "stuffed" in the paddr as bits [4:0] since the
227  * orig 5 bits of paddr were anyways ignored by CDU line ops, as they
228  * represent the offset within cache-line. The adv of using this "clumsy"
229  * interface for additional info was no new reg was needed in CDU programming
230  * model.
231  *
232  * 17:13 represented the max num of bits passable, actual bits needed were
233  * fewer, based on the num-of-aliases possible.
234  * -for 2 alias possibility, only bit 13 needed (32K cache)
235  * -for 4 alias possibility, bits 14:13 needed (64K cache)
236  *
237  * ------------------
238  * MMU v3
239  * ------------------
240  * This ver of MMU supports variable page sizes (1k-16k): although Linux will
241  * only support 8k (default), 16k and 4k.
242  * However from hardware perspective, smaller page sizes aggravate aliasing
243  * meaning more vaddr bits needed to disambiguate the cache-line-op ;
244  * the existing scheme of piggybacking won't work for certain configurations.
245  * Two new registers IC_PTAG and DC_PTAG inttoduced.
246  * "tag" bits are provided in PTAG, index bits in existing IVIL/IVDL/FLDL regs
247  */
248 
249 static inline
250 void __cache_line_loop_v2(phys_addr_t paddr, unsigned long vaddr,
251                           unsigned long sz, const int op, const int full_page)
252 {
253         unsigned int aux_cmd;
254         int num_lines;
255 
256         if (op == OP_INV_IC) {
257                 aux_cmd = ARC_REG_IC_IVIL;
258         } else {
259                 /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */
260                 aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL;
261         }
262 
263         /* Ensure we properly floor/ceil the non-line aligned/sized requests
264          * and have @paddr - aligned to cache line and integral @num_lines.
265          * This however can be avoided for page sized since:
266          *  -@paddr will be cache-line aligned already (being page aligned)
267          *  -@sz will be integral multiple of line size (being page sized).
268          */
269         if (!full_page) {
270                 sz += paddr & ~CACHE_LINE_MASK;
271                 paddr &= CACHE_LINE_MASK;
272                 vaddr &= CACHE_LINE_MASK;
273         }
274 
275         num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES);
276 
277         /* MMUv2 and before: paddr contains stuffed vaddrs bits */
278         paddr |= (vaddr >> PAGE_SHIFT) & 0x1F;
279 
280         while (num_lines-- > 0) {
281                 write_aux_reg(aux_cmd, paddr);
282                 paddr += L1_CACHE_BYTES;
283         }
284 }
285 
286 /*
287  * For ARC700 MMUv3 I-cache and D-cache flushes
288  *  - ARC700 programming model requires paddr and vaddr be passed in seperate
289  *    AUX registers (*_IV*L and *_PTAG respectively) irrespective of whether the
290  *    caches actually alias or not.
291  * -  For HS38, only the aliasing I-cache configuration uses the PTAG reg
292  *    (non aliasing I-cache version doesn't; while D-cache can't possibly alias)
293  */
294 static inline
295 void __cache_line_loop_v3(phys_addr_t paddr, unsigned long vaddr,
296                           unsigned long sz, const int op, const int full_page)
297 {
298         unsigned int aux_cmd, aux_tag;
299         int num_lines;
300 
301         if (op == OP_INV_IC) {
302                 aux_cmd = ARC_REG_IC_IVIL;
303                 aux_tag = ARC_REG_IC_PTAG;
304         } else {
305                 aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL;
306                 aux_tag = ARC_REG_DC_PTAG;
307         }
308 
309         /* Ensure we properly floor/ceil the non-line aligned/sized requests
310          * and have @paddr - aligned to cache line and integral @num_lines.
311          * This however can be avoided for page sized since:
312          *  -@paddr will be cache-line aligned already (being page aligned)
313          *  -@sz will be integral multiple of line size (being page sized).
314          */
315         if (!full_page) {
316                 sz += paddr & ~CACHE_LINE_MASK;
317                 paddr &= CACHE_LINE_MASK;
318                 vaddr &= CACHE_LINE_MASK;
319         }
320         num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES);
321 
322         /*
323          * MMUv3, cache ops require paddr in PTAG reg
324          * if V-P const for loop, PTAG can be written once outside loop
325          */
326         if (full_page)
327                 write_aux_reg(aux_tag, paddr);
328 
329         /*
330          * This is technically for MMU v4, using the MMU v3 programming model
331          * Special work for HS38 aliasing I-cache configuration with PAE40
332          *   - upper 8 bits of paddr need to be written into PTAG_HI
333          *   - (and needs to be written before the lower 32 bits)
334          * Note that PTAG_HI is hoisted outside the line loop
335          */
336         if (is_pae40_enabled() && op == OP_INV_IC)
337                 write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32);
338 
339         while (num_lines-- > 0) {
340                 if (!full_page) {
341                         write_aux_reg(aux_tag, paddr);
342                         paddr += L1_CACHE_BYTES;
343                 }
344 
345                 write_aux_reg(aux_cmd, vaddr);
346                 vaddr += L1_CACHE_BYTES;
347         }
348 }
349 
350 #ifndef USE_RGN_FLSH
351 
352 /*
353  * In HS38x (MMU v4), I-cache is VIPT (can alias), D-cache is PIPT
354  * Here's how cache ops are implemented
355  *
356  *  - D-cache: only paddr needed (in DC_IVDL/DC_FLDL)
357  *  - I-cache Non Aliasing: Despite VIPT, only paddr needed (in IC_IVIL)
358  *  - I-cache Aliasing: Both vaddr and paddr needed (in IC_IVIL, IC_PTAG
359  *    respectively, similar to MMU v3 programming model, hence
360  *    __cache_line_loop_v3() is used)
361  *
362  * If PAE40 is enabled, independent of aliasing considerations, the higher bits
363  * needs to be written into PTAG_HI
364  */
365 static inline
366 void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr,
367                           unsigned long sz, const int op, const int full_page)
368 {
369         unsigned int aux_cmd;
370         int num_lines;
371 
372         if (op == OP_INV_IC) {
373                 aux_cmd = ARC_REG_IC_IVIL;
374         } else {
375                 /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */
376                 aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL;
377         }
378 
379         /* Ensure we properly floor/ceil the non-line aligned/sized requests
380          * and have @paddr - aligned to cache line and integral @num_lines.
381          * This however can be avoided for page sized since:
382          *  -@paddr will be cache-line aligned already (being page aligned)
383          *  -@sz will be integral multiple of line size (being page sized).
384          */
385         if (!full_page) {
386                 sz += paddr & ~CACHE_LINE_MASK;
387                 paddr &= CACHE_LINE_MASK;
388         }
389 
390         num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES);
391 
392         /*
393          * For HS38 PAE40 configuration
394          *   - upper 8 bits of paddr need to be written into PTAG_HI
395          *   - (and needs to be written before the lower 32 bits)
396          */
397         if (is_pae40_enabled()) {
398                 if (op == OP_INV_IC)
399                         /*
400                          * Non aliasing I-cache in HS38,
401                          * aliasing I-cache handled in __cache_line_loop_v3()
402                          */
403                         write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32);
404                 else
405                         write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32);
406         }
407 
408         while (num_lines-- > 0) {
409                 write_aux_reg(aux_cmd, paddr);
410                 paddr += L1_CACHE_BYTES;
411         }
412 }
413 
414 #else
415 
416 /*
417  * optimized flush operation which takes a region as opposed to iterating per line
418  */
419 static inline
420 void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr,
421                           unsigned long sz, const int op, const int full_page)
422 {
423         unsigned int s, e;
424 
425         /* Only for Non aliasing I-cache in HS38 */
426         if (op == OP_INV_IC) {
427                 s = ARC_REG_IC_IVIR;
428                 e = ARC_REG_IC_ENDR;
429         } else {
430                 s = ARC_REG_DC_STARTR;
431                 e = ARC_REG_DC_ENDR;
432         }
433 
434         if (!full_page) {
435                 /* for any leading gap between @paddr and start of cache line */
436                 sz += paddr & ~CACHE_LINE_MASK;
437                 paddr &= CACHE_LINE_MASK;
438 
439                 /*
440                  *  account for any trailing gap to end of cache line
441                  *  this is equivalent to DIV_ROUND_UP() in line ops above
442                  */
443                 sz += L1_CACHE_BYTES - 1;
444         }
445 
446         if (is_pae40_enabled()) {
447                 /* TBD: check if crossing 4TB boundary */
448                 if (op == OP_INV_IC)
449                         write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32);
450                 else
451                         write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32);
452         }
453 
454         /* ENDR needs to be set ahead of START */
455         write_aux_reg(e, paddr + sz);   /* ENDR is exclusive */
456         write_aux_reg(s, paddr);
457 
458         /* caller waits on DC_CTRL.FS */
459 }
460 
461 #endif
462 
463 #if (CONFIG_ARC_MMU_VER < 3)
464 #define __cache_line_loop       __cache_line_loop_v2
465 #elif (CONFIG_ARC_MMU_VER == 3)
466 #define __cache_line_loop       __cache_line_loop_v3
467 #elif (CONFIG_ARC_MMU_VER > 3)
468 #define __cache_line_loop       __cache_line_loop_v4
469 #endif
470 
471 #ifdef CONFIG_ARC_HAS_DCACHE
472 
473 /***************************************************************
474  * Machine specific helpers for Entire D-Cache or Per Line ops
475  */
476 
477 #ifndef USE_RGN_FLSH
478 /*
479  * this version avoids extra read/write of DC_CTRL for flush or invalid ops
480  * in the non region flush regime (such as for ARCompact)
481  */
482 static inline void __before_dc_op(const int op)
483 {
484         if (op == OP_FLUSH_N_INV) {
485                 /* Dcache provides 2 cmd: FLUSH or INV
486                  * INV inturn has sub-modes: DISCARD or FLUSH-BEFORE
487                  * flush-n-inv is achieved by INV cmd but with IM=1
488                  * So toggle INV sub-mode depending on op request and default
489                  */
490                 const unsigned int ctl = ARC_REG_DC_CTRL;
491                 write_aux_reg(ctl, read_aux_reg(ctl) | DC_CTRL_INV_MODE_FLUSH);
492         }
493 }
494 
495 #else
496 
497 static inline void __before_dc_op(const int op)
498 {
499         const unsigned int ctl = ARC_REG_DC_CTRL;
500         unsigned int val = read_aux_reg(ctl);
501 
502         if (op == OP_FLUSH_N_INV) {
503                 val |= DC_CTRL_INV_MODE_FLUSH;
504         }
505 
506         if (op != OP_INV_IC) {
507                 /*
508                  * Flush / Invalidate is provided by DC_CTRL.RNG_OP 0 or 1
509                  * combined Flush-n-invalidate uses DC_CTRL.IM = 1 set above
510                  */
511                 val &= ~DC_CTRL_RGN_OP_MSK;
512                 if (op & OP_INV)
513                         val |= DC_CTRL_RGN_OP_INV;
514         }
515         write_aux_reg(ctl, val);
516 }
517 
518 #endif
519 
520 
521 static inline void __after_dc_op(const int op)
522 {
523         if (op & OP_FLUSH) {
524                 const unsigned int ctl = ARC_REG_DC_CTRL;
525                 unsigned int reg;
526 
527                 /* flush / flush-n-inv both wait */
528                 while ((reg = read_aux_reg(ctl)) & DC_CTRL_FLUSH_STATUS)
529                         ;
530 
531                 /* Switch back to default Invalidate mode */
532                 if (op == OP_FLUSH_N_INV)
533                         write_aux_reg(ctl, reg & ~DC_CTRL_INV_MODE_FLUSH);
534         }
535 }
536 
537 /*
538  * Operation on Entire D-Cache
539  * @op = {OP_INV, OP_FLUSH, OP_FLUSH_N_INV}
540  * Note that constant propagation ensures all the checks are gone
541  * in generated code
542  */
543 static inline void __dc_entire_op(const int op)
544 {
545         int aux;
546 
547         __before_dc_op(op);
548 
549         if (op & OP_INV)        /* Inv or flush-n-inv use same cmd reg */
550                 aux = ARC_REG_DC_IVDC;
551         else
552                 aux = ARC_REG_DC_FLSH;
553 
554         write_aux_reg(aux, 0x1);
555 
556         __after_dc_op(op);
557 }
558 
559 static inline void __dc_disable(void)
560 {
561         const int r = ARC_REG_DC_CTRL;
562 
563         __dc_entire_op(OP_FLUSH_N_INV);
564         write_aux_reg(r, read_aux_reg(r) | DC_CTRL_DIS);
565 }
566 
567 static void __dc_enable(void)
568 {
569         const int r = ARC_REG_DC_CTRL;
570 
571         write_aux_reg(r, read_aux_reg(r) & ~DC_CTRL_DIS);
572 }
573 
574 /* For kernel mappings cache operation: index is same as paddr */
575 #define __dc_line_op_k(p, sz, op)       __dc_line_op(p, p, sz, op)
576 
577 /*
578  * D-Cache Line ops: Per Line INV (discard or wback+discard) or FLUSH (wback)
579  */
580 static inline void __dc_line_op(phys_addr_t paddr, unsigned long vaddr,
581                                 unsigned long sz, const int op)
582 {
583         const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE;
584         unsigned long flags;
585 
586         local_irq_save(flags);
587 
588         __before_dc_op(op);
589 
590         __cache_line_loop(paddr, vaddr, sz, op, full_page);
591 
592         __after_dc_op(op);
593 
594         local_irq_restore(flags);
595 }
596 
597 #else
598 
599 #define __dc_entire_op(op)
600 #define __dc_disable()
601 #define __dc_enable()
602 #define __dc_line_op(paddr, vaddr, sz, op)
603 #define __dc_line_op_k(paddr, sz, op)
604 
605 #endif /* CONFIG_ARC_HAS_DCACHE */
606 
607 #ifdef CONFIG_ARC_HAS_ICACHE
608 
609 static inline void __ic_entire_inv(void)
610 {
611         write_aux_reg(ARC_REG_IC_IVIC, 1);
612         read_aux_reg(ARC_REG_IC_CTRL);  /* blocks */
613 }
614 
615 static inline void
616 __ic_line_inv_vaddr_local(phys_addr_t paddr, unsigned long vaddr,
617                           unsigned long sz)
618 {
619         const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE;
620         unsigned long flags;
621 
622         local_irq_save(flags);
623         (*_cache_line_loop_ic_fn)(paddr, vaddr, sz, OP_INV_IC, full_page);
624         local_irq_restore(flags);
625 }
626 
627 #ifndef CONFIG_SMP
628 
629 #define __ic_line_inv_vaddr(p, v, s)    __ic_line_inv_vaddr_local(p, v, s)
630 
631 #else
632 
633 struct ic_inv_args {
634         phys_addr_t paddr, vaddr;
635         int sz;
636 };
637 
638 static void __ic_line_inv_vaddr_helper(void *info)
639 {
640         struct ic_inv_args *ic_inv = info;
641 
642         __ic_line_inv_vaddr_local(ic_inv->paddr, ic_inv->vaddr, ic_inv->sz);
643 }
644 
645 static void __ic_line_inv_vaddr(phys_addr_t paddr, unsigned long vaddr,
646                                 unsigned long sz)
647 {
648         struct ic_inv_args ic_inv = {
649                 .paddr = paddr,
650                 .vaddr = vaddr,
651                 .sz    = sz
652         };
653 
654         on_each_cpu(__ic_line_inv_vaddr_helper, &ic_inv, 1);
655 }
656 
657 #endif  /* CONFIG_SMP */
658 
659 #else   /* !CONFIG_ARC_HAS_ICACHE */
660 
661 #define __ic_entire_inv()
662 #define __ic_line_inv_vaddr(pstart, vstart, sz)
663 
664 #endif /* CONFIG_ARC_HAS_ICACHE */
665 
666 noinline void slc_op_rgn(phys_addr_t paddr, unsigned long sz, const int op)
667 {
668 #ifdef CONFIG_ISA_ARCV2
669         /*
670          * SLC is shared between all cores and concurrent aux operations from
671          * multiple cores need to be serialized using a spinlock
672          * A concurrent operation can be silently ignored and/or the old/new
673          * operation can remain incomplete forever (lockup in SLC_CTRL_BUSY loop
674          * below)
675          */
676         static DEFINE_SPINLOCK(lock);
677         unsigned long flags;
678         unsigned int ctrl;
679         phys_addr_t end;
680 
681         spin_lock_irqsave(&lock, flags);
682 
683         /*
684          * The Region Flush operation is specified by CTRL.RGN_OP[11..9]
685          *  - b'000 (default) is Flush,
686          *  - b'001 is Invalidate if CTRL.IM == 0
687          *  - b'001 is Flush-n-Invalidate if CTRL.IM == 1
688          */
689         ctrl = read_aux_reg(ARC_REG_SLC_CTRL);
690 
691         /* Don't rely on default value of IM bit */
692         if (!(op & OP_FLUSH))           /* i.e. OP_INV */
693                 ctrl &= ~SLC_CTRL_IM;   /* clear IM: Disable flush before Inv */
694         else
695                 ctrl |= SLC_CTRL_IM;
696 
697         if (op & OP_INV)
698                 ctrl |= SLC_CTRL_RGN_OP_INV;    /* Inv or flush-n-inv */
699         else
700                 ctrl &= ~SLC_CTRL_RGN_OP_INV;
701 
702         write_aux_reg(ARC_REG_SLC_CTRL, ctrl);
703 
704         /*
705          * Lower bits are ignored, no need to clip
706          * END needs to be setup before START (latter triggers the operation)
707          * END can't be same as START, so add (l2_line_sz - 1) to sz
708          */
709         end = paddr + sz + l2_line_sz - 1;
710         if (is_pae40_enabled())
711                 write_aux_reg(ARC_REG_SLC_RGN_END1, upper_32_bits(end));
712 
713         write_aux_reg(ARC_REG_SLC_RGN_END, lower_32_bits(end));
714 
715         if (is_pae40_enabled())
716                 write_aux_reg(ARC_REG_SLC_RGN_START1, upper_32_bits(paddr));
717 
718         write_aux_reg(ARC_REG_SLC_RGN_START, lower_32_bits(paddr));
719 
720         /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */
721         read_aux_reg(ARC_REG_SLC_CTRL);
722 
723         while (read_aux_reg(ARC_REG_SLC_CTRL) & SLC_CTRL_BUSY);
724 
725         spin_unlock_irqrestore(&lock, flags);
726 #endif
727 }
728 
729 noinline void slc_op_line(phys_addr_t paddr, unsigned long sz, const int op)
730 {
731 #ifdef CONFIG_ISA_ARCV2
732         /*
733          * SLC is shared between all cores and concurrent aux operations from
734          * multiple cores need to be serialized using a spinlock
735          * A concurrent operation can be silently ignored and/or the old/new
736          * operation can remain incomplete forever (lockup in SLC_CTRL_BUSY loop
737          * below)
738          */
739         static DEFINE_SPINLOCK(lock);
740 
741         const unsigned long SLC_LINE_MASK = ~(l2_line_sz - 1);
742         unsigned int ctrl, cmd;
743         unsigned long flags;
744         int num_lines;
745 
746         spin_lock_irqsave(&lock, flags);
747 
748         ctrl = read_aux_reg(ARC_REG_SLC_CTRL);
749 
750         /* Don't rely on default value of IM bit */
751         if (!(op & OP_FLUSH))           /* i.e. OP_INV */
752                 ctrl &= ~SLC_CTRL_IM;   /* clear IM: Disable flush before Inv */
753         else
754                 ctrl |= SLC_CTRL_IM;
755 
756         write_aux_reg(ARC_REG_SLC_CTRL, ctrl);
757 
758         cmd = op & OP_INV ? ARC_AUX_SLC_IVDL : ARC_AUX_SLC_FLDL;
759 
760         sz += paddr & ~SLC_LINE_MASK;
761         paddr &= SLC_LINE_MASK;
762 
763         num_lines = DIV_ROUND_UP(sz, l2_line_sz);
764 
765         while (num_lines-- > 0) {
766                 write_aux_reg(cmd, paddr);
767                 paddr += l2_line_sz;
768         }
769 
770         /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */
771         read_aux_reg(ARC_REG_SLC_CTRL);
772 
773         while (read_aux_reg(ARC_REG_SLC_CTRL) & SLC_CTRL_BUSY);
774 
775         spin_unlock_irqrestore(&lock, flags);
776 #endif
777 }
778 
779 #define slc_op(paddr, sz, op)   slc_op_rgn(paddr, sz, op)
780 
781 noinline static void slc_entire_op(const int op)
782 {
783         unsigned int ctrl, r = ARC_REG_SLC_CTRL;
784 
785         ctrl = read_aux_reg(r);
786 
787         if (!(op & OP_FLUSH))           /* i.e. OP_INV */
788                 ctrl &= ~SLC_CTRL_IM;   /* clear IM: Disable flush before Inv */
789         else
790                 ctrl |= SLC_CTRL_IM;
791 
792         write_aux_reg(r, ctrl);
793 
794         if (op & OP_INV)        /* Inv or flush-n-inv use same cmd reg */
795                 write_aux_reg(ARC_REG_SLC_INVALIDATE, 0x1);
796         else
797                 write_aux_reg(ARC_REG_SLC_FLUSH, 0x1);
798 
799         /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */
800         read_aux_reg(r);
801 
802         /* Important to wait for flush to complete */
803         while (read_aux_reg(r) & SLC_CTRL_BUSY);
804 }
805 
806 static inline void arc_slc_disable(void)
807 {
808         const int r = ARC_REG_SLC_CTRL;
809 
810         slc_entire_op(OP_FLUSH_N_INV);
811         write_aux_reg(r, read_aux_reg(r) | SLC_CTRL_DIS);
812 }
813 
814 static inline void arc_slc_enable(void)
815 {
816         const int r = ARC_REG_SLC_CTRL;
817 
818         write_aux_reg(r, read_aux_reg(r) & ~SLC_CTRL_DIS);
819 }
820 
821 /***********************************************************
822  * Exported APIs
823  */
824 
825 /*
826  * Handle cache congruency of kernel and userspace mappings of page when kernel
827  * writes-to/reads-from
828  *
829  * The idea is to defer flushing of kernel mapping after a WRITE, possible if:
830  *  -dcache is NOT aliasing, hence any U/K-mappings of page are congruent
831  *  -U-mapping doesn't exist yet for page (finalised in update_mmu_cache)
832  *  -In SMP, if hardware caches are coherent
833  *
834  * There's a corollary case, where kernel READs from a userspace mapped page.
835  * If the U-mapping is not congruent to to K-mapping, former needs flushing.
836  */
837 void flush_dcache_page(struct page *page)
838 {
839         struct address_space *mapping;
840 
841         if (!cache_is_vipt_aliasing()) {
842                 clear_bit(PG_dc_clean, &page->flags);
843                 return;
844         }
845 
846         /* don't handle anon pages here */
847         mapping = page_mapping_file(page);
848         if (!mapping)
849                 return;
850 
851         /*
852          * pagecache page, file not yet mapped to userspace
853          * Make a note that K-mapping is dirty
854          */
855         if (!mapping_mapped(mapping)) {
856                 clear_bit(PG_dc_clean, &page->flags);
857         } else if (page_mapcount(page)) {
858 
859                 /* kernel reading from page with U-mapping */
860                 phys_addr_t paddr = (unsigned long)page_address(page);
861                 unsigned long vaddr = page->index << PAGE_SHIFT;
862 
863                 if (addr_not_cache_congruent(paddr, vaddr))
864                         __flush_dcache_page(paddr, vaddr);
865         }
866 }
867 EXPORT_SYMBOL(flush_dcache_page);
868 
869 /*
870  * DMA ops for systems with L1 cache only
871  * Make memory coherent with L1 cache by flushing/invalidating L1 lines
872  */
873 static void __dma_cache_wback_inv_l1(phys_addr_t start, unsigned long sz)
874 {
875         __dc_line_op_k(start, sz, OP_FLUSH_N_INV);
876 }
877 
878 static void __dma_cache_inv_l1(phys_addr_t start, unsigned long sz)
879 {
880         __dc_line_op_k(start, sz, OP_INV);
881 }
882 
883 static void __dma_cache_wback_l1(phys_addr_t start, unsigned long sz)
884 {
885         __dc_line_op_k(start, sz, OP_FLUSH);
886 }
887 
888 /*
889  * DMA ops for systems with both L1 and L2 caches, but without IOC
890  * Both L1 and L2 lines need to be explicitly flushed/invalidated
891  */
892 static void __dma_cache_wback_inv_slc(phys_addr_t start, unsigned long sz)
893 {
894         __dc_line_op_k(start, sz, OP_FLUSH_N_INV);
895         slc_op(start, sz, OP_FLUSH_N_INV);
896 }
897 
898 static void __dma_cache_inv_slc(phys_addr_t start, unsigned long sz)
899 {
900         __dc_line_op_k(start, sz, OP_INV);
901         slc_op(start, sz, OP_INV);
902 }
903 
904 static void __dma_cache_wback_slc(phys_addr_t start, unsigned long sz)
905 {
906         __dc_line_op_k(start, sz, OP_FLUSH);
907         slc_op(start, sz, OP_FLUSH);
908 }
909 
910 /*
911  * Exported DMA API
912  */
913 void dma_cache_wback_inv(phys_addr_t start, unsigned long sz)
914 {
915         __dma_cache_wback_inv(start, sz);
916 }
917 EXPORT_SYMBOL(dma_cache_wback_inv);
918 
919 void dma_cache_inv(phys_addr_t start, unsigned long sz)
920 {
921         __dma_cache_inv(start, sz);
922 }
923 EXPORT_SYMBOL(dma_cache_inv);
924 
925 void dma_cache_wback(phys_addr_t start, unsigned long sz)
926 {
927         __dma_cache_wback(start, sz);
928 }
929 EXPORT_SYMBOL(dma_cache_wback);
930 
931 /*
932  * This is API for making I/D Caches consistent when modifying
933  * kernel code (loadable modules, kprobes, kgdb...)
934  * This is called on insmod, with kernel virtual address for CODE of
935  * the module. ARC cache maintenance ops require PHY address thus we
936  * need to convert vmalloc addr to PHY addr
937  */
938 void flush_icache_range(unsigned long kstart, unsigned long kend)
939 {
940         unsigned int tot_sz;
941 
942         WARN(kstart < TASK_SIZE, "%s() can't handle user vaddr", __func__);
943 
944         /* Shortcut for bigger flush ranges.
945          * Here we don't care if this was kernel virtual or phy addr
946          */
947         tot_sz = kend - kstart;
948         if (tot_sz > PAGE_SIZE) {
949                 flush_cache_all();
950                 return;
951         }
952 
953         /* Case: Kernel Phy addr (0x8000_0000 onwards) */
954         if (likely(kstart > PAGE_OFFSET)) {
955                 /*
956                  * The 2nd arg despite being paddr will be used to index icache
957                  * This is OK since no alternate virtual mappings will exist
958                  * given the callers for this case: kprobe/kgdb in built-in
959                  * kernel code only.
960                  */
961                 __sync_icache_dcache(kstart, kstart, kend - kstart);
962                 return;
963         }
964 
965         /*
966          * Case: Kernel Vaddr (0x7000_0000 to 0x7fff_ffff)
967          * (1) ARC Cache Maintenance ops only take Phy addr, hence special
968          *     handling of kernel vaddr.
969          *
970          * (2) Despite @tot_sz being < PAGE_SIZE (bigger cases handled already),
971          *     it still needs to handle  a 2 page scenario, where the range
972          *     straddles across 2 virtual pages and hence need for loop
973          */
974         while (tot_sz > 0) {
975                 unsigned int off, sz;
976                 unsigned long phy, pfn;
977 
978                 off = kstart % PAGE_SIZE;
979                 pfn = vmalloc_to_pfn((void *)kstart);
980                 phy = (pfn << PAGE_SHIFT) + off;
981                 sz = min_t(unsigned int, tot_sz, PAGE_SIZE - off);
982                 __sync_icache_dcache(phy, kstart, sz);
983                 kstart += sz;
984                 tot_sz -= sz;
985         }
986 }
987 EXPORT_SYMBOL(flush_icache_range);
988 
989 /*
990  * General purpose helper to make I and D cache lines consistent.
991  * @paddr is phy addr of region
992  * @vaddr is typically user vaddr (breakpoint) or kernel vaddr (vmalloc)
993  *    However in one instance, when called by kprobe (for a breakpt in
994  *    builtin kernel code) @vaddr will be paddr only, meaning CDU operation will
995  *    use a paddr to index the cache (despite VIPT). This is fine since since a
996  *    builtin kernel page will not have any virtual mappings.
997  *    kprobe on loadable module will be kernel vaddr.
998  */
999 void __sync_icache_dcache(phys_addr_t paddr, unsigned long vaddr, int len)
1000 {
1001         __dc_line_op(paddr, vaddr, len, OP_FLUSH_N_INV);
1002         __ic_line_inv_vaddr(paddr, vaddr, len);
1003 }
1004 
1005 /* wrapper to compile time eliminate alignment checks in flush loop */
1006 void __inv_icache_page(phys_addr_t paddr, unsigned long vaddr)
1007 {
1008         __ic_line_inv_vaddr(paddr, vaddr, PAGE_SIZE);
1009 }
1010 
1011 /*
1012  * wrapper to clearout kernel or userspace mappings of a page
1013  * For kernel mappings @vaddr == @paddr
1014  */
1015 void __flush_dcache_page(phys_addr_t paddr, unsigned long vaddr)
1016 {
1017         __dc_line_op(paddr, vaddr & PAGE_MASK, PAGE_SIZE, OP_FLUSH_N_INV);
1018 }
1019 
1020 noinline void flush_cache_all(void)
1021 {
1022         unsigned long flags;
1023 
1024         local_irq_save(flags);
1025 
1026         __ic_entire_inv();
1027         __dc_entire_op(OP_FLUSH_N_INV);
1028 
1029         local_irq_restore(flags);
1030 
1031 }
1032 
1033 #ifdef CONFIG_ARC_CACHE_VIPT_ALIASING
1034 
1035 void flush_cache_mm(struct mm_struct *mm)
1036 {
1037         flush_cache_all();
1038 }
1039 
1040 void flush_cache_page(struct vm_area_struct *vma, unsigned long u_vaddr,
1041                       unsigned long pfn)
1042 {
1043         phys_addr_t paddr = pfn << PAGE_SHIFT;
1044 
1045         u_vaddr &= PAGE_MASK;
1046 
1047         __flush_dcache_page(paddr, u_vaddr);
1048 
1049         if (vma->vm_flags & VM_EXEC)
1050                 __inv_icache_page(paddr, u_vaddr);
1051 }
1052 
1053 void flush_cache_range(struct vm_area_struct *vma, unsigned long start,
1054                        unsigned long end)
1055 {
1056         flush_cache_all();
1057 }
1058 
1059 void flush_anon_page(struct vm_area_struct *vma, struct page *page,
1060                      unsigned long u_vaddr)
1061 {
1062         /* TBD: do we really need to clear the kernel mapping */
1063         __flush_dcache_page((phys_addr_t)page_address(page), u_vaddr);
1064         __flush_dcache_page((phys_addr_t)page_address(page),
1065                             (phys_addr_t)page_address(page));
1066 
1067 }
1068 
1069 #endif
1070 
1071 void copy_user_highpage(struct page *to, struct page *from,
1072         unsigned long u_vaddr, struct vm_area_struct *vma)
1073 {
1074         void *kfrom = kmap_atomic(from);
1075         void *kto = kmap_atomic(to);
1076         int clean_src_k_mappings = 0;
1077 
1078         /*
1079          * If SRC page was already mapped in userspace AND it's U-mapping is
1080          * not congruent with K-mapping, sync former to physical page so that
1081          * K-mapping in memcpy below, sees the right data
1082          *
1083          * Note that while @u_vaddr refers to DST page's userspace vaddr, it is
1084          * equally valid for SRC page as well
1085          *
1086          * For !VIPT cache, all of this gets compiled out as
1087          * addr_not_cache_congruent() is 0
1088          */
1089         if (page_mapcount(from) && addr_not_cache_congruent(kfrom, u_vaddr)) {
1090                 __flush_dcache_page((unsigned long)kfrom, u_vaddr);
1091                 clean_src_k_mappings = 1;
1092         }
1093 
1094         copy_page(kto, kfrom);
1095 
1096         /*
1097          * Mark DST page K-mapping as dirty for a later finalization by
1098          * update_mmu_cache(). Although the finalization could have been done
1099          * here as well (given that both vaddr/paddr are available).
1100          * But update_mmu_cache() already has code to do that for other
1101          * non copied user pages (e.g. read faults which wire in pagecache page
1102          * directly).
1103          */
1104         clear_bit(PG_dc_clean, &to->flags);
1105 
1106         /*
1107          * if SRC was already usermapped and non-congruent to kernel mapping
1108          * sync the kernel mapping back to physical page
1109          */
1110         if (clean_src_k_mappings) {
1111                 __flush_dcache_page((unsigned long)kfrom, (unsigned long)kfrom);
1112                 set_bit(PG_dc_clean, &from->flags);
1113         } else {
1114                 clear_bit(PG_dc_clean, &from->flags);
1115         }
1116 
1117         kunmap_atomic(kto);
1118         kunmap_atomic(kfrom);
1119 }
1120 
1121 void clear_user_page(void *to, unsigned long u_vaddr, struct page *page)
1122 {
1123         clear_page(to);
1124         clear_bit(PG_dc_clean, &page->flags);
1125 }
1126 
1127 
1128 /**********************************************************************
1129  * Explicit Cache flush request from user space via syscall
1130  * Needed for JITs which generate code on the fly
1131  */
1132 SYSCALL_DEFINE3(cacheflush, uint32_t, start, uint32_t, sz, uint32_t, flags)
1133 {
1134         /* TBD: optimize this */
1135         flush_cache_all();
1136         return 0;
1137 }
1138 
1139 /*
1140  * IO-Coherency (IOC) setup rules:
1141  *
1142  * 1. Needs to be at system level, so only once by Master core
1143  *    Non-Masters need not be accessing caches at that time
1144  *    - They are either HALT_ON_RESET and kick started much later or
1145  *    - if run on reset, need to ensure that arc_platform_smp_wait_to_boot()
1146  *      doesn't perturb caches or coherency unit
1147  *
1148  * 2. caches (L1 and SLC) need to be purged (flush+inv) before setting up IOC,
1149  *    otherwise any straggler data might behave strangely post IOC enabling
1150  *
1151  * 3. All Caches need to be disabled when setting up IOC to elide any in-flight
1152  *    Coherency transactions
1153  */
1154 noinline void __init arc_ioc_setup(void)
1155 {
1156         unsigned int ioc_base, mem_sz;
1157 
1158         /*
1159          * If IOC was already enabled (due to bootloader) it technically needs to
1160          * be reconfigured with aperture base,size corresponding to Linux memory map
1161          * which will certainly be different than uboot's. But disabling and
1162          * reenabling IOC when DMA might be potentially active is tricky business.
1163          * To avoid random memory issues later, just panic here and ask user to
1164          * upgrade bootloader to one which doesn't enable IOC
1165          */
1166         if (read_aux_reg(ARC_REG_IO_COH_ENABLE) & ARC_IO_COH_ENABLE_BIT)
1167                 panic("IOC already enabled, please upgrade bootloader!\n");
1168 
1169         if (!ioc_enable)
1170                 return;
1171 
1172         /* Flush + invalidate + disable L1 dcache */
1173         __dc_disable();
1174 
1175         /* Flush + invalidate SLC */
1176         if (read_aux_reg(ARC_REG_SLC_BCR))
1177                 slc_entire_op(OP_FLUSH_N_INV);
1178 
1179         /*
1180          * currently IOC Aperture covers entire DDR
1181          * TBD: fix for PGU + 1GB of low mem
1182          * TBD: fix for PAE
1183          */
1184         mem_sz = arc_get_mem_sz();
1185 
1186         if (!is_power_of_2(mem_sz) || mem_sz < 4096)
1187                 panic("IOC Aperture size must be power of 2 larger than 4KB");
1188 
1189         /*
1190          * IOC Aperture size decoded as 2 ^ (SIZE + 2) KB,
1191          * so setting 0x11 implies 512MB, 0x12 implies 1GB...
1192          */
1193         write_aux_reg(ARC_REG_IO_COH_AP0_SIZE, order_base_2(mem_sz >> 10) - 2);
1194 
1195         /* for now assume kernel base is start of IOC aperture */
1196         ioc_base = CONFIG_LINUX_RAM_BASE;
1197 
1198         if (ioc_base % mem_sz != 0)
1199                 panic("IOC Aperture start must be aligned to the size of the aperture");
1200 
1201         write_aux_reg(ARC_REG_IO_COH_AP0_BASE, ioc_base >> 12);
1202         write_aux_reg(ARC_REG_IO_COH_PARTIAL, ARC_IO_COH_PARTIAL_BIT);
1203         write_aux_reg(ARC_REG_IO_COH_ENABLE, ARC_IO_COH_ENABLE_BIT);
1204 
1205         /* Re-enable L1 dcache */
1206         __dc_enable();
1207 }
1208 
1209 /*
1210  * Cache related boot time checks/setups only needed on master CPU:
1211  *  - Geometry checks (kernel build and hardware agree: e.g. L1_CACHE_BYTES)
1212  *    Assume SMP only, so all cores will have same cache config. A check on
1213  *    one core suffices for all
1214  *  - IOC setup / dma callbacks only need to be done once
1215  */
1216 void __init arc_cache_init_master(void)
1217 {
1218         unsigned int __maybe_unused cpu = smp_processor_id();
1219 
1220         if (IS_ENABLED(CONFIG_ARC_HAS_ICACHE)) {
1221                 struct cpuinfo_arc_cache *ic = &cpuinfo_arc700[cpu].icache;
1222 
1223                 if (!ic->line_len)
1224                         panic("cache support enabled but non-existent cache\n");
1225 
1226                 if (ic->line_len != L1_CACHE_BYTES)
1227                         panic("ICache line [%d] != kernel Config [%d]",
1228                               ic->line_len, L1_CACHE_BYTES);
1229 
1230                 /*
1231                  * In MMU v4 (HS38x) the aliasing icache config uses IVIL/PTAG
1232                  * pair to provide vaddr/paddr respectively, just as in MMU v3
1233                  */
1234                 if (is_isa_arcv2() && ic->alias)
1235                         _cache_line_loop_ic_fn = __cache_line_loop_v3;
1236                 else
1237                         _cache_line_loop_ic_fn = __cache_line_loop;
1238         }
1239 
1240         if (IS_ENABLED(CONFIG_ARC_HAS_DCACHE)) {
1241                 struct cpuinfo_arc_cache *dc = &cpuinfo_arc700[cpu].dcache;
1242 
1243                 if (!dc->line_len)
1244                         panic("cache support enabled but non-existent cache\n");
1245 
1246                 if (dc->line_len != L1_CACHE_BYTES)
1247                         panic("DCache line [%d] != kernel Config [%d]",
1248                               dc->line_len, L1_CACHE_BYTES);
1249 
1250                 /* check for D-Cache aliasing on ARCompact: ARCv2 has PIPT */
1251                 if (is_isa_arcompact()) {
1252                         int handled = IS_ENABLED(CONFIG_ARC_CACHE_VIPT_ALIASING);
1253                         int num_colors = dc->sz_k/dc->assoc/TO_KB(PAGE_SIZE);
1254 
1255                         if (dc->alias) {
1256                                 if (!handled)
1257                                         panic("Enable CONFIG_ARC_CACHE_VIPT_ALIASING\n");
1258                                 if (CACHE_COLORS_NUM != num_colors)
1259                                         panic("CACHE_COLORS_NUM not optimized for config\n");
1260                         } else if (!dc->alias && handled) {
1261                                 panic("Disable CONFIG_ARC_CACHE_VIPT_ALIASING\n");
1262                         }
1263                 }
1264         }
1265 
1266         /*
1267          * Check that SMP_CACHE_BYTES (and hence ARCH_DMA_MINALIGN) is larger
1268          * or equal to any cache line length.
1269          */
1270         BUILD_BUG_ON_MSG(L1_CACHE_BYTES > SMP_CACHE_BYTES,
1271                          "SMP_CACHE_BYTES must be >= any cache line length");
1272         if (is_isa_arcv2() && (l2_line_sz > SMP_CACHE_BYTES))
1273                 panic("L2 Cache line [%d] > kernel Config [%d]\n",
1274                       l2_line_sz, SMP_CACHE_BYTES);
1275 
1276         /* Note that SLC disable not formally supported till HS 3.0 */
1277         if (is_isa_arcv2() && l2_line_sz && !slc_enable)
1278                 arc_slc_disable();
1279 
1280         if (is_isa_arcv2() && ioc_exists)
1281                 arc_ioc_setup();
1282 
1283         if (is_isa_arcv2() && l2_line_sz && slc_enable) {
1284                 __dma_cache_wback_inv = __dma_cache_wback_inv_slc;
1285                 __dma_cache_inv = __dma_cache_inv_slc;
1286                 __dma_cache_wback = __dma_cache_wback_slc;
1287         } else {
1288                 __dma_cache_wback_inv = __dma_cache_wback_inv_l1;
1289                 __dma_cache_inv = __dma_cache_inv_l1;
1290                 __dma_cache_wback = __dma_cache_wback_l1;
1291         }
1292         /*
1293          * In case of IOC (say IOC+SLC case), pointers above could still be set
1294          * but end up not being relevant as the first function in chain is not
1295          * called at all for devices using coherent DMA.
1296          *     arch_sync_dma_for_cpu() -> dma_cache_*() -> __dma_cache_*()
1297          */
1298 }
1299 
1300 void __ref arc_cache_init(void)
1301 {
1302         unsigned int __maybe_unused cpu = smp_processor_id();
1303         char str[256];
1304 
1305         pr_info("%s", arc_cache_mumbojumbo(0, str, sizeof(str)));
1306 
1307         if (!cpu)
1308                 arc_cache_init_master();
1309 
1310         /*
1311          * In PAE regime, TLB and cache maintenance ops take wider addresses
1312          * And even if PAE is not enabled in kernel, the upper 32-bits still need
1313          * to be zeroed to keep the ops sane.
1314          * As an optimization for more common !PAE enabled case, zero them out
1315          * once at init, rather than checking/setting to 0 for every runtime op
1316          */
1317         if (is_isa_arcv2() && pae40_exist_but_not_enab()) {
1318 
1319                 if (IS_ENABLED(CONFIG_ARC_HAS_ICACHE))
1320                         write_aux_reg(ARC_REG_IC_PTAG_HI, 0);
1321 
1322                 if (IS_ENABLED(CONFIG_ARC_HAS_DCACHE))
1323                         write_aux_reg(ARC_REG_DC_PTAG_HI, 0);
1324 
1325                 if (l2_line_sz) {
1326                         write_aux_reg(ARC_REG_SLC_RGN_END1, 0);
1327                         write_aux_reg(ARC_REG_SLC_RGN_START1, 0);
1328                 }
1329         }
1330 }
1331 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp