~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/mips/include/asm/octeon/cvmx-pow.h

Version: ~ [ linux-5.4-rc7 ] ~ [ linux-5.3.10 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.83 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.153 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.200 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.200 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.76 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /***********************license start***************
  2  * Author: Cavium Networks
  3  *
  4  * Contact: support@caviumnetworks.com
  5  * This file is part of the OCTEON SDK
  6  *
  7  * Copyright (c) 2003-2008 Cavium Networks
  8  *
  9  * This file is free software; you can redistribute it and/or modify
 10  * it under the terms of the GNU General Public License, Version 2, as
 11  * published by the Free Software Foundation.
 12  *
 13  * This file is distributed in the hope that it will be useful, but
 14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
 15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
 16  * NONINFRINGEMENT.  See the GNU General Public License for more
 17  * details.
 18  *
 19  * You should have received a copy of the GNU General Public License
 20  * along with this file; if not, write to the Free Software
 21  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
 22  * or visit http://www.gnu.org/licenses/.
 23  *
 24  * This file may also be available under a different license from Cavium.
 25  * Contact Cavium Networks for more information
 26  ***********************license end**************************************/
 27 
 28 /**
 29  * Interface to the hardware Packet Order / Work unit.
 30  *
 31  * New, starting with SDK 1.7.0, cvmx-pow supports a number of
 32  * extended consistency checks. The define
 33  * CVMX_ENABLE_POW_CHECKS controls the runtime insertion of POW
 34  * internal state checks to find common programming errors. If
 35  * CVMX_ENABLE_POW_CHECKS is not defined, checks are by default
 36  * enabled. For example, cvmx-pow will check for the following
 37  * program errors or POW state inconsistency.
 38  * - Requesting a POW operation with an active tag switch in
 39  *   progress.
 40  * - Waiting for a tag switch to complete for an excessively
 41  *   long period. This is normally a sign of an error in locking
 42  *   causing deadlock.
 43  * - Illegal tag switches from NULL_NULL.
 44  * - Illegal tag switches from NULL.
 45  * - Illegal deschedule request.
 46  * - WQE pointer not matching the one attached to the core by
 47  *   the POW.
 48  *
 49  */
 50 
 51 #ifndef __CVMX_POW_H__
 52 #define __CVMX_POW_H__
 53 
 54 #include <asm/octeon/cvmx-pow-defs.h>
 55 
 56 #include <asm/octeon/cvmx-scratch.h>
 57 #include <asm/octeon/cvmx-wqe.h>
 58 
 59 /* Default to having all POW constancy checks turned on */
 60 #ifndef CVMX_ENABLE_POW_CHECKS
 61 #define CVMX_ENABLE_POW_CHECKS 1
 62 #endif
 63 
 64 enum cvmx_pow_tag_type {
 65         /* Tag ordering is maintained */
 66         CVMX_POW_TAG_TYPE_ORDERED   = 0L,
 67         /* Tag ordering is maintained, and at most one PP has the tag */
 68         CVMX_POW_TAG_TYPE_ATOMIC    = 1L,
 69         /*
 70          * The work queue entry from the order - NEVER tag switch from
 71          * NULL to NULL
 72          */
 73         CVMX_POW_TAG_TYPE_NULL      = 2L,
 74         /* A tag switch to NULL, and there is no space reserved in POW
 75          * - NEVER tag switch to NULL_NULL
 76          * - NEVER tag switch from NULL_NULL
 77          * - NULL_NULL is entered at the beginning of time and on a deschedule.
 78          * - NULL_NULL can be exited by a new work request. A NULL_SWITCH
 79          * load can also switch the state to NULL
 80          */
 81         CVMX_POW_TAG_TYPE_NULL_NULL = 3L
 82 };
 83 
 84 /**
 85  * Wait flag values for pow functions.
 86  */
 87 typedef enum {
 88         CVMX_POW_WAIT = 1,
 89         CVMX_POW_NO_WAIT = 0,
 90 } cvmx_pow_wait_t;
 91 
 92 /**
 93  *  POW tag operations.  These are used in the data stored to the POW.
 94  */
 95 typedef enum {
 96         /*
 97          * switch the tag (only) for this PP
 98          * - the previous tag should be non-NULL in this case
 99          * - tag switch response required
100          * - fields used: op, type, tag
101          */
102         CVMX_POW_TAG_OP_SWTAG = 0L,
103         /*
104          * switch the tag for this PP, with full information
105          * - this should be used when the previous tag is NULL
106          * - tag switch response required
107          * - fields used: address, op, grp, type, tag
108          */
109         CVMX_POW_TAG_OP_SWTAG_FULL = 1L,
110         /*
111          * switch the tag (and/or group) for this PP and de-schedule
112          * - OK to keep the tag the same and only change the group
113          * - fields used: op, no_sched, grp, type, tag
114          */
115         CVMX_POW_TAG_OP_SWTAG_DESCH = 2L,
116         /*
117          * just de-schedule
118          * - fields used: op, no_sched
119          */
120         CVMX_POW_TAG_OP_DESCH = 3L,
121         /*
122          * create an entirely new work queue entry
123          * - fields used: address, op, qos, grp, type, tag
124          */
125         CVMX_POW_TAG_OP_ADDWQ = 4L,
126         /*
127          * just update the work queue pointer and grp for this PP
128          * - fields used: address, op, grp
129          */
130         CVMX_POW_TAG_OP_UPDATE_WQP_GRP = 5L,
131         /*
132          * set the no_sched bit on the de-schedule list
133          *
134          * - does nothing if the selected entry is not on the
135          *   de-schedule list
136          *
137          * - does nothing if the stored work queue pointer does not
138          *   match the address field
139          *
140          * - fields used: address, index, op
141          *
142          *  Before issuing a *_NSCHED operation, SW must guarantee
143          *  that all prior deschedules and set/clr NSCHED operations
144          *  are complete and all prior switches are complete. The
145          *  hardware provides the opsdone bit and swdone bit for SW
146          *  polling. After issuing a *_NSCHED operation, SW must
147          *  guarantee that the set/clr NSCHED is complete before any
148          *  subsequent operations.
149          */
150         CVMX_POW_TAG_OP_SET_NSCHED = 6L,
151         /*
152          * clears the no_sched bit on the de-schedule list
153          *
154          * - does nothing if the selected entry is not on the
155          *   de-schedule list
156          *
157          * - does nothing if the stored work queue pointer does not
158          *   match the address field
159          *
160          * - fields used: address, index, op
161          *
162          * Before issuing a *_NSCHED operation, SW must guarantee that
163          * all prior deschedules and set/clr NSCHED operations are
164          * complete and all prior switches are complete. The hardware
165          * provides the opsdone bit and swdone bit for SW
166          * polling. After issuing a *_NSCHED operation, SW must
167          * guarantee that the set/clr NSCHED is complete before any
168          * subsequent operations.
169          */
170         CVMX_POW_TAG_OP_CLR_NSCHED = 7L,
171         /* do nothing */
172         CVMX_POW_TAG_OP_NOP = 15L
173 } cvmx_pow_tag_op_t;
174 
175 /**
176  * This structure defines the store data on a store to POW
177  */
178 typedef union {
179         uint64_t u64;
180         struct {
181                 /*
182                  * Don't reschedule this entry. no_sched is used for
183                  * CVMX_POW_TAG_OP_SWTAG_DESCH and
184                  * CVMX_POW_TAG_OP_DESCH
185                  */
186                 uint64_t no_sched:1;
187                 uint64_t unused:2;
188                 /* Tontains index of entry for a CVMX_POW_TAG_OP_*_NSCHED */
189                 uint64_t index:13;
190                 /* The operation to perform */
191                 cvmx_pow_tag_op_t op:4;
192                 uint64_t unused2:2;
193                 /*
194                  * The QOS level for the packet. qos is only used for
195                  * CVMX_POW_TAG_OP_ADDWQ
196                  */
197                 uint64_t qos:3;
198                 /*
199                  * The group that the work queue entry will be
200                  * scheduled to grp is used for CVMX_POW_TAG_OP_ADDWQ,
201                  * CVMX_POW_TAG_OP_SWTAG_FULL,
202                  * CVMX_POW_TAG_OP_SWTAG_DESCH, and
203                  * CVMX_POW_TAG_OP_UPDATE_WQP_GRP
204                  */
205                 uint64_t grp:4;
206                 /*
207                  * The type of the tag. type is used for everything
208                  * except CVMX_POW_TAG_OP_DESCH,
209                  * CVMX_POW_TAG_OP_UPDATE_WQP_GRP, and
210                  * CVMX_POW_TAG_OP_*_NSCHED
211                  */
212                 uint64_t type:3;
213                 /*
214                  * The actual tag. tag is used for everything except
215                  * CVMX_POW_TAG_OP_DESCH,
216                  * CVMX_POW_TAG_OP_UPDATE_WQP_GRP, and
217                  * CVMX_POW_TAG_OP_*_NSCHED
218                  */
219                 uint64_t tag:32;
220         } s;
221 } cvmx_pow_tag_req_t;
222 
223 /**
224  * This structure describes the address to load stuff from POW
225  */
226 typedef union {
227         uint64_t u64;
228 
229     /**
230      * Address for new work request loads (did<2:0> == 0)
231      */
232         struct {
233                 /* Mips64 address region. Should be CVMX_IO_SEG */
234                 uint64_t mem_region:2;
235                 /* Must be zero */
236                 uint64_t reserved_49_61:13;
237                 /* Must be one */
238                 uint64_t is_io:1;
239                 /* the ID of POW -- did<2:0> == 0 in this case */
240                 uint64_t did:8;
241                 /* Must be zero */
242                 uint64_t reserved_4_39:36;
243                 /*
244                  * If set, don't return load response until work is
245                  * available.
246                  */
247                 uint64_t wait:1;
248                 /* Must be zero */
249                 uint64_t reserved_0_2:3;
250         } swork;
251 
252     /**
253      * Address for loads to get POW internal status
254      */
255         struct {
256                 /* Mips64 address region. Should be CVMX_IO_SEG */
257                 uint64_t mem_region:2;
258                 /* Must be zero */
259                 uint64_t reserved_49_61:13;
260                 /* Must be one */
261                 uint64_t is_io:1;
262                 /* the ID of POW -- did<2:0> == 1 in this case */
263                 uint64_t did:8;
264                 /* Must be zero */
265                 uint64_t reserved_10_39:30;
266                 /* The core id to get status for */
267                 uint64_t coreid:4;
268                 /*
269                  * If set and get_cur is set, return reverse tag-list
270                  * pointer rather than forward tag-list pointer.
271                  */
272                 uint64_t get_rev:1;
273                 /*
274                  * If set, return current status rather than pending
275                  * status.
276                  */
277                 uint64_t get_cur:1;
278                 /*
279                  * If set, get the work-queue pointer rather than
280                  * tag/type.
281                  */
282                 uint64_t get_wqp:1;
283                 /* Must be zero */
284                 uint64_t reserved_0_2:3;
285         } sstatus;
286 
287     /**
288      * Address for memory loads to get POW internal state
289      */
290         struct {
291                 /* Mips64 address region. Should be CVMX_IO_SEG */
292                 uint64_t mem_region:2;
293                 /* Must be zero */
294                 uint64_t reserved_49_61:13;
295                 /* Must be one */
296                 uint64_t is_io:1;
297                 /* the ID of POW -- did<2:0> == 2 in this case */
298                 uint64_t did:8;
299                 /* Must be zero */
300                 uint64_t reserved_16_39:24;
301                 /* POW memory index */
302                 uint64_t index:11;
303                 /*
304                  * If set, return deschedule information rather than
305                  * the standard response for work-queue index (invalid
306                  * if the work-queue entry is not on the deschedule
307                  * list).
308                  */
309                 uint64_t get_des:1;
310                 /*
311                  * If set, get the work-queue pointer rather than
312                  * tag/type (no effect when get_des set).
313                  */
314                 uint64_t get_wqp:1;
315                 /* Must be zero */
316                 uint64_t reserved_0_2:3;
317         } smemload;
318 
319     /**
320      * Address for index/pointer loads
321      */
322         struct {
323                 /* Mips64 address region. Should be CVMX_IO_SEG */
324                 uint64_t mem_region:2;
325                 /* Must be zero */
326                 uint64_t reserved_49_61:13;
327                 /* Must be one */
328                 uint64_t is_io:1;
329                 /* the ID of POW -- did<2:0> == 3 in this case */
330                 uint64_t did:8;
331                 /* Must be zero */
332                 uint64_t reserved_9_39:31;
333                 /*
334                  * when {get_rmt ==0 AND get_des_get_tail == 0}, this
335                  * field selects one of eight POW internal-input
336                  * queues (0-7), one per QOS level; values 8-15 are
337                  * illegal in this case; when {get_rmt ==0 AND
338                  * get_des_get_tail == 1}, this field selects one of
339                  * 16 deschedule lists (per group); when get_rmt ==1,
340                  * this field selects one of 16 memory-input queue
341                  * lists.  The two memory-input queue lists associated
342                  * with each QOS level are:
343                  *
344                  * - qosgrp = 0, qosgrp = 8:      QOS0
345                  * - qosgrp = 1, qosgrp = 9:      QOS1
346                  * - qosgrp = 2, qosgrp = 10:     QOS2
347                  * - qosgrp = 3, qosgrp = 11:     QOS3
348                  * - qosgrp = 4, qosgrp = 12:     QOS4
349                  * - qosgrp = 5, qosgrp = 13:     QOS5
350                  * - qosgrp = 6, qosgrp = 14:     QOS6
351                  * - qosgrp = 7, qosgrp = 15:     QOS7
352                  */
353                 uint64_t qosgrp:4;
354                 /*
355                  * If set and get_rmt is clear, return deschedule list
356                  * indexes rather than indexes for the specified qos
357                  * level; if set and get_rmt is set, return the tail
358                  * pointer rather than the head pointer for the
359                  * specified qos level.
360                  */
361                 uint64_t get_des_get_tail:1;
362                 /*
363                  * If set, return remote pointers rather than the
364                  * local indexes for the specified qos level.
365                  */
366                 uint64_t get_rmt:1;
367                 /* Must be zero */
368                 uint64_t reserved_0_2:3;
369         } sindexload;
370 
371     /**
372      * address for NULL_RD request (did<2:0> == 4) when this is read,
373      * HW attempts to change the state to NULL if it is NULL_NULL (the
374      * hardware cannot switch from NULL_NULL to NULL if a POW entry is
375      * not available - software may need to recover by finishing
376      * another piece of work before a POW entry can ever become
377      * available.)
378      */
379         struct {
380                 /* Mips64 address region. Should be CVMX_IO_SEG */
381                 uint64_t mem_region:2;
382                 /* Must be zero */
383                 uint64_t reserved_49_61:13;
384                 /* Must be one */
385                 uint64_t is_io:1;
386                 /* the ID of POW -- did<2:0> == 4 in this case */
387                 uint64_t did:8;
388                 /* Must be zero */
389                 uint64_t reserved_0_39:40;
390         } snull_rd;
391 } cvmx_pow_load_addr_t;
392 
393 /**
394  * This structure defines the response to a load/SENDSINGLE to POW
395  * (except CSR reads)
396  */
397 typedef union {
398         uint64_t u64;
399 
400     /**
401      * Response to new work request loads
402      */
403         struct {
404                 /*
405                  * Set when no new work queue entry was returned.  *
406                  * If there was de-scheduled work, the HW will
407                  * definitely return it. When this bit is set, it
408                  * could mean either mean:
409                  *
410                  * - There was no work, or
411                  *
412                  * - There was no work that the HW could find. This
413                  *   case can happen, regardless of the wait bit value
414                  *   in the original request, when there is work in
415                  *   the IQ's that is too deep down the list.
416                  */
417                 uint64_t no_work:1;
418                 /* Must be zero */
419                 uint64_t reserved_40_62:23;
420                 /* 36 in O1 -- the work queue pointer */
421                 uint64_t addr:40;
422         } s_work;
423 
424     /**
425      * Result for a POW Status Load (when get_cur==0 and get_wqp==0)
426      */
427         struct {
428                 uint64_t reserved_62_63:2;
429                 /* Set when there is a pending non-NULL SWTAG or
430                  * SWTAG_FULL, and the POW entry has not left the list
431                  * for the original tag. */
432                 uint64_t pend_switch:1;
433                 /* Set when SWTAG_FULL and pend_switch is set. */
434                 uint64_t pend_switch_full:1;
435                 /*
436                  * Set when there is a pending NULL SWTAG, or an
437                  * implicit switch to NULL.
438                  */
439                 uint64_t pend_switch_null:1;
440                 /* Set when there is a pending DESCHED or SWTAG_DESCHED. */
441                 uint64_t pend_desched:1;
442                 /*
443                  * Set when there is a pending SWTAG_DESCHED and
444                  * pend_desched is set.
445                  */
446                 uint64_t pend_desched_switch:1;
447                 /* Set when nosched is desired and pend_desched is set. */
448                 uint64_t pend_nosched:1;
449                 /* Set when there is a pending GET_WORK. */
450                 uint64_t pend_new_work:1;
451                 /*
452                  * When pend_new_work is set, this bit indicates that
453                  * the wait bit was set.
454                  */
455                 uint64_t pend_new_work_wait:1;
456                 /* Set when there is a pending NULL_RD. */
457                 uint64_t pend_null_rd:1;
458                 /* Set when there is a pending CLR_NSCHED. */
459                 uint64_t pend_nosched_clr:1;
460                 uint64_t reserved_51:1;
461                 /* This is the index when pend_nosched_clr is set. */
462                 uint64_t pend_index:11;
463                 /*
464                  * This is the new_grp when (pend_desched AND
465                  * pend_desched_switch) is set.
466                  */
467                 uint64_t pend_grp:4;
468                 uint64_t reserved_34_35:2;
469                 /*
470                  * This is the tag type when pend_switch or
471                  * (pend_desched AND pend_desched_switch) are set.
472                  */
473                 uint64_t pend_type:2;
474                 /*
475                  * - this is the tag when pend_switch or (pend_desched
476                  *    AND pend_desched_switch) are set.
477                  */
478                 uint64_t pend_tag:32;
479         } s_sstatus0;
480 
481     /**
482      * Result for a POW Status Load (when get_cur==0 and get_wqp==1)
483      */
484         struct {
485                 uint64_t reserved_62_63:2;
486                 /*
487                  * Set when there is a pending non-NULL SWTAG or
488                  * SWTAG_FULL, and the POW entry has not left the list
489                  * for the original tag.
490                  */
491                 uint64_t pend_switch:1;
492                 /* Set when SWTAG_FULL and pend_switch is set. */
493                 uint64_t pend_switch_full:1;
494                 /*
495                  * Set when there is a pending NULL SWTAG, or an
496                  * implicit switch to NULL.
497                  */
498                 uint64_t pend_switch_null:1;
499                 /*
500                  * Set when there is a pending DESCHED or
501                  * SWTAG_DESCHED.
502                  */
503                 uint64_t pend_desched:1;
504                 /*
505                  * Set when there is a pending SWTAG_DESCHED and
506                  * pend_desched is set.
507                  */
508                 uint64_t pend_desched_switch:1;
509                 /* Set when nosched is desired and pend_desched is set. */
510                 uint64_t pend_nosched:1;
511                 /* Set when there is a pending GET_WORK. */
512                 uint64_t pend_new_work:1;
513                 /*
514                  * When pend_new_work is set, this bit indicates that
515                  * the wait bit was set.
516                  */
517                 uint64_t pend_new_work_wait:1;
518                 /* Set when there is a pending NULL_RD. */
519                 uint64_t pend_null_rd:1;
520                 /* Set when there is a pending CLR_NSCHED. */
521                 uint64_t pend_nosched_clr:1;
522                 uint64_t reserved_51:1;
523                 /* This is the index when pend_nosched_clr is set. */
524                 uint64_t pend_index:11;
525                 /*
526                  * This is the new_grp when (pend_desched AND
527                  * pend_desched_switch) is set.
528                  */
529                 uint64_t pend_grp:4;
530                 /* This is the wqp when pend_nosched_clr is set. */
531                 uint64_t pend_wqp:36;
532         } s_sstatus1;
533 
534     /**
535      * Result for a POW Status Load (when get_cur==1, get_wqp==0, and
536      * get_rev==0)
537      */
538         struct {
539                 uint64_t reserved_62_63:2;
540                 /*
541                  * Points to the next POW entry in the tag list when
542                  * tail == 0 (and tag_type is not NULL or NULL_NULL).
543                  */
544                 uint64_t link_index:11;
545                 /* The POW entry attached to the core. */
546                 uint64_t index:11;
547                 /*
548                  * The group attached to the core (updated when new
549                  * tag list entered on SWTAG_FULL).
550                  */
551                 uint64_t grp:4;
552                 /*
553                  * Set when this POW entry is at the head of its tag
554                  * list (also set when in the NULL or NULL_NULL
555                  * state).
556                  */
557                 uint64_t head:1;
558                 /*
559                  * Set when this POW entry is at the tail of its tag
560                  * list (also set when in the NULL or NULL_NULL
561                  * state).
562                  */
563                 uint64_t tail:1;
564                 /*
565                  * The tag type attached to the core (updated when new
566                  * tag list entered on SWTAG, SWTAG_FULL, or
567                  * SWTAG_DESCHED).
568                  */
569                 uint64_t tag_type:2;
570                 /*
571                  * The tag attached to the core (updated when new tag
572                  * list entered on SWTAG, SWTAG_FULL, or
573                  * SWTAG_DESCHED).
574                  */
575                 uint64_t tag:32;
576         } s_sstatus2;
577 
578     /**
579      * Result for a POW Status Load (when get_cur==1, get_wqp==0, and get_rev==1)
580      */
581         struct {
582                 uint64_t reserved_62_63:2;
583                 /*
584                  * Points to the prior POW entry in the tag list when
585                  * head == 0 (and tag_type is not NULL or
586                  * NULL_NULL). This field is unpredictable when the
587                  * core's state is NULL or NULL_NULL.
588                  */
589                 uint64_t revlink_index:11;
590                 /* The POW entry attached to the core. */
591                 uint64_t index:11;
592                 /*
593                  * The group attached to the core (updated when new
594                  * tag list entered on SWTAG_FULL).
595                  */
596                 uint64_t grp:4;
597                 /* Set when this POW entry is at the head of its tag
598                  * list (also set when in the NULL or NULL_NULL
599                  * state).
600                  */
601                 uint64_t head:1;
602                 /*
603                  * Set when this POW entry is at the tail of its tag
604                  * list (also set when in the NULL or NULL_NULL
605                  * state).
606                  */
607                 uint64_t tail:1;
608                 /*
609                  * The tag type attached to the core (updated when new
610                  * tag list entered on SWTAG, SWTAG_FULL, or
611                  * SWTAG_DESCHED).
612                  */
613                 uint64_t tag_type:2;
614                 /*
615                  * The tag attached to the core (updated when new tag
616                  * list entered on SWTAG, SWTAG_FULL, or
617                  * SWTAG_DESCHED).
618                  */
619                 uint64_t tag:32;
620         } s_sstatus3;
621 
622     /**
623      * Result for a POW Status Load (when get_cur==1, get_wqp==1, and
624      * get_rev==0)
625      */
626         struct {
627                 uint64_t reserved_62_63:2;
628                 /*
629                  * Points to the next POW entry in the tag list when
630                  * tail == 0 (and tag_type is not NULL or NULL_NULL).
631                  */
632                 uint64_t link_index:11;
633                 /* The POW entry attached to the core. */
634                 uint64_t index:11;
635                 /*
636                  * The group attached to the core (updated when new
637                  * tag list entered on SWTAG_FULL).
638                  */
639                 uint64_t grp:4;
640                 /*
641                  * The wqp attached to the core (updated when new tag
642                  * list entered on SWTAG_FULL).
643                  */
644                 uint64_t wqp:36;
645         } s_sstatus4;
646 
647     /**
648      * Result for a POW Status Load (when get_cur==1, get_wqp==1, and
649      * get_rev==1)
650      */
651         struct {
652                 uint64_t reserved_62_63:2;
653                 /*
654                  * Points to the prior POW entry in the tag list when
655                  * head == 0 (and tag_type is not NULL or
656                  * NULL_NULL). This field is unpredictable when the
657                  * core's state is NULL or NULL_NULL.
658                  */
659                 uint64_t revlink_index:11;
660                 /* The POW entry attached to the core. */
661                 uint64_t index:11;
662                 /*
663                  * The group attached to the core (updated when new
664                  * tag list entered on SWTAG_FULL).
665                  */
666                 uint64_t grp:4;
667                 /*
668                  * The wqp attached to the core (updated when new tag
669                  * list entered on SWTAG_FULL).
670                  */
671                 uint64_t wqp:36;
672         } s_sstatus5;
673 
674     /**
675      * Result For POW Memory Load (get_des == 0 and get_wqp == 0)
676      */
677         struct {
678                 uint64_t reserved_51_63:13;
679                 /*
680                  * The next entry in the input, free, descheduled_head
681                  * list (unpredictable if entry is the tail of the
682                  * list).
683                  */
684                 uint64_t next_index:11;
685                 /* The group of the POW entry. */
686                 uint64_t grp:4;
687                 uint64_t reserved_35:1;
688                 /*
689                  * Set when this POW entry is at the tail of its tag
690                  * list (also set when in the NULL or NULL_NULL
691                  * state).
692                  */
693                 uint64_t tail:1;
694                 /* The tag type of the POW entry. */
695                 uint64_t tag_type:2;
696                 /* The tag of the POW entry. */
697                 uint64_t tag:32;
698         } s_smemload0;
699 
700     /**
701      * Result For POW Memory Load (get_des == 0 and get_wqp == 1)
702      */
703         struct {
704                 uint64_t reserved_51_63:13;
705                 /*
706                  * The next entry in the input, free, descheduled_head
707                  * list (unpredictable if entry is the tail of the
708                  * list).
709                  */
710                 uint64_t next_index:11;
711                 /* The group of the POW entry. */
712                 uint64_t grp:4;
713                 /* The WQP held in the POW entry. */
714                 uint64_t wqp:36;
715         } s_smemload1;
716 
717     /**
718      * Result For POW Memory Load (get_des == 1)
719      */
720         struct {
721                 uint64_t reserved_51_63:13;
722                 /*
723                  * The next entry in the tag list connected to the
724                  * descheduled head.
725                  */
726                 uint64_t fwd_index:11;
727                 /* The group of the POW entry. */
728                 uint64_t grp:4;
729                 /* The nosched bit for the POW entry. */
730                 uint64_t nosched:1;
731                 /* There is a pending tag switch */
732                 uint64_t pend_switch:1;
733                 /*
734                  * The next tag type for the new tag list when
735                  * pend_switch is set.
736                  */
737                 uint64_t pend_type:2;
738                 /*
739                  * The next tag for the new tag list when pend_switch
740                  * is set.
741                  */
742                 uint64_t pend_tag:32;
743         } s_smemload2;
744 
745     /**
746      * Result For POW Index/Pointer Load (get_rmt == 0/get_des_get_tail == 0)
747      */
748         struct {
749                 uint64_t reserved_52_63:12;
750                 /*
751                  * set when there is one or more POW entries on the
752                  * free list.
753                  */
754                 uint64_t free_val:1;
755                 /*
756                  * set when there is exactly one POW entry on the free
757                  * list.
758                  */
759                 uint64_t free_one:1;
760                 uint64_t reserved_49:1;
761                 /*
762                  * when free_val is set, indicates the first entry on
763                  * the free list.
764                  */
765                 uint64_t free_head:11;
766                 uint64_t reserved_37:1;
767                 /*
768                  * when free_val is set, indicates the last entry on
769                  * the free list.
770                  */
771                 uint64_t free_tail:11;
772                 /*
773                  * set when there is one or more POW entries on the
774                  * input Q list selected by qosgrp.
775                  */
776                 uint64_t loc_val:1;
777                 /*
778                  * set when there is exactly one POW entry on the
779                  * input Q list selected by qosgrp.
780                  */
781                 uint64_t loc_one:1;
782                 uint64_t reserved_23:1;
783                 /*
784                  * when loc_val is set, indicates the first entry on
785                  * the input Q list selected by qosgrp.
786                  */
787                 uint64_t loc_head:11;
788                 uint64_t reserved_11:1;
789                 /*
790                  * when loc_val is set, indicates the last entry on
791                  * the input Q list selected by qosgrp.
792                  */
793                 uint64_t loc_tail:11;
794         } sindexload0;
795 
796     /**
797      * Result For POW Index/Pointer Load (get_rmt == 0/get_des_get_tail == 1)
798      */
799         struct {
800                 uint64_t reserved_52_63:12;
801                 /*
802                  * set when there is one or more POW entries on the
803                  * nosched list.
804                  */
805                 uint64_t nosched_val:1;
806                 /*
807                  * set when there is exactly one POW entry on the
808                  * nosched list.
809                  */
810                 uint64_t nosched_one:1;
811                 uint64_t reserved_49:1;
812                 /*
813                  * when nosched_val is set, indicates the first entry
814                  * on the nosched list.
815                  */
816                 uint64_t nosched_head:11;
817                 uint64_t reserved_37:1;
818                 /*
819                  * when nosched_val is set, indicates the last entry
820                  * on the nosched list.
821                  */
822                 uint64_t nosched_tail:11;
823                 /*
824                  * set when there is one or more descheduled heads on
825                  * the descheduled list selected by qosgrp.
826                  */
827                 uint64_t des_val:1;
828                 /*
829                  * set when there is exactly one descheduled head on
830                  * the descheduled list selected by qosgrp.
831                  */
832                 uint64_t des_one:1;
833                 uint64_t reserved_23:1;
834                 /*
835                  * when des_val is set, indicates the first
836                  * descheduled head on the descheduled list selected
837                  * by qosgrp.
838                  */
839                 uint64_t des_head:11;
840                 uint64_t reserved_11:1;
841                 /*
842                  * when des_val is set, indicates the last descheduled
843                  * head on the descheduled list selected by qosgrp.
844                  */
845                 uint64_t des_tail:11;
846         } sindexload1;
847 
848     /**
849      * Result For POW Index/Pointer Load (get_rmt == 1/get_des_get_tail == 0)
850      */
851         struct {
852                 uint64_t reserved_39_63:25;
853                 /*
854                  * Set when this DRAM list is the current head
855                  * (i.e. is the next to be reloaded when the POW
856                  * hardware reloads a POW entry from DRAM). The POW
857                  * hardware alternates between the two DRAM lists
858                  * associated with a QOS level when it reloads work
859                  * from DRAM into the POW unit.
860                  */
861                 uint64_t rmt_is_head:1;
862                 /*
863                  * Set when the DRAM portion of the input Q list
864                  * selected by qosgrp contains one or more pieces of
865                  * work.
866                  */
867                 uint64_t rmt_val:1;
868                 /*
869                  * Set when the DRAM portion of the input Q list
870                  * selected by qosgrp contains exactly one piece of
871                  * work.
872                  */
873                 uint64_t rmt_one:1;
874                 /*
875                  * When rmt_val is set, indicates the first piece of
876                  * work on the DRAM input Q list selected by
877                  * qosgrp.
878                  */
879                 uint64_t rmt_head:36;
880         } sindexload2;
881 
882     /**
883      * Result For POW Index/Pointer Load (get_rmt ==
884      * 1/get_des_get_tail == 1)
885      */
886         struct {
887                 uint64_t reserved_39_63:25;
888                 /*
889                  * set when this DRAM list is the current head
890                  * (i.e. is the next to be reloaded when the POW
891                  * hardware reloads a POW entry from DRAM). The POW
892                  * hardware alternates between the two DRAM lists
893                  * associated with a QOS level when it reloads work
894                  * from DRAM into the POW unit.
895                  */
896                 uint64_t rmt_is_head:1;
897                 /*
898                  * set when the DRAM portion of the input Q list
899                  * selected by qosgrp contains one or more pieces of
900                  * work.
901                  */
902                 uint64_t rmt_val:1;
903                 /*
904                  * set when the DRAM portion of the input Q list
905                  * selected by qosgrp contains exactly one piece of
906                  * work.
907                  */
908                 uint64_t rmt_one:1;
909                 /*
910                  * when rmt_val is set, indicates the last piece of
911                  * work on the DRAM input Q list selected by
912                  * qosgrp.
913                  */
914                 uint64_t rmt_tail:36;
915         } sindexload3;
916 
917     /**
918      * Response to NULL_RD request loads
919      */
920         struct {
921                 uint64_t unused:62;
922                 /* of type cvmx_pow_tag_type_t. state is one of the
923                  * following:
924                  *
925                  * - CVMX_POW_TAG_TYPE_ORDERED
926                  * - CVMX_POW_TAG_TYPE_ATOMIC
927                  * - CVMX_POW_TAG_TYPE_NULL
928                  * - CVMX_POW_TAG_TYPE_NULL_NULL
929                  */
930                 uint64_t state:2;
931         } s_null_rd;
932 
933 } cvmx_pow_tag_load_resp_t;
934 
935 /**
936  * This structure describes the address used for stores to the POW.
937  *  The store address is meaningful on stores to the POW.  The
938  *  hardware assumes that an aligned 64-bit store was used for all
939  *  these stores.  Note the assumption that the work queue entry is
940  *  aligned on an 8-byte boundary (since the low-order 3 address bits
941  *  must be zero).  Note that not all fields are used by all
942  *  operations.
943  *
944  *  NOTE: The following is the behavior of the pending switch bit at the PP
945  *       for POW stores (i.e. when did<7:3> == 0xc)
946  *     - did<2:0> == 0      => pending switch bit is set
947  *     - did<2:0> == 1      => no affect on the pending switch bit
948  *     - did<2:0> == 3      => pending switch bit is cleared
949  *     - did<2:0> == 7      => no affect on the pending switch bit
950  *     - did<2:0> == others => must not be used
951  *     - No other loads/stores have an affect on the pending switch bit
952  *     - The switch bus from POW can clear the pending switch bit
953  *
954  *  NOTE: did<2:0> == 2 is used by the HW for a special single-cycle
955  *  ADDWQ command that only contains the pointer). SW must never use
956  *  did<2:0> == 2.
957  */
958 typedef union {
959     /**
960      * Unsigned 64 bit integer representation of store address
961      */
962         uint64_t u64;
963 
964         struct {
965                 /* Memory region.  Should be CVMX_IO_SEG in most cases */
966                 uint64_t mem_reg:2;
967                 uint64_t reserved_49_61:13;     /* Must be zero */
968                 uint64_t is_io:1;       /* Must be one */
969                 /* Device ID of POW.  Note that different sub-dids are used. */
970                 uint64_t did:8;
971                 uint64_t reserved_36_39:4;      /* Must be zero */
972                 /* Address field. addr<2:0> must be zero */
973                 uint64_t addr:36;
974         } stag;
975 } cvmx_pow_tag_store_addr_t;
976 
977 /**
978  * decode of the store data when an IOBDMA SENDSINGLE is sent to POW
979  */
980 typedef union {
981         uint64_t u64;
982 
983         struct {
984                 /*
985                  * the (64-bit word) location in scratchpad to write
986                  * to (if len != 0)
987                  */
988                 uint64_t scraddr:8;
989                 /* the number of words in the response (0 => no response) */
990                 uint64_t len:8;
991                 /* the ID of the device on the non-coherent bus */
992                 uint64_t did:8;
993                 uint64_t unused:36;
994                 /* if set, don't return load response until work is available */
995                 uint64_t wait:1;
996                 uint64_t unused2:3;
997         } s;
998 
999 } cvmx_pow_iobdma_store_t;
1000 
1001 /* CSR typedefs have been moved to cvmx-csr-*.h */
1002 
1003 /**
1004  * Get the POW tag for this core. This returns the current
1005  * tag type, tag, group, and POW entry index associated with
1006  * this core. Index is only valid if the tag type isn't NULL_NULL.
1007  * If a tag switch is pending this routine returns the tag before
1008  * the tag switch, not after.
1009  *
1010  * Returns Current tag
1011  */
1012 static inline cvmx_pow_tag_req_t cvmx_pow_get_current_tag(void)
1013 {
1014         cvmx_pow_load_addr_t load_addr;
1015         cvmx_pow_tag_load_resp_t load_resp;
1016         cvmx_pow_tag_req_t result;
1017 
1018         load_addr.u64 = 0;
1019         load_addr.sstatus.mem_region = CVMX_IO_SEG;
1020         load_addr.sstatus.is_io = 1;
1021         load_addr.sstatus.did = CVMX_OCT_DID_TAG_TAG1;
1022         load_addr.sstatus.coreid = cvmx_get_core_num();
1023         load_addr.sstatus.get_cur = 1;
1024         load_resp.u64 = cvmx_read_csr(load_addr.u64);
1025         result.u64 = 0;
1026         result.s.grp = load_resp.s_sstatus2.grp;
1027         result.s.index = load_resp.s_sstatus2.index;
1028         result.s.type = load_resp.s_sstatus2.tag_type;
1029         result.s.tag = load_resp.s_sstatus2.tag;
1030         return result;
1031 }
1032 
1033 /**
1034  * Get the POW WQE for this core. This returns the work queue
1035  * entry currently associated with this core.
1036  *
1037  * Returns WQE pointer
1038  */
1039 static inline cvmx_wqe_t *cvmx_pow_get_current_wqp(void)
1040 {
1041         cvmx_pow_load_addr_t load_addr;
1042         cvmx_pow_tag_load_resp_t load_resp;
1043 
1044         load_addr.u64 = 0;
1045         load_addr.sstatus.mem_region = CVMX_IO_SEG;
1046         load_addr.sstatus.is_io = 1;
1047         load_addr.sstatus.did = CVMX_OCT_DID_TAG_TAG1;
1048         load_addr.sstatus.coreid = cvmx_get_core_num();
1049         load_addr.sstatus.get_cur = 1;
1050         load_addr.sstatus.get_wqp = 1;
1051         load_resp.u64 = cvmx_read_csr(load_addr.u64);
1052         return (cvmx_wqe_t *) cvmx_phys_to_ptr(load_resp.s_sstatus4.wqp);
1053 }
1054 
1055 #ifndef CVMX_MF_CHORD
1056 #define CVMX_MF_CHORD(dest)         CVMX_RDHWR(dest, 30)
1057 #endif
1058 
1059 /**
1060  * Print a warning if a tag switch is pending for this core
1061  *
1062  * @function: Function name checking for a pending tag switch
1063  */
1064 static inline void __cvmx_pow_warn_if_pending_switch(const char *function)
1065 {
1066         uint64_t switch_complete;
1067         CVMX_MF_CHORD(switch_complete);
1068         if (!switch_complete)
1069                 pr_warning("%s called with tag switch in progress\n", function);
1070 }
1071 
1072 /**
1073  * Waits for a tag switch to complete by polling the completion bit.
1074  * Note that switches to NULL complete immediately and do not need
1075  * to be waited for.
1076  */
1077 static inline void cvmx_pow_tag_sw_wait(void)
1078 {
1079         const uint64_t MAX_CYCLES = 1ull << 31;
1080         uint64_t switch_complete;
1081         uint64_t start_cycle = cvmx_get_cycle();
1082         while (1) {
1083                 CVMX_MF_CHORD(switch_complete);
1084                 if (unlikely(switch_complete))
1085                         break;
1086                 if (unlikely(cvmx_get_cycle() > start_cycle + MAX_CYCLES)) {
1087                         pr_warning("Tag switch is taking a long time, "
1088                                    "possible deadlock\n");
1089                         start_cycle = -MAX_CYCLES - 1;
1090                 }
1091         }
1092 }
1093 
1094 /**
1095  * Synchronous work request.  Requests work from the POW.
1096  * This function does NOT wait for previous tag switches to complete,
1097  * so the caller must ensure that there is not a pending tag switch.
1098  *
1099  * @wait:   When set, call stalls until work becomes avaiable, or times out.
1100  *               If not set, returns immediately.
1101  *
1102  * Returns Returns the WQE pointer from POW. Returns NULL if no work
1103  * was available.
1104  */
1105 static inline cvmx_wqe_t *cvmx_pow_work_request_sync_nocheck(cvmx_pow_wait_t
1106                                                              wait)
1107 {
1108         cvmx_pow_load_addr_t ptr;
1109         cvmx_pow_tag_load_resp_t result;
1110 
1111         if (CVMX_ENABLE_POW_CHECKS)
1112                 __cvmx_pow_warn_if_pending_switch(__func__);
1113 
1114         ptr.u64 = 0;
1115         ptr.swork.mem_region = CVMX_IO_SEG;
1116         ptr.swork.is_io = 1;
1117         ptr.swork.did = CVMX_OCT_DID_TAG_SWTAG;
1118         ptr.swork.wait = wait;
1119 
1120         result.u64 = cvmx_read_csr(ptr.u64);
1121 
1122         if (result.s_work.no_work)
1123                 return NULL;
1124         else
1125                 return (cvmx_wqe_t *) cvmx_phys_to_ptr(result.s_work.addr);
1126 }
1127 
1128 /**
1129  * Synchronous work request.  Requests work from the POW.
1130  * This function waits for any previous tag switch to complete before
1131  * requesting the new work.
1132  *
1133  * @wait:   When set, call stalls until work becomes avaiable, or times out.
1134  *               If not set, returns immediately.
1135  *
1136  * Returns Returns the WQE pointer from POW. Returns NULL if no work
1137  * was available.
1138  */
1139 static inline cvmx_wqe_t *cvmx_pow_work_request_sync(cvmx_pow_wait_t wait)
1140 {
1141         if (CVMX_ENABLE_POW_CHECKS)
1142                 __cvmx_pow_warn_if_pending_switch(__func__);
1143 
1144         /* Must not have a switch pending when requesting work */
1145         cvmx_pow_tag_sw_wait();
1146         return cvmx_pow_work_request_sync_nocheck(wait);
1147 
1148 }
1149 
1150 /**
1151  * Synchronous null_rd request.  Requests a switch out of NULL_NULL POW state.
1152  * This function waits for any previous tag switch to complete before
1153  * requesting the null_rd.
1154  *
1155  * Returns Returns the POW state of type cvmx_pow_tag_type_t.
1156  */
1157 static inline enum cvmx_pow_tag_type cvmx_pow_work_request_null_rd(void)
1158 {
1159         cvmx_pow_load_addr_t ptr;
1160         cvmx_pow_tag_load_resp_t result;
1161 
1162         if (CVMX_ENABLE_POW_CHECKS)
1163                 __cvmx_pow_warn_if_pending_switch(__func__);
1164 
1165         /* Must not have a switch pending when requesting work */
1166         cvmx_pow_tag_sw_wait();
1167 
1168         ptr.u64 = 0;
1169         ptr.snull_rd.mem_region = CVMX_IO_SEG;
1170         ptr.snull_rd.is_io = 1;
1171         ptr.snull_rd.did = CVMX_OCT_DID_TAG_NULL_RD;
1172 
1173         result.u64 = cvmx_read_csr(ptr.u64);
1174 
1175         return (enum cvmx_pow_tag_type) result.s_null_rd.state;
1176 }
1177 
1178 /**
1179  * Asynchronous work request.  Work is requested from the POW unit,
1180  * and should later be checked with function
1181  * cvmx_pow_work_response_async.  This function does NOT wait for
1182  * previous tag switches to complete, so the caller must ensure that
1183  * there is not a pending tag switch.
1184  *
1185  * @scr_addr: Scratch memory address that response will be returned
1186  *            to, which is either a valid WQE, or a response with the
1187  *            invalid bit set.  Byte address, must be 8 byte aligned.
1188  *
1189  * @wait: 1 to cause response to wait for work to become available (or
1190  *        timeout), 0 to cause response to return immediately
1191  */
1192 static inline void cvmx_pow_work_request_async_nocheck(int scr_addr,
1193                                                        cvmx_pow_wait_t wait)
1194 {
1195         cvmx_pow_iobdma_store_t data;
1196 
1197         if (CVMX_ENABLE_POW_CHECKS)
1198                 __cvmx_pow_warn_if_pending_switch(__func__);
1199 
1200         /* scr_addr must be 8 byte aligned */
1201         data.s.scraddr = scr_addr >> 3;
1202         data.s.len = 1;
1203         data.s.did = CVMX_OCT_DID_TAG_SWTAG;
1204         data.s.wait = wait;
1205         cvmx_send_single(data.u64);
1206 }
1207 
1208 /**
1209  * Asynchronous work request.  Work is requested from the POW unit,
1210  * and should later be checked with function
1211  * cvmx_pow_work_response_async.  This function waits for any previous
1212  * tag switch to complete before requesting the new work.
1213  *
1214  * @scr_addr: Scratch memory address that response will be returned
1215  *            to, which is either a valid WQE, or a response with the
1216  *            invalid bit set.  Byte address, must be 8 byte aligned.
1217  *
1218  * @wait: 1 to cause response to wait for work to become available (or
1219  *                  timeout), 0 to cause response to return immediately
1220  */
1221 static inline void cvmx_pow_work_request_async(int scr_addr,
1222                                                cvmx_pow_wait_t wait)
1223 {
1224         if (CVMX_ENABLE_POW_CHECKS)
1225                 __cvmx_pow_warn_if_pending_switch(__func__);
1226 
1227         /* Must not have a switch pending when requesting work */
1228         cvmx_pow_tag_sw_wait();
1229         cvmx_pow_work_request_async_nocheck(scr_addr, wait);
1230 }
1231 
1232 /**
1233  * Gets result of asynchronous work request.  Performs a IOBDMA sync
1234  * to wait for the response.
1235  *
1236  * @scr_addr: Scratch memory address to get result from Byte address,
1237  *            must be 8 byte aligned.
1238  *
1239  * Returns Returns the WQE from the scratch register, or NULL if no
1240  * work was available.
1241  */
1242 static inline cvmx_wqe_t *cvmx_pow_work_response_async(int scr_addr)
1243 {
1244         cvmx_pow_tag_load_resp_t result;
1245 
1246         CVMX_SYNCIOBDMA;
1247         result.u64 = cvmx_scratch_read64(scr_addr);
1248 
1249         if (result.s_work.no_work)
1250                 return NULL;
1251         else
1252                 return (cvmx_wqe_t *) cvmx_phys_to_ptr(result.s_work.addr);
1253 }
1254 
1255 /**
1256  * Checks if a work queue entry pointer returned by a work
1257  * request is valid.  It may be invalid due to no work
1258  * being available or due to a timeout.
1259  *
1260  * @wqe_ptr: pointer to a work queue entry returned by the POW
1261  *
1262  * Returns 0 if pointer is valid
1263  *         1 if invalid (no work was returned)
1264  */
1265 static inline uint64_t cvmx_pow_work_invalid(cvmx_wqe_t *wqe_ptr)
1266 {
1267         return wqe_ptr == NULL;
1268 }
1269 
1270 /**
1271  * Starts a tag switch to the provided tag value and tag type.
1272  * Completion for the tag switch must be checked for separately.  This
1273  * function does NOT update the work queue entry in dram to match tag
1274  * value and type, so the application must keep track of these if they
1275  * are important to the application.  This tag switch command must not
1276  * be used for switches to NULL, as the tag switch pending bit will be
1277  * set by the switch request, but never cleared by the hardware.
1278  *
1279  * NOTE: This should not be used when switching from a NULL tag.  Use
1280  * cvmx_pow_tag_sw_full() instead.
1281  *
1282  * This function does no checks, so the caller must ensure that any
1283  * previous tag switch has completed.
1284  *
1285  * @tag:      new tag value
1286  * @tag_type: new tag type (ordered or atomic)
1287  */
1288 static inline void cvmx_pow_tag_sw_nocheck(uint32_t tag,
1289                                            enum cvmx_pow_tag_type tag_type)
1290 {
1291         cvmx_addr_t ptr;
1292         cvmx_pow_tag_req_t tag_req;
1293 
1294         if (CVMX_ENABLE_POW_CHECKS) {
1295                 cvmx_pow_tag_req_t current_tag;
1296                 __cvmx_pow_warn_if_pending_switch(__func__);
1297                 current_tag = cvmx_pow_get_current_tag();
1298                 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL_NULL)
1299                         pr_warning("%s called with NULL_NULL tag\n",
1300                                    __func__);
1301                 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL)
1302                         pr_warning("%s called with NULL tag\n", __func__);
1303                 if ((current_tag.s.type == tag_type)
1304                    && (current_tag.s.tag == tag))
1305                         pr_warning("%s called to perform a tag switch to the "
1306                                    "same tag\n",
1307                              __func__);
1308                 if (tag_type == CVMX_POW_TAG_TYPE_NULL)
1309                         pr_warning("%s called to perform a tag switch to "
1310                                    "NULL. Use cvmx_pow_tag_sw_null() instead\n",
1311                              __func__);
1312         }
1313 
1314         /*
1315          * Note that WQE in DRAM is not updated here, as the POW does
1316          * not read from DRAM once the WQE is in flight.  See hardware
1317          * manual for complete details.  It is the application's
1318          * responsibility to keep track of the current tag value if
1319          * that is important.
1320          */
1321 
1322         tag_req.u64 = 0;
1323         tag_req.s.op = CVMX_POW_TAG_OP_SWTAG;
1324         tag_req.s.tag = tag;
1325         tag_req.s.type = tag_type;
1326 
1327         ptr.u64 = 0;
1328         ptr.sio.mem_region = CVMX_IO_SEG;
1329         ptr.sio.is_io = 1;
1330         ptr.sio.did = CVMX_OCT_DID_TAG_SWTAG;
1331 
1332         /* once this store arrives at POW, it will attempt the switch
1333            software must wait for the switch to complete separately */
1334         cvmx_write_io(ptr.u64, tag_req.u64);
1335 }
1336 
1337 /**
1338  * Starts a tag switch to the provided tag value and tag type.
1339  * Completion for the tag switch must be checked for separately.  This
1340  * function does NOT update the work queue entry in dram to match tag
1341  * value and type, so the application must keep track of these if they
1342  * are important to the application.  This tag switch command must not
1343  * be used for switches to NULL, as the tag switch pending bit will be
1344  * set by the switch request, but never cleared by the hardware.
1345  *
1346  * NOTE: This should not be used when switching from a NULL tag.  Use
1347  * cvmx_pow_tag_sw_full() instead.
1348  *
1349  * This function waits for any previous tag switch to complete, and also
1350  * displays an error on tag switches to NULL.
1351  *
1352  * @tag:      new tag value
1353  * @tag_type: new tag type (ordered or atomic)
1354  */
1355 static inline void cvmx_pow_tag_sw(uint32_t tag,
1356                                    enum cvmx_pow_tag_type tag_type)
1357 {
1358         if (CVMX_ENABLE_POW_CHECKS)
1359                 __cvmx_pow_warn_if_pending_switch(__func__);
1360 
1361         /*
1362          * Note that WQE in DRAM is not updated here, as the POW does
1363          * not read from DRAM once the WQE is in flight.  See hardware
1364          * manual for complete details.  It is the application's
1365          * responsibility to keep track of the current tag value if
1366          * that is important.
1367          */
1368 
1369         /*
1370          * Ensure that there is not a pending tag switch, as a tag
1371          * switch cannot be started if a previous switch is still
1372          * pending.
1373          */
1374         cvmx_pow_tag_sw_wait();
1375         cvmx_pow_tag_sw_nocheck(tag, tag_type);
1376 }
1377 
1378 /**
1379  * Starts a tag switch to the provided tag value and tag type.
1380  * Completion for the tag switch must be checked for separately.  This
1381  * function does NOT update the work queue entry in dram to match tag
1382  * value and type, so the application must keep track of these if they
1383  * are important to the application.  This tag switch command must not
1384  * be used for switches to NULL, as the tag switch pending bit will be
1385  * set by the switch request, but never cleared by the hardware.
1386  *
1387  * This function must be used for tag switches from NULL.
1388  *
1389  * This function does no checks, so the caller must ensure that any
1390  * previous tag switch has completed.
1391  *
1392  * @wqp:      pointer to work queue entry to submit.  This entry is
1393  *            updated to match the other parameters
1394  * @tag:      tag value to be assigned to work queue entry
1395  * @tag_type: type of tag
1396  * @group:    group value for the work queue entry.
1397  */
1398 static inline void cvmx_pow_tag_sw_full_nocheck(cvmx_wqe_t *wqp, uint32_t tag,
1399                                                 enum cvmx_pow_tag_type tag_type,
1400                                                 uint64_t group)
1401 {
1402         cvmx_addr_t ptr;
1403         cvmx_pow_tag_req_t tag_req;
1404 
1405         if (CVMX_ENABLE_POW_CHECKS) {
1406                 cvmx_pow_tag_req_t current_tag;
1407                 __cvmx_pow_warn_if_pending_switch(__func__);
1408                 current_tag = cvmx_pow_get_current_tag();
1409                 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL_NULL)
1410                         pr_warning("%s called with NULL_NULL tag\n",
1411                                    __func__);
1412                 if ((current_tag.s.type == tag_type)
1413                    && (current_tag.s.tag == tag))
1414                         pr_warning("%s called to perform a tag switch to "
1415                                    "the same tag\n",
1416                              __func__);
1417                 if (tag_type == CVMX_POW_TAG_TYPE_NULL)
1418                         pr_warning("%s called to perform a tag switch to "
1419                                    "NULL. Use cvmx_pow_tag_sw_null() instead\n",
1420                              __func__);
1421                 if (wqp != cvmx_phys_to_ptr(0x80))
1422                         if (wqp != cvmx_pow_get_current_wqp())
1423                                 pr_warning("%s passed WQE(%p) doesn't match "
1424                                            "the address in the POW(%p)\n",
1425                                      __func__, wqp,
1426                                      cvmx_pow_get_current_wqp());
1427         }
1428 
1429         /*
1430          * Note that WQE in DRAM is not updated here, as the POW does
1431          * not read from DRAM once the WQE is in flight.  See hardware
1432          * manual for complete details.  It is the application's
1433          * responsibility to keep track of the current tag value if
1434          * that is important.
1435          */
1436 
1437         tag_req.u64 = 0;
1438         tag_req.s.op = CVMX_POW_TAG_OP_SWTAG_FULL;
1439         tag_req.s.tag = tag;
1440         tag_req.s.type = tag_type;
1441         tag_req.s.grp = group;
1442 
1443         ptr.u64 = 0;
1444         ptr.sio.mem_region = CVMX_IO_SEG;
1445         ptr.sio.is_io = 1;
1446         ptr.sio.did = CVMX_OCT_DID_TAG_SWTAG;
1447         ptr.sio.offset = CAST64(wqp);
1448 
1449         /*
1450          * once this store arrives at POW, it will attempt the switch
1451          * software must wait for the switch to complete separately.
1452          */
1453         cvmx_write_io(ptr.u64, tag_req.u64);
1454 }
1455 
1456 /**
1457  * Starts a tag switch to the provided tag value and tag type.
1458  * Completion for the tag switch must be checked for separately.  This
1459  * function does NOT update the work queue entry in dram to match tag
1460  * value and type, so the application must keep track of these if they
1461  * are important to the application.  This tag switch command must not
1462  * be used for switches to NULL, as the tag switch pending bit will be
1463  * set by the switch request, but never cleared by the hardware.
1464  *
1465  * This function must be used for tag switches from NULL.
1466  *
1467  * This function waits for any pending tag switches to complete
1468  * before requesting the tag switch.
1469  *
1470  * @wqp:      pointer to work queue entry to submit.  This entry is updated
1471  *            to match the other parameters
1472  * @tag:      tag value to be assigned to work queue entry
1473  * @tag_type: type of tag
1474  * @group:      group value for the work queue entry.
1475  */
1476 static inline void cvmx_pow_tag_sw_full(cvmx_wqe_t *wqp, uint32_t tag,
1477                                         enum cvmx_pow_tag_type tag_type,
1478                                         uint64_t group)
1479 {
1480         if (CVMX_ENABLE_POW_CHECKS)
1481                 __cvmx_pow_warn_if_pending_switch(__func__);
1482 
1483         /*
1484          * Ensure that there is not a pending tag switch, as a tag
1485          * switch cannot be started if a previous switch is still
1486          * pending.
1487          */
1488         cvmx_pow_tag_sw_wait();
1489         cvmx_pow_tag_sw_full_nocheck(wqp, tag, tag_type, group);
1490 }
1491 
1492 /**
1493  * Switch to a NULL tag, which ends any ordering or
1494  * synchronization provided by the POW for the current
1495  * work queue entry.  This operation completes immediately,
1496  * so completion should not be waited for.
1497  * This function does NOT wait for previous tag switches to complete,
1498  * so the caller must ensure that any previous tag switches have completed.
1499  */
1500 static inline void cvmx_pow_tag_sw_null_nocheck(void)
1501 {
1502         cvmx_addr_t ptr;
1503         cvmx_pow_tag_req_t tag_req;
1504 
1505         if (CVMX_ENABLE_POW_CHECKS) {
1506                 cvmx_pow_tag_req_t current_tag;
1507                 __cvmx_pow_warn_if_pending_switch(__func__);
1508                 current_tag = cvmx_pow_get_current_tag();
1509                 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL_NULL)
1510                         pr_warning("%s called with NULL_NULL tag\n",
1511                                    __func__);
1512                 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL)
1513                         pr_warning("%s called when we already have a "
1514                                    "NULL tag\n",
1515                              __func__);
1516         }
1517 
1518         tag_req.u64 = 0;
1519         tag_req.s.op = CVMX_POW_TAG_OP_SWTAG;
1520         tag_req.s.type = CVMX_POW_TAG_TYPE_NULL;
1521 
1522         ptr.u64 = 0;
1523         ptr.sio.mem_region = CVMX_IO_SEG;
1524         ptr.sio.is_io = 1;
1525         ptr.sio.did = CVMX_OCT_DID_TAG_TAG1;
1526 
1527         cvmx_write_io(ptr.u64, tag_req.u64);
1528 
1529         /* switch to NULL completes immediately */
1530 }
1531 
1532 /**
1533  * Switch to a NULL tag, which ends any ordering or
1534  * synchronization provided by the POW for the current
1535  * work queue entry.  This operation completes immediately,
1536  * so completion should not be waited for.
1537  * This function waits for any pending tag switches to complete
1538  * before requesting the switch to NULL.
1539  */
1540 static inline void cvmx_pow_tag_sw_null(void)
1541 {
1542         if (CVMX_ENABLE_POW_CHECKS)
1543                 __cvmx_pow_warn_if_pending_switch(__func__);
1544 
1545         /*
1546          * Ensure that there is not a pending tag switch, as a tag
1547          * switch cannot be started if a previous switch is still
1548          * pending.
1549          */
1550         cvmx_pow_tag_sw_wait();
1551         cvmx_pow_tag_sw_null_nocheck();
1552 
1553         /* switch to NULL completes immediately */
1554 }
1555 
1556 /**
1557  * Submits work to an input queue.  This function updates the work
1558  * queue entry in DRAM to match the arguments given.  Note that the
1559  * tag provided is for the work queue entry submitted, and is
1560  * unrelated to the tag that the core currently holds.
1561  *
1562  * @wqp:      pointer to work queue entry to submit.  This entry is
1563  *            updated to match the other parameters
1564  * @tag:      tag value to be assigned to work queue entry
1565  * @tag_type: type of tag
1566  * @qos:      Input queue to add to.
1567  * @grp:      group value for the work queue entry.
1568  */
1569 static inline void cvmx_pow_work_submit(cvmx_wqe_t *wqp, uint32_t tag,
1570                                         enum cvmx_pow_tag_type tag_type,
1571                                         uint64_t qos, uint64_t grp)
1572 {
1573         cvmx_addr_t ptr;
1574         cvmx_pow_tag_req_t tag_req;
1575 
1576         wqp->qos = qos;
1577         wqp->tag = tag;
1578         wqp->tag_type = tag_type;
1579         wqp->grp = grp;
1580 
1581         tag_req.u64 = 0;
1582         tag_req.s.op = CVMX_POW_TAG_OP_ADDWQ;
1583         tag_req.s.type = tag_type;
1584         tag_req.s.tag = tag;
1585         tag_req.s.qos = qos;
1586         tag_req.s.grp = grp;
1587 
1588         ptr.u64 = 0;
1589         ptr.sio.mem_region = CVMX_IO_SEG;
1590         ptr.sio.is_io = 1;
1591         ptr.sio.did = CVMX_OCT_DID_TAG_TAG1;
1592         ptr.sio.offset = cvmx_ptr_to_phys(wqp);
1593 
1594         /*
1595          * SYNC write to memory before the work submit.  This is
1596          * necessary as POW may read values from DRAM at this time.
1597          */
1598         CVMX_SYNCWS;
1599         cvmx_write_io(ptr.u64, tag_req.u64);
1600 }
1601 
1602 /**
1603  * This function sets the group mask for a core.  The group mask
1604  * indicates which groups each core will accept work from. There are
1605  * 16 groups.
1606  *
1607  * @core_num:   core to apply mask to
1608  * @mask:   Group mask. There are 16 groups, so only bits 0-15 are valid,
1609  *               representing groups 0-15.
1610  *               Each 1 bit in the mask enables the core to accept work from
1611  *               the corresponding group.
1612  */
1613 static inline void cvmx_pow_set_group_mask(uint64_t core_num, uint64_t mask)
1614 {
1615         union cvmx_pow_pp_grp_mskx grp_msk;
1616 
1617         grp_msk.u64 = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(core_num));
1618         grp_msk.s.grp_msk = mask;
1619         cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(core_num), grp_msk.u64);
1620 }
1621 
1622 /**
1623  * This function sets POW static priorities for a core. Each input queue has
1624  * an associated priority value.
1625  *
1626  * @core_num:   core to apply priorities to
1627  * @priority:   Vector of 8 priorities, one per POW Input Queue (0-7).
1628  *                   Highest priority is 0 and lowest is 7. A priority value
1629  *                   of 0xF instructs POW to skip the Input Queue when
1630  *                   scheduling to this specific core.
1631  *                   NOTE: priorities should not have gaps in values, meaning
1632  *                         {0,1,1,1,1,1,1,1} is a valid configuration while
1633  *                         {0,2,2,2,2,2,2,2} is not.
1634  */
1635 static inline void cvmx_pow_set_priority(uint64_t core_num,
1636                                          const uint8_t priority[])
1637 {
1638         /* POW priorities are supported on CN5xxx and later */
1639         if (!OCTEON_IS_MODEL(OCTEON_CN3XXX)) {
1640                 union cvmx_pow_pp_grp_mskx grp_msk;
1641 
1642                 grp_msk.u64 = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(core_num));
1643                 grp_msk.s.qos0_pri = priority[0];
1644                 grp_msk.s.qos1_pri = priority[1];
1645                 grp_msk.s.qos2_pri = priority[2];
1646                 grp_msk.s.qos3_pri = priority[3];
1647                 grp_msk.s.qos4_pri = priority[4];
1648                 grp_msk.s.qos5_pri = priority[5];
1649                 grp_msk.s.qos6_pri = priority[6];
1650                 grp_msk.s.qos7_pri = priority[7];
1651 
1652                 /* Detect gaps between priorities and flag error */
1653                 {
1654                         int i;
1655                         uint32_t prio_mask = 0;
1656 
1657                         for (i = 0; i < 8; i++)
1658                                 if (priority[i] != 0xF)
1659                                         prio_mask |= 1 << priority[i];
1660 
1661                         if (prio_mask ^ ((1 << cvmx_pop(prio_mask)) - 1)) {
1662                                 pr_err("POW static priorities should be "
1663                                        "contiguous (0x%llx)\n",
1664                                      (unsigned long long)prio_mask);
1665                                 return;
1666                         }
1667                 }
1668 
1669                 cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(core_num), grp_msk.u64);
1670         }
1671 }
1672 
1673 /**
1674  * Performs a tag switch and then an immediate deschedule. This completes
1675  * immediately, so completion must not be waited for.  This function does NOT
1676  * update the wqe in DRAM to match arguments.
1677  *
1678  * This function does NOT wait for any prior tag switches to complete, so the
1679  * calling code must do this.
1680  *
1681  * Note the following CAVEAT of the Octeon HW behavior when
1682  * re-scheduling DE-SCHEDULEd items whose (next) state is
1683  * ORDERED:
1684  *   - If there are no switches pending at the time that the
1685  *     HW executes the de-schedule, the HW will only re-schedule
1686  *     the head of the FIFO associated with the given tag. This
1687  *     means that in many respects, the HW treats this ORDERED
1688  *     tag as an ATOMIC tag. Note that in the SWTAG_DESCH
1689  *     case (to an ORDERED tag), the HW will do the switch
1690  *     before the deschedule whenever it is possible to do
1691  *     the switch immediately, so it may often look like
1692  *     this case.
1693  *   - If there is a pending switch to ORDERED at the time
1694  *     the HW executes the de-schedule, the HW will perform
1695  *     the switch at the time it re-schedules, and will be
1696  *     able to reschedule any/all of the entries with the
1697  *     same tag.
1698  * Due to this behavior, the RECOMMENDATION to software is
1699  * that they have a (next) state of ATOMIC when they
1700  * DE-SCHEDULE. If an ORDERED tag is what was really desired,
1701  * SW can choose to immediately switch to an ORDERED tag
1702  * after the work (that has an ATOMIC tag) is re-scheduled.
1703  * Note that since there are never any tag switches pending
1704  * when the HW re-schedules, this switch can be IMMEDIATE upon
1705  * the reception of the pointer during the re-schedule.
1706  *
1707  * @tag:      New tag value
1708  * @tag_type: New tag type
1709  * @group:    New group value
1710  * @no_sched: Control whether this work queue entry will be rescheduled.
1711  *                 - 1 : don't schedule this work
1712  *                 - 0 : allow this work to be scheduled.
1713  */
1714 static inline void cvmx_pow_tag_sw_desched_nocheck(
1715         uint32_t tag,
1716         enum cvmx_pow_tag_type tag_type,
1717         uint64_t group,
1718         uint64_t no_sched)
1719 {
1720         cvmx_addr_t ptr;
1721         cvmx_pow_tag_req_t tag_req;
1722 
1723         if (CVMX_ENABLE_POW_CHECKS) {
1724                 cvmx_pow_tag_req_t current_tag;
1725                 __cvmx_pow_warn_if_pending_switch(__func__);
1726                 current_tag = cvmx_pow_get_current_tag();
1727                 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL_NULL)
1728                         pr_warning("%s called with NULL_NULL tag\n",
1729                                    __func__);
1730                 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL)
1731                         pr_warning("%s called with NULL tag. Deschedule not "
1732                                    "allowed from NULL state\n",
1733                              __func__);
1734                 if ((current_tag.s.type != CVMX_POW_TAG_TYPE_ATOMIC)
1735                         && (tag_type != CVMX_POW_TAG_TYPE_ATOMIC))
1736                         pr_warning("%s called where neither the before or "
1737                                    "after tag is ATOMIC\n",
1738                              __func__);
1739         }
1740 
1741         tag_req.u64 = 0;
1742         tag_req.s.op = CVMX_POW_TAG_OP_SWTAG_DESCH;
1743         tag_req.s.tag = tag;
1744         tag_req.s.type = tag_type;
1745         tag_req.s.grp = group;
1746         tag_req.s.no_sched = no_sched;
1747 
1748         ptr.u64 = 0;
1749         ptr.sio.mem_region = CVMX_IO_SEG;
1750         ptr.sio.is_io = 1;
1751         ptr.sio.did = CVMX_OCT_DID_TAG_TAG3;
1752         /*
1753          * since TAG3 is used, this store will clear the local pending
1754          * switch bit.
1755          */
1756         cvmx_write_io(ptr.u64, tag_req.u64);
1757 }
1758 
1759 /**
1760  * Performs a tag switch and then an immediate deschedule. This completes
1761  * immediately, so completion must not be waited for.  This function does NOT
1762  * update the wqe in DRAM to match arguments.
1763  *
1764  * This function waits for any prior tag switches to complete, so the
1765  * calling code may call this function with a pending tag switch.
1766  *
1767  * Note the following CAVEAT of the Octeon HW behavior when
1768  * re-scheduling DE-SCHEDULEd items whose (next) state is
1769  * ORDERED:
1770  *   - If there are no switches pending at the time that the
1771  *     HW executes the de-schedule, the HW will only re-schedule
1772  *     the head of the FIFO associated with the given tag. This
1773  *     means that in many respects, the HW treats this ORDERED
1774  *     tag as an ATOMIC tag. Note that in the SWTAG_DESCH
1775  *     case (to an ORDERED tag), the HW will do the switch
1776  *     before the deschedule whenever it is possible to do
1777  *     the switch immediately, so it may often look like
1778  *     this case.
1779  *   - If there is a pending switch to ORDERED at the time
1780  *     the HW executes the de-schedule, the HW will perform
1781  *     the switch at the time it re-schedules, and will be
1782  *     able to reschedule any/all of the entries with the
1783  *     same tag.
1784  * Due to this behavior, the RECOMMENDATION to software is
1785  * that they have a (next) state of ATOMIC when they
1786  * DE-SCHEDULE. If an ORDERED tag is what was really desired,
1787  * SW can choose to immediately switch to an ORDERED tag
1788  * after the work (that has an ATOMIC tag) is re-scheduled.
1789  * Note that since there are never any tag switches pending
1790  * when the HW re-schedules, this switch can be IMMEDIATE upon
1791  * the reception of the pointer during the re-schedule.
1792  *
1793  * @tag:      New tag value
1794  * @tag_type: New tag type
1795  * @group:    New group value
1796  * @no_sched: Control whether this work queue entry will be rescheduled.
1797  *                 - 1 : don't schedule this work
1798  *                 - 0 : allow this work to be scheduled.
1799  */
1800 static inline void cvmx_pow_tag_sw_desched(uint32_t tag,
1801                                            enum cvmx_pow_tag_type tag_type,
1802                                            uint64_t group, uint64_t no_sched)
1803 {
1804         if (CVMX_ENABLE_POW_CHECKS)
1805                 __cvmx_pow_warn_if_pending_switch(__func__);
1806 
1807         /* Need to make sure any writes to the work queue entry are complete */
1808         CVMX_SYNCWS;
1809         /*
1810          * Ensure that there is not a pending tag switch, as a tag
1811          * switch cannot be started if a previous switch is still
1812          * pending.
1813          */
1814         cvmx_pow_tag_sw_wait();
1815         cvmx_pow_tag_sw_desched_nocheck(tag, tag_type, group, no_sched);
1816 }
1817 
1818 /**
1819  * Descchedules the current work queue entry.
1820  *
1821  * @no_sched: no schedule flag value to be set on the work queue
1822  *            entry.  If this is set the entry will not be
1823  *            rescheduled.
1824  */
1825 static inline void cvmx_pow_desched(uint64_t no_sched)
1826 {
1827         cvmx_addr_t ptr;
1828         cvmx_pow_tag_req_t tag_req;
1829 
1830         if (CVMX_ENABLE_POW_CHECKS) {
1831                 cvmx_pow_tag_req_t current_tag;
1832                 __cvmx_pow_warn_if_pending_switch(__func__);
1833                 current_tag = cvmx_pow_get_current_tag();
1834                 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL_NULL)
1835                         pr_warning("%s called with NULL_NULL tag\n",
1836                                    __func__);
1837                 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL)
1838                         pr_warning("%s called with NULL tag. Deschedule not "
1839                                    "expected from NULL state\n",
1840                              __func__);
1841         }
1842 
1843         /* Need to make sure any writes to the work queue entry are complete */
1844         CVMX_SYNCWS;
1845 
1846         tag_req.u64 = 0;
1847         tag_req.s.op = CVMX_POW_TAG_OP_DESCH;
1848         tag_req.s.no_sched = no_sched;
1849 
1850         ptr.u64 = 0;
1851         ptr.sio.mem_region = CVMX_IO_SEG;
1852         ptr.sio.is_io = 1;
1853         ptr.sio.did = CVMX_OCT_DID_TAG_TAG3;
1854         /*
1855          * since TAG3 is used, this store will clear the local pending
1856          * switch bit.
1857          */
1858         cvmx_write_io(ptr.u64, tag_req.u64);
1859 }
1860 
1861 /****************************************************
1862 * Define usage of bits within the 32 bit tag values.
1863 *****************************************************/
1864 
1865 /*
1866  * Number of bits of the tag used by software.  The SW bits are always
1867  * a contiguous block of the high starting at bit 31.  The hardware
1868  * bits are always the low bits.  By default, the top 8 bits of the
1869  * tag are reserved for software, and the low 24 are set by the IPD
1870  * unit.
1871  */
1872 #define CVMX_TAG_SW_BITS    (8)
1873 #define CVMX_TAG_SW_SHIFT   (32 - CVMX_TAG_SW_BITS)
1874 
1875 /* Below is the list of values for the top 8 bits of the tag. */
1876 /*
1877  * Tag values with top byte of this value are reserved for internal
1878  * executive uses.
1879  */
1880 #define CVMX_TAG_SW_BITS_INTERNAL  0x1
1881 /* The executive divides the remaining 24 bits as follows:
1882  *  - the upper 8 bits (bits 23 - 16 of the tag) define a subgroup
1883  *
1884  *  - the lower 16 bits (bits 15 - 0 of the tag) define are the value
1885  *    with the subgroup
1886  *
1887  * Note that this section describes the format of tags generated by
1888  * software - refer to the hardware documentation for a description of
1889  * the tags values generated by the packet input hardware.  Subgroups
1890  * are defined here.
1891  */
1892 /* Mask for the value portion of the tag */
1893 #define CVMX_TAG_SUBGROUP_MASK  0xFFFF
1894 #define CVMX_TAG_SUBGROUP_SHIFT 16
1895 #define CVMX_TAG_SUBGROUP_PKO  0x1
1896 
1897 /* End of executive tag subgroup definitions */
1898 
1899 /*
1900  * The remaining values software bit values 0x2 - 0xff are available
1901  * for application use.
1902  */
1903 
1904 /**
1905  * This function creates a 32 bit tag value from the two values provided.
1906  *
1907  * @sw_bits: The upper bits (number depends on configuration) are set
1908  *           to this value.  The remainder of bits are set by the
1909  *           hw_bits parameter.
1910  *
1911  * @hw_bits: The lower bits (number depends on configuration) are set
1912  *           to this value.  The remainder of bits are set by the
1913  *           sw_bits parameter.
1914  *
1915  * Returns 32 bit value of the combined hw and sw bits.
1916  */
1917 static inline uint32_t cvmx_pow_tag_compose(uint64_t sw_bits, uint64_t hw_bits)
1918 {
1919         return ((sw_bits & cvmx_build_mask(CVMX_TAG_SW_BITS)) <<
1920                         CVMX_TAG_SW_SHIFT) |
1921                 (hw_bits & cvmx_build_mask(32 - CVMX_TAG_SW_BITS));
1922 }
1923 
1924 /**
1925  * Extracts the bits allocated for software use from the tag
1926  *
1927  * @tag:    32 bit tag value
1928  *
1929  * Returns N bit software tag value, where N is configurable with the
1930  * CVMX_TAG_SW_BITS define
1931  */
1932 static inline uint32_t cvmx_pow_tag_get_sw_bits(uint64_t tag)
1933 {
1934         return (tag >> (32 - CVMX_TAG_SW_BITS)) &
1935                 cvmx_build_mask(CVMX_TAG_SW_BITS);
1936 }
1937 
1938 /**
1939  *
1940  * Extracts the bits allocated for hardware use from the tag
1941  *
1942  * @tag:    32 bit tag value
1943  *
1944  * Returns (32 - N) bit software tag value, where N is configurable
1945  * with the CVMX_TAG_SW_BITS define
1946  */
1947 static inline uint32_t cvmx_pow_tag_get_hw_bits(uint64_t tag)
1948 {
1949         return tag & cvmx_build_mask(32 - CVMX_TAG_SW_BITS);
1950 }
1951 
1952 /**
1953  * Store the current POW internal state into the supplied
1954  * buffer. It is recommended that you pass a buffer of at least
1955  * 128KB. The format of the capture may change based on SDK
1956  * version and Octeon chip.
1957  *
1958  * @buffer: Buffer to store capture into
1959  * @buffer_size:
1960  *               The size of the supplied buffer
1961  *
1962  * Returns Zero on success, negative on failure
1963  */
1964 extern int cvmx_pow_capture(void *buffer, int buffer_size);
1965 
1966 /**
1967  * Dump a POW capture to the console in a human readable format.
1968  *
1969  * @buffer: POW capture from cvmx_pow_capture()
1970  * @buffer_size:
1971  *               Size of the buffer
1972  */
1973 extern void cvmx_pow_display(void *buffer, int buffer_size);
1974 
1975 /**
1976  * Return the number of POW entries supported by this chip
1977  *
1978  * Returns Number of POW entries
1979  */
1980 extern int cvmx_pow_get_num_entries(void);
1981 
1982 #endif /* __CVMX_POW_H__ */
1983 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp