~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/mips/include/asm/octeon/cvmx-pow.h

Version: ~ [ linux-5.3 ] ~ [ linux-5.2.14 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.72 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.143 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.192 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.192 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.73 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /***********************license start***************
  2  * Author: Cavium Networks
  3  *
  4  * Contact: support@caviumnetworks.com
  5  * This file is part of the OCTEON SDK
  6  *
  7  * Copyright (c) 2003-2008 Cavium Networks
  8  *
  9  * This file is free software; you can redistribute it and/or modify
 10  * it under the terms of the GNU General Public License, Version 2, as
 11  * published by the Free Software Foundation.
 12  *
 13  * This file is distributed in the hope that it will be useful, but
 14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
 15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
 16  * NONINFRINGEMENT.  See the GNU General Public License for more
 17  * details.
 18  *
 19  * You should have received a copy of the GNU General Public License
 20  * along with this file; if not, write to the Free Software
 21  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
 22  * or visit http://www.gnu.org/licenses/.
 23  *
 24  * This file may also be available under a different license from Cavium.
 25  * Contact Cavium Networks for more information
 26  ***********************license end**************************************/
 27 
 28 /**
 29  * Interface to the hardware Packet Order / Work unit.
 30  *
 31  * New, starting with SDK 1.7.0, cvmx-pow supports a number of
 32  * extended consistency checks. The define
 33  * CVMX_ENABLE_POW_CHECKS controls the runtime insertion of POW
 34  * internal state checks to find common programming errors. If
 35  * CVMX_ENABLE_POW_CHECKS is not defined, checks are by default
 36  * enabled. For example, cvmx-pow will check for the following
 37  * program errors or POW state inconsistency.
 38  * - Requesting a POW operation with an active tag switch in
 39  *   progress.
 40  * - Waiting for a tag switch to complete for an excessively
 41  *   long period. This is normally a sign of an error in locking
 42  *   causing deadlock.
 43  * - Illegal tag switches from NULL_NULL.
 44  * - Illegal tag switches from NULL.
 45  * - Illegal deschedule request.
 46  * - WQE pointer not matching the one attached to the core by
 47  *   the POW.
 48  *
 49  */
 50 
 51 #ifndef __CVMX_POW_H__
 52 #define __CVMX_POW_H__
 53 
 54 #include <asm/octeon/cvmx-pow-defs.h>
 55 
 56 #include <asm/octeon/cvmx-scratch.h>
 57 #include <asm/octeon/cvmx-wqe.h>
 58 
 59 /* Default to having all POW constancy checks turned on */
 60 #ifndef CVMX_ENABLE_POW_CHECKS
 61 #define CVMX_ENABLE_POW_CHECKS 1
 62 #endif
 63 
 64 enum cvmx_pow_tag_type {
 65         /* Tag ordering is maintained */
 66         CVMX_POW_TAG_TYPE_ORDERED   = 0L,
 67         /* Tag ordering is maintained, and at most one PP has the tag */
 68         CVMX_POW_TAG_TYPE_ATOMIC    = 1L,
 69         /*
 70          * The work queue entry from the order - NEVER tag switch from
 71          * NULL to NULL
 72          */
 73         CVMX_POW_TAG_TYPE_NULL      = 2L,
 74         /* A tag switch to NULL, and there is no space reserved in POW
 75          * - NEVER tag switch to NULL_NULL
 76          * - NEVER tag switch from NULL_NULL
 77          * - NULL_NULL is entered at the beginning of time and on a deschedule.
 78          * - NULL_NULL can be exited by a new work request. A NULL_SWITCH
 79          * load can also switch the state to NULL
 80          */
 81         CVMX_POW_TAG_TYPE_NULL_NULL = 3L
 82 };
 83 
 84 /**
 85  * Wait flag values for pow functions.
 86  */
 87 typedef enum {
 88         CVMX_POW_WAIT = 1,
 89         CVMX_POW_NO_WAIT = 0,
 90 } cvmx_pow_wait_t;
 91 
 92 /**
 93  *  POW tag operations.  These are used in the data stored to the POW.
 94  */
 95 typedef enum {
 96         /*
 97          * switch the tag (only) for this PP
 98          * - the previous tag should be non-NULL in this case
 99          * - tag switch response required
100          * - fields used: op, type, tag
101          */
102         CVMX_POW_TAG_OP_SWTAG = 0L,
103         /*
104          * switch the tag for this PP, with full information
105          * - this should be used when the previous tag is NULL
106          * - tag switch response required
107          * - fields used: address, op, grp, type, tag
108          */
109         CVMX_POW_TAG_OP_SWTAG_FULL = 1L,
110         /*
111          * switch the tag (and/or group) for this PP and de-schedule
112          * - OK to keep the tag the same and only change the group
113          * - fields used: op, no_sched, grp, type, tag
114          */
115         CVMX_POW_TAG_OP_SWTAG_DESCH = 2L,
116         /*
117          * just de-schedule
118          * - fields used: op, no_sched
119          */
120         CVMX_POW_TAG_OP_DESCH = 3L,
121         /*
122          * create an entirely new work queue entry
123          * - fields used: address, op, qos, grp, type, tag
124          */
125         CVMX_POW_TAG_OP_ADDWQ = 4L,
126         /*
127          * just update the work queue pointer and grp for this PP
128          * - fields used: address, op, grp
129          */
130         CVMX_POW_TAG_OP_UPDATE_WQP_GRP = 5L,
131         /*
132          * set the no_sched bit on the de-schedule list
133          *
134          * - does nothing if the selected entry is not on the
135          *   de-schedule list
136          *
137          * - does nothing if the stored work queue pointer does not
138          *   match the address field
139          *
140          * - fields used: address, index, op
141          *
142          *  Before issuing a *_NSCHED operation, SW must guarantee
143          *  that all prior deschedules and set/clr NSCHED operations
144          *  are complete and all prior switches are complete. The
145          *  hardware provides the opsdone bit and swdone bit for SW
146          *  polling. After issuing a *_NSCHED operation, SW must
147          *  guarantee that the set/clr NSCHED is complete before any
148          *  subsequent operations.
149          */
150         CVMX_POW_TAG_OP_SET_NSCHED = 6L,
151         /*
152          * clears the no_sched bit on the de-schedule list
153          *
154          * - does nothing if the selected entry is not on the
155          *   de-schedule list
156          *
157          * - does nothing if the stored work queue pointer does not
158          *   match the address field
159          *
160          * - fields used: address, index, op
161          *
162          * Before issuing a *_NSCHED operation, SW must guarantee that
163          * all prior deschedules and set/clr NSCHED operations are
164          * complete and all prior switches are complete. The hardware
165          * provides the opsdone bit and swdone bit for SW
166          * polling. After issuing a *_NSCHED operation, SW must
167          * guarantee that the set/clr NSCHED is complete before any
168          * subsequent operations.
169          */
170         CVMX_POW_TAG_OP_CLR_NSCHED = 7L,
171         /* do nothing */
172         CVMX_POW_TAG_OP_NOP = 15L
173 } cvmx_pow_tag_op_t;
174 
175 /**
176  * This structure defines the store data on a store to POW
177  */
178 typedef union {
179         uint64_t u64;
180         struct {
181 #ifdef __BIG_ENDIAN_BITFIELD
182                 /*
183                  * Don't reschedule this entry. no_sched is used for
184                  * CVMX_POW_TAG_OP_SWTAG_DESCH and
185                  * CVMX_POW_TAG_OP_DESCH
186                  */
187                 uint64_t no_sched:1;
188                 uint64_t unused:2;
189                 /* Tontains index of entry for a CVMX_POW_TAG_OP_*_NSCHED */
190                 uint64_t index:13;
191                 /* The operation to perform */
192                 cvmx_pow_tag_op_t op:4;
193                 uint64_t unused2:2;
194                 /*
195                  * The QOS level for the packet. qos is only used for
196                  * CVMX_POW_TAG_OP_ADDWQ
197                  */
198                 uint64_t qos:3;
199                 /*
200                  * The group that the work queue entry will be
201                  * scheduled to grp is used for CVMX_POW_TAG_OP_ADDWQ,
202                  * CVMX_POW_TAG_OP_SWTAG_FULL,
203                  * CVMX_POW_TAG_OP_SWTAG_DESCH, and
204                  * CVMX_POW_TAG_OP_UPDATE_WQP_GRP
205                  */
206                 uint64_t grp:4;
207                 /*
208                  * The type of the tag. type is used for everything
209                  * except CVMX_POW_TAG_OP_DESCH,
210                  * CVMX_POW_TAG_OP_UPDATE_WQP_GRP, and
211                  * CVMX_POW_TAG_OP_*_NSCHED
212                  */
213                 uint64_t type:3;
214                 /*
215                  * The actual tag. tag is used for everything except
216                  * CVMX_POW_TAG_OP_DESCH,
217                  * CVMX_POW_TAG_OP_UPDATE_WQP_GRP, and
218                  * CVMX_POW_TAG_OP_*_NSCHED
219                  */
220                 uint64_t tag:32;
221 #else
222                 uint64_t tag:32;
223                 uint64_t type:3;
224                 uint64_t grp:4;
225                 uint64_t qos:3;
226                 uint64_t unused2:2;
227                 cvmx_pow_tag_op_t op:4;
228                 uint64_t index:13;
229                 uint64_t unused:2;
230                 uint64_t no_sched:1;
231 #endif
232         } s;
233 } cvmx_pow_tag_req_t;
234 
235 /**
236  * This structure describes the address to load stuff from POW
237  */
238 typedef union {
239         uint64_t u64;
240 
241     /**
242      * Address for new work request loads (did<2:0> == 0)
243      */
244         struct {
245 #ifdef __BIG_ENDIAN_BITFIELD
246                 /* Mips64 address region. Should be CVMX_IO_SEG */
247                 uint64_t mem_region:2;
248                 /* Must be zero */
249                 uint64_t reserved_49_61:13;
250                 /* Must be one */
251                 uint64_t is_io:1;
252                 /* the ID of POW -- did<2:0> == 0 in this case */
253                 uint64_t did:8;
254                 /* Must be zero */
255                 uint64_t reserved_4_39:36;
256                 /*
257                  * If set, don't return load response until work is
258                  * available.
259                  */
260                 uint64_t wait:1;
261                 /* Must be zero */
262                 uint64_t reserved_0_2:3;
263 #else
264                 uint64_t reserved_0_2:3;
265                 uint64_t wait:1;
266                 uint64_t reserved_4_39:36;
267                 uint64_t did:8;
268                 uint64_t is_io:1;
269                 uint64_t reserved_49_61:13;
270                 uint64_t mem_region:2;
271 #endif
272         } swork;
273 
274     /**
275      * Address for loads to get POW internal status
276      */
277         struct {
278 #ifdef __BIG_ENDIAN_BITFIELD
279                 /* Mips64 address region. Should be CVMX_IO_SEG */
280                 uint64_t mem_region:2;
281                 /* Must be zero */
282                 uint64_t reserved_49_61:13;
283                 /* Must be one */
284                 uint64_t is_io:1;
285                 /* the ID of POW -- did<2:0> == 1 in this case */
286                 uint64_t did:8;
287                 /* Must be zero */
288                 uint64_t reserved_10_39:30;
289                 /* The core id to get status for */
290                 uint64_t coreid:4;
291                 /*
292                  * If set and get_cur is set, return reverse tag-list
293                  * pointer rather than forward tag-list pointer.
294                  */
295                 uint64_t get_rev:1;
296                 /*
297                  * If set, return current status rather than pending
298                  * status.
299                  */
300                 uint64_t get_cur:1;
301                 /*
302                  * If set, get the work-queue pointer rather than
303                  * tag/type.
304                  */
305                 uint64_t get_wqp:1;
306                 /* Must be zero */
307                 uint64_t reserved_0_2:3;
308 #else
309                 uint64_t reserved_0_2:3;
310                 uint64_t get_wqp:1;
311                 uint64_t get_cur:1;
312                 uint64_t get_rev:1;
313                 uint64_t coreid:4;
314                 uint64_t reserved_10_39:30;
315                 uint64_t did:8;
316                 uint64_t is_io:1;
317                 uint64_t reserved_49_61:13;
318                 uint64_t mem_region:2;
319 #endif
320         } sstatus;
321 
322     /**
323      * Address for memory loads to get POW internal state
324      */
325         struct {
326 #ifdef __BIG_ENDIAN_BITFIELD
327                 /* Mips64 address region. Should be CVMX_IO_SEG */
328                 uint64_t mem_region:2;
329                 /* Must be zero */
330                 uint64_t reserved_49_61:13;
331                 /* Must be one */
332                 uint64_t is_io:1;
333                 /* the ID of POW -- did<2:0> == 2 in this case */
334                 uint64_t did:8;
335                 /* Must be zero */
336                 uint64_t reserved_16_39:24;
337                 /* POW memory index */
338                 uint64_t index:11;
339                 /*
340                  * If set, return deschedule information rather than
341                  * the standard response for work-queue index (invalid
342                  * if the work-queue entry is not on the deschedule
343                  * list).
344                  */
345                 uint64_t get_des:1;
346                 /*
347                  * If set, get the work-queue pointer rather than
348                  * tag/type (no effect when get_des set).
349                  */
350                 uint64_t get_wqp:1;
351                 /* Must be zero */
352                 uint64_t reserved_0_2:3;
353 #else
354                 uint64_t reserved_0_2:3;
355                 uint64_t get_wqp:1;
356                 uint64_t get_des:1;
357                 uint64_t index:11;
358                 uint64_t reserved_16_39:24;
359                 uint64_t did:8;
360                 uint64_t is_io:1;
361                 uint64_t reserved_49_61:13;
362                 uint64_t mem_region:2;
363 #endif
364         } smemload;
365 
366     /**
367      * Address for index/pointer loads
368      */
369         struct {
370 #ifdef __BIG_ENDIAN_BITFIELD
371                 /* Mips64 address region. Should be CVMX_IO_SEG */
372                 uint64_t mem_region:2;
373                 /* Must be zero */
374                 uint64_t reserved_49_61:13;
375                 /* Must be one */
376                 uint64_t is_io:1;
377                 /* the ID of POW -- did<2:0> == 3 in this case */
378                 uint64_t did:8;
379                 /* Must be zero */
380                 uint64_t reserved_9_39:31;
381                 /*
382                  * when {get_rmt ==0 AND get_des_get_tail == 0}, this
383                  * field selects one of eight POW internal-input
384                  * queues (0-7), one per QOS level; values 8-15 are
385                  * illegal in this case; when {get_rmt ==0 AND
386                  * get_des_get_tail == 1}, this field selects one of
387                  * 16 deschedule lists (per group); when get_rmt ==1,
388                  * this field selects one of 16 memory-input queue
389                  * lists.  The two memory-input queue lists associated
390                  * with each QOS level are:
391                  *
392                  * - qosgrp = 0, qosgrp = 8:      QOS0
393                  * - qosgrp = 1, qosgrp = 9:      QOS1
394                  * - qosgrp = 2, qosgrp = 10:     QOS2
395                  * - qosgrp = 3, qosgrp = 11:     QOS3
396                  * - qosgrp = 4, qosgrp = 12:     QOS4
397                  * - qosgrp = 5, qosgrp = 13:     QOS5
398                  * - qosgrp = 6, qosgrp = 14:     QOS6
399                  * - qosgrp = 7, qosgrp = 15:     QOS7
400                  */
401                 uint64_t qosgrp:4;
402                 /*
403                  * If set and get_rmt is clear, return deschedule list
404                  * indexes rather than indexes for the specified qos
405                  * level; if set and get_rmt is set, return the tail
406                  * pointer rather than the head pointer for the
407                  * specified qos level.
408                  */
409                 uint64_t get_des_get_tail:1;
410                 /*
411                  * If set, return remote pointers rather than the
412                  * local indexes for the specified qos level.
413                  */
414                 uint64_t get_rmt:1;
415                 /* Must be zero */
416                 uint64_t reserved_0_2:3;
417 #else
418                 uint64_t reserved_0_2:3;
419                 uint64_t get_rmt:1;
420                 uint64_t get_des_get_tail:1;
421                 uint64_t qosgrp:4;
422                 uint64_t reserved_9_39:31;
423                 uint64_t did:8;
424                 uint64_t is_io:1;
425                 uint64_t reserved_49_61:13;
426                 uint64_t mem_region:2;
427 #endif
428         } sindexload;
429 
430     /**
431      * address for NULL_RD request (did<2:0> == 4) when this is read,
432      * HW attempts to change the state to NULL if it is NULL_NULL (the
433      * hardware cannot switch from NULL_NULL to NULL if a POW entry is
434      * not available - software may need to recover by finishing
435      * another piece of work before a POW entry can ever become
436      * available.)
437      */
438         struct {
439 #ifdef __BIG_ENDIAN_BITFIELD
440                 /* Mips64 address region. Should be CVMX_IO_SEG */
441                 uint64_t mem_region:2;
442                 /* Must be zero */
443                 uint64_t reserved_49_61:13;
444                 /* Must be one */
445                 uint64_t is_io:1;
446                 /* the ID of POW -- did<2:0> == 4 in this case */
447                 uint64_t did:8;
448                 /* Must be zero */
449                 uint64_t reserved_0_39:40;
450 #else
451                 uint64_t reserved_0_39:40;
452                 uint64_t did:8;
453                 uint64_t is_io:1;
454                 uint64_t reserved_49_61:13;
455                 uint64_t mem_region:2;
456 #endif
457         } snull_rd;
458 } cvmx_pow_load_addr_t;
459 
460 /**
461  * This structure defines the response to a load/SENDSINGLE to POW
462  * (except CSR reads)
463  */
464 typedef union {
465         uint64_t u64;
466 
467     /**
468      * Response to new work request loads
469      */
470         struct {
471 #ifdef __BIG_ENDIAN_BITFIELD
472                 /*
473                  * Set when no new work queue entry was returned.  *
474                  * If there was de-scheduled work, the HW will
475                  * definitely return it. When this bit is set, it
476                  * could mean either mean:
477                  *
478                  * - There was no work, or
479                  *
480                  * - There was no work that the HW could find. This
481                  *   case can happen, regardless of the wait bit value
482                  *   in the original request, when there is work in
483                  *   the IQ's that is too deep down the list.
484                  */
485                 uint64_t no_work:1;
486                 /* Must be zero */
487                 uint64_t reserved_40_62:23;
488                 /* 36 in O1 -- the work queue pointer */
489                 uint64_t addr:40;
490 #else
491                 uint64_t addr:40;
492                 uint64_t reserved_40_62:23;
493                 uint64_t no_work:1;
494 #endif
495         } s_work;
496 
497     /**
498      * Result for a POW Status Load (when get_cur==0 and get_wqp==0)
499      */
500         struct {
501 #ifdef __BIG_ENDIAN_BITFIELD
502                 uint64_t reserved_62_63:2;
503                 /* Set when there is a pending non-NULL SWTAG or
504                  * SWTAG_FULL, and the POW entry has not left the list
505                  * for the original tag. */
506                 uint64_t pend_switch:1;
507                 /* Set when SWTAG_FULL and pend_switch is set. */
508                 uint64_t pend_switch_full:1;
509                 /*
510                  * Set when there is a pending NULL SWTAG, or an
511                  * implicit switch to NULL.
512                  */
513                 uint64_t pend_switch_null:1;
514                 /* Set when there is a pending DESCHED or SWTAG_DESCHED. */
515                 uint64_t pend_desched:1;
516                 /*
517                  * Set when there is a pending SWTAG_DESCHED and
518                  * pend_desched is set.
519                  */
520                 uint64_t pend_desched_switch:1;
521                 /* Set when nosched is desired and pend_desched is set. */
522                 uint64_t pend_nosched:1;
523                 /* Set when there is a pending GET_WORK. */
524                 uint64_t pend_new_work:1;
525                 /*
526                  * When pend_new_work is set, this bit indicates that
527                  * the wait bit was set.
528                  */
529                 uint64_t pend_new_work_wait:1;
530                 /* Set when there is a pending NULL_RD. */
531                 uint64_t pend_null_rd:1;
532                 /* Set when there is a pending CLR_NSCHED. */
533                 uint64_t pend_nosched_clr:1;
534                 uint64_t reserved_51:1;
535                 /* This is the index when pend_nosched_clr is set. */
536                 uint64_t pend_index:11;
537                 /*
538                  * This is the new_grp when (pend_desched AND
539                  * pend_desched_switch) is set.
540                  */
541                 uint64_t pend_grp:4;
542                 uint64_t reserved_34_35:2;
543                 /*
544                  * This is the tag type when pend_switch or
545                  * (pend_desched AND pend_desched_switch) are set.
546                  */
547                 uint64_t pend_type:2;
548                 /*
549                  * - this is the tag when pend_switch or (pend_desched
550                  *    AND pend_desched_switch) are set.
551                  */
552                 uint64_t pend_tag:32;
553 #else
554                 uint64_t pend_tag:32;
555                 uint64_t pend_type:2;
556                 uint64_t reserved_34_35:2;
557                 uint64_t pend_grp:4;
558                 uint64_t pend_index:11;
559                 uint64_t reserved_51:1;
560                 uint64_t pend_nosched_clr:1;
561                 uint64_t pend_null_rd:1;
562                 uint64_t pend_new_work_wait:1;
563                 uint64_t pend_new_work:1;
564                 uint64_t pend_nosched:1;
565                 uint64_t pend_desched_switch:1;
566                 uint64_t pend_desched:1;
567                 uint64_t pend_switch_null:1;
568                 uint64_t pend_switch_full:1;
569                 uint64_t pend_switch:1;
570                 uint64_t reserved_62_63:2;
571 #endif
572         } s_sstatus0;
573 
574     /**
575      * Result for a POW Status Load (when get_cur==0 and get_wqp==1)
576      */
577         struct {
578 #ifdef __BIG_ENDIAN_BITFIELD
579                 uint64_t reserved_62_63:2;
580                 /*
581                  * Set when there is a pending non-NULL SWTAG or
582                  * SWTAG_FULL, and the POW entry has not left the list
583                  * for the original tag.
584                  */
585                 uint64_t pend_switch:1;
586                 /* Set when SWTAG_FULL and pend_switch is set. */
587                 uint64_t pend_switch_full:1;
588                 /*
589                  * Set when there is a pending NULL SWTAG, or an
590                  * implicit switch to NULL.
591                  */
592                 uint64_t pend_switch_null:1;
593                 /*
594                  * Set when there is a pending DESCHED or
595                  * SWTAG_DESCHED.
596                  */
597                 uint64_t pend_desched:1;
598                 /*
599                  * Set when there is a pending SWTAG_DESCHED and
600                  * pend_desched is set.
601                  */
602                 uint64_t pend_desched_switch:1;
603                 /* Set when nosched is desired and pend_desched is set. */
604                 uint64_t pend_nosched:1;
605                 /* Set when there is a pending GET_WORK. */
606                 uint64_t pend_new_work:1;
607                 /*
608                  * When pend_new_work is set, this bit indicates that
609                  * the wait bit was set.
610                  */
611                 uint64_t pend_new_work_wait:1;
612                 /* Set when there is a pending NULL_RD. */
613                 uint64_t pend_null_rd:1;
614                 /* Set when there is a pending CLR_NSCHED. */
615                 uint64_t pend_nosched_clr:1;
616                 uint64_t reserved_51:1;
617                 /* This is the index when pend_nosched_clr is set. */
618                 uint64_t pend_index:11;
619                 /*
620                  * This is the new_grp when (pend_desched AND
621                  * pend_desched_switch) is set.
622                  */
623                 uint64_t pend_grp:4;
624                 /* This is the wqp when pend_nosched_clr is set. */
625                 uint64_t pend_wqp:36;
626 #else
627                 uint64_t pend_wqp:36;
628                 uint64_t pend_grp:4;
629                 uint64_t pend_index:11;
630                 uint64_t reserved_51:1;
631                 uint64_t pend_nosched_clr:1;
632                 uint64_t pend_null_rd:1;
633                 uint64_t pend_new_work_wait:1;
634                 uint64_t pend_new_work:1;
635                 uint64_t pend_nosched:1;
636                 uint64_t pend_desched_switch:1;
637                 uint64_t pend_desched:1;
638                 uint64_t pend_switch_null:1;
639                 uint64_t pend_switch_full:1;
640                 uint64_t pend_switch:1;
641                 uint64_t reserved_62_63:2;
642 #endif
643         } s_sstatus1;
644 
645     /**
646      * Result for a POW Status Load (when get_cur==1, get_wqp==0, and
647      * get_rev==0)
648      */
649         struct {
650 #ifdef __BIG_ENDIAN_BITFIELD
651                 uint64_t reserved_62_63:2;
652                 /*
653                  * Points to the next POW entry in the tag list when
654                  * tail == 0 (and tag_type is not NULL or NULL_NULL).
655                  */
656                 uint64_t link_index:11;
657                 /* The POW entry attached to the core. */
658                 uint64_t index:11;
659                 /*
660                  * The group attached to the core (updated when new
661                  * tag list entered on SWTAG_FULL).
662                  */
663                 uint64_t grp:4;
664                 /*
665                  * Set when this POW entry is at the head of its tag
666                  * list (also set when in the NULL or NULL_NULL
667                  * state).
668                  */
669                 uint64_t head:1;
670                 /*
671                  * Set when this POW entry is at the tail of its tag
672                  * list (also set when in the NULL or NULL_NULL
673                  * state).
674                  */
675                 uint64_t tail:1;
676                 /*
677                  * The tag type attached to the core (updated when new
678                  * tag list entered on SWTAG, SWTAG_FULL, or
679                  * SWTAG_DESCHED).
680                  */
681                 uint64_t tag_type:2;
682                 /*
683                  * The tag attached to the core (updated when new tag
684                  * list entered on SWTAG, SWTAG_FULL, or
685                  * SWTAG_DESCHED).
686                  */
687                 uint64_t tag:32;
688 #else
689                 uint64_t tag:32;
690                 uint64_t tag_type:2;
691                 uint64_t tail:1;
692                 uint64_t head:1;
693                 uint64_t grp:4;
694                 uint64_t index:11;
695                 uint64_t link_index:11;
696                 uint64_t reserved_62_63:2;
697 #endif
698         } s_sstatus2;
699 
700     /**
701      * Result for a POW Status Load (when get_cur==1, get_wqp==0, and get_rev==1)
702      */
703         struct {
704 #ifdef __BIG_ENDIAN_BITFIELD
705                 uint64_t reserved_62_63:2;
706                 /*
707                  * Points to the prior POW entry in the tag list when
708                  * head == 0 (and tag_type is not NULL or
709                  * NULL_NULL). This field is unpredictable when the
710                  * core's state is NULL or NULL_NULL.
711                  */
712                 uint64_t revlink_index:11;
713                 /* The POW entry attached to the core. */
714                 uint64_t index:11;
715                 /*
716                  * The group attached to the core (updated when new
717                  * tag list entered on SWTAG_FULL).
718                  */
719                 uint64_t grp:4;
720                 /* Set when this POW entry is at the head of its tag
721                  * list (also set when in the NULL or NULL_NULL
722                  * state).
723                  */
724                 uint64_t head:1;
725                 /*
726                  * Set when this POW entry is at the tail of its tag
727                  * list (also set when in the NULL or NULL_NULL
728                  * state).
729                  */
730                 uint64_t tail:1;
731                 /*
732                  * The tag type attached to the core (updated when new
733                  * tag list entered on SWTAG, SWTAG_FULL, or
734                  * SWTAG_DESCHED).
735                  */
736                 uint64_t tag_type:2;
737                 /*
738                  * The tag attached to the core (updated when new tag
739                  * list entered on SWTAG, SWTAG_FULL, or
740                  * SWTAG_DESCHED).
741                  */
742                 uint64_t tag:32;
743 #else
744                 uint64_t tag:32;
745                 uint64_t tag_type:2;
746                 uint64_t tail:1;
747                 uint64_t head:1;
748                 uint64_t grp:4;
749                 uint64_t index:11;
750                 uint64_t revlink_index:11;
751                 uint64_t reserved_62_63:2;
752 #endif
753         } s_sstatus3;
754 
755     /**
756      * Result for a POW Status Load (when get_cur==1, get_wqp==1, and
757      * get_rev==0)
758      */
759         struct {
760 #ifdef __BIG_ENDIAN_BITFIELD
761                 uint64_t reserved_62_63:2;
762                 /*
763                  * Points to the next POW entry in the tag list when
764                  * tail == 0 (and tag_type is not NULL or NULL_NULL).
765                  */
766                 uint64_t link_index:11;
767                 /* The POW entry attached to the core. */
768                 uint64_t index:11;
769                 /*
770                  * The group attached to the core (updated when new
771                  * tag list entered on SWTAG_FULL).
772                  */
773                 uint64_t grp:4;
774                 /*
775                  * The wqp attached to the core (updated when new tag
776                  * list entered on SWTAG_FULL).
777                  */
778                 uint64_t wqp:36;
779 #else
780                 uint64_t wqp:36;
781                 uint64_t grp:4;
782                 uint64_t index:11;
783                 uint64_t link_index:11;
784                 uint64_t reserved_62_63:2;
785 #endif
786         } s_sstatus4;
787 
788     /**
789      * Result for a POW Status Load (when get_cur==1, get_wqp==1, and
790      * get_rev==1)
791      */
792         struct {
793 #ifdef __BIG_ENDIAN_BITFIELD
794                 uint64_t reserved_62_63:2;
795                 /*
796                  * Points to the prior POW entry in the tag list when
797                  * head == 0 (and tag_type is not NULL or
798                  * NULL_NULL). This field is unpredictable when the
799                  * core's state is NULL or NULL_NULL.
800                  */
801                 uint64_t revlink_index:11;
802                 /* The POW entry attached to the core. */
803                 uint64_t index:11;
804                 /*
805                  * The group attached to the core (updated when new
806                  * tag list entered on SWTAG_FULL).
807                  */
808                 uint64_t grp:4;
809                 /*
810                  * The wqp attached to the core (updated when new tag
811                  * list entered on SWTAG_FULL).
812                  */
813                 uint64_t wqp:36;
814 #else
815                 uint64_t wqp:36;
816                 uint64_t grp:4;
817                 uint64_t index:11;
818                 uint64_t revlink_index:11;
819                 uint64_t reserved_62_63:2;
820 #endif
821         } s_sstatus5;
822 
823     /**
824      * Result For POW Memory Load (get_des == 0 and get_wqp == 0)
825      */
826         struct {
827 #ifdef __BIG_ENDIAN_BITFIELD
828                 uint64_t reserved_51_63:13;
829                 /*
830                  * The next entry in the input, free, descheduled_head
831                  * list (unpredictable if entry is the tail of the
832                  * list).
833                  */
834                 uint64_t next_index:11;
835                 /* The group of the POW entry. */
836                 uint64_t grp:4;
837                 uint64_t reserved_35:1;
838                 /*
839                  * Set when this POW entry is at the tail of its tag
840                  * list (also set when in the NULL or NULL_NULL
841                  * state).
842                  */
843                 uint64_t tail:1;
844                 /* The tag type of the POW entry. */
845                 uint64_t tag_type:2;
846                 /* The tag of the POW entry. */
847                 uint64_t tag:32;
848 #else
849                 uint64_t tag:32;
850                 uint64_t tag_type:2;
851                 uint64_t tail:1;
852                 uint64_t reserved_35:1;
853                 uint64_t grp:4;
854                 uint64_t next_index:11;
855                 uint64_t reserved_51_63:13;
856 #endif
857         } s_smemload0;
858 
859     /**
860      * Result For POW Memory Load (get_des == 0 and get_wqp == 1)
861      */
862         struct {
863 #ifdef __BIG_ENDIAN_BITFIELD
864                 uint64_t reserved_51_63:13;
865                 /*
866                  * The next entry in the input, free, descheduled_head
867                  * list (unpredictable if entry is the tail of the
868                  * list).
869                  */
870                 uint64_t next_index:11;
871                 /* The group of the POW entry. */
872                 uint64_t grp:4;
873                 /* The WQP held in the POW entry. */
874                 uint64_t wqp:36;
875 #else
876                 uint64_t wqp:36;
877                 uint64_t grp:4;
878                 uint64_t next_index:11;
879                 uint64_t reserved_51_63:13;
880 #endif
881         } s_smemload1;
882 
883     /**
884      * Result For POW Memory Load (get_des == 1)
885      */
886         struct {
887 #ifdef __BIG_ENDIAN_BITFIELD
888                 uint64_t reserved_51_63:13;
889                 /*
890                  * The next entry in the tag list connected to the
891                  * descheduled head.
892                  */
893                 uint64_t fwd_index:11;
894                 /* The group of the POW entry. */
895                 uint64_t grp:4;
896                 /* The nosched bit for the POW entry. */
897                 uint64_t nosched:1;
898                 /* There is a pending tag switch */
899                 uint64_t pend_switch:1;
900                 /*
901                  * The next tag type for the new tag list when
902                  * pend_switch is set.
903                  */
904                 uint64_t pend_type:2;
905                 /*
906                  * The next tag for the new tag list when pend_switch
907                  * is set.
908                  */
909                 uint64_t pend_tag:32;
910 #else
911                 uint64_t pend_tag:32;
912                 uint64_t pend_type:2;
913                 uint64_t pend_switch:1;
914                 uint64_t nosched:1;
915                 uint64_t grp:4;
916                 uint64_t fwd_index:11;
917                 uint64_t reserved_51_63:13;
918 #endif
919         } s_smemload2;
920 
921     /**
922      * Result For POW Index/Pointer Load (get_rmt == 0/get_des_get_tail == 0)
923      */
924         struct {
925 #ifdef __BIG_ENDIAN_BITFIELD
926                 uint64_t reserved_52_63:12;
927                 /*
928                  * set when there is one or more POW entries on the
929                  * free list.
930                  */
931                 uint64_t free_val:1;
932                 /*
933                  * set when there is exactly one POW entry on the free
934                  * list.
935                  */
936                 uint64_t free_one:1;
937                 uint64_t reserved_49:1;
938                 /*
939                  * when free_val is set, indicates the first entry on
940                  * the free list.
941                  */
942                 uint64_t free_head:11;
943                 uint64_t reserved_37:1;
944                 /*
945                  * when free_val is set, indicates the last entry on
946                  * the free list.
947                  */
948                 uint64_t free_tail:11;
949                 /*
950                  * set when there is one or more POW entries on the
951                  * input Q list selected by qosgrp.
952                  */
953                 uint64_t loc_val:1;
954                 /*
955                  * set when there is exactly one POW entry on the
956                  * input Q list selected by qosgrp.
957                  */
958                 uint64_t loc_one:1;
959                 uint64_t reserved_23:1;
960                 /*
961                  * when loc_val is set, indicates the first entry on
962                  * the input Q list selected by qosgrp.
963                  */
964                 uint64_t loc_head:11;
965                 uint64_t reserved_11:1;
966                 /*
967                  * when loc_val is set, indicates the last entry on
968                  * the input Q list selected by qosgrp.
969                  */
970                 uint64_t loc_tail:11;
971 #else
972                 uint64_t loc_tail:11;
973                 uint64_t reserved_11:1;
974                 uint64_t loc_head:11;
975                 uint64_t reserved_23:1;
976                 uint64_t loc_one:1;
977                 uint64_t loc_val:1;
978                 uint64_t free_tail:11;
979                 uint64_t reserved_37:1;
980                 uint64_t free_head:11;
981                 uint64_t reserved_49:1;
982                 uint64_t free_one:1;
983                 uint64_t free_val:1;
984                 uint64_t reserved_52_63:12;
985 #endif
986         } sindexload0;
987 
988     /**
989      * Result For POW Index/Pointer Load (get_rmt == 0/get_des_get_tail == 1)
990      */
991         struct {
992 #ifdef __BIG_ENDIAN_BITFIELD
993                 uint64_t reserved_52_63:12;
994                 /*
995                  * set when there is one or more POW entries on the
996                  * nosched list.
997                  */
998                 uint64_t nosched_val:1;
999                 /*
1000                  * set when there is exactly one POW entry on the
1001                  * nosched list.
1002                  */
1003                 uint64_t nosched_one:1;
1004                 uint64_t reserved_49:1;
1005                 /*
1006                  * when nosched_val is set, indicates the first entry
1007                  * on the nosched list.
1008                  */
1009                 uint64_t nosched_head:11;
1010                 uint64_t reserved_37:1;
1011                 /*
1012                  * when nosched_val is set, indicates the last entry
1013                  * on the nosched list.
1014                  */
1015                 uint64_t nosched_tail:11;
1016                 /*
1017                  * set when there is one or more descheduled heads on
1018                  * the descheduled list selected by qosgrp.
1019                  */
1020                 uint64_t des_val:1;
1021                 /*
1022                  * set when there is exactly one descheduled head on
1023                  * the descheduled list selected by qosgrp.
1024                  */
1025                 uint64_t des_one:1;
1026                 uint64_t reserved_23:1;
1027                 /*
1028                  * when des_val is set, indicates the first
1029                  * descheduled head on the descheduled list selected
1030                  * by qosgrp.
1031                  */
1032                 uint64_t des_head:11;
1033                 uint64_t reserved_11:1;
1034                 /*
1035                  * when des_val is set, indicates the last descheduled
1036                  * head on the descheduled list selected by qosgrp.
1037                  */
1038                 uint64_t des_tail:11;
1039 #else
1040                 uint64_t des_tail:11;
1041                 uint64_t reserved_11:1;
1042                 uint64_t des_head:11;
1043                 uint64_t reserved_23:1;
1044                 uint64_t des_one:1;
1045                 uint64_t des_val:1;
1046                 uint64_t nosched_tail:11;
1047                 uint64_t reserved_37:1;
1048                 uint64_t nosched_head:11;
1049                 uint64_t reserved_49:1;
1050                 uint64_t nosched_one:1;
1051                 uint64_t nosched_val:1;
1052                 uint64_t reserved_52_63:12;
1053 #endif
1054         } sindexload1;
1055 
1056     /**
1057      * Result For POW Index/Pointer Load (get_rmt == 1/get_des_get_tail == 0)
1058      */
1059         struct {
1060 #ifdef __BIG_ENDIAN_BITFIELD
1061                 uint64_t reserved_39_63:25;
1062                 /*
1063                  * Set when this DRAM list is the current head
1064                  * (i.e. is the next to be reloaded when the POW
1065                  * hardware reloads a POW entry from DRAM). The POW
1066                  * hardware alternates between the two DRAM lists
1067                  * associated with a QOS level when it reloads work
1068                  * from DRAM into the POW unit.
1069                  */
1070                 uint64_t rmt_is_head:1;
1071                 /*
1072                  * Set when the DRAM portion of the input Q list
1073                  * selected by qosgrp contains one or more pieces of
1074                  * work.
1075                  */
1076                 uint64_t rmt_val:1;
1077                 /*
1078                  * Set when the DRAM portion of the input Q list
1079                  * selected by qosgrp contains exactly one piece of
1080                  * work.
1081                  */
1082                 uint64_t rmt_one:1;
1083                 /*
1084                  * When rmt_val is set, indicates the first piece of
1085                  * work on the DRAM input Q list selected by
1086                  * qosgrp.
1087                  */
1088                 uint64_t rmt_head:36;
1089 #else
1090                 uint64_t rmt_head:36;
1091                 uint64_t rmt_one:1;
1092                 uint64_t rmt_val:1;
1093                 uint64_t rmt_is_head:1;
1094                 uint64_t reserved_39_63:25;
1095 #endif
1096         } sindexload2;
1097 
1098     /**
1099      * Result For POW Index/Pointer Load (get_rmt ==
1100      * 1/get_des_get_tail == 1)
1101      */
1102         struct {
1103 #ifdef __BIG_ENDIAN_BITFIELD
1104                 uint64_t reserved_39_63:25;
1105                 /*
1106                  * set when this DRAM list is the current head
1107                  * (i.e. is the next to be reloaded when the POW
1108                  * hardware reloads a POW entry from DRAM). The POW
1109                  * hardware alternates between the two DRAM lists
1110                  * associated with a QOS level when it reloads work
1111                  * from DRAM into the POW unit.
1112                  */
1113                 uint64_t rmt_is_head:1;
1114                 /*
1115                  * set when the DRAM portion of the input Q list
1116                  * selected by qosgrp contains one or more pieces of
1117                  * work.
1118                  */
1119                 uint64_t rmt_val:1;
1120                 /*
1121                  * set when the DRAM portion of the input Q list
1122                  * selected by qosgrp contains exactly one piece of
1123                  * work.
1124                  */
1125                 uint64_t rmt_one:1;
1126                 /*
1127                  * when rmt_val is set, indicates the last piece of
1128                  * work on the DRAM input Q list selected by
1129                  * qosgrp.
1130                  */
1131                 uint64_t rmt_tail:36;
1132 #else
1133                 uint64_t rmt_tail:36;
1134                 uint64_t rmt_one:1;
1135                 uint64_t rmt_val:1;
1136                 uint64_t rmt_is_head:1;
1137                 uint64_t reserved_39_63:25;
1138 #endif
1139         } sindexload3;
1140 
1141     /**
1142      * Response to NULL_RD request loads
1143      */
1144         struct {
1145 #ifdef __BIG_ENDIAN_BITFIELD
1146                 uint64_t unused:62;
1147                 /* of type cvmx_pow_tag_type_t. state is one of the
1148                  * following:
1149                  *
1150                  * - CVMX_POW_TAG_TYPE_ORDERED
1151                  * - CVMX_POW_TAG_TYPE_ATOMIC
1152                  * - CVMX_POW_TAG_TYPE_NULL
1153                  * - CVMX_POW_TAG_TYPE_NULL_NULL
1154                  */
1155                 uint64_t state:2;
1156 #else
1157                 uint64_t state:2;
1158                 uint64_t unused:62;
1159 #endif
1160         } s_null_rd;
1161 
1162 } cvmx_pow_tag_load_resp_t;
1163 
1164 /**
1165  * This structure describes the address used for stores to the POW.
1166  *  The store address is meaningful on stores to the POW.  The
1167  *  hardware assumes that an aligned 64-bit store was used for all
1168  *  these stores.  Note the assumption that the work queue entry is
1169  *  aligned on an 8-byte boundary (since the low-order 3 address bits
1170  *  must be zero).  Note that not all fields are used by all
1171  *  operations.
1172  *
1173  *  NOTE: The following is the behavior of the pending switch bit at the PP
1174  *       for POW stores (i.e. when did<7:3> == 0xc)
1175  *     - did<2:0> == 0      => pending switch bit is set
1176  *     - did<2:0> == 1      => no affect on the pending switch bit
1177  *     - did<2:0> == 3      => pending switch bit is cleared
1178  *     - did<2:0> == 7      => no affect on the pending switch bit
1179  *     - did<2:0> == others => must not be used
1180  *     - No other loads/stores have an affect on the pending switch bit
1181  *     - The switch bus from POW can clear the pending switch bit
1182  *
1183  *  NOTE: did<2:0> == 2 is used by the HW for a special single-cycle
1184  *  ADDWQ command that only contains the pointer). SW must never use
1185  *  did<2:0> == 2.
1186  */
1187 typedef union {
1188     /**
1189      * Unsigned 64 bit integer representation of store address
1190      */
1191         uint64_t u64;
1192 
1193         struct {
1194 #ifdef __BIG_ENDIAN_BITFIELD
1195                 /* Memory region.  Should be CVMX_IO_SEG in most cases */
1196                 uint64_t mem_reg:2;
1197                 uint64_t reserved_49_61:13;     /* Must be zero */
1198                 uint64_t is_io:1;       /* Must be one */
1199                 /* Device ID of POW.  Note that different sub-dids are used. */
1200                 uint64_t did:8;
1201                 uint64_t reserved_36_39:4;      /* Must be zero */
1202                 /* Address field. addr<2:0> must be zero */
1203                 uint64_t addr:36;
1204 #else
1205                 uint64_t addr:36;
1206                 uint64_t reserved_36_39:4;
1207                 uint64_t did:8;
1208                 uint64_t is_io:1;
1209                 uint64_t reserved_49_61:13;
1210                 uint64_t mem_reg:2;
1211 #endif
1212         } stag;
1213 } cvmx_pow_tag_store_addr_t;
1214 
1215 /**
1216  * decode of the store data when an IOBDMA SENDSINGLE is sent to POW
1217  */
1218 typedef union {
1219         uint64_t u64;
1220 
1221         struct {
1222 #ifdef __BIG_ENDIAN_BITFIELD
1223                 /*
1224                  * the (64-bit word) location in scratchpad to write
1225                  * to (if len != 0)
1226                  */
1227                 uint64_t scraddr:8;
1228                 /* the number of words in the response (0 => no response) */
1229                 uint64_t len:8;
1230                 /* the ID of the device on the non-coherent bus */
1231                 uint64_t did:8;
1232                 uint64_t unused:36;
1233                 /* if set, don't return load response until work is available */
1234                 uint64_t wait:1;
1235                 uint64_t unused2:3;
1236 #else
1237                 uint64_t unused2:3;
1238                 uint64_t wait:1;
1239                 uint64_t unused:36;
1240                 uint64_t did:8;
1241                 uint64_t len:8;
1242                 uint64_t scraddr:8;
1243 #endif
1244         } s;
1245 
1246 } cvmx_pow_iobdma_store_t;
1247 
1248 /* CSR typedefs have been moved to cvmx-csr-*.h */
1249 
1250 /**
1251  * Get the POW tag for this core. This returns the current
1252  * tag type, tag, group, and POW entry index associated with
1253  * this core. Index is only valid if the tag type isn't NULL_NULL.
1254  * If a tag switch is pending this routine returns the tag before
1255  * the tag switch, not after.
1256  *
1257  * Returns Current tag
1258  */
1259 static inline cvmx_pow_tag_req_t cvmx_pow_get_current_tag(void)
1260 {
1261         cvmx_pow_load_addr_t load_addr;
1262         cvmx_pow_tag_load_resp_t load_resp;
1263         cvmx_pow_tag_req_t result;
1264 
1265         load_addr.u64 = 0;
1266         load_addr.sstatus.mem_region = CVMX_IO_SEG;
1267         load_addr.sstatus.is_io = 1;
1268         load_addr.sstatus.did = CVMX_OCT_DID_TAG_TAG1;
1269         load_addr.sstatus.coreid = cvmx_get_core_num();
1270         load_addr.sstatus.get_cur = 1;
1271         load_resp.u64 = cvmx_read_csr(load_addr.u64);
1272         result.u64 = 0;
1273         result.s.grp = load_resp.s_sstatus2.grp;
1274         result.s.index = load_resp.s_sstatus2.index;
1275         result.s.type = load_resp.s_sstatus2.tag_type;
1276         result.s.tag = load_resp.s_sstatus2.tag;
1277         return result;
1278 }
1279 
1280 /**
1281  * Get the POW WQE for this core. This returns the work queue
1282  * entry currently associated with this core.
1283  *
1284  * Returns WQE pointer
1285  */
1286 static inline cvmx_wqe_t *cvmx_pow_get_current_wqp(void)
1287 {
1288         cvmx_pow_load_addr_t load_addr;
1289         cvmx_pow_tag_load_resp_t load_resp;
1290 
1291         load_addr.u64 = 0;
1292         load_addr.sstatus.mem_region = CVMX_IO_SEG;
1293         load_addr.sstatus.is_io = 1;
1294         load_addr.sstatus.did = CVMX_OCT_DID_TAG_TAG1;
1295         load_addr.sstatus.coreid = cvmx_get_core_num();
1296         load_addr.sstatus.get_cur = 1;
1297         load_addr.sstatus.get_wqp = 1;
1298         load_resp.u64 = cvmx_read_csr(load_addr.u64);
1299         return (cvmx_wqe_t *) cvmx_phys_to_ptr(load_resp.s_sstatus4.wqp);
1300 }
1301 
1302 #ifndef CVMX_MF_CHORD
1303 #define CVMX_MF_CHORD(dest)         CVMX_RDHWR(dest, 30)
1304 #endif
1305 
1306 /**
1307  * Print a warning if a tag switch is pending for this core
1308  *
1309  * @function: Function name checking for a pending tag switch
1310  */
1311 static inline void __cvmx_pow_warn_if_pending_switch(const char *function)
1312 {
1313         uint64_t switch_complete;
1314         CVMX_MF_CHORD(switch_complete);
1315         if (!switch_complete)
1316                 pr_warn("%s called with tag switch in progress\n", function);
1317 }
1318 
1319 /**
1320  * Waits for a tag switch to complete by polling the completion bit.
1321  * Note that switches to NULL complete immediately and do not need
1322  * to be waited for.
1323  */
1324 static inline void cvmx_pow_tag_sw_wait(void)
1325 {
1326         const uint64_t MAX_CYCLES = 1ull << 31;
1327         uint64_t switch_complete;
1328         uint64_t start_cycle = cvmx_get_cycle();
1329         while (1) {
1330                 CVMX_MF_CHORD(switch_complete);
1331                 if (unlikely(switch_complete))
1332                         break;
1333                 if (unlikely(cvmx_get_cycle() > start_cycle + MAX_CYCLES)) {
1334                         pr_warn("Tag switch is taking a long time, possible deadlock\n");
1335                         start_cycle = -MAX_CYCLES - 1;
1336                 }
1337         }
1338 }
1339 
1340 /**
1341  * Synchronous work request.  Requests work from the POW.
1342  * This function does NOT wait for previous tag switches to complete,
1343  * so the caller must ensure that there is not a pending tag switch.
1344  *
1345  * @wait:   When set, call stalls until work becomes avaiable, or times out.
1346  *               If not set, returns immediately.
1347  *
1348  * Returns Returns the WQE pointer from POW. Returns NULL if no work
1349  * was available.
1350  */
1351 static inline cvmx_wqe_t *cvmx_pow_work_request_sync_nocheck(cvmx_pow_wait_t
1352                                                              wait)
1353 {
1354         cvmx_pow_load_addr_t ptr;
1355         cvmx_pow_tag_load_resp_t result;
1356 
1357         if (CVMX_ENABLE_POW_CHECKS)
1358                 __cvmx_pow_warn_if_pending_switch(__func__);
1359 
1360         ptr.u64 = 0;
1361         ptr.swork.mem_region = CVMX_IO_SEG;
1362         ptr.swork.is_io = 1;
1363         ptr.swork.did = CVMX_OCT_DID_TAG_SWTAG;
1364         ptr.swork.wait = wait;
1365 
1366         result.u64 = cvmx_read_csr(ptr.u64);
1367 
1368         if (result.s_work.no_work)
1369                 return NULL;
1370         else
1371                 return (cvmx_wqe_t *) cvmx_phys_to_ptr(result.s_work.addr);
1372 }
1373 
1374 /**
1375  * Synchronous work request.  Requests work from the POW.
1376  * This function waits for any previous tag switch to complete before
1377  * requesting the new work.
1378  *
1379  * @wait:   When set, call stalls until work becomes avaiable, or times out.
1380  *               If not set, returns immediately.
1381  *
1382  * Returns Returns the WQE pointer from POW. Returns NULL if no work
1383  * was available.
1384  */
1385 static inline cvmx_wqe_t *cvmx_pow_work_request_sync(cvmx_pow_wait_t wait)
1386 {
1387         if (CVMX_ENABLE_POW_CHECKS)
1388                 __cvmx_pow_warn_if_pending_switch(__func__);
1389 
1390         /* Must not have a switch pending when requesting work */
1391         cvmx_pow_tag_sw_wait();
1392         return cvmx_pow_work_request_sync_nocheck(wait);
1393 
1394 }
1395 
1396 /**
1397  * Synchronous null_rd request.  Requests a switch out of NULL_NULL POW state.
1398  * This function waits for any previous tag switch to complete before
1399  * requesting the null_rd.
1400  *
1401  * Returns Returns the POW state of type cvmx_pow_tag_type_t.
1402  */
1403 static inline enum cvmx_pow_tag_type cvmx_pow_work_request_null_rd(void)
1404 {
1405         cvmx_pow_load_addr_t ptr;
1406         cvmx_pow_tag_load_resp_t result;
1407 
1408         if (CVMX_ENABLE_POW_CHECKS)
1409                 __cvmx_pow_warn_if_pending_switch(__func__);
1410 
1411         /* Must not have a switch pending when requesting work */
1412         cvmx_pow_tag_sw_wait();
1413 
1414         ptr.u64 = 0;
1415         ptr.snull_rd.mem_region = CVMX_IO_SEG;
1416         ptr.snull_rd.is_io = 1;
1417         ptr.snull_rd.did = CVMX_OCT_DID_TAG_NULL_RD;
1418 
1419         result.u64 = cvmx_read_csr(ptr.u64);
1420 
1421         return (enum cvmx_pow_tag_type) result.s_null_rd.state;
1422 }
1423 
1424 /**
1425  * Asynchronous work request.  Work is requested from the POW unit,
1426  * and should later be checked with function
1427  * cvmx_pow_work_response_async.  This function does NOT wait for
1428  * previous tag switches to complete, so the caller must ensure that
1429  * there is not a pending tag switch.
1430  *
1431  * @scr_addr: Scratch memory address that response will be returned
1432  *            to, which is either a valid WQE, or a response with the
1433  *            invalid bit set.  Byte address, must be 8 byte aligned.
1434  *
1435  * @wait: 1 to cause response to wait for work to become available (or
1436  *        timeout), 0 to cause response to return immediately
1437  */
1438 static inline void cvmx_pow_work_request_async_nocheck(int scr_addr,
1439                                                        cvmx_pow_wait_t wait)
1440 {
1441         cvmx_pow_iobdma_store_t data;
1442 
1443         if (CVMX_ENABLE_POW_CHECKS)
1444                 __cvmx_pow_warn_if_pending_switch(__func__);
1445 
1446         /* scr_addr must be 8 byte aligned */
1447         data.s.scraddr = scr_addr >> 3;
1448         data.s.len = 1;
1449         data.s.did = CVMX_OCT_DID_TAG_SWTAG;
1450         data.s.wait = wait;
1451         cvmx_send_single(data.u64);
1452 }
1453 
1454 /**
1455  * Asynchronous work request.  Work is requested from the POW unit,
1456  * and should later be checked with function
1457  * cvmx_pow_work_response_async.  This function waits for any previous
1458  * tag switch to complete before requesting the new work.
1459  *
1460  * @scr_addr: Scratch memory address that response will be returned
1461  *            to, which is either a valid WQE, or a response with the
1462  *            invalid bit set.  Byte address, must be 8 byte aligned.
1463  *
1464  * @wait: 1 to cause response to wait for work to become available (or
1465  *                  timeout), 0 to cause response to return immediately
1466  */
1467 static inline void cvmx_pow_work_request_async(int scr_addr,
1468                                                cvmx_pow_wait_t wait)
1469 {
1470         if (CVMX_ENABLE_POW_CHECKS)
1471                 __cvmx_pow_warn_if_pending_switch(__func__);
1472 
1473         /* Must not have a switch pending when requesting work */
1474         cvmx_pow_tag_sw_wait();
1475         cvmx_pow_work_request_async_nocheck(scr_addr, wait);
1476 }
1477 
1478 /**
1479  * Gets result of asynchronous work request.  Performs a IOBDMA sync
1480  * to wait for the response.
1481  *
1482  * @scr_addr: Scratch memory address to get result from Byte address,
1483  *            must be 8 byte aligned.
1484  *
1485  * Returns Returns the WQE from the scratch register, or NULL if no
1486  * work was available.
1487  */
1488 static inline cvmx_wqe_t *cvmx_pow_work_response_async(int scr_addr)
1489 {
1490         cvmx_pow_tag_load_resp_t result;
1491 
1492         CVMX_SYNCIOBDMA;
1493         result.u64 = cvmx_scratch_read64(scr_addr);
1494 
1495         if (result.s_work.no_work)
1496                 return NULL;
1497         else
1498                 return (cvmx_wqe_t *) cvmx_phys_to_ptr(result.s_work.addr);
1499 }
1500 
1501 /**
1502  * Checks if a work queue entry pointer returned by a work
1503  * request is valid.  It may be invalid due to no work
1504  * being available or due to a timeout.
1505  *
1506  * @wqe_ptr: pointer to a work queue entry returned by the POW
1507  *
1508  * Returns 0 if pointer is valid
1509  *         1 if invalid (no work was returned)
1510  */
1511 static inline uint64_t cvmx_pow_work_invalid(cvmx_wqe_t *wqe_ptr)
1512 {
1513         return wqe_ptr == NULL;
1514 }
1515 
1516 /**
1517  * Starts a tag switch to the provided tag value and tag type.
1518  * Completion for the tag switch must be checked for separately.  This
1519  * function does NOT update the work queue entry in dram to match tag
1520  * value and type, so the application must keep track of these if they
1521  * are important to the application.  This tag switch command must not
1522  * be used for switches to NULL, as the tag switch pending bit will be
1523  * set by the switch request, but never cleared by the hardware.
1524  *
1525  * NOTE: This should not be used when switching from a NULL tag.  Use
1526  * cvmx_pow_tag_sw_full() instead.
1527  *
1528  * This function does no checks, so the caller must ensure that any
1529  * previous tag switch has completed.
1530  *
1531  * @tag:      new tag value
1532  * @tag_type: new tag type (ordered or atomic)
1533  */
1534 static inline void cvmx_pow_tag_sw_nocheck(uint32_t tag,
1535                                            enum cvmx_pow_tag_type tag_type)
1536 {
1537         cvmx_addr_t ptr;
1538         cvmx_pow_tag_req_t tag_req;
1539 
1540         if (CVMX_ENABLE_POW_CHECKS) {
1541                 cvmx_pow_tag_req_t current_tag;
1542                 __cvmx_pow_warn_if_pending_switch(__func__);
1543                 current_tag = cvmx_pow_get_current_tag();
1544                 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL_NULL)
1545                         pr_warn("%s called with NULL_NULL tag\n", __func__);
1546                 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL)
1547                         pr_warn("%s called with NULL tag\n", __func__);
1548                 if ((current_tag.s.type == tag_type)
1549                    && (current_tag.s.tag == tag))
1550                         pr_warn("%s called to perform a tag switch to the same tag\n",
1551                                 __func__);
1552                 if (tag_type == CVMX_POW_TAG_TYPE_NULL)
1553                         pr_warn("%s called to perform a tag switch to NULL. Use cvmx_pow_tag_sw_null() instead\n",
1554                                 __func__);
1555         }
1556 
1557         /*
1558          * Note that WQE in DRAM is not updated here, as the POW does
1559          * not read from DRAM once the WQE is in flight.  See hardware
1560          * manual for complete details.  It is the application's
1561          * responsibility to keep track of the current tag value if
1562          * that is important.
1563          */
1564 
1565         tag_req.u64 = 0;
1566         tag_req.s.op = CVMX_POW_TAG_OP_SWTAG;
1567         tag_req.s.tag = tag;
1568         tag_req.s.type = tag_type;
1569 
1570         ptr.u64 = 0;
1571         ptr.sio.mem_region = CVMX_IO_SEG;
1572         ptr.sio.is_io = 1;
1573         ptr.sio.did = CVMX_OCT_DID_TAG_SWTAG;
1574 
1575         /* once this store arrives at POW, it will attempt the switch
1576            software must wait for the switch to complete separately */
1577         cvmx_write_io(ptr.u64, tag_req.u64);
1578 }
1579 
1580 /**
1581  * Starts a tag switch to the provided tag value and tag type.
1582  * Completion for the tag switch must be checked for separately.  This
1583  * function does NOT update the work queue entry in dram to match tag
1584  * value and type, so the application must keep track of these if they
1585  * are important to the application.  This tag switch command must not
1586  * be used for switches to NULL, as the tag switch pending bit will be
1587  * set by the switch request, but never cleared by the hardware.
1588  *
1589  * NOTE: This should not be used when switching from a NULL tag.  Use
1590  * cvmx_pow_tag_sw_full() instead.
1591  *
1592  * This function waits for any previous tag switch to complete, and also
1593  * displays an error on tag switches to NULL.
1594  *
1595  * @tag:      new tag value
1596  * @tag_type: new tag type (ordered or atomic)
1597  */
1598 static inline void cvmx_pow_tag_sw(uint32_t tag,
1599                                    enum cvmx_pow_tag_type tag_type)
1600 {
1601         if (CVMX_ENABLE_POW_CHECKS)
1602                 __cvmx_pow_warn_if_pending_switch(__func__);
1603 
1604         /*
1605          * Note that WQE in DRAM is not updated here, as the POW does
1606          * not read from DRAM once the WQE is in flight.  See hardware
1607          * manual for complete details.  It is the application's
1608          * responsibility to keep track of the current tag value if
1609          * that is important.
1610          */
1611 
1612         /*
1613          * Ensure that there is not a pending tag switch, as a tag
1614          * switch cannot be started if a previous switch is still
1615          * pending.
1616          */
1617         cvmx_pow_tag_sw_wait();
1618         cvmx_pow_tag_sw_nocheck(tag, tag_type);
1619 }
1620 
1621 /**
1622  * Starts a tag switch to the provided tag value and tag type.
1623  * Completion for the tag switch must be checked for separately.  This
1624  * function does NOT update the work queue entry in dram to match tag
1625  * value and type, so the application must keep track of these if they
1626  * are important to the application.  This tag switch command must not
1627  * be used for switches to NULL, as the tag switch pending bit will be
1628  * set by the switch request, but never cleared by the hardware.
1629  *
1630  * This function must be used for tag switches from NULL.
1631  *
1632  * This function does no checks, so the caller must ensure that any
1633  * previous tag switch has completed.
1634  *
1635  * @wqp:      pointer to work queue entry to submit.  This entry is
1636  *            updated to match the other parameters
1637  * @tag:      tag value to be assigned to work queue entry
1638  * @tag_type: type of tag
1639  * @group:    group value for the work queue entry.
1640  */
1641 static inline void cvmx_pow_tag_sw_full_nocheck(cvmx_wqe_t *wqp, uint32_t tag,
1642                                                 enum cvmx_pow_tag_type tag_type,
1643                                                 uint64_t group)
1644 {
1645         cvmx_addr_t ptr;
1646         cvmx_pow_tag_req_t tag_req;
1647 
1648         if (CVMX_ENABLE_POW_CHECKS) {
1649                 cvmx_pow_tag_req_t current_tag;
1650                 __cvmx_pow_warn_if_pending_switch(__func__);
1651                 current_tag = cvmx_pow_get_current_tag();
1652                 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL_NULL)
1653                         pr_warn("%s called with NULL_NULL tag\n", __func__);
1654                 if ((current_tag.s.type == tag_type)
1655                    && (current_tag.s.tag == tag))
1656                         pr_warn("%s called to perform a tag switch to the same tag\n",
1657                                 __func__);
1658                 if (tag_type == CVMX_POW_TAG_TYPE_NULL)
1659                         pr_warn("%s called to perform a tag switch to NULL. Use cvmx_pow_tag_sw_null() instead\n",
1660                                 __func__);
1661                 if (wqp != cvmx_phys_to_ptr(0x80))
1662                         if (wqp != cvmx_pow_get_current_wqp())
1663                                 pr_warn("%s passed WQE(%p) doesn't match the address in the POW(%p)\n",
1664                                         __func__, wqp,
1665                                         cvmx_pow_get_current_wqp());
1666         }
1667 
1668         /*
1669          * Note that WQE in DRAM is not updated here, as the POW does
1670          * not read from DRAM once the WQE is in flight.  See hardware
1671          * manual for complete details.  It is the application's
1672          * responsibility to keep track of the current tag value if
1673          * that is important.
1674          */
1675 
1676         tag_req.u64 = 0;
1677         tag_req.s.op = CVMX_POW_TAG_OP_SWTAG_FULL;
1678         tag_req.s.tag = tag;
1679         tag_req.s.type = tag_type;
1680         tag_req.s.grp = group;
1681 
1682         ptr.u64 = 0;
1683         ptr.sio.mem_region = CVMX_IO_SEG;
1684         ptr.sio.is_io = 1;
1685         ptr.sio.did = CVMX_OCT_DID_TAG_SWTAG;
1686         ptr.sio.offset = CAST64(wqp);
1687 
1688         /*
1689          * once this store arrives at POW, it will attempt the switch
1690          * software must wait for the switch to complete separately.
1691          */
1692         cvmx_write_io(ptr.u64, tag_req.u64);
1693 }
1694 
1695 /**
1696  * Starts a tag switch to the provided tag value and tag type.
1697  * Completion for the tag switch must be checked for separately.  This
1698  * function does NOT update the work queue entry in dram to match tag
1699  * value and type, so the application must keep track of these if they
1700  * are important to the application.  This tag switch command must not
1701  * be used for switches to NULL, as the tag switch pending bit will be
1702  * set by the switch request, but never cleared by the hardware.
1703  *
1704  * This function must be used for tag switches from NULL.
1705  *
1706  * This function waits for any pending tag switches to complete
1707  * before requesting the tag switch.
1708  *
1709  * @wqp:      pointer to work queue entry to submit.  This entry is updated
1710  *            to match the other parameters
1711  * @tag:      tag value to be assigned to work queue entry
1712  * @tag_type: type of tag
1713  * @group:      group value for the work queue entry.
1714  */
1715 static inline void cvmx_pow_tag_sw_full(cvmx_wqe_t *wqp, uint32_t tag,
1716                                         enum cvmx_pow_tag_type tag_type,
1717                                         uint64_t group)
1718 {
1719         if (CVMX_ENABLE_POW_CHECKS)
1720                 __cvmx_pow_warn_if_pending_switch(__func__);
1721 
1722         /*
1723          * Ensure that there is not a pending tag switch, as a tag
1724          * switch cannot be started if a previous switch is still
1725          * pending.
1726          */
1727         cvmx_pow_tag_sw_wait();
1728         cvmx_pow_tag_sw_full_nocheck(wqp, tag, tag_type, group);
1729 }
1730 
1731 /**
1732  * Switch to a NULL tag, which ends any ordering or
1733  * synchronization provided by the POW for the current
1734  * work queue entry.  This operation completes immediately,
1735  * so completion should not be waited for.
1736  * This function does NOT wait for previous tag switches to complete,
1737  * so the caller must ensure that any previous tag switches have completed.
1738  */
1739 static inline void cvmx_pow_tag_sw_null_nocheck(void)
1740 {
1741         cvmx_addr_t ptr;
1742         cvmx_pow_tag_req_t tag_req;
1743 
1744         if (CVMX_ENABLE_POW_CHECKS) {
1745                 cvmx_pow_tag_req_t current_tag;
1746                 __cvmx_pow_warn_if_pending_switch(__func__);
1747                 current_tag = cvmx_pow_get_current_tag();
1748                 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL_NULL)
1749                         pr_warn("%s called with NULL_NULL tag\n", __func__);
1750                 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL)
1751                         pr_warn("%s called when we already have a NULL tag\n",
1752                                 __func__);
1753         }
1754 
1755         tag_req.u64 = 0;
1756         tag_req.s.op = CVMX_POW_TAG_OP_SWTAG;
1757         tag_req.s.type = CVMX_POW_TAG_TYPE_NULL;
1758 
1759         ptr.u64 = 0;
1760         ptr.sio.mem_region = CVMX_IO_SEG;
1761         ptr.sio.is_io = 1;
1762         ptr.sio.did = CVMX_OCT_DID_TAG_TAG1;
1763 
1764         cvmx_write_io(ptr.u64, tag_req.u64);
1765 
1766         /* switch to NULL completes immediately */
1767 }
1768 
1769 /**
1770  * Switch to a NULL tag, which ends any ordering or
1771  * synchronization provided by the POW for the current
1772  * work queue entry.  This operation completes immediately,
1773  * so completion should not be waited for.
1774  * This function waits for any pending tag switches to complete
1775  * before requesting the switch to NULL.
1776  */
1777 static inline void cvmx_pow_tag_sw_null(void)
1778 {
1779         if (CVMX_ENABLE_POW_CHECKS)
1780                 __cvmx_pow_warn_if_pending_switch(__func__);
1781 
1782         /*
1783          * Ensure that there is not a pending tag switch, as a tag
1784          * switch cannot be started if a previous switch is still
1785          * pending.
1786          */
1787         cvmx_pow_tag_sw_wait();
1788         cvmx_pow_tag_sw_null_nocheck();
1789 
1790         /* switch to NULL completes immediately */
1791 }
1792 
1793 /**
1794  * Submits work to an input queue.  This function updates the work
1795  * queue entry in DRAM to match the arguments given.  Note that the
1796  * tag provided is for the work queue entry submitted, and is
1797  * unrelated to the tag that the core currently holds.
1798  *
1799  * @wqp:      pointer to work queue entry to submit.  This entry is
1800  *            updated to match the other parameters
1801  * @tag:      tag value to be assigned to work queue entry
1802  * @tag_type: type of tag
1803  * @qos:      Input queue to add to.
1804  * @grp:      group value for the work queue entry.
1805  */
1806 static inline void cvmx_pow_work_submit(cvmx_wqe_t *wqp, uint32_t tag,
1807                                         enum cvmx_pow_tag_type tag_type,
1808                                         uint64_t qos, uint64_t grp)
1809 {
1810         cvmx_addr_t ptr;
1811         cvmx_pow_tag_req_t tag_req;
1812 
1813         wqp->word1.tag = tag;
1814         wqp->word1.tag_type = tag_type;
1815 
1816         cvmx_wqe_set_qos(wqp, qos);
1817         cvmx_wqe_set_grp(wqp, grp);
1818 
1819         tag_req.u64 = 0;
1820         tag_req.s.op = CVMX_POW_TAG_OP_ADDWQ;
1821         tag_req.s.type = tag_type;
1822         tag_req.s.tag = tag;
1823         tag_req.s.qos = qos;
1824         tag_req.s.grp = grp;
1825 
1826         ptr.u64 = 0;
1827         ptr.sio.mem_region = CVMX_IO_SEG;
1828         ptr.sio.is_io = 1;
1829         ptr.sio.did = CVMX_OCT_DID_TAG_TAG1;
1830         ptr.sio.offset = cvmx_ptr_to_phys(wqp);
1831 
1832         /*
1833          * SYNC write to memory before the work submit.  This is
1834          * necessary as POW may read values from DRAM at this time.
1835          */
1836         CVMX_SYNCWS;
1837         cvmx_write_io(ptr.u64, tag_req.u64);
1838 }
1839 
1840 /**
1841  * This function sets the group mask for a core.  The group mask
1842  * indicates which groups each core will accept work from. There are
1843  * 16 groups.
1844  *
1845  * @core_num:   core to apply mask to
1846  * @mask:   Group mask. There are 16 groups, so only bits 0-15 are valid,
1847  *               representing groups 0-15.
1848  *               Each 1 bit in the mask enables the core to accept work from
1849  *               the corresponding group.
1850  */
1851 static inline void cvmx_pow_set_group_mask(uint64_t core_num, uint64_t mask)
1852 {
1853         union cvmx_pow_pp_grp_mskx grp_msk;
1854 
1855         grp_msk.u64 = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(core_num));
1856         grp_msk.s.grp_msk = mask;
1857         cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(core_num), grp_msk.u64);
1858 }
1859 
1860 /**
1861  * This function sets POW static priorities for a core. Each input queue has
1862  * an associated priority value.
1863  *
1864  * @core_num:   core to apply priorities to
1865  * @priority:   Vector of 8 priorities, one per POW Input Queue (0-7).
1866  *                   Highest priority is 0 and lowest is 7. A priority value
1867  *                   of 0xF instructs POW to skip the Input Queue when
1868  *                   scheduling to this specific core.
1869  *                   NOTE: priorities should not have gaps in values, meaning
1870  *                         {0,1,1,1,1,1,1,1} is a valid configuration while
1871  *                         {0,2,2,2,2,2,2,2} is not.
1872  */
1873 static inline void cvmx_pow_set_priority(uint64_t core_num,
1874                                          const uint8_t priority[])
1875 {
1876         /* POW priorities are supported on CN5xxx and later */
1877         if (!OCTEON_IS_MODEL(OCTEON_CN3XXX)) {
1878                 union cvmx_pow_pp_grp_mskx grp_msk;
1879 
1880                 grp_msk.u64 = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(core_num));
1881                 grp_msk.s.qos0_pri = priority[0];
1882                 grp_msk.s.qos1_pri = priority[1];
1883                 grp_msk.s.qos2_pri = priority[2];
1884                 grp_msk.s.qos3_pri = priority[3];
1885                 grp_msk.s.qos4_pri = priority[4];
1886                 grp_msk.s.qos5_pri = priority[5];
1887                 grp_msk.s.qos6_pri = priority[6];
1888                 grp_msk.s.qos7_pri = priority[7];
1889 
1890                 /* Detect gaps between priorities and flag error */
1891                 {
1892                         int i;
1893                         uint32_t prio_mask = 0;
1894 
1895                         for (i = 0; i < 8; i++)
1896                                 if (priority[i] != 0xF)
1897                                         prio_mask |= 1 << priority[i];
1898 
1899                         if (prio_mask ^ ((1 << cvmx_pop(prio_mask)) - 1)) {
1900                                 pr_err("POW static priorities should be "
1901                                        "contiguous (0x%llx)\n",
1902                                      (unsigned long long)prio_mask);
1903                                 return;
1904                         }
1905                 }
1906 
1907                 cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(core_num), grp_msk.u64);
1908         }
1909 }
1910 
1911 /**
1912  * Performs a tag switch and then an immediate deschedule. This completes
1913  * immediately, so completion must not be waited for.  This function does NOT
1914  * update the wqe in DRAM to match arguments.
1915  *
1916  * This function does NOT wait for any prior tag switches to complete, so the
1917  * calling code must do this.
1918  *
1919  * Note the following CAVEAT of the Octeon HW behavior when
1920  * re-scheduling DE-SCHEDULEd items whose (next) state is
1921  * ORDERED:
1922  *   - If there are no switches pending at the time that the
1923  *     HW executes the de-schedule, the HW will only re-schedule
1924  *     the head of the FIFO associated with the given tag. This
1925  *     means that in many respects, the HW treats this ORDERED
1926  *     tag as an ATOMIC tag. Note that in the SWTAG_DESCH
1927  *     case (to an ORDERED tag), the HW will do the switch
1928  *     before the deschedule whenever it is possible to do
1929  *     the switch immediately, so it may often look like
1930  *     this case.
1931  *   - If there is a pending switch to ORDERED at the time
1932  *     the HW executes the de-schedule, the HW will perform
1933  *     the switch at the time it re-schedules, and will be
1934  *     able to reschedule any/all of the entries with the
1935  *     same tag.
1936  * Due to this behavior, the RECOMMENDATION to software is
1937  * that they have a (next) state of ATOMIC when they
1938  * DE-SCHEDULE. If an ORDERED tag is what was really desired,
1939  * SW can choose to immediately switch to an ORDERED tag
1940  * after the work (that has an ATOMIC tag) is re-scheduled.
1941  * Note that since there are never any tag switches pending
1942  * when the HW re-schedules, this switch can be IMMEDIATE upon
1943  * the reception of the pointer during the re-schedule.
1944  *
1945  * @tag:      New tag value
1946  * @tag_type: New tag type
1947  * @group:    New group value
1948  * @no_sched: Control whether this work queue entry will be rescheduled.
1949  *                 - 1 : don't schedule this work
1950  *                 - 0 : allow this work to be scheduled.
1951  */
1952 static inline void cvmx_pow_tag_sw_desched_nocheck(
1953         uint32_t tag,
1954         enum cvmx_pow_tag_type tag_type,
1955         uint64_t group,
1956         uint64_t no_sched)
1957 {
1958         cvmx_addr_t ptr;
1959         cvmx_pow_tag_req_t tag_req;
1960 
1961         if (CVMX_ENABLE_POW_CHECKS) {
1962                 cvmx_pow_tag_req_t current_tag;
1963                 __cvmx_pow_warn_if_pending_switch(__func__);
1964                 current_tag = cvmx_pow_get_current_tag();
1965                 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL_NULL)
1966                         pr_warn("%s called with NULL_NULL tag\n", __func__);
1967                 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL)
1968                         pr_warn("%s called with NULL tag. Deschedule not allowed from NULL state\n",
1969                                 __func__);
1970                 if ((current_tag.s.type != CVMX_POW_TAG_TYPE_ATOMIC)
1971                         && (tag_type != CVMX_POW_TAG_TYPE_ATOMIC))
1972                         pr_warn("%s called where neither the before or after tag is ATOMIC\n",
1973                                 __func__);
1974         }
1975 
1976         tag_req.u64 = 0;
1977         tag_req.s.op = CVMX_POW_TAG_OP_SWTAG_DESCH;
1978         tag_req.s.tag = tag;
1979         tag_req.s.type = tag_type;
1980         tag_req.s.grp = group;
1981         tag_req.s.no_sched = no_sched;
1982 
1983         ptr.u64 = 0;
1984         ptr.sio.mem_region = CVMX_IO_SEG;
1985         ptr.sio.is_io = 1;
1986         ptr.sio.did = CVMX_OCT_DID_TAG_TAG3;
1987         /*
1988          * since TAG3 is used, this store will clear the local pending
1989          * switch bit.
1990          */
1991         cvmx_write_io(ptr.u64, tag_req.u64);
1992 }
1993 
1994 /**
1995  * Performs a tag switch and then an immediate deschedule. This completes
1996  * immediately, so completion must not be waited for.  This function does NOT
1997  * update the wqe in DRAM to match arguments.
1998  *
1999  * This function waits for any prior tag switches to complete, so the
2000  * calling code may call this function with a pending tag switch.
2001  *
2002  * Note the following CAVEAT of the Octeon HW behavior when
2003  * re-scheduling DE-SCHEDULEd items whose (next) state is
2004  * ORDERED:
2005  *   - If there are no switches pending at the time that the
2006  *     HW executes the de-schedule, the HW will only re-schedule
2007  *     the head of the FIFO associated with the given tag. This
2008  *     means that in many respects, the HW treats this ORDERED
2009  *     tag as an ATOMIC tag. Note that in the SWTAG_DESCH
2010  *     case (to an ORDERED tag), the HW will do the switch
2011  *     before the deschedule whenever it is possible to do
2012  *     the switch immediately, so it may often look like
2013  *     this case.
2014  *   - If there is a pending switch to ORDERED at the time
2015  *     the HW executes the de-schedule, the HW will perform
2016  *     the switch at the time it re-schedules, and will be
2017  *     able to reschedule any/all of the entries with the
2018  *     same tag.
2019  * Due to this behavior, the RECOMMENDATION to software is
2020  * that they have a (next) state of ATOMIC when they
2021  * DE-SCHEDULE. If an ORDERED tag is what was really desired,
2022  * SW can choose to immediately switch to an ORDERED tag
2023  * after the work (that has an ATOMIC tag) is re-scheduled.
2024  * Note that since there are never any tag switches pending
2025  * when the HW re-schedules, this switch can be IMMEDIATE upon
2026  * the reception of the pointer during the re-schedule.
2027  *
2028  * @tag:      New tag value
2029  * @tag_type: New tag type
2030  * @group:    New group value
2031  * @no_sched: Control whether this work queue entry will be rescheduled.
2032  *                 - 1 : don't schedule this work
2033  *                 - 0 : allow this work to be scheduled.
2034  */
2035 static inline void cvmx_pow_tag_sw_desched(uint32_t tag,
2036                                            enum cvmx_pow_tag_type tag_type,
2037                                            uint64_t group, uint64_t no_sched)
2038 {
2039         if (CVMX_ENABLE_POW_CHECKS)
2040                 __cvmx_pow_warn_if_pending_switch(__func__);
2041 
2042         /* Need to make sure any writes to the work queue entry are complete */
2043         CVMX_SYNCWS;
2044         /*
2045          * Ensure that there is not a pending tag switch, as a tag
2046          * switch cannot be started if a previous switch is still
2047          * pending.
2048          */
2049         cvmx_pow_tag_sw_wait();
2050         cvmx_pow_tag_sw_desched_nocheck(tag, tag_type, group, no_sched);
2051 }
2052 
2053 /**
2054  * Descchedules the current work queue entry.
2055  *
2056  * @no_sched: no schedule flag value to be set on the work queue
2057  *            entry.  If this is set the entry will not be
2058  *            rescheduled.
2059  */
2060 static inline void cvmx_pow_desched(uint64_t no_sched)
2061 {
2062         cvmx_addr_t ptr;
2063         cvmx_pow_tag_req_t tag_req;
2064 
2065         if (CVMX_ENABLE_POW_CHECKS) {
2066                 cvmx_pow_tag_req_t current_tag;
2067                 __cvmx_pow_warn_if_pending_switch(__func__);
2068                 current_tag = cvmx_pow_get_current_tag();
2069                 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL_NULL)
2070                         pr_warn("%s called with NULL_NULL tag\n", __func__);
2071                 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL)
2072                         pr_warn("%s called with NULL tag. Deschedule not expected from NULL state\n",
2073                                 __func__);
2074         }
2075 
2076         /* Need to make sure any writes to the work queue entry are complete */
2077         CVMX_SYNCWS;
2078 
2079         tag_req.u64 = 0;
2080         tag_req.s.op = CVMX_POW_TAG_OP_DESCH;
2081         tag_req.s.no_sched = no_sched;
2082 
2083         ptr.u64 = 0;
2084         ptr.sio.mem_region = CVMX_IO_SEG;
2085         ptr.sio.is_io = 1;
2086         ptr.sio.did = CVMX_OCT_DID_TAG_TAG3;
2087         /*
2088          * since TAG3 is used, this store will clear the local pending
2089          * switch bit.
2090          */
2091         cvmx_write_io(ptr.u64, tag_req.u64);
2092 }
2093 
2094 /****************************************************
2095 * Define usage of bits within the 32 bit tag values.
2096 *****************************************************/
2097 
2098 /*
2099  * Number of bits of the tag used by software.  The SW bits are always
2100  * a contiguous block of the high starting at bit 31.  The hardware
2101  * bits are always the low bits.  By default, the top 8 bits of the
2102  * tag are reserved for software, and the low 24 are set by the IPD
2103  * unit.
2104  */
2105 #define CVMX_TAG_SW_BITS    (8)
2106 #define CVMX_TAG_SW_SHIFT   (32 - CVMX_TAG_SW_BITS)
2107 
2108 /* Below is the list of values for the top 8 bits of the tag. */
2109 /*
2110  * Tag values with top byte of this value are reserved for internal
2111  * executive uses.
2112  */
2113 #define CVMX_TAG_SW_BITS_INTERNAL  0x1
2114 /* The executive divides the remaining 24 bits as follows:
2115  *  - the upper 8 bits (bits 23 - 16 of the tag) define a subgroup
2116  *
2117  *  - the lower 16 bits (bits 15 - 0 of the tag) define are the value
2118  *    with the subgroup
2119  *
2120  * Note that this section describes the format of tags generated by
2121  * software - refer to the hardware documentation for a description of
2122  * the tags values generated by the packet input hardware.  Subgroups
2123  * are defined here.
2124  */
2125 /* Mask for the value portion of the tag */
2126 #define CVMX_TAG_SUBGROUP_MASK  0xFFFF
2127 #define CVMX_TAG_SUBGROUP_SHIFT 16
2128 #define CVMX_TAG_SUBGROUP_PKO  0x1
2129 
2130 /* End of executive tag subgroup definitions */
2131 
2132 /*
2133  * The remaining values software bit values 0x2 - 0xff are available
2134  * for application use.
2135  */
2136 
2137 /**
2138  * This function creates a 32 bit tag value from the two values provided.
2139  *
2140  * @sw_bits: The upper bits (number depends on configuration) are set
2141  *           to this value.  The remainder of bits are set by the
2142  *           hw_bits parameter.
2143  *
2144  * @hw_bits: The lower bits (number depends on configuration) are set
2145  *           to this value.  The remainder of bits are set by the
2146  *           sw_bits parameter.
2147  *
2148  * Returns 32 bit value of the combined hw and sw bits.
2149  */
2150 static inline uint32_t cvmx_pow_tag_compose(uint64_t sw_bits, uint64_t hw_bits)
2151 {
2152         return ((sw_bits & cvmx_build_mask(CVMX_TAG_SW_BITS)) <<
2153                         CVMX_TAG_SW_SHIFT) |
2154                 (hw_bits & cvmx_build_mask(32 - CVMX_TAG_SW_BITS));
2155 }
2156 
2157 /**
2158  * Extracts the bits allocated for software use from the tag
2159  *
2160  * @tag:    32 bit tag value
2161  *
2162  * Returns N bit software tag value, where N is configurable with the
2163  * CVMX_TAG_SW_BITS define
2164  */
2165 static inline uint32_t cvmx_pow_tag_get_sw_bits(uint64_t tag)
2166 {
2167         return (tag >> (32 - CVMX_TAG_SW_BITS)) &
2168                 cvmx_build_mask(CVMX_TAG_SW_BITS);
2169 }
2170 
2171 /**
2172  *
2173  * Extracts the bits allocated for hardware use from the tag
2174  *
2175  * @tag:    32 bit tag value
2176  *
2177  * Returns (32 - N) bit software tag value, where N is configurable
2178  * with the CVMX_TAG_SW_BITS define
2179  */
2180 static inline uint32_t cvmx_pow_tag_get_hw_bits(uint64_t tag)
2181 {
2182         return tag & cvmx_build_mask(32 - CVMX_TAG_SW_BITS);
2183 }
2184 
2185 /**
2186  * Store the current POW internal state into the supplied
2187  * buffer. It is recommended that you pass a buffer of at least
2188  * 128KB. The format of the capture may change based on SDK
2189  * version and Octeon chip.
2190  *
2191  * @buffer: Buffer to store capture into
2192  * @buffer_size:
2193  *               The size of the supplied buffer
2194  *
2195  * Returns Zero on success, negative on failure
2196  */
2197 extern int cvmx_pow_capture(void *buffer, int buffer_size);
2198 
2199 /**
2200  * Dump a POW capture to the console in a human readable format.
2201  *
2202  * @buffer: POW capture from cvmx_pow_capture()
2203  * @buffer_size:
2204  *               Size of the buffer
2205  */
2206 extern void cvmx_pow_display(void *buffer, int buffer_size);
2207 
2208 /**
2209  * Return the number of POW entries supported by this chip
2210  *
2211  * Returns Number of POW entries
2212  */
2213 extern int cvmx_pow_get_num_entries(void);
2214 
2215 #endif /* __CVMX_POW_H__ */
2216 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp