~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/cris/arch-v32/drivers/cryptocop.c

Version: ~ [ linux-5.4 ] ~ [ linux-5.3.13 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.86 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.156 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.203 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.202 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.78 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Stream co-processor driver for the ETRAX FS
  3  *
  4  *    Copyright (C) 2003-2007  Axis Communications AB
  5  */
  6 
  7 #include <linux/init.h>
  8 #include <linux/sched.h>
  9 #include <linux/module.h>
 10 #include <linux/slab.h>
 11 #include <linux/string.h>
 12 #include <linux/fs.h>
 13 #include <linux/mm.h>
 14 #include <linux/spinlock.h>
 15 #include <linux/stddef.h>
 16 
 17 #include <asm/uaccess.h>
 18 #include <asm/io.h>
 19 #include <linux/atomic.h>
 20 
 21 #include <linux/list.h>
 22 #include <linux/interrupt.h>
 23 
 24 #include <asm/signal.h>
 25 #include <asm/irq.h>
 26 
 27 #include <dma.h>
 28 #include <hwregs/dma.h>
 29 #include <hwregs/reg_map.h>
 30 #include <hwregs/reg_rdwr.h>
 31 #include <hwregs/intr_vect_defs.h>
 32 
 33 #include <hwregs/strcop.h>
 34 #include <hwregs/strcop_defs.h>
 35 #include <cryptocop.h>
 36 
 37 #ifdef CONFIG_ETRAXFS
 38 #define IN_DMA 9
 39 #define OUT_DMA 8
 40 #define IN_DMA_INST regi_dma9
 41 #define OUT_DMA_INST regi_dma8
 42 #define DMA_IRQ DMA9_INTR_VECT
 43 #else
 44 #define IN_DMA 3
 45 #define OUT_DMA 2
 46 #define IN_DMA_INST regi_dma3
 47 #define OUT_DMA_INST regi_dma2
 48 #define DMA_IRQ DMA3_INTR_VECT
 49 #endif
 50 
 51 #define DESCR_ALLOC_PAD  (31)
 52 
 53 struct cryptocop_dma_desc {
 54         char *free_buf; /* If non-null will be kfreed in free_cdesc() */
 55         dma_descr_data *dma_descr;
 56 
 57         unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
 58 
 59         unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
 60         struct cryptocop_dma_desc *next;
 61 };
 62 
 63 
 64 struct cryptocop_int_operation{
 65         void                        *alloc_ptr;
 66         cryptocop_session_id        sid;
 67 
 68         dma_descr_context           ctx_out;
 69         dma_descr_context           ctx_in;
 70 
 71         /* DMA descriptors allocated by driver. */
 72         struct cryptocop_dma_desc   *cdesc_out;
 73         struct cryptocop_dma_desc   *cdesc_in;
 74 
 75         /* Strcop config to use. */
 76         cryptocop_3des_mode         tdes_mode;
 77         cryptocop_csum_type         csum_mode;
 78 
 79         /* DMA descrs provided by consumer. */
 80         dma_descr_data              *ddesc_out;
 81         dma_descr_data              *ddesc_in;
 82 };
 83 
 84 
 85 struct cryptocop_tfrm_ctx {
 86         cryptocop_tfrm_id tid;
 87         unsigned int blocklength;
 88 
 89         unsigned int start_ix;
 90 
 91         struct cryptocop_tfrm_cfg *tcfg;
 92         struct cryptocop_transform_ctx *tctx;
 93 
 94         unsigned char previous_src;
 95         unsigned char current_src;
 96 
 97         /* Values to use in metadata out. */
 98         unsigned char hash_conf;
 99         unsigned char hash_mode;
100         unsigned char ciph_conf;
101         unsigned char cbcmode;
102         unsigned char decrypt;
103 
104         unsigned int requires_padding:1;
105         unsigned int strict_block_length:1;
106         unsigned int active:1;
107         unsigned int done:1;
108         size_t consumed;
109         size_t produced;
110 
111         /* Pad (input) descriptors to put in the DMA out list when the transform
112          * output is put on the DMA in list. */
113         struct cryptocop_dma_desc *pad_descs;
114 
115         struct cryptocop_tfrm_ctx *prev_src;
116         struct cryptocop_tfrm_ctx *curr_src;
117 
118         /* Mapping to HW. */
119         unsigned char unit_no;
120 };
121 
122 
123 struct cryptocop_private{
124         cryptocop_session_id sid;
125         struct cryptocop_private *next;
126 };
127 
128 /* Session list. */
129 
130 struct cryptocop_transform_ctx{
131         struct cryptocop_transform_init init;
132         unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
133         unsigned int dec_key_set:1;
134 
135         struct cryptocop_transform_ctx *next;
136 };
137 
138 
139 struct cryptocop_session{
140         cryptocop_session_id sid;
141 
142         struct cryptocop_transform_ctx *tfrm_ctx;
143 
144         struct cryptocop_session *next;
145 };
146 
147 /* Priority levels for jobs sent to the cryptocop.  Checksum operations from
148    kernel have highest priority since TCPIP stack processing must not
149    be a bottleneck. */
150 typedef enum {
151         cryptocop_prio_kernel_csum = 0,
152         cryptocop_prio_kernel = 1,
153         cryptocop_prio_user = 2,
154         cryptocop_prio_no_prios = 3
155 } cryptocop_queue_priority;
156 
157 struct cryptocop_prio_queue{
158         struct list_head jobs;
159         cryptocop_queue_priority prio;
160 };
161 
162 struct cryptocop_prio_job{
163         struct list_head node;
164         cryptocop_queue_priority prio;
165 
166         struct cryptocop_operation *oper;
167         struct cryptocop_int_operation *iop;
168 };
169 
170 struct ioctl_job_cb_ctx {
171         unsigned int processed:1;
172 };
173 
174 
175 static struct cryptocop_session *cryptocop_sessions = NULL;
176 spinlock_t cryptocop_sessions_lock;
177 
178 /* Next Session ID to assign. */
179 static cryptocop_session_id next_sid = 1;
180 
181 /* Pad for checksum. */
182 static const char csum_zero_pad[1] = {0x00};
183 
184 /* Trash buffer for mem2mem operations. */
185 #define MEM2MEM_DISCARD_BUF_LENGTH  (512)
186 static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
187 
188 /* Descriptor pool. */
189 /* FIXME Tweak this value. */
190 #define CRYPTOCOP_DESCRIPTOR_POOL_SIZE   (100)
191 static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
192 static struct cryptocop_dma_desc *descr_pool_free_list;
193 static int descr_pool_no_free;
194 static spinlock_t descr_pool_lock;
195 
196 /* Lock to stop cryptocop to start processing of a new operation. The holder
197    of this lock MUST call cryptocop_start_job() after it is unlocked. */
198 spinlock_t cryptocop_process_lock;
199 
200 static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
201 static spinlock_t cryptocop_job_queue_lock;
202 static struct cryptocop_prio_job *cryptocop_running_job = NULL;
203 static spinlock_t running_job_lock;
204 
205 /* The interrupt handler appends completed jobs to this list. The scehduled
206  * tasklet removes them upon sending the response to the crypto consumer. */
207 static struct list_head cryptocop_completed_jobs;
208 static spinlock_t cryptocop_completed_jobs_lock;
209 
210 DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
211 
212 
213 /** Local functions. **/
214 
215 static int cryptocop_open(struct inode *, struct file *);
216 
217 static int cryptocop_release(struct inode *, struct file *);
218 
219 static long cryptocop_ioctl(struct file *file,
220                            unsigned int cmd, unsigned long arg);
221 
222 static void cryptocop_start_job(void);
223 
224 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
225 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
226 
227 static int cryptocop_job_queue_init(void);
228 static void cryptocop_job_queue_close(void);
229 
230 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
231 
232 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
233 
234 static int transform_ok(struct cryptocop_transform_init *tinit);
235 
236 static struct cryptocop_session *get_session(cryptocop_session_id sid);
237 
238 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
239 
240 static void delete_internal_operation(struct cryptocop_int_operation *iop);
241 
242 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength);
243 
244 static int init_stream_coprocessor(void);
245 
246 static void __exit exit_stream_coprocessor(void);
247 
248 /*#define LDEBUG*/
249 #ifdef LDEBUG
250 #define DEBUG(s) s
251 #define DEBUG_API(s) s
252 static void print_cryptocop_operation(struct cryptocop_operation *cop);
253 static void print_dma_descriptors(struct cryptocop_int_operation *iop);
254 static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
255 static void print_lock_status(void);
256 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
257 #define assert(s) do{if (!(s)) panic(#s);} while(0);
258 #else
259 #define DEBUG(s)
260 #define DEBUG_API(s)
261 #define assert(s)
262 #endif
263 
264 
265 /* Transform constants. */
266 #define DES_BLOCK_LENGTH   (8)
267 #define AES_BLOCK_LENGTH   (16)
268 #define MD5_BLOCK_LENGTH   (64)
269 #define SHA1_BLOCK_LENGTH  (64)
270 #define CSUM_BLOCK_LENGTH  (2)
271 #define MD5_STATE_LENGTH   (16)
272 #define SHA1_STATE_LENGTH  (20)
273 
274 /* The device number. */
275 #define CRYPTOCOP_MAJOR    (254)
276 #define CRYPTOCOP_MINOR    (0)
277 
278 
279 
280 const struct file_operations cryptocop_fops = {
281         .owner          = THIS_MODULE,
282         .open           = cryptocop_open,
283         .release        = cryptocop_release,
284         .unlocked_ioctl = cryptocop_ioctl,
285         .llseek         = noop_llseek,
286 };
287 
288 
289 static void free_cdesc(struct cryptocop_dma_desc *cdesc)
290 {
291         DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
292         kfree(cdesc->free_buf);
293 
294         if (cdesc->from_pool) {
295                 unsigned long int flags;
296                 spin_lock_irqsave(&descr_pool_lock, flags);
297                 cdesc->next = descr_pool_free_list;
298                 descr_pool_free_list = cdesc;
299                 ++descr_pool_no_free;
300                 spin_unlock_irqrestore(&descr_pool_lock, flags);
301         } else {
302                 kfree(cdesc);
303         }
304 }
305 
306 
307 static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
308 {
309         int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
310         struct cryptocop_dma_desc *cdesc;
311 
312         if (use_pool) {
313                 unsigned long int flags;
314                 spin_lock_irqsave(&descr_pool_lock, flags);
315                 if (!descr_pool_free_list) {
316                         spin_unlock_irqrestore(&descr_pool_lock, flags);
317                         DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
318                         return NULL;
319                 }
320                 cdesc = descr_pool_free_list;
321                 descr_pool_free_list = descr_pool_free_list->next;
322                 --descr_pool_no_free;
323                 spin_unlock_irqrestore(&descr_pool_lock, flags);
324                 cdesc->from_pool = 1;
325         } else {
326                 cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
327                 if (!cdesc) {
328                         DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
329                         return NULL;
330                 }
331                 cdesc->from_pool = 0;
332         }
333         cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
334 
335         cdesc->next = NULL;
336 
337         cdesc->free_buf = NULL;
338         cdesc->dma_descr->out_eop = 0;
339         cdesc->dma_descr->in_eop = 0;
340         cdesc->dma_descr->intr = 0;
341         cdesc->dma_descr->eol = 0;
342         cdesc->dma_descr->wait = 0;
343         cdesc->dma_descr->buf = NULL;
344         cdesc->dma_descr->after = NULL;
345 
346         DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
347         return cdesc;
348 }
349 
350 
351 static void setup_descr_chain(struct cryptocop_dma_desc *cd)
352 {
353         DEBUG(printk("setup_descr_chain: entering\n"));
354         while (cd) {
355                 if (cd->next) {
356                         cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
357                 } else {
358                         cd->dma_descr->next = NULL;
359                 }
360                 cd = cd->next;
361         }
362         DEBUG(printk("setup_descr_chain: exit\n"));
363 }
364 
365 
366 /* Create a pad descriptor for the transform.
367  * Return -1 for error, 0 if pad created. */
368 static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
369 {
370         struct cryptocop_dma_desc        *cdesc = NULL;
371         int                              error = 0;
372         struct strcop_meta_out           mo = {
373                 .ciphsel = src_none,
374                 .hashsel = src_none,
375                 .csumsel = src_none
376         };
377         char                             *pad;
378         size_t                           plen;
379 
380         DEBUG(printk("create_pad_descriptor: start.\n"));
381         /* Setup pad descriptor. */
382 
383         DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
384         cdesc = alloc_cdesc(alloc_flag);
385         if (!cdesc){
386                 DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
387                 goto error_cleanup;
388         }
389         switch (tc->unit_no) {
390         case src_md5:
391                 error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
392                 if (error){
393                         DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
394                         goto error_cleanup;
395                 }
396                 cdesc->free_buf = pad;
397                 mo.hashsel = src_dma;
398                 mo.hashconf = tc->hash_conf;
399                 mo.hashmode = tc->hash_mode;
400                 break;
401         case src_sha1:
402                 error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
403                 if (error){
404                         DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
405                         goto error_cleanup;
406                 }
407                 cdesc->free_buf = pad;
408                 mo.hashsel = src_dma;
409                 mo.hashconf = tc->hash_conf;
410                 mo.hashmode = tc->hash_mode;
411                 break;
412         case src_csum:
413                 if (tc->consumed % tc->blocklength){
414                         pad = (char*)csum_zero_pad;
415                         plen = 1;
416                 } else {
417                         pad = (char*)cdesc; /* Use any pointer. */
418                         plen = 0;
419                 }
420                 mo.csumsel = src_dma;
421                 break;
422         }
423         cdesc->dma_descr->wait = 1;
424         cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed.  EOP is ok here since the padded unit is the only one active. */
425         cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
426         cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
427 
428         cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
429         *pad_desc = cdesc;
430 
431         return 0;
432 
433  error_cleanup:
434         if (cdesc) free_cdesc(cdesc);
435         return -1;
436 }
437 
438 
439 static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
440 {
441         struct cryptocop_dma_desc  *key_desc = alloc_cdesc(alloc_flag);
442         struct strcop_meta_out     mo = {0};
443 
444         DEBUG(printk("setup_key_dl_desc\n"));
445 
446         if (!key_desc) {
447                 DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
448                 return -ENOMEM;
449         }
450 
451         /* Download key. */
452         if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
453                 /* Precook the AES decrypt key. */
454                 if (!tc->tctx->dec_key_set){
455                         get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
456                         tc->tctx->dec_key_set = 1;
457                 }
458                 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
459                 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
460         } else {
461                 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
462                 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
463         }
464         /* Setup metadata. */
465         mo.dlkey = 1;
466         switch (tc->tctx->init.keylen) {
467         case 64:
468                 mo.decrypt = 0;
469                 mo.hashmode = 0;
470                 break;
471         case 128:
472                 mo.decrypt = 0;
473                 mo.hashmode = 1;
474                 break;
475         case 192:
476                 mo.decrypt = 1;
477                 mo.hashmode = 0;
478                 break;
479         case 256:
480                 mo.decrypt = 1;
481                 mo.hashmode = 1;
482                 break;
483         default:
484                 break;
485         }
486         mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
487         key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
488 
489         key_desc->dma_descr->out_eop = 1;
490         key_desc->dma_descr->wait = 1;
491         key_desc->dma_descr->intr = 0;
492 
493         *kd = key_desc;
494         return 0;
495 }
496 
497 static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
498 {
499         struct cryptocop_dma_desc  *iv_desc = alloc_cdesc(alloc_flag);
500         struct strcop_meta_out     mo = {0};
501 
502         DEBUG(printk("setup_cipher_iv_desc\n"));
503 
504         if (!iv_desc) {
505                 DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
506                 return -ENOMEM;
507         }
508         /* Download IV. */
509         iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
510         iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
511 
512         /* Setup metadata. */
513         mo.hashsel = mo.csumsel = src_none;
514         mo.ciphsel = src_dma;
515         mo.ciphconf = tc->ciph_conf;
516         mo.cbcmode = tc->cbcmode;
517 
518         iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
519 
520         iv_desc->dma_descr->out_eop = 0;
521         iv_desc->dma_descr->wait = 1;
522         iv_desc->dma_descr->intr = 0;
523 
524         *id = iv_desc;
525         return 0;
526 }
527 
528 /* Map the ouput length of the transform to operation output starting on the inject index. */
529 static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
530 {
531         int                        err = 0;
532         struct cryptocop_dma_desc  head = {0};
533         struct cryptocop_dma_desc  *outdesc = &head;
534         size_t                     iov_offset = 0;
535         size_t                     out_ix = 0;
536         int                        outiov_ix = 0;
537         struct strcop_meta_in      mi = {0};
538 
539         size_t                     out_length = tc->produced;
540         int                        rem_length;
541         int                        dlength;
542 
543         assert(out_length != 0);
544         if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
545                 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
546                 return -EINVAL;
547         }
548         /* Traverse the out iovec until the result inject index is reached. */
549         while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
550                 out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
551                 outiov_ix++;
552         }
553         if (outiov_ix >= operation->tfrm_op.outcount){
554                 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
555                 return -EINVAL;
556         }
557         iov_offset = tc->tcfg->inject_ix - out_ix;
558         mi.dmasel = tc->unit_no;
559 
560         /* Setup the output descriptors. */
561         while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
562                 outdesc->next = alloc_cdesc(alloc_flag);
563                 if (!outdesc->next) {
564                         DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
565                         err = -ENOMEM;
566                         goto error_cleanup;
567                 }
568                 outdesc = outdesc->next;
569                 rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
570                 dlength = (out_length < rem_length) ? out_length : rem_length;
571 
572                 DEBUG(printk("create_input_descriptors:\n"
573                              "outiov_ix=%d, rem_length=%d, dlength=%d\n"
574                              "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
575                              "outcount=%d, outiov_ix=%d\n",
576                              outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
577 
578                 outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
579                 outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
580                 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
581 
582                 out_length -= dlength;
583                 iov_offset += dlength;
584                 if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
585                         iov_offset = 0;
586                         ++outiov_ix;
587                 }
588         }
589         if (out_length > 0){
590                 DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
591                 err = -EINVAL;
592                 goto error_cleanup;
593         }
594         /* Set sync in last descriptor. */
595         mi.sync = 1;
596         outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
597 
598         *id = head.next;
599         return 0;
600 
601  error_cleanup:
602         while (head.next) {
603                 outdesc = head.next->next;
604                 free_cdesc(head.next);
605                 head.next = outdesc;
606         }
607         return err;
608 }
609 
610 
611 static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
612 {
613         while (desc_len != 0) {
614                 struct cryptocop_dma_desc  *cdesc;
615                 int                        rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
616                 int                        dlength = (desc_len < rem_length) ? desc_len : rem_length;
617 
618                 cdesc = alloc_cdesc(alloc_flag);
619                 if (!cdesc) {
620                         DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
621                         return -ENOMEM;
622                 }
623                 (*current_out_cdesc)->next = cdesc;
624                 (*current_out_cdesc) = cdesc;
625 
626                 cdesc->free_buf = NULL;
627 
628                 cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
629                 cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
630 
631                 assert(desc_len >= dlength);
632                 desc_len -= dlength;
633                 *iniov_offset += dlength;
634                 if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
635                         *iniov_offset = 0;
636                         ++(*iniov_ix);
637                         if (*iniov_ix > operation->tfrm_op.incount) {
638                                 DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
639                                 return  -EINVAL;
640                         }
641                 }
642                 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
643         } /* while (desc_len != 0) */
644         /* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
645         (*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
646 
647         return 0;
648 }
649 
650 
651 static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
652 {
653         DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
654         if (tc->tcfg) {
655                 int                        failed = 0;
656                 struct cryptocop_dma_desc  *idescs = NULL;
657                 DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
658                 if (tc->pad_descs) {
659                         DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
660                         while (tc->pad_descs) {
661                                 DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
662                                 (*current_out_cdesc)->next = tc->pad_descs;
663                                 tc->pad_descs = tc->pad_descs->next;
664                                 (*current_out_cdesc) = (*current_out_cdesc)->next;
665                         }
666                 }
667 
668                 /* Setup and append output descriptors to DMA in list. */
669                 if (tc->unit_no == src_dma){
670                         /* mem2mem.  Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
671                         struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
672                         unsigned int start_ix = tc->start_ix;
673                         while (start_ix){
674                                 unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
675                                 (*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
676                                 if (!(*current_in_cdesc)->next){
677                                         DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
678                                         return -ENOMEM;
679                                 }
680                                 (*current_in_cdesc) = (*current_in_cdesc)->next;
681                                 (*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
682                                 (*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
683                                 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
684                                 start_ix -= desclen;
685                         }
686                         mi.sync = 1;
687                         (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
688                 }
689 
690                 failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
691                 if (failed){
692                         DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
693                         return failed;
694                 }
695                 DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
696                 while (idescs) {
697                         DEBUG(printk("append descriptor 0x%p\n", idescs));
698                         (*current_in_cdesc)->next = idescs;
699                         idescs = idescs->next;
700                         (*current_in_cdesc) = (*current_in_cdesc)->next;
701                 }
702         }
703         return 0;
704 }
705 
706 
707 
708 static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
709 {
710         struct cryptocop_session *sess;
711         struct cryptocop_transform_ctx *tctx;
712 
713         struct cryptocop_tfrm_ctx digest_ctx = {
714                 .previous_src = src_none,
715                 .current_src = src_none,
716                 .start_ix = 0,
717                 .requires_padding = 1,
718                 .strict_block_length = 0,
719                 .hash_conf = 0,
720                 .hash_mode = 0,
721                 .ciph_conf = 0,
722                 .cbcmode = 0,
723                 .decrypt = 0,
724                 .consumed = 0,
725                 .produced = 0,
726                 .pad_descs = NULL,
727                 .active = 0,
728                 .done = 0,
729                 .prev_src = NULL,
730                 .curr_src = NULL,
731                 .tcfg = NULL};
732         struct cryptocop_tfrm_ctx cipher_ctx = {
733                 .previous_src = src_none,
734                 .current_src = src_none,
735                 .start_ix = 0,
736                 .requires_padding = 0,
737                 .strict_block_length = 1,
738                 .hash_conf = 0,
739                 .hash_mode = 0,
740                 .ciph_conf = 0,
741                 .cbcmode = 0,
742                 .decrypt = 0,
743                 .consumed = 0,
744                 .produced = 0,
745                 .pad_descs = NULL,
746                 .active = 0,
747                 .done = 0,
748                 .prev_src = NULL,
749                 .curr_src = NULL,
750                 .tcfg = NULL};
751         struct cryptocop_tfrm_ctx csum_ctx = {
752                 .previous_src = src_none,
753                 .current_src = src_none,
754                 .start_ix = 0,
755                 .blocklength = 2,
756                 .requires_padding = 1,
757                 .strict_block_length = 0,
758                 .hash_conf = 0,
759                 .hash_mode = 0,
760                 .ciph_conf = 0,
761                 .cbcmode = 0,
762                 .decrypt = 0,
763                 .consumed = 0,
764                 .produced = 0,
765                 .pad_descs = NULL,
766                 .active = 0,
767                 .done = 0,
768                 .tcfg = NULL,
769                 .prev_src = NULL,
770                 .curr_src = NULL,
771                 .unit_no = src_csum};
772         struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
773 
774         unsigned int indata_ix = 0;
775 
776         /* iovec accounting. */
777         int iniov_ix = 0;
778         int iniov_offset = 0;
779 
780         /* Operation descriptor cfg traversal pointer. */
781         struct cryptocop_desc *odsc;
782 
783         int failed = 0;
784         /* List heads for allocated descriptors. */
785         struct cryptocop_dma_desc out_cdesc_head = {0};
786         struct cryptocop_dma_desc in_cdesc_head = {0};
787 
788         struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
789         struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
790 
791         struct cryptocop_tfrm_ctx *output_tc = NULL;
792         void                      *iop_alloc_ptr;
793 
794         assert(operation != NULL);
795         assert(int_op != NULL);
796 
797         DEBUG(printk("cryptocop_setup_dma_list: start\n"));
798         DEBUG(print_cryptocop_operation(operation));
799 
800         sess = get_session(operation->sid);
801         if (!sess) {
802                 DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
803                 failed = -EINVAL;
804                 goto error_cleanup;
805         }
806         iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
807         if (!iop_alloc_ptr) {
808                 DEBUG_API(printk("cryptocop_setup_dma_list:  kmalloc cryptocop_int_operation\n"));
809                 failed = -ENOMEM;
810                 goto error_cleanup;
811         }
812         (*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
813         DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
814         (*int_op)->alloc_ptr = iop_alloc_ptr;
815         DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
816 
817         (*int_op)->sid = operation->sid;
818         (*int_op)->cdesc_out = NULL;
819         (*int_op)->cdesc_in = NULL;
820         (*int_op)->tdes_mode = cryptocop_3des_ede;
821         (*int_op)->csum_mode = cryptocop_csum_le;
822         (*int_op)->ddesc_out = NULL;
823         (*int_op)->ddesc_in = NULL;
824 
825         /* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
826         if (!tcfg) {
827                 DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
828                 failed = -EINVAL;
829                 goto error_cleanup;
830         }
831         while (tcfg) {
832                 tctx = get_transform_ctx(sess, tcfg->tid);
833                 if (!tctx) {
834                         DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
835                         failed = -EINVAL;
836                         goto error_cleanup;
837                 }
838                 if (tcfg->inject_ix > operation->tfrm_op.outlen){
839                         DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
840                         failed = -EINVAL;
841                         goto error_cleanup;
842                 }
843                 switch (tctx->init.alg){
844                 case cryptocop_alg_mem2mem:
845                         if (cipher_ctx.tcfg != NULL){
846                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
847                                 failed = -EINVAL;
848                                 goto error_cleanup;
849                         }
850                         /* mem2mem is handled as a NULL cipher. */
851                         cipher_ctx.cbcmode = 0;
852                         cipher_ctx.decrypt = 0;
853                         cipher_ctx.blocklength = 1;
854                         cipher_ctx.ciph_conf = 0;
855                         cipher_ctx.unit_no = src_dma;
856                         cipher_ctx.tcfg = tcfg;
857                         cipher_ctx.tctx = tctx;
858                         break;
859                 case cryptocop_alg_des:
860                 case cryptocop_alg_3des:
861                 case cryptocop_alg_aes:
862                         /* cipher */
863                         if (cipher_ctx.tcfg != NULL){
864                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
865                                 failed = -EINVAL;
866                                 goto error_cleanup;
867                         }
868                         cipher_ctx.tcfg = tcfg;
869                         cipher_ctx.tctx = tctx;
870                         if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
871                                 cipher_ctx.decrypt = 1;
872                         }
873                         switch (tctx->init.cipher_mode) {
874                         case cryptocop_cipher_mode_ecb:
875                                 cipher_ctx.cbcmode = 0;
876                                 break;
877                         case cryptocop_cipher_mode_cbc:
878                                 cipher_ctx.cbcmode = 1;
879                                 break;
880                         default:
881                                 DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
882                                 failed = -EINVAL;
883                                 goto error_cleanup;
884                         }
885                         DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
886                         switch (tctx->init.alg){
887                         case cryptocop_alg_des:
888                                 cipher_ctx.ciph_conf = 0;
889                                 cipher_ctx.unit_no = src_des;
890                                 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
891                                 break;
892                         case cryptocop_alg_3des:
893                                 cipher_ctx.ciph_conf = 1;
894                                 cipher_ctx.unit_no = src_des;
895                                 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
896                                 break;
897                         case cryptocop_alg_aes:
898                                 cipher_ctx.ciph_conf = 2;
899                                 cipher_ctx.unit_no = src_aes;
900                                 cipher_ctx.blocklength = AES_BLOCK_LENGTH;
901                                 break;
902                         default:
903                                 panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
904                         }
905                         (*int_op)->tdes_mode = tctx->init.tdes_mode;
906                         break;
907                 case cryptocop_alg_md5:
908                 case cryptocop_alg_sha1:
909                         /* digest */
910                         if (digest_ctx.tcfg != NULL){
911                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
912                                 failed = -EINVAL;
913                                 goto error_cleanup;
914                         }
915                         digest_ctx.tcfg = tcfg;
916                         digest_ctx.tctx = tctx;
917                         digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
918                         switch (tctx->init.alg){
919                         case cryptocop_alg_md5:
920                                 digest_ctx.blocklength = MD5_BLOCK_LENGTH;
921                                 digest_ctx.unit_no = src_md5;
922                                 digest_ctx.hash_conf = 1; /* 1 => MD-5 */
923                                 break;
924                         case cryptocop_alg_sha1:
925                                 digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
926                                 digest_ctx.unit_no = src_sha1;
927                                 digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
928                                 break;
929                         default:
930                                 panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
931                         }
932                         break;
933                 case cryptocop_alg_csum:
934                         /* digest */
935                         if (csum_ctx.tcfg != NULL){
936                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
937                                 failed = -EINVAL;
938                                 goto error_cleanup;
939                         }
940                         (*int_op)->csum_mode = tctx->init.csum_mode;
941                         csum_ctx.tcfg = tcfg;
942                         csum_ctx.tctx = tctx;
943                         break;
944                 default:
945                         /* no algorithm. */
946                         DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
947                         failed = -EINVAL;
948                         goto error_cleanup;
949                 }
950                 tcfg = tcfg->next;
951         }
952         /* Download key if a cipher is used. */
953         if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
954                 struct cryptocop_dma_desc  *key_desc = NULL;
955 
956                 failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
957                 if (failed) {
958                         DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
959                         goto error_cleanup;
960                 }
961                 current_out_cdesc->next = key_desc;
962                 current_out_cdesc = key_desc;
963                 indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
964 
965                 /* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
966                 if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
967                         struct cryptocop_dma_desc  *iv_desc = NULL;
968 
969                         DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
970 
971                         failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
972                         if (failed) {
973                                 DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
974                                 goto error_cleanup;
975                         }
976                         current_out_cdesc->next = iv_desc;
977                         current_out_cdesc = iv_desc;
978                         indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
979                 }
980         }
981 
982         /* Process descriptors. */
983         odsc = operation->tfrm_op.desc;
984         while (odsc) {
985                 struct cryptocop_desc_cfg   *dcfg = odsc->cfg;
986                 struct strcop_meta_out      meta_out = {0};
987                 size_t                      desc_len = odsc->length;
988                 int                         active_count, eop_needed_count;
989 
990                 output_tc = NULL;
991 
992                 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
993 
994                 while (dcfg) {
995                         struct cryptocop_tfrm_ctx  *tc = NULL;
996 
997                         DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
998                         /* Get the local context for the transform and mark it as the output unit if it produces output. */
999                         if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
1000                                 tc = &digest_ctx;
1001                         } else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
1002                                 tc = &cipher_ctx;
1003                         } else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
1004                                 tc = &csum_ctx;
1005                         }
1006                         if (!tc) {
1007                                 DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
1008                                 failed = -EINVAL;
1009                                 goto error_cleanup;
1010                         }
1011                         if (tc->done) {
1012                                 DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1013                                 failed = -EINVAL;
1014                                 goto error_cleanup;
1015                         }
1016                         if (!tc->active) {
1017                                 tc->start_ix = indata_ix;
1018                                 tc->active = 1;
1019                         }
1020 
1021                         tc->previous_src = tc->current_src;
1022                         tc->prev_src = tc->curr_src;
1023                         /* Map source unit id to DMA source config. */
1024                         switch (dcfg->src){
1025                         case cryptocop_source_dma:
1026                                 tc->current_src = src_dma;
1027                                 break;
1028                         case cryptocop_source_des:
1029                                 tc->current_src = src_des;
1030                                 break;
1031                         case cryptocop_source_3des:
1032                                 tc->current_src = src_des;
1033                                 break;
1034                         case cryptocop_source_aes:
1035                                 tc->current_src = src_aes;
1036                                 break;
1037                         case cryptocop_source_md5:
1038                         case cryptocop_source_sha1:
1039                         case cryptocop_source_csum:
1040                         case cryptocop_source_none:
1041                         default:
1042                                 /* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1043                                  */
1044                                 DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1045                                 failed = -EINVAL;
1046                                 goto error_cleanup;
1047                         }
1048                         if (tc->current_src != src_dma) {
1049                                 /* Find the unit we are sourcing from. */
1050                                 if (digest_ctx.unit_no == tc->current_src){
1051                                         tc->curr_src = &digest_ctx;
1052                                 } else if (cipher_ctx.unit_no == tc->current_src){
1053                                         tc->curr_src = &cipher_ctx;
1054                                 } else if (csum_ctx.unit_no == tc->current_src){
1055                                         tc->curr_src = &csum_ctx;
1056                                 }
1057                                 if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1058                                         DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1059                                         failed = -EINVAL;
1060                                         goto error_cleanup;
1061                                 }
1062                         } else {
1063                                 tc->curr_src = NULL;
1064                         }
1065 
1066                         /* Detect source switch. */
1067                         DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1068                         if (tc->active && (tc->current_src != tc->previous_src)) {
1069                                 /* Only allow source switch when both the old source unit and the new one have
1070                                  * no pending data to process (i.e. the consumed length must be a multiple of the
1071                                  * transform blocklength). */
1072                                 /* Note: if the src == NULL we are actually sourcing from DMA out. */
1073                                 if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1074                                     ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1075                                 {
1076                                         DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1077                                         failed = -EINVAL;
1078                                         goto error_cleanup;
1079                                 }
1080                         }
1081                         /* Detect unit deactivation. */
1082                         if (dcfg->last) {
1083                                 /* Length check of this is handled below. */
1084                                 tc->done = 1;
1085                         }
1086                         dcfg = dcfg->next;
1087                 } /* while (dcfg) */
1088                 DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1089 
1090                 if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1091                         DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1092                         failed = -EINVAL;
1093                         goto error_cleanup;
1094                 }
1095                 if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1096                         DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1097                         failed = -EINVAL;
1098                         goto error_cleanup;
1099                 }
1100                 if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1101                         DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1102                         failed = -EINVAL;
1103                         goto error_cleanup;
1104                 }
1105 
1106                 /* Update consumed and produced lengths.
1107 
1108                    The consumed length accounting here is actually cheating.  If a unit source from DMA (or any
1109                    other unit that process data in blocks of one octet) it is correct, but if it source from a
1110                    block processing unit, i.e. a cipher, it will be temporarily incorrect at some times.  However
1111                    since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1112                    unit has processed an exact multiple of its block length the end result will be correct.
1113                    Beware that if the source change restriction change this code will need to be (much) reworked.
1114                 */
1115                 DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1116 
1117                 if (csum_ctx.active) {
1118                         csum_ctx.consumed += desc_len;
1119                         if (csum_ctx.done) {
1120                                 csum_ctx.produced = 2;
1121                         }
1122                         DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1123                 }
1124                 if (digest_ctx.active) {
1125                         digest_ctx.consumed += desc_len;
1126                         if (digest_ctx.done) {
1127                                 if (digest_ctx.unit_no == src_md5) {
1128                                         digest_ctx.produced = MD5_STATE_LENGTH;
1129                                 } else {
1130                                         digest_ctx.produced = SHA1_STATE_LENGTH;
1131                                 }
1132                         }
1133                         DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1134                 }
1135                 if (cipher_ctx.active) {
1136                         /* Ciphers are allowed only to source from DMA out.  That is filtered above. */
1137                         assert(cipher_ctx.current_src == src_dma);
1138                         cipher_ctx.consumed += desc_len;
1139                         cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1140                         if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1141                                 cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1142                         }
1143                         DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1144                 }
1145 
1146                 /* Setup the DMA out descriptors. */
1147                 /* Configure the metadata. */
1148                 active_count = 0;
1149                 eop_needed_count = 0;
1150                 if (cipher_ctx.active) {
1151                         ++active_count;
1152                         if (cipher_ctx.unit_no == src_dma){
1153                                 /* mem2mem */
1154                                 meta_out.ciphsel = src_none;
1155                         } else {
1156                                 meta_out.ciphsel = cipher_ctx.current_src;
1157                         }
1158                         meta_out.ciphconf = cipher_ctx.ciph_conf;
1159                         meta_out.cbcmode = cipher_ctx.cbcmode;
1160                         meta_out.decrypt = cipher_ctx.decrypt;
1161                         DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1162                         if (cipher_ctx.done) ++eop_needed_count;
1163                 } else {
1164                         meta_out.ciphsel = src_none;
1165                 }
1166 
1167                 if (digest_ctx.active) {
1168                         ++active_count;
1169                         meta_out.hashsel = digest_ctx.current_src;
1170                         meta_out.hashconf = digest_ctx.hash_conf;
1171                         meta_out.hashmode = 0; /* Explicit mode is not used here. */
1172                         DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1173                         if (digest_ctx.done) {
1174                                 assert(digest_ctx.pad_descs == NULL);
1175                                 failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1176                                 if (failed) {
1177                                         DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1178                                         goto error_cleanup;
1179                                 }
1180                         }
1181                 } else {
1182                         meta_out.hashsel = src_none;
1183                 }
1184 
1185                 if (csum_ctx.active) {
1186                         ++active_count;
1187                         meta_out.csumsel = csum_ctx.current_src;
1188                         if (csum_ctx.done) {
1189                                 assert(csum_ctx.pad_descs == NULL);
1190                                 failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1191                                 if (failed) {
1192                                         DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1193                                         goto error_cleanup;
1194                                 }
1195                         }
1196                 } else {
1197                         meta_out.csumsel = src_none;
1198                 }
1199                 DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1200                 /* Setup DMA out descriptors for the indata. */
1201                 failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1202                 if (failed) {
1203                         DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1204                         goto error_cleanup;
1205                 }
1206                 /* Setup out EOP.  If there are active units that are not done here they cannot get an EOP
1207                  * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1208                  * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1209                  */
1210                 assert(active_count >= eop_needed_count);
1211                 assert((eop_needed_count == 0) || (eop_needed_count == 1));
1212                 if (eop_needed_count) {
1213                         /* This means that the bulk operation (cipeher/m2m) is terminated. */
1214                         if (active_count > 1) {
1215                                 /* Use zero length EOP descriptor. */
1216                                 struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1217                                 struct strcop_meta_out    ed_mo = {0};
1218                                 if (!ed) {
1219                                         DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1220                                         failed = -ENOMEM;
1221                                         goto error_cleanup;
1222                                 }
1223 
1224                                 assert(cipher_ctx.active && cipher_ctx.done);
1225 
1226                                 if (cipher_ctx.unit_no == src_dma){
1227                                         /* mem2mem */
1228                                         ed_mo.ciphsel = src_none;
1229                                 } else {
1230                                         ed_mo.ciphsel = cipher_ctx.current_src;
1231                                 }
1232                                 ed_mo.ciphconf = cipher_ctx.ciph_conf;
1233                                 ed_mo.cbcmode = cipher_ctx.cbcmode;
1234                                 ed_mo.decrypt = cipher_ctx.decrypt;
1235 
1236                                 ed->free_buf = NULL;
1237                                 ed->dma_descr->wait = 1;
1238                                 ed->dma_descr->out_eop = 1;
1239 
1240                                 ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1241                                 ed->dma_descr->after = ed->dma_descr->buf;
1242                                 ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1243                                 current_out_cdesc->next = ed;
1244                                 current_out_cdesc = ed;
1245                         } else {
1246                                 /* Set EOP in the current out descriptor since the only active module is
1247                                  * the one needing the EOP. */
1248 
1249                                 current_out_cdesc->dma_descr->out_eop = 1;
1250                         }
1251                 }
1252 
1253                 if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1254                 if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1255                 if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1256                 indata_ix += odsc->length;
1257                 odsc = odsc->next;
1258         } /* while (odsc) */ /* Process descriptors. */
1259         DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1260         if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1261                 DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1262                 failed = -EINVAL;
1263                 goto error_cleanup;
1264         }
1265         if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1266                 DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1267                 failed = -EINVAL;
1268                 goto error_cleanup;
1269         }
1270         if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1271                 DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1272                 failed = -EINVAL;
1273                 goto error_cleanup;
1274         }
1275 
1276         failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1277         if (failed){
1278                 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1279                 goto error_cleanup;
1280         }
1281         failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1282         if (failed){
1283                 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1284                 goto error_cleanup;
1285         }
1286         failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1287         if (failed){
1288                 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1289                 goto error_cleanup;
1290         }
1291 
1292         DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1293         (*int_op)->cdesc_out = out_cdesc_head.next;
1294         (*int_op)->cdesc_in = in_cdesc_head.next;
1295         DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1296 
1297         setup_descr_chain(out_cdesc_head.next);
1298         setup_descr_chain(in_cdesc_head.next);
1299 
1300         /* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1301          * last DMA out descriptor for EOL.
1302          */
1303         current_in_cdesc->dma_descr->intr = 1;
1304         current_in_cdesc->dma_descr->eol = 1;
1305         current_out_cdesc->dma_descr->eol = 1;
1306 
1307         /* Setup DMA contexts. */
1308         (*int_op)->ctx_out.next = NULL;
1309         (*int_op)->ctx_out.eol = 1;
1310         (*int_op)->ctx_out.intr = 0;
1311         (*int_op)->ctx_out.store_mode = 0;
1312         (*int_op)->ctx_out.en = 0;
1313         (*int_op)->ctx_out.dis = 0;
1314         (*int_op)->ctx_out.md0 = 0;
1315         (*int_op)->ctx_out.md1 = 0;
1316         (*int_op)->ctx_out.md2 = 0;
1317         (*int_op)->ctx_out.md3 = 0;
1318         (*int_op)->ctx_out.md4 = 0;
1319         (*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1320         (*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1321 
1322         (*int_op)->ctx_in.next = NULL;
1323         (*int_op)->ctx_in.eol = 1;
1324         (*int_op)->ctx_in.intr = 0;
1325         (*int_op)->ctx_in.store_mode = 0;
1326         (*int_op)->ctx_in.en = 0;
1327         (*int_op)->ctx_in.dis = 0;
1328         (*int_op)->ctx_in.md0 = 0;
1329         (*int_op)->ctx_in.md1 = 0;
1330         (*int_op)->ctx_in.md2 = 0;
1331         (*int_op)->ctx_in.md3 = 0;
1332         (*int_op)->ctx_in.md4 = 0;
1333 
1334         (*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1335         (*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1336 
1337         DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1338         return 0;
1339 
1340 error_cleanup:
1341         {
1342                 /* Free all allocated resources. */
1343                 struct cryptocop_dma_desc *tmp_cdesc;
1344                 while (digest_ctx.pad_descs){
1345                         tmp_cdesc = digest_ctx.pad_descs->next;
1346                         free_cdesc(digest_ctx.pad_descs);
1347                         digest_ctx.pad_descs = tmp_cdesc;
1348                 }
1349                 while (csum_ctx.pad_descs){
1350                         tmp_cdesc = csum_ctx.pad_descs->next;
1351                         free_cdesc(csum_ctx.pad_descs);
1352                         csum_ctx.pad_descs = tmp_cdesc;
1353                 }
1354                 assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1355 
1356                 if (*int_op != NULL) delete_internal_operation(*int_op);
1357         }
1358         DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1359         return failed;
1360 }
1361 
1362 
1363 static void delete_internal_operation(struct cryptocop_int_operation *iop)
1364 {
1365         void                      *ptr = iop->alloc_ptr;
1366         struct cryptocop_dma_desc *cd = iop->cdesc_out;
1367         struct cryptocop_dma_desc *next;
1368 
1369         DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1370 
1371         while (cd) {
1372                 next = cd->next;
1373                 free_cdesc(cd);
1374                 cd = next;
1375         }
1376         cd = iop->cdesc_in;
1377         while (cd) {
1378                 next = cd->next;
1379                 free_cdesc(cd);
1380                 cd = next;
1381         }
1382         kfree(ptr);
1383 }
1384 
1385 #define MD5_MIN_PAD_LENGTH (9)
1386 #define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1387 
1388 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1389 {
1390         size_t                  padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1391         unsigned char           *p;
1392         int                     i;
1393         unsigned long long int  bit_length = hashed_length << 3;
1394 
1395         if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1396 
1397         p = kzalloc(padlen, alloc_flag);
1398         if (!p) return -ENOMEM;
1399 
1400         *p = 0x80;
1401 
1402         DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1403 
1404         i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1405         while (bit_length != 0){
1406                 p[i++] = bit_length % 0x100;
1407                 bit_length >>= 8;
1408         }
1409 
1410         *pad = (char*)p;
1411         *pad_length = padlen;
1412 
1413         return 0;
1414 }
1415 
1416 #define SHA1_MIN_PAD_LENGTH (9)
1417 #define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1418 
1419 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1420 {
1421         size_t                  padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1422         unsigned char           *p;
1423         int                     i;
1424         unsigned long long int  bit_length = hashed_length << 3;
1425 
1426         if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1427 
1428         p = kzalloc(padlen, alloc_flag);
1429         if (!p) return -ENOMEM;
1430 
1431         *p = 0x80;
1432 
1433         DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1434 
1435         i = padlen - 1;
1436         while (bit_length != 0){
1437                 p[i--] = bit_length % 0x100;
1438                 bit_length >>= 8;
1439         }
1440 
1441         *pad = (char*)p;
1442         *pad_length = padlen;
1443 
1444         return 0;
1445 }
1446 
1447 
1448 static int transform_ok(struct cryptocop_transform_init *tinit)
1449 {
1450         switch (tinit->alg){
1451         case cryptocop_alg_csum:
1452                 switch (tinit->csum_mode){
1453                 case cryptocop_csum_le:
1454                 case cryptocop_csum_be:
1455                         break;
1456                 default:
1457                         DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1458                         return -EINVAL;
1459                 }
1460         case cryptocop_alg_mem2mem:
1461         case cryptocop_alg_md5:
1462         case cryptocop_alg_sha1:
1463                 if (tinit->keylen != 0) {
1464                         DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1465                         return -EINVAL; /* This check is a bit strict. */
1466                 }
1467                 break;
1468         case cryptocop_alg_des:
1469                 if (tinit->keylen != 64) {
1470                         DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1471                         return -EINVAL;
1472                 }
1473                 break;
1474         case cryptocop_alg_3des:
1475                 if (tinit->keylen != 192) {
1476                         DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1477                         return -EINVAL;
1478                 }
1479                 break;
1480         case cryptocop_alg_aes:
1481                 if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1482                         DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1483                         return -EINVAL;
1484                 }
1485                 break;
1486         case cryptocop_no_alg:
1487         default:
1488                 DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1489                 return -EINVAL;
1490         }
1491 
1492         switch (tinit->alg){
1493         case cryptocop_alg_des:
1494         case cryptocop_alg_3des:
1495         case cryptocop_alg_aes:
1496                 if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1497         default:
1498                  break;
1499         }
1500         return 0;
1501 }
1502 
1503 
1504 int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1505 {
1506         struct cryptocop_session         *sess;
1507         struct cryptocop_transform_init  *tfrm_in = tinit;
1508         struct cryptocop_transform_init  *tmp_in;
1509         int                              no_tfrms = 0;
1510         int                              i;
1511         unsigned long int                flags;
1512 
1513         init_stream_coprocessor(); /* For safety if we are called early */
1514 
1515         while (tfrm_in){
1516                 int err;
1517                 ++no_tfrms;
1518                 if ((err = transform_ok(tfrm_in))) {
1519                         DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1520                         return err;
1521                 }
1522                 tfrm_in = tfrm_in->next;
1523         }
1524         if (0 == no_tfrms) {
1525                 DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1526                 return -EINVAL;
1527         }
1528 
1529         sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1530         if (!sess){
1531                 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1532                 return -ENOMEM;
1533         }
1534 
1535         sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1536         if (!sess->tfrm_ctx) {
1537                 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1538                 kfree(sess);
1539                 return -ENOMEM;
1540         }
1541 
1542         tfrm_in = tinit;
1543         for (i = 0; i < no_tfrms; i++){
1544                 tmp_in = tfrm_in->next;
1545                 while (tmp_in){
1546                         if (tmp_in->tid == tfrm_in->tid) {
1547                                 DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1548                                 kfree(sess->tfrm_ctx);
1549                                 kfree(sess);
1550                                 return -EINVAL;
1551                         }
1552                         tmp_in = tmp_in->next;
1553                 }
1554                 memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1555                 sess->tfrm_ctx[i].dec_key_set = 0;
1556                 sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1557 
1558                 tfrm_in = tfrm_in->next;
1559         }
1560         sess->tfrm_ctx[i-1].next = NULL;
1561 
1562         spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1563         sess->sid = next_sid;
1564         next_sid++;
1565         /* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1566          *      OTOH 2^64 is a really large number of session. */
1567         if (next_sid == 0) next_sid = 1;
1568 
1569         /* Prepend to session list. */
1570         sess->next = cryptocop_sessions;
1571         cryptocop_sessions = sess;
1572         spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1573         *sid = sess->sid;
1574         return 0;
1575 }
1576 
1577 
1578 int cryptocop_free_session(cryptocop_session_id sid)
1579 {
1580         struct cryptocop_transform_ctx    *tc;
1581         struct cryptocop_session          *sess = NULL;
1582         struct cryptocop_session          *psess = NULL;
1583         unsigned long int                 flags;
1584         int                               i;
1585         LIST_HEAD(remove_list);
1586         struct list_head                  *node, *tmp;
1587         struct cryptocop_prio_job         *pj;
1588 
1589         DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1590 
1591         spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1592         sess = cryptocop_sessions;
1593         while (sess && sess->sid != sid){
1594                 psess = sess;
1595                 sess = sess->next;
1596         }
1597         if (sess){
1598                 if (psess){
1599                         psess->next = sess->next;
1600                 } else {
1601                         cryptocop_sessions = sess->next;
1602                 }
1603         }
1604         spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1605 
1606         if (!sess) return -EINVAL;
1607 
1608         /* Remove queued jobs. */
1609         spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1610 
1611         for (i = 0; i < cryptocop_prio_no_prios; i++){
1612                 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1613                         list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1614                                 pj = list_entry(node, struct cryptocop_prio_job, node);
1615                                 if (pj->oper->sid == sid) {
1616                                         list_move_tail(node, &remove_list);
1617                                 }
1618                         }
1619                 }
1620         }
1621         spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1622 
1623         list_for_each_safe(node, tmp, &remove_list) {
1624                 list_del(node);
1625                 pj = list_entry(node, struct cryptocop_prio_job, node);
1626                 pj->oper->operation_status = -EAGAIN;  /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1627                 DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1628                 pj->oper->cb(pj->oper, pj->oper->cb_data);
1629                 delete_internal_operation(pj->iop);
1630                 kfree(pj);
1631         }
1632 
1633         tc = sess->tfrm_ctx;
1634         /* Erase keying data. */
1635         while (tc){
1636                 DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1637                 memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1638                 memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1639                 tc = tc->next;
1640         }
1641         kfree(sess->tfrm_ctx);
1642         kfree(sess);
1643 
1644         return 0;
1645 }
1646 
1647 static struct cryptocop_session *get_session(cryptocop_session_id sid)
1648 {
1649         struct cryptocop_session    *sess;
1650         unsigned long int           flags;
1651 
1652         spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1653         sess = cryptocop_sessions;
1654         while (sess && (sess->sid != sid)){
1655                 sess = sess->next;
1656         }
1657         spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1658 
1659         return sess;
1660 }
1661 
1662 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1663 {
1664         struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1665 
1666         DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1667         assert(sess != NULL);
1668         while (tc && tc->init.tid != tid){
1669                 DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1670                 tc = tc->next;
1671         }
1672         DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1673         return tc;
1674 }
1675 
1676 
1677 
1678 /* The AES s-transform matrix (s-box). */
1679 static const u8 aes_sbox[256] = {
1680         99,  124, 119, 123, 242, 107, 111, 197, 48,  1,   103, 43,  254, 215, 171, 118,
1681         202, 130, 201, 125, 250, 89,  71,  240, 173, 212, 162, 175, 156, 164, 114, 192,
1682         183, 253, 147, 38,  54,  63,  247, 204, 52,  165, 229, 241, 113, 216, 49,  21,
1683         4,   199, 35,  195, 24,  150, 5,   154, 7,   18,  128, 226, 235, 39,  178, 117,
1684         9,   131, 44,  26,  27,  110, 90,  160, 82,  59,  214, 179, 41,  227, 47,  132,
1685         83,  209, 0,   237, 32,  252, 177, 91,  106, 203, 190, 57,  74,  76,  88,  207,
1686         208, 239, 170, 251, 67,  77,  51,  133, 69,  249, 2,   127, 80,  60,  159, 168,
1687         81,  163, 64,  143, 146, 157, 56,  245, 188, 182, 218, 33,  16,  255, 243, 210,
1688         205, 12,  19,  236, 95,  151, 68,  23,  196, 167, 126, 61,  100, 93,  25,  115,
1689         96,  129, 79,  220, 34,  42,  144, 136, 70,  238, 184, 20,  222, 94,  11,  219,
1690         224, 50,  58,  10,  73,  6,   36,  92,  194, 211, 172, 98,  145, 149, 228, 121,
1691         231, 200, 55,  109, 141, 213, 78,  169, 108, 86,  244, 234, 101, 122, 174, 8,
1692         186, 120, 37,  46,  28,  166, 180, 198, 232, 221, 116, 31,  75,  189, 139, 138,
1693         112, 62,  181, 102, 72,  3,   246, 14,  97,  53,  87,  185, 134, 193, 29,  158,
1694         225, 248, 152, 17,  105, 217, 142, 148, 155, 30,  135, 233, 206, 85,  40,  223,
1695         140, 161, 137, 13,  191, 230, 66,  104, 65,  153, 45,  15,  176, 84,  187, 22
1696 };
1697 
1698 /* AES has a 32 bit word round constants for each round in the
1699  * key schedule.  round_constant[i] is really Rcon[i+1] in FIPS187.
1700  */
1701 static u32 round_constant[11] = {
1702         0x01000000, 0x02000000, 0x04000000, 0x08000000,
1703         0x10000000, 0x20000000, 0x40000000, 0x80000000,
1704         0x1B000000, 0x36000000, 0x6C000000
1705 };
1706 
1707 /* Apply the s-box to each of the four occtets in w. */
1708 static u32 aes_ks_subword(const u32 w)
1709 {
1710         u8 bytes[4];
1711 
1712         *(u32*)(&bytes[0]) = w;
1713         bytes[0] = aes_sbox[bytes[0]];
1714         bytes[1] = aes_sbox[bytes[1]];
1715         bytes[2] = aes_sbox[bytes[2]];
1716         bytes[3] = aes_sbox[bytes[3]];
1717         return *(u32*)(&bytes[0]);
1718 }
1719 
1720 /* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1721  * (Note that AES words are 32 bit long)
1722  *
1723  * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1724  * word temp
1725  * i = 0
1726  * while (i < Nk) {
1727  *   w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1728  *   i = i + 1
1729  * }
1730  * i = Nk
1731  *
1732  * while (i < (Nb * (Nr + 1))) {
1733  *   temp = w[i - 1]
1734  *   if ((i mod Nk) == 0) {
1735  *     temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1736  *   }
1737  *   else if ((Nk > 6) && ((i mod Nk) == 4)) {
1738  *     temp = SubWord(temp)
1739  *   }
1740  *   w[i] = w[i - Nk] xor temp
1741  * }
1742  * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1743  * SubWord(t) applies the AES s-box individually to each octet
1744  * in a 32 bit word.
1745  *
1746  * For AES Nk can have the values 4, 6, and 8 (corresponding to
1747  * values for Nr of 10, 12, and 14).  Nb is always 4.
1748  *
1749  * To construct w[i], w[i - 1] and w[i - Nk] must be
1750  * available.  Consequently we must keep a state of the last Nk words
1751  * to be able to create the last round keys.
1752  */
1753 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength)
1754 {
1755         u32 temp;
1756         u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1757         u8  w_last_ix;
1758         int i;
1759         u8  nr, nk;
1760 
1761         switch (keylength){
1762         case 128:
1763                 nk = 4;
1764                 nr = 10;
1765                 break;
1766         case 192:
1767                 nk = 6;
1768                 nr = 12;
1769                 break;
1770         case 256:
1771                 nk = 8;
1772                 nr = 14;
1773                 break;
1774         default:
1775                 panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1776         };
1777 
1778         /* Need to do host byte order correction here since key is byte oriented and the
1779          * kx algorithm is word (u32) oriented. */
1780         for (i = 0; i < nk; i+=1) {
1781                 w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1782         }
1783 
1784         i = (int)nk;
1785         w_last_ix = i - 1;
1786         while (i < (4 * (nr + 2))) {
1787                 temp = w_ring[w_last_ix];
1788                 if (!(i % nk)) {
1789                         /* RotWord(temp) */
1790                         temp = (temp << 8) | (temp >> 24);
1791                         temp = aes_ks_subword(temp);
1792                         temp ^= round_constant[i/nk - 1];
1793                 } else if ((nk > 6) && ((i % nk) == 4)) {
1794                         temp = aes_ks_subword(temp);
1795                 }
1796                 w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1797                 temp ^= w_ring[w_last_ix];
1798                 w_ring[w_last_ix] = temp;
1799 
1800                 /* We need the round keys for round Nr+1 and Nr+2 (round key
1801                  * Nr+2 is the round key beyond the last one used when
1802                  * encrypting).  Rounds are numbered starting from 0, Nr=10
1803                  * implies 11 rounds are used in encryption/decryption.
1804                  */
1805                 if (i >= (4 * nr)) {
1806                         /* Need to do host byte order correction here, the key
1807                          * is byte oriented. */
1808                         *(u32*)dec_key = cpu_to_be32(temp);
1809                         dec_key += 4;
1810                 }
1811                 ++i;
1812         }
1813 }
1814 
1815 
1816 /**** Job/operation management. ****/
1817 
1818 int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1819 {
1820         return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1821 }
1822 
1823 int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1824 {
1825         return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1826 }
1827 
1828 int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1829 {
1830         return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1831 }
1832 
1833 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1834 {
1835         int                           ret;
1836         struct cryptocop_prio_job     *pj = NULL;
1837         unsigned long int             flags;
1838 
1839         DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1840 
1841         if (!operation || !operation->cb){
1842                 DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1843                 return -EINVAL;
1844         }
1845 
1846         if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1847                 DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1848                 return ret;
1849         }
1850         assert(pj != NULL);
1851 
1852         spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1853         list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1854         spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1855 
1856         /* Make sure a job is running */
1857         cryptocop_start_job();
1858         return 0;
1859 }
1860 
1861 static void cryptocop_do_tasklet(unsigned long unused);
1862 DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1863 
1864 static void cryptocop_do_tasklet(unsigned long unused)
1865 {
1866         struct list_head             *node;
1867         struct cryptocop_prio_job    *pj = NULL;
1868         unsigned long                flags;
1869 
1870         DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1871 
1872         do {
1873                 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1874                 if (!list_empty(&cryptocop_completed_jobs)){
1875                         node = cryptocop_completed_jobs.next;
1876                         list_del(node);
1877                         pj = list_entry(node, struct cryptocop_prio_job, node);
1878                 } else {
1879                         pj = NULL;
1880                 }
1881                 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1882                 if (pj) {
1883                         assert(pj->oper != NULL);
1884 
1885                         /* Notify consumer of operation completeness. */
1886                         DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1887 
1888                         pj->oper->operation_status = 0; /* Job is completed. */
1889                         pj->oper->cb(pj->oper, pj->oper->cb_data);
1890                         delete_internal_operation(pj->iop);
1891                         kfree(pj);
1892                 }
1893         } while (pj != NULL);
1894 
1895         DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1896 }
1897 
1898 static irqreturn_t
1899 dma_done_interrupt(int irq, void *dev_id)
1900 {
1901         struct cryptocop_prio_job *done_job;
1902         reg_dma_rw_ack_intr ack_intr = {
1903                 .data = 1,
1904         };
1905 
1906         REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1907 
1908         DEBUG(printk("cryptocop DMA done\n"));
1909 
1910         spin_lock(&running_job_lock);
1911         if (cryptocop_running_job == NULL){
1912                 printk("stream co-processor got interrupt when not busy\n");
1913                 spin_unlock(&running_job_lock);
1914                 return IRQ_HANDLED;
1915         }
1916         done_job = cryptocop_running_job;
1917         cryptocop_running_job = NULL;
1918         spin_unlock(&running_job_lock);
1919 
1920         /* Start processing a job. */
1921         if (!spin_trylock(&cryptocop_process_lock)){
1922                 DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1923         } else {
1924                 cryptocop_start_job();
1925                 spin_unlock(&cryptocop_process_lock);
1926         }
1927 
1928         done_job->oper->operation_status = 0; /* Job is completed. */
1929         if (done_job->oper->fast_callback){
1930                 /* This operation wants callback from interrupt. */
1931                 done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1932                 delete_internal_operation(done_job->iop);
1933                 kfree(done_job);
1934         } else {
1935                 spin_lock(&cryptocop_completed_jobs_lock);
1936                 list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1937                 spin_unlock(&cryptocop_completed_jobs_lock);
1938                 tasklet_schedule(&cryptocop_tasklet);
1939         }
1940 
1941         DEBUG(printk("cryptocop leave irq handler\n"));
1942         return IRQ_HANDLED;
1943 }
1944 
1945 
1946 /* Setup interrupts and DMA channels. */
1947 static int init_cryptocop(void)
1948 {
1949         unsigned long          flags;
1950         reg_dma_rw_cfg         dma_cfg = {.en = 1};
1951         reg_dma_rw_intr_mask   intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1952         reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
1953         reg_strcop_rw_cfg      strcop_cfg = {
1954                 .ipend = regk_strcop_little,
1955                 .td1 = regk_strcop_e,
1956                 .td2 = regk_strcop_d,
1957                 .td3 = regk_strcop_e,
1958                 .ignore_sync = 0,
1959                 .en = 1
1960         };
1961 
1962         if (request_irq(DMA_IRQ, dma_done_interrupt, 0,
1963                         "stream co-processor DMA", NULL))
1964                 panic("request_irq stream co-processor irq dma9");
1965 
1966         (void)crisv32_request_dma(OUT_DMA, "strcop", DMA_PANIC_ON_ERROR,
1967                 0, dma_strp);
1968         (void)crisv32_request_dma(IN_DMA, "strcop", DMA_PANIC_ON_ERROR,
1969                 0, dma_strp);
1970 
1971         local_irq_save(flags);
1972 
1973         /* Reset and enable the cryptocop. */
1974         strcop_cfg.en = 0;
1975         REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1976         strcop_cfg.en = 1;
1977         REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1978 
1979         /* Enable DMAs. */
1980         REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
1981         REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
1982 
1983         /* Set up wordsize = 4 for DMAs. */
1984         DMA_WR_CMD(OUT_DMA_INST, regk_dma_set_w_size4);
1985         DMA_WR_CMD(IN_DMA_INST, regk_dma_set_w_size4);
1986 
1987         /* Enable interrupts. */
1988         REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
1989 
1990         /* Clear intr ack. */
1991         REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1992 
1993         local_irq_restore(flags);
1994 
1995         return 0;
1996 }
1997 
1998 /* Free used cryptocop hw resources (interrupt and DMA channels). */
1999 static void release_cryptocop(void)
2000 {
2001         unsigned long          flags;
2002         reg_dma_rw_cfg         dma_cfg = {.en = 0};
2003         reg_dma_rw_intr_mask   intr_mask_in = {0};
2004         reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
2005 
2006         local_irq_save(flags);
2007 
2008         /* Clear intr ack. */
2009         REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
2010 
2011         /* Disable DMAs. */
2012         REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
2013         REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
2014 
2015         /* Disable interrupts. */
2016         REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
2017 
2018         local_irq_restore(flags);
2019 
2020         free_irq(DMA_IRQ, NULL);
2021 
2022         (void)crisv32_free_dma(OUT_DMA);
2023         (void)crisv32_free_dma(IN_DMA);
2024 }
2025 
2026 
2027 /* Init job queue. */
2028 static int cryptocop_job_queue_init(void)
2029 {
2030         int i;
2031 
2032         INIT_LIST_HEAD(&cryptocop_completed_jobs);
2033 
2034         for (i = 0; i < cryptocop_prio_no_prios; i++){
2035                 cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2036                 INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2037         }
2038         return 0;
2039 }
2040 
2041 
2042 static void cryptocop_job_queue_close(void)
2043 {
2044         struct list_head               *node, *tmp;
2045         struct cryptocop_prio_job      *pj = NULL;
2046         unsigned long int              process_flags, flags;
2047         int                            i;
2048 
2049         /* FIXME: This is as yet untested code. */
2050 
2051         /* Stop strcop from getting an operation to process while we are closing the
2052            module. */
2053         spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2054 
2055         /* Empty the job queue. */
2056         for (i = 0; i < cryptocop_prio_no_prios; i++){
2057                 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2058                         list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2059                                 pj = list_entry(node, struct cryptocop_prio_job, node);
2060                                 list_del(node);
2061 
2062                                 /* Call callback to notify consumer of job removal. */
2063                                 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2064                                 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2065                                 pj->oper->cb(pj->oper, pj->oper->cb_data);
2066 
2067                                 delete_internal_operation(pj->iop);
2068                                 kfree(pj);
2069                         }
2070                 }
2071         }
2072         spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2073 
2074         /* Remove the running job, if any. */
2075         spin_lock_irqsave(&running_job_lock, flags);
2076         if (cryptocop_running_job){
2077                 reg_strcop_rw_cfg rw_cfg;
2078                 reg_dma_rw_cfg    dma_out_cfg, dma_in_cfg;
2079 
2080                 /* Stop DMA. */
2081                 dma_out_cfg = REG_RD(dma, OUT_DMA_INST, rw_cfg);
2082                 dma_out_cfg.en = regk_dma_no;
2083                 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_out_cfg);
2084 
2085                 dma_in_cfg = REG_RD(dma, IN_DMA_INST, rw_cfg);
2086                 dma_in_cfg.en = regk_dma_no;
2087                 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_in_cfg);
2088 
2089                 /* Disble the cryptocop. */
2090                 rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2091                 rw_cfg.en = 0;
2092                 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2093 
2094                 pj = cryptocop_running_job;
2095                 cryptocop_running_job = NULL;
2096 
2097                 /* Call callback to notify consumer of job removal. */
2098                 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2099                 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2100                 pj->oper->cb(pj->oper, pj->oper->cb_data);
2101 
2102                 delete_internal_operation(pj->iop);
2103                 kfree(pj);
2104         }
2105         spin_unlock_irqrestore(&running_job_lock, flags);
2106 
2107         /* Remove completed jobs, if any. */
2108         spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2109 
2110         list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2111                 pj = list_entry(node, struct cryptocop_prio_job, node);
2112                 list_del(node);
2113                 /* Call callback to notify consumer of job removal. */
2114                 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2115                 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2116                 pj->oper->cb(pj->oper, pj->oper->cb_data);
2117 
2118                 delete_internal_operation(pj->iop);
2119                 kfree(pj);
2120         }
2121         spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2122 }
2123 
2124 
2125 static void cryptocop_start_job(void)
2126 {
2127         int                          i;
2128         struct cryptocop_prio_job    *pj;
2129         unsigned long int            flags;
2130         unsigned long int            running_job_flags;
2131         reg_strcop_rw_cfg            rw_cfg = {.en = 1, .ignore_sync = 0};
2132 
2133         DEBUG(printk("cryptocop_start_job: entering\n"));
2134 
2135         spin_lock_irqsave(&running_job_lock, running_job_flags);
2136         if (cryptocop_running_job != NULL){
2137                 /* Already running. */
2138                 DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2139                 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2140                 return;
2141         }
2142         spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2143 
2144         /* Check the queues in priority order. */
2145         for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2146         if (i == cryptocop_prio_no_prios) {
2147                 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2148                 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2149                 DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2150                 return; /* No jobs to run */
2151         }
2152         DEBUG(printk("starting job for prio %d\n", i));
2153 
2154         /* TODO: Do not starve lower priority jobs.  Let in a lower
2155          * prio job for every N-th processed higher prio job or some
2156          * other scheduling policy.  This could reasonably be
2157          * tweakable since the optimal balance would depend on the
2158          * type of load on the system. */
2159 
2160         /* Pull the DMA lists from the job and start the DMA client. */
2161         pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2162         list_del(&pj->node);
2163         spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2164         cryptocop_running_job = pj;
2165 
2166         /* Set config register (3DES and CSUM modes). */
2167         switch (pj->iop->tdes_mode){
2168         case cryptocop_3des_eee:
2169                 rw_cfg.td1 = regk_strcop_e;
2170                 rw_cfg.td2 = regk_strcop_e;
2171                 rw_cfg.td3 = regk_strcop_e;
2172                 break;
2173         case cryptocop_3des_eed:
2174                 rw_cfg.td1 = regk_strcop_e;
2175                 rw_cfg.td2 = regk_strcop_e;
2176                 rw_cfg.td3 = regk_strcop_d;
2177                 break;
2178         case cryptocop_3des_ede:
2179                 rw_cfg.td1 = regk_strcop_e;
2180                 rw_cfg.td2 = regk_strcop_d;
2181                 rw_cfg.td3 = regk_strcop_e;
2182                 break;
2183         case cryptocop_3des_edd:
2184                 rw_cfg.td1 = regk_strcop_e;
2185                 rw_cfg.td2 = regk_strcop_d;
2186                 rw_cfg.td3 = regk_strcop_d;
2187                 break;
2188         case cryptocop_3des_dee:
2189                 rw_cfg.td1 = regk_strcop_d;
2190                 rw_cfg.td2 = regk_strcop_e;
2191                 rw_cfg.td3 = regk_strcop_e;
2192                 break;
2193         case cryptocop_3des_ded:
2194                 rw_cfg.td1 = regk_strcop_d;
2195                 rw_cfg.td2 = regk_strcop_e;
2196                 rw_cfg.td3 = regk_strcop_d;
2197                 break;
2198         case cryptocop_3des_dde:
2199                 rw_cfg.td1 = regk_strcop_d;
2200                 rw_cfg.td2 = regk_strcop_d;
2201                 rw_cfg.td3 = regk_strcop_e;
2202                 break;
2203         case cryptocop_3des_ddd:
2204                 rw_cfg.td1 = regk_strcop_d;
2205                 rw_cfg.td2 = regk_strcop_d;
2206                 rw_cfg.td3 = regk_strcop_d;
2207                 break;
2208         default:
2209                 DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2210         }
2211         switch (pj->iop->csum_mode){
2212         case cryptocop_csum_le:
2213                 rw_cfg.ipend = regk_strcop_little;
2214                 break;
2215         case cryptocop_csum_be:
2216                 rw_cfg.ipend = regk_strcop_big;
2217                 break;
2218         default:
2219                 DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2220         }
2221         REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2222 
2223         DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2224                      "ctx_in: 0x%p, phys: 0x%p\n"
2225                      "ctx_out: 0x%p, phys: 0x%p\n",
2226                      pj,
2227                      &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2228                      &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2229 
2230         /* Start input DMA. */
2231         flush_dma_context(&pj->iop->ctx_in);
2232         DMA_START_CONTEXT(IN_DMA_INST, virt_to_phys(&pj->iop->ctx_in));
2233 
2234         /* Start output DMA. */
2235         DMA_START_CONTEXT(OUT_DMA_INST, virt_to_phys(&pj->iop->ctx_out));
2236 
2237         spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2238         DEBUG(printk("cryptocop_start_job: exiting\n"));
2239 }
2240 
2241 
2242 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2243 {
2244         int  err;
2245         int  alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2246         void *iop_alloc_ptr = NULL;
2247 
2248         *pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2249         if (!*pj) return -ENOMEM;
2250 
2251         DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2252 
2253         (*pj)->oper = operation;
2254         DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n",  (*pj)->oper->cb, (*pj)->oper->cb_data));
2255 
2256         if (operation->use_dmalists) {
2257                 DEBUG(print_user_dma_lists(&operation->list_op));
2258                 if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2259                         DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2260                         kfree(*pj);
2261                         return -EINVAL;
2262                 }
2263                 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2264                 if (!iop_alloc_ptr) {
2265                         DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2266                         kfree(*pj);
2267                         return -ENOMEM;
2268                 }
2269                 (*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2270                 DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2271                 (*pj)->iop->alloc_ptr = iop_alloc_ptr;
2272                 (*pj)->iop->sid = operation->sid;
2273                 (*pj)->iop->cdesc_out = NULL;
2274                 (*pj)->iop->cdesc_in = NULL;
2275                 (*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2276                 (*pj)->iop->csum_mode = operation->list_op.csum_mode;
2277                 (*pj)->iop->ddesc_out = operation->list_op.outlist;
2278                 (*pj)->iop->ddesc_in = operation->list_op.inlist;
2279 
2280                 /* Setup DMA contexts. */
2281                 (*pj)->iop->ctx_out.next = NULL;
2282                 (*pj)->iop->ctx_out.eol = 1;
2283                 (*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2284                 (*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2285 
2286                 (*pj)->iop->ctx_in.next = NULL;
2287                 (*pj)->iop->ctx_in.eol = 1;
2288                 (*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2289                 (*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2290         } else {
2291                 if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2292                         DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2293                         kfree(*pj);
2294                         return err;
2295                 }
2296         }
2297         DEBUG(print_dma_descriptors((*pj)->iop));
2298 
2299         DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2300 
2301         return 0;
2302 }
2303 
2304 static int cryptocop_open(struct inode *inode, struct file *filp)
2305 {
2306         int p = iminor(inode);
2307 
2308         if (p != CRYPTOCOP_MINOR) return -EINVAL;
2309 
2310         filp->private_data = NULL;
2311         return 0;
2312 }
2313 
2314 
2315 static int cryptocop_release(struct inode *inode, struct file *filp)
2316 {
2317         struct cryptocop_private *dev = filp->private_data;
2318         struct cryptocop_private *dev_next;
2319 
2320         while (dev){
2321                 dev_next = dev->next;
2322                 if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2323                         (void)cryptocop_free_session(dev->sid);
2324                 }
2325                 kfree(dev);
2326                 dev = dev_next;
2327         }
2328 
2329         return 0;
2330 }
2331 
2332 
2333 static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2334                                          unsigned int cmd, unsigned long arg)
2335 {
2336         struct cryptocop_private  *dev = filp->private_data;
2337         struct cryptocop_private  *prev_dev = NULL;
2338         struct strcop_session_op  *sess_op = (struct strcop_session_op *)arg;
2339         struct strcop_session_op  sop;
2340         int                       err;
2341 
2342         DEBUG(printk("cryptocop_ioctl_close_session\n"));
2343 
2344         if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2345                 return -EFAULT;
2346         err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2347         if (err) return -EFAULT;
2348 
2349         while (dev && (dev->sid != sop.ses_id)) {
2350                 prev_dev = dev;
2351                 dev = dev->next;
2352         }
2353         if (dev){
2354                 if (prev_dev){
2355                         prev_dev->next = dev->next;
2356                 } else {
2357                         filp->private_data = dev->next;
2358                 }
2359                 err = cryptocop_free_session(dev->sid);
2360                 if (err) return -EFAULT;
2361         } else {
2362                 DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2363                 return -EINVAL;
2364         }
2365         return 0;
2366 }
2367 
2368 
2369 static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2370 {
2371         struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2372 
2373         DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2374 
2375         jc->processed = 1;
2376         wake_up(&cryptocop_ioc_process_wq);
2377 }
2378 
2379 
2380 #define CRYPTOCOP_IOCTL_CIPHER_TID  (1)
2381 #define CRYPTOCOP_IOCTL_DIGEST_TID  (2)
2382 #define CRYPTOCOP_IOCTL_CSUM_TID    (3)
2383 
2384 static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2385 {
2386         size_t ch_ix = 0;
2387 
2388         if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2389         if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2390         if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2391 
2392         DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2393         return ch_ix;
2394 }
2395 
2396 
2397 static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2398 {
2399         size_t ch_ix = INT_MAX;
2400         size_t tmp_ix = 0;
2401 
2402         if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2403                 if (crp_op->cipher_start > ix) {
2404                         ch_ix = crp_op->cipher_start;
2405                 } else {
2406                         ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2407                 }
2408         }
2409         if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2410                 if (crp_op->digest_start > ix) {
2411                         tmp_ix = crp_op->digest_start;
2412                 } else {
2413                         tmp_ix = crp_op->digest_start + crp_op->digest_len;
2414                 }
2415                 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2416         }
2417         if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2418                 if (crp_op->csum_start > ix) {
2419                         tmp_ix = crp_op->csum_start;
2420                 } else {
2421                         tmp_ix = crp_op->csum_start + crp_op->csum_len;
2422                 }
2423                 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2424         }
2425         if (ch_ix == INT_MAX) ch_ix = ix;
2426         DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2427         return ch_ix;
2428 }
2429 
2430 
2431 /* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2432  * Return -1 for ok, 0 for fail. */
2433 static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2434 {
2435         int tmplen;
2436 
2437         assert(iov != NULL);
2438         assert(iovix != NULL);
2439         assert(pages != NULL);
2440         assert(pageix != NULL);
2441         assert(pageoffset != NULL);
2442 
2443         DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2444 
2445         while (map_length > 0){
2446                 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2447                 if (*iovix >= iovlen){
2448                         DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2449                         return 0;
2450                 }
2451                 if (*pageix >= nopages){
2452                         DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2453                         return 0;
2454                 }
2455                 iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2456                 tmplen = PAGE_SIZE - *pageoffset;
2457                 if (tmplen < map_length){
2458                         (*pageoffset) = 0;
2459                         (*pageix)++;
2460                 } else {
2461                         tmplen = map_length;
2462                         (*pageoffset) += map_length;
2463                 }
2464                 DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2465                 iov[*iovix].iov_len = tmplen;
2466                 map_length -= tmplen;
2467                 (*iovix)++;
2468         }
2469         DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2470         return -1;
2471 }
2472 
2473 
2474 
2475 static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2476 {
2477         int                             i;
2478         struct cryptocop_private        *dev = filp->private_data;
2479         struct strcop_crypto_op         *crp_oper = (struct strcop_crypto_op *)arg;
2480         struct strcop_crypto_op         oper = {0};
2481         int                             err = 0;
2482         struct cryptocop_operation      *cop = NULL;
2483 
2484         struct ioctl_job_cb_ctx         *jc = NULL;
2485 
2486         struct page                     **inpages = NULL;
2487         struct page                     **outpages = NULL;
2488         int                             noinpages = 0;
2489         int                             nooutpages = 0;
2490 
2491         struct cryptocop_desc           descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2492                                                    * can get connected/disconnected on different places in the indata. */
2493         struct cryptocop_desc_cfg       dcfgs[5*3];
2494         int                             desc_ix = 0;
2495         int                             dcfg_ix = 0;
2496         struct cryptocop_tfrm_cfg       ciph_tcfg = {0};
2497         struct cryptocop_tfrm_cfg       digest_tcfg = {0};
2498         struct cryptocop_tfrm_cfg       csum_tcfg = {0};
2499 
2500         unsigned char                   *digest_result = NULL;
2501         int                             digest_length = 0;
2502         int                             cblocklen = 0;
2503         unsigned char                   csum_result[CSUM_BLOCK_LENGTH];
2504         struct cryptocop_session        *sess;
2505 
2506         int    iovlen = 0;
2507         int    iovix = 0;
2508         int    pageix = 0;
2509         int    pageoffset = 0;
2510 
2511         size_t prev_ix = 0;
2512         size_t next_ix;
2513 
2514         int    cipher_active, digest_active, csum_active;
2515         int    end_digest, end_csum;
2516         int    digest_done = 0;
2517         int    cipher_done = 0;
2518         int    csum_done = 0;
2519 
2520         DEBUG(printk("cryptocop_ioctl_process\n"));
2521 
2522         if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2523                 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2524                 return -EFAULT;
2525         }
2526         if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2527                 DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2528                 return -EFAULT;
2529         }
2530         DEBUG(print_strcop_crypto_op(&oper));
2531 
2532         while (dev && dev->sid != oper.ses_id) dev = dev->next;
2533         if (!dev){
2534                 DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2535                 return -EINVAL;
2536         }
2537 
2538         /* Check buffers. */
2539         if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2540                 DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2541                 return -EINVAL;
2542         }
2543 
2544         if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2545                 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2546                 return -EFAULT;
2547         }
2548         if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2549                 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2550                 return -EFAULT;
2551         }
2552 
2553         cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2554         if (!cop) {
2555                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2556                 return -ENOMEM;
2557         }
2558         jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2559         if (!jc) {
2560                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2561                 err = -ENOMEM;
2562                 goto error_cleanup;
2563         }
2564         jc->processed = 0;
2565 
2566         cop->cb_data = jc;
2567         cop->cb = ioctl_process_job_callback;
2568         cop->operation_status = 0;
2569         cop->use_dmalists = 0;
2570         cop->in_interrupt = 0;
2571         cop->fast_callback = 0;
2572         cop->tfrm_op.tfrm_cfg = NULL;
2573         cop->tfrm_op.desc = NULL;
2574         cop->tfrm_op.indata = NULL;
2575         cop->tfrm_op.incount = 0;
2576         cop->tfrm_op.inlen = 0;
2577         cop->tfrm_op.outdata = NULL;
2578         cop->tfrm_op.outcount = 0;
2579         cop->tfrm_op.outlen = 0;
2580 
2581         sess = get_session(oper.ses_id);
2582         if (!sess){
2583                 DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2584                 kfree(cop);
2585                 kfree(jc);
2586                 return -EINVAL;
2587         }
2588 
2589         if (oper.do_cipher) {
2590                 unsigned int                    cipher_outlen = 0;
2591                 struct cryptocop_transform_ctx  *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2592                 if (!tc) {
2593                         DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2594                         err = -EINVAL;
2595                         goto error_cleanup;
2596                 }
2597                 ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2598                 ciph_tcfg.inject_ix = 0;
2599                 ciph_tcfg.flags = 0;
2600                 if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2601                         DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2602                         kfree(cop);
2603                         kfree(jc);
2604                         return -EINVAL;
2605                 }
2606                 cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2607                 if (oper.cipher_len % cblocklen) {
2608                         kfree(cop);
2609                         kfree(jc);
2610                         DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2611                         return -EINVAL;
2612                 }
2613                 cipher_outlen = oper.cipher_len;
2614                 if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2615                         if (oper.cipher_explicit) {
2616                                 ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2617                                 memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2618                         } else {
2619                                 cipher_outlen = oper.cipher_len - cblocklen;
2620                         }
2621                 } else {
2622                         if (oper.cipher_explicit){
2623                                 kfree(cop);
2624                                 kfree(jc);
2625                                 DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2626                                 return -EINVAL;
2627                         }
2628                 }
2629                 if (oper.cipher_outlen != cipher_outlen) {
2630                         kfree(cop);
2631                         kfree(jc);
2632                         DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2633                         return -EINVAL;
2634                 }
2635 
2636                 if (oper.decrypt){
2637                         ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2638                 } else {
2639                         ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2640                 }
2641                 ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2642                 cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2643         }
2644         if (oper.do_digest){
2645                 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2646                 if (!tc) {
2647                         DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2648                         err = -EINVAL;
2649                         goto error_cleanup;
2650                 }
2651                 digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2652                 digest_result = kmalloc(digest_length, GFP_KERNEL);
2653                 if (!digest_result) {
2654                         DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2655                         err = -EINVAL;
2656                         goto error_cleanup;
2657                 }
2658                 DEBUG(memset(digest_result, 0xff, digest_length));
2659 
2660                 digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2661                 digest_tcfg.inject_ix = 0;
2662                 ciph_tcfg.inject_ix += digest_length;
2663                 if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2664                         DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2665                         err = -EINVAL;
2666                         goto error_cleanup;
2667                 }
2668 
2669                 digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2670                 cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2671         }
2672         if (oper.do_csum){
2673                 csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2674                 csum_tcfg.inject_ix = digest_length;
2675                 ciph_tcfg.inject_ix += 2;
2676 
2677                 if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2678                         DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2679                         kfree(cop);
2680                         kfree(jc);
2681                         return -EINVAL;
2682                 }
2683 
2684                 csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2685                 cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2686         }
2687 
2688         prev_ix = first_cfg_change_ix(&oper);
2689         if (prev_ix > oper.inlen) {
2690                 DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2691                 nooutpages = noinpages = 0;
2692                 err = -EINVAL;
2693                 goto error_cleanup;
2694         }
2695         DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2696 
2697         /* Map user pages for in and out data of the operation. */
2698         noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2699         DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2700         inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2701         if (!inpages){
2702                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2703                 nooutpages = noinpages = 0;
2704                 err = -ENOMEM;
2705                 goto error_cleanup;
2706         }
2707         if (oper.do_cipher){
2708                 nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2709                 DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2710                 outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2711                 if (!outpages){
2712                         DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2713                         nooutpages = noinpages = 0;
2714                         err = -ENOMEM;
2715                         goto error_cleanup;
2716                 }
2717         }
2718 
2719         /* Acquire the mm page semaphore. */
2720         down_read(&current->mm->mmap_sem);
2721 
2722         err = get_user_pages(current,
2723                              current->mm,
2724                              (unsigned long int)(oper.indata + prev_ix),
2725                              noinpages,
2726                              0,  /* read access only for in data */
2727                              0, /* no force */
2728                              inpages,
2729                              NULL);
2730 
2731         if (err < 0) {
2732                 up_read(&current->mm->mmap_sem);
2733                 nooutpages = noinpages = 0;
2734                 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2735                 goto error_cleanup;
2736         }
2737         noinpages = err;
2738         if (oper.do_cipher){
2739                 err = get_user_pages(current,
2740                                      current->mm,
2741                                      (unsigned long int)oper.cipher_outdata,
2742                                      nooutpages,
2743                                      1, /* write access for out data */
2744                                      0, /* no force */
2745                                      outpages,
2746                                      NULL);
2747                 up_read(&current->mm->mmap_sem);
2748                 if (err < 0) {
2749                         nooutpages = 0;
2750                         DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2751                         goto error_cleanup;
2752                 }
2753                 nooutpages = err;
2754         } else {
2755                 up_read(&current->mm->mmap_sem);
2756         }
2757 
2758         /* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2759          * csum output and splits when units are (dis-)connected. */
2760         cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2761         cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2762         if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2763                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2764                 err = -ENOMEM;
2765                 goto error_cleanup;
2766         }
2767 
2768         cop->tfrm_op.inlen = oper.inlen - prev_ix;
2769         cop->tfrm_op.outlen = 0;
2770         if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2771         if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2772         if (oper.do_csum) cop->tfrm_op.outlen += 2;
2773 
2774         /* Setup the in iovecs. */
2775         cop->tfrm_op.incount = noinpages;
2776         if (noinpages > 1){
2777                 size_t tmplen = cop->tfrm_op.inlen;
2778 
2779                 cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2780                 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2781                 tmplen -= cop->tfrm_op.indata[0].iov_len;
2782                 for (i = 1; i<noinpages; i++){
2783                         cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2784                         cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2785                         tmplen -= PAGE_SIZE;
2786                 }
2787         } else {
2788                 cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2789                 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2790         }
2791 
2792         iovlen = nooutpages + 6;
2793         pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2794 
2795         next_ix = next_cfg_change_ix(&oper, prev_ix);
2796         if (prev_ix == next_ix){
2797                 DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2798                 err = -EINVAL;  /* This should be impossible barring bugs. */
2799                 goto error_cleanup;
2800         }
2801         while (prev_ix != next_ix){
2802                 end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2803                 descs[desc_ix].cfg = NULL;
2804                 descs[desc_ix].length = next_ix - prev_ix;
2805 
2806                 if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2807                         dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2808                         dcfgs[dcfg_ix].src = cryptocop_source_dma;
2809                         cipher_active = 1;
2810 
2811                         if (next_ix == (oper.cipher_start + oper.cipher_len)){
2812                                 cipher_done = 1;
2813                                 dcfgs[dcfg_ix].last = 1;
2814                         } else {
2815                                 dcfgs[dcfg_ix].last = 0;
2816                         }
2817                         dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2818                         descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2819                         ++dcfg_ix;
2820                 }
2821                 if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2822                         digest_active = 1;
2823                         dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2824                         dcfgs[dcfg_ix].src = cryptocop_source_dma;
2825                         if (next_ix == (oper.digest_start + oper.digest_len)){
2826                                 assert(!digest_done);
2827                                 digest_done = 1;
2828                                 dcfgs[dcfg_ix].last = 1;
2829                         } else {
2830                                 dcfgs[dcfg_ix].last = 0;
2831                         }
2832                         dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2833                         descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2834                         ++dcfg_ix;
2835                 }
2836                 if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2837                         csum_active = 1;
2838                         dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2839                         dcfgs[dcfg_ix].src = cryptocop_source_dma;
2840                         if (next_ix == (oper.csum_start + oper.csum_len)){
2841                                 csum_done = 1;
2842                                 dcfgs[dcfg_ix].last = 1;
2843                         } else {
2844                                 dcfgs[dcfg_ix].last = 0;
2845                         }
2846                         dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2847                         descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2848                         ++dcfg_ix;
2849                 }
2850                 if (!descs[desc_ix].cfg){
2851                         DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2852                         err = -EINVAL;
2853                         goto error_cleanup;
2854                 }
2855                 descs[desc_ix].next = &(descs[desc_ix]) + 1;
2856                 ++desc_ix;
2857                 prev_ix = next_ix;
2858                 next_ix = next_cfg_change_ix(&oper, prev_ix);
2859         }
2860         if (desc_ix > 0){
2861                 descs[desc_ix-1].next = NULL;
2862         } else {
2863                 descs[0].next = NULL;
2864         }
2865         if (oper.do_digest) {
2866                 DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2867                 /* Add outdata iovec, length == <length of type of digest> */
2868                 cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2869                 cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2870                 ++iovix;
2871         }
2872         if (oper.do_csum) {
2873                 /* Add outdata iovec, length == 2, the length of csum. */
2874                 DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2875                 /* Add outdata iovec, length == <length of type of digest> */
2876                 cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2877                 cop->tfrm_op.outdata[iovix].iov_len = 2;
2878                 ++iovix;
2879         }
2880         if (oper.do_cipher) {
2881                 if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2882                         DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2883                         err = -ENOSYS; /* This should be impossible barring bugs. */
2884                         goto error_cleanup;
2885                 }
2886         }
2887         DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2888         cop->tfrm_op.outcount = iovix;
2889         assert(iovix <= (nooutpages + 6));
2890 
2891         cop->sid = oper.ses_id;
2892         cop->tfrm_op.desc = &descs[0];
2893 
2894         DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2895 
2896         if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2897                 DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2898                 err = -EINVAL;
2899                 goto error_cleanup;
2900         }
2901 
2902         DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2903 
2904         wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2905         DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2906         if (!jc->processed){
2907                 printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2908                 err = -EIO;
2909                 goto error_cleanup;
2910         }
2911 
2912         /* Job process done.  Cipher output should already be correct in job so no post processing of outdata. */
2913         DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2914         if (cop->operation_status == 0){
2915                 if (oper.do_digest){
2916                         DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2917                         err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2918                         if (0 != err){
2919                                 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2920                                 err = -EFAULT;
2921                                 goto error_cleanup;
2922                         }
2923                 }
2924                 if (oper.do_csum){
2925                         DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2926                         err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2927                         if (0 != err){
2928                                 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2929                                 err = -EFAULT;
2930                                 goto error_cleanup;
2931                         }
2932                 }
2933                 err = 0;
2934         } else {
2935                 DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2936                 err = cop->operation_status;
2937         }
2938 
2939  error_cleanup:
2940         /* Release page caches. */
2941         for (i = 0; i < noinpages; i++){
2942                 put_page(inpages[i]);
2943         }
2944         for (i = 0; i < nooutpages; i++){
2945                 int spdl_err;
2946                 /* Mark output pages dirty. */
2947                 spdl_err = set_page_dirty_lock(outpages[i]);
2948                 DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
2949         }
2950         for (i = 0; i < nooutpages; i++){
2951                 put_page(outpages[i]);
2952         }
2953 
2954         kfree(digest_result);
2955         kfree(inpages);
2956         kfree(outpages);
2957         if (cop){
2958                 kfree(cop->tfrm_op.indata);
2959                 kfree(cop->tfrm_op.outdata);
2960                 kfree(cop);
2961         }
2962         kfree(jc);
2963 
2964         DEBUG(print_lock_status());
2965 
2966         return err;
2967 }
2968 
2969 
2970 static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2971 {
2972         cryptocop_session_id             sid;
2973         int                              err;
2974         struct cryptocop_private         *dev;
2975         struct strcop_session_op         *sess_op = (struct strcop_session_op *)arg;
2976         struct strcop_session_op         sop;
2977         struct cryptocop_transform_init  *tis = NULL;
2978         struct cryptocop_transform_init  ti_cipher = {0};
2979         struct cryptocop_transform_init  ti_digest = {0};
2980         struct cryptocop_transform_init  ti_csum = {0};
2981 
2982         if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2983                 return -EFAULT;
2984         err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2985         if (err) return -EFAULT;
2986         if (sop.cipher != cryptocop_cipher_none) {
2987                 if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2988         }
2989         DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2990 
2991         DEBUG(printk("\tcipher:%d\n"
2992                      "\tcipher_mode:%d\n"
2993                      "\tdigest:%d\n"
2994                      "\tcsum:%d\n",
2995                      (int)sop.cipher,
2996                      (int)sop.cmode,
2997                      (int)sop.digest,
2998                      (int)sop.csum));
2999 
3000         if (sop.cipher != cryptocop_cipher_none){
3001                 /* Init the cipher. */
3002                 switch (sop.cipher){
3003                 case cryptocop_cipher_des:
3004                         ti_cipher.alg = cryptocop_alg_des;
3005                         break;
3006                 case cryptocop_cipher_3des:
3007                         ti_cipher.alg = cryptocop_alg_3des;
3008                         break;
3009                 case cryptocop_cipher_aes:
3010                         ti_cipher.alg = cryptocop_alg_aes;
3011                         break;
3012                 default:
3013                         DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3014                         return -EINVAL;
3015                 };
3016                 DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3017                 copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3018                 ti_cipher.keylen = sop.keylen;
3019                 switch (sop.cmode){
3020                 case cryptocop_cipher_mode_cbc:
3021                 case cryptocop_cipher_mode_ecb:
3022                         ti_cipher.cipher_mode = sop.cmode;
3023                         break;
3024                 default:
3025                         DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3026                         return -EINVAL;
3027                 }
3028                 DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3029                 switch (sop.des3_mode){
3030                 case cryptocop_3des_eee:
3031                 case cryptocop_3des_eed:
3032                 case cryptocop_3des_ede:
3033                 case cryptocop_3des_edd:
3034                 case cryptocop_3des_dee:
3035                 case cryptocop_3des_ded:
3036                 case cryptocop_3des_dde:
3037                 case cryptocop_3des_ddd:
3038                         ti_cipher.tdes_mode = sop.des3_mode;
3039                         break;
3040                 default:
3041                         DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3042                         return -EINVAL;
3043                 }
3044                 ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3045                 ti_cipher.next = tis;
3046                 tis = &ti_cipher;
3047         } /* if (sop.cipher != cryptocop_cipher_none) */
3048         if (sop.digest != cryptocop_digest_none){
3049                 DEBUG(printk("setting digest transform\n"));
3050                 switch (sop.digest){
3051                 case cryptocop_digest_md5:
3052                         ti_digest.alg = cryptocop_alg_md5;
3053                         break;
3054                 case cryptocop_digest_sha1:
3055                         ti_digest.alg = cryptocop_alg_sha1;
3056                         break;
3057                 default:
3058                         DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3059                         return -EINVAL;
3060                 }
3061                 ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3062                 ti_digest.next = tis;
3063                 tis = &ti_digest;
3064         } /* if (sop.digest != cryptocop_digest_none) */
3065         if (sop.csum != cryptocop_csum_none){
3066                 DEBUG(printk("setting csum transform\n"));
3067                 switch (sop.csum){
3068                 case cryptocop_csum_le:
3069                 case cryptocop_csum_be:
3070                         ti_csum.csum_mode = sop.csum;
3071                         break;
3072                 default:
3073                         DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3074                         return -EINVAL;
3075                 }
3076                 ti_csum.alg = cryptocop_alg_csum;
3077                 ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3078                 ti_csum.next = tis;
3079                 tis = &ti_csum;
3080         } /* (sop.csum != cryptocop_csum_none) */
3081         dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3082         if (!dev){
3083                 DEBUG_API(printk("create session, alloc dev\n"));
3084                 return -ENOMEM;
3085         }
3086 
3087         err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3088         DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3089 
3090         if (err) {
3091                 kfree(dev);
3092                 return err;
3093         }
3094         sess_op->ses_id = sid;
3095         dev->sid = sid;
3096         dev->next = filp->private_data;
3097         filp->private_data = dev;
3098 
3099         return 0;
3100 }
3101 
3102 static long cryptocop_ioctl_unlocked(struct inode *inode,
3103         struct file *filp, unsigned int cmd, unsigned long arg)
3104 {
3105         int err = 0;
3106         if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3107                 DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3108                 return -ENOTTY;
3109         }
3110         if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3111                 return -ENOTTY;
3112         }
3113         /* Access check of the argument.  Some commands, e.g. create session and process op,
3114            needs additional checks.  Those are handled in the command handling functions. */
3115         if (_IOC_DIR(cmd) & _IOC_READ)
3116                 err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3117         else if (_IOC_DIR(cmd) & _IOC_WRITE)
3118                 err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3119         if (err) return -EFAULT;
3120 
3121         switch (cmd) {
3122         case CRYPTOCOP_IO_CREATE_SESSION:
3123                 return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3124         case CRYPTOCOP_IO_CLOSE_SESSION:
3125                 return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3126         case CRYPTOCOP_IO_PROCESS_OP:
3127                 return cryptocop_ioctl_process(inode, filp, cmd, arg);
3128         default:
3129                 DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3130                 return -ENOTTY;
3131         }
3132         return 0;
3133 }
3134 
3135 static long
3136 cryptocop_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
3137 {
3138        long ret;
3139 
3140        mutex_lock(&cryptocop_mutex);
3141        ret = cryptocop_ioctl_unlocked(file_inode(filp), filp, cmd, arg);
3142        mutex_unlock(&cryptocop_mutex);
3143 
3144        return ret;
3145 }
3146 
3147 
3148 #ifdef LDEBUG
3149 static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3150 {
3151         struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3152         struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3153         int                       i;
3154 
3155         printk("print_dma_descriptors start\n");
3156 
3157         printk("iop:\n");
3158         printk("\tsid: 0x%lld\n", iop->sid);
3159 
3160         printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3161         printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3162         printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3163         printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3164 
3165         printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3166         printk("\tnext: 0x%p\n"
3167                "\tsaved_data: 0x%p\n"
3168                "\tsaved_data_buf: 0x%p\n",
3169                iop->ctx_out.next,
3170                iop->ctx_out.saved_data,
3171                iop->ctx_out.saved_data_buf);
3172 
3173         printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3174         printk("\tnext: 0x%p\n"
3175                "\tsaved_data: 0x%p\n"
3176                "\tsaved_data_buf: 0x%p\n",
3177                iop->ctx_in.next,
3178                iop->ctx_in.saved_data,
3179                iop->ctx_in.saved_data_buf);
3180 
3181         i = 0;
3182         while (cdesc_out) {
3183                 dma_descr_data *td;
3184                 printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3185                 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3186                 td = cdesc_out->dma_descr;
3187                 printk("\n\tbuf: 0x%p\n"
3188                        "\tafter: 0x%p\n"
3189                        "\tmd: 0x%04x\n"
3190                        "\tnext: 0x%p\n",
3191                        td->buf,
3192                        td->after,
3193                        td->md,
3194                        td->next);
3195                 printk("flags:\n"
3196                        "\twait:\t%d\n"
3197                        "\teol:\t%d\n"
3198                        "\touteop:\t%d\n"
3199                        "\tineop:\t%d\n"
3200                        "\tintr:\t%d\n",
3201                        td->wait,
3202                        td->eol,
3203                        td->out_eop,
3204                        td->in_eop,
3205                        td->intr);
3206                 cdesc_out = cdesc_out->next;
3207                 i++;
3208         }
3209         i = 0;
3210         while (cdesc_in) {
3211                 dma_descr_data *td;
3212                 printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3213                 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3214                 td = cdesc_in->dma_descr;
3215                 printk("\n\tbuf: 0x%p\n"
3216                        "\tafter: 0x%p\n"
3217                        "\tmd: 0x%04x\n"
3218                        "\tnext: 0x%p\n",
3219                        td->buf,
3220                        td->after,
3221                        td->md,
3222                        td->next);
3223                 printk("flags:\n"
3224                        "\twait:\t%d\n"
3225                        "\teol:\t%d\n"
3226                        "\touteop:\t%d\n"
3227                        "\tineop:\t%d\n"
3228                        "\tintr:\t%d\n",
3229                        td->wait,
3230                        td->eol,
3231                        td->out_eop,
3232                        td->in_eop,
3233                        td->intr);
3234                 cdesc_in = cdesc_in->next;
3235                 i++;
3236         }
3237 
3238         printk("print_dma_descriptors end\n");
3239 }
3240 
3241 
3242 static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3243 {
3244         printk("print_strcop_crypto_op, 0x%p\n", cop);
3245 
3246         /* Indata. */
3247         printk("indata=0x%p\n"
3248                "inlen=%d\n"
3249                "do_cipher=%d\n"
3250                "decrypt=%d\n"
3251                "cipher_explicit=%d\n"
3252                "cipher_start=%d\n"
3253                "cipher_len=%d\n"
3254                "outdata=0x%p\n"
3255                "outlen=%d\n",
3256                cop->indata,
3257                cop->inlen,
3258                cop->do_cipher,
3259                cop->decrypt,
3260                cop->cipher_explicit,
3261                cop->cipher_start,
3262                cop->cipher_len,
3263                cop->cipher_outdata,
3264                cop->cipher_outlen);
3265 
3266         printk("do_digest=%d\n"
3267                "digest_start=%d\n"
3268                "digest_len=%d\n",
3269                cop->do_digest,
3270                cop->digest_start,
3271                cop->digest_len);
3272 
3273         printk("do_csum=%d\n"
3274                "csum_start=%d\n"
3275                "csum_len=%d\n",
3276                cop->do_csum,
3277                cop->csum_start,
3278                cop->csum_len);
3279 }
3280 
3281 static void print_cryptocop_operation(struct cryptocop_operation *cop)
3282 {
3283         struct cryptocop_desc      *d;
3284         struct cryptocop_tfrm_cfg  *tc;
3285         struct cryptocop_desc_cfg  *dc;
3286         int                        i;
3287 
3288         printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3289         printk("sid: %lld\n", cop->sid);
3290         printk("operation_status=%d\n"
3291                "use_dmalists=%d\n"
3292                "in_interrupt=%d\n"
3293                "fast_callback=%d\n",
3294                cop->operation_status,
3295                cop->use_dmalists,
3296                cop->in_interrupt,
3297                cop->fast_callback);
3298 
3299         if (cop->use_dmalists){
3300                 print_user_dma_lists(&cop->list_op);
3301         } else {
3302                 printk("cop->tfrm_op\n"
3303                        "tfrm_cfg=0x%p\n"
3304                        "desc=0x%p\n"
3305                        "indata=0x%p\n"
3306                        "incount=%d\n"
3307                        "inlen=%d\n"
3308                        "outdata=0x%p\n"
3309                        "outcount=%d\n"
3310                        "outlen=%d\n\n",
3311                        cop->tfrm_op.tfrm_cfg,
3312                        cop->tfrm_op.desc,
3313                        cop->tfrm_op.indata,
3314                        cop->tfrm_op.incount,
3315                        cop->tfrm_op.inlen,
3316                        cop->tfrm_op.outdata,
3317                        cop->tfrm_op.outcount,
3318                        cop->tfrm_op.outlen);
3319 
3320                 tc = cop->tfrm_op.tfrm_cfg;
3321                 while (tc){
3322                         printk("tfrm_cfg, 0x%p\n"
3323                                "tid=%d\n"
3324                                "flags=%d\n"
3325                                "inject_ix=%d\n"
3326                                "next=0x%p\n",
3327                                tc,
3328                                tc->tid,
3329                                tc->flags,
3330                                tc->inject_ix,
3331                                tc->next);
3332                         tc = tc->next;
3333                 }
3334                 d = cop->tfrm_op.desc;
3335                 while (d){
3336                         printk("\n======================desc, 0x%p\n"
3337                                "length=%d\n"
3338                                "cfg=0x%p\n"
3339                                "next=0x%p\n",
3340                                d,
3341                                d->length,
3342                                d->cfg,
3343                                d->next);
3344                         dc = d->cfg;
3345                         while (dc){
3346                                 printk("=========desc_cfg, 0x%p\n"
3347                                        "tid=%d\n"
3348                                        "src=%d\n"
3349                                        "last=%d\n"
3350                                        "next=0x%p\n",
3351                                        dc,
3352                                        dc->tid,
3353                                        dc->src,
3354                                        dc->last,
3355                                        dc->next);
3356                                 dc = dc->next;
3357                         }
3358                         d = d->next;
3359                 }
3360                 printk("\n====iniov\n");
3361                 for (i = 0; i < cop->tfrm_op.incount; i++){
3362                         printk("indata[%d]\n"
3363                                "base=0x%p\n"
3364                                "len=%d\n",
3365                                i,
3366                                cop->tfrm_op.indata[i].iov_base,
3367                                cop->tfrm_op.indata[i].iov_len);
3368                 }
3369                 printk("\n====outiov\n");
3370                 for (i = 0; i < cop->tfrm_op.outcount; i++){
3371                         printk("outdata[%d]\n"
3372                                "base=0x%p\n"
3373                                "len=%d\n",
3374                                i,
3375                                cop->tfrm_op.outdata[i].iov_base,
3376                                cop->tfrm_op.outdata[i].iov_len);
3377                 }
3378         }
3379         printk("------------end print_cryptocop_operation\n");
3380 }
3381 
3382 
3383 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3384 {
3385         dma_descr_data *dd;
3386         int i;
3387 
3388         printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3389 
3390         printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3391         printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3392 
3393         printk("##############outlist\n");
3394         dd = phys_to_virt((unsigned long int)dma_op->outlist);
3395         i = 0;
3396         while (dd != NULL) {
3397                 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3398                 printk("\n\tbuf: 0x%p\n"
3399                        "\tafter: 0x%p\n"
3400                        "\tmd: 0x%04x\n"
3401                        "\tnext: 0x%p\n",
3402                        dd->buf,
3403                        dd->after,
3404                        dd->md,
3405                        dd->next);
3406                 printk("flags:\n"
3407                        "\twait:\t%d\n"
3408                        "\teol:\t%d\n"
3409                        "\touteop:\t%d\n"
3410                        "\tineop:\t%d\n"
3411                        "\tintr:\t%d\n",
3412                        dd->wait,
3413                        dd->eol,
3414                        dd->out_eop,
3415                        dd->in_eop,
3416                        dd->intr);
3417                 if (dd->eol)
3418                         dd = NULL;
3419                 else
3420                         dd = phys_to_virt((unsigned long int)dd->next);
3421                 ++i;
3422         }
3423 
3424         printk("##############inlist\n");
3425         dd = phys_to_virt((unsigned long int)dma_op->inlist);
3426         i = 0;
3427         while (dd != NULL) {
3428                 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3429                 printk("\n\tbuf: 0x%p\n"
3430                        "\tafter: 0x%p\n"
3431                        "\tmd: 0x%04x\n"
3432                        "\tnext: 0x%p\n",
3433                        dd->buf,
3434                        dd->after,
3435                        dd->md,
3436                        dd->next);
3437                 printk("flags:\n"
3438                        "\twait:\t%d\n"
3439                        "\teol:\t%d\n"
3440                        "\touteop:\t%d\n"
3441                        "\tineop:\t%d\n"
3442                        "\tintr:\t%d\n",
3443                        dd->wait,
3444                        dd->eol,
3445                        dd->out_eop,
3446                        dd->in_eop,
3447                        dd->intr);
3448                 if (dd->eol)
3449                         dd = NULL;
3450                 else
3451                         dd = phys_to_virt((unsigned long int)dd->next);
3452                 ++i;
3453         }
3454 }
3455 
3456 
3457 static void print_lock_status(void)
3458 {
3459         printk("**********************print_lock_status\n");
3460         printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3461         printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3462         printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3463         printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3464         printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3465         printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3466 }
3467 #endif /* LDEBUG */
3468 
3469 
3470 static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3471 
3472 static int init_stream_coprocessor(void)
3473 {
3474         int err;
3475         int i;
3476         static int initialized = 0;
3477 
3478         if (initialized)
3479                 return 0;
3480 
3481         initialized = 1;
3482 
3483         printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3484 
3485         err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3486         if (err < 0) {
3487                 printk(KERN_ERR "stream co-processor: could not get major number.\n");
3488                 return err;
3489         }
3490 
3491         err = init_cryptocop();
3492         if (err) {
3493                 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3494                 return err;
3495         }
3496         err = cryptocop_job_queue_init();
3497         if (err) {
3498                 release_cryptocop();
3499                 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3500                 return err;
3501         }
3502         /* Init the descriptor pool. */
3503         for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3504                 descr_pool[i].from_pool = 1;
3505                 descr_pool[i].next = &descr_pool[i + 1];
3506         }
3507         descr_pool[i].from_pool = 1;
3508         descr_pool[i].next = NULL;
3509         descr_pool_free_list = &descr_pool[0];
3510         descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3511 
3512         spin_lock_init(&cryptocop_completed_jobs_lock);
3513         spin_lock_init(&cryptocop_job_queue_lock);
3514         spin_lock_init(&descr_pool_lock);
3515         spin_lock_init(&cryptocop_sessions_lock);
3516         spin_lock_init(&running_job_lock);
3517         spin_lock_init(&cryptocop_process_lock);
3518 
3519         cryptocop_sessions = NULL;
3520         next_sid = 1;
3521 
3522         cryptocop_running_job = NULL;
3523 
3524         printk("stream co-processor: init done.\n");
3525         return 0;
3526 }
3527 
3528 static void __exit exit_stream_coprocessor(void)
3529 {
3530         release_cryptocop();
3531         cryptocop_job_queue_close();
3532 }
3533 
3534 module_init(init_stream_coprocessor);
3535 module_exit(exit_stream_coprocessor);
3536 
3537 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp