~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/fs/crypto/crypto.c

Version: ~ [ linux-5.1-rc2 ] ~ [ linux-5.0.4 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.31 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.108 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.165 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.177 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.137 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.63 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * This contains encryption functions for per-file encryption.
  3  *
  4  * Copyright (C) 2015, Google, Inc.
  5  * Copyright (C) 2015, Motorola Mobility
  6  *
  7  * Written by Michael Halcrow, 2014.
  8  *
  9  * Filename encryption additions
 10  *      Uday Savagaonkar, 2014
 11  * Encryption policy handling additions
 12  *      Ildar Muslukhov, 2014
 13  * Add fscrypt_pullback_bio_page()
 14  *      Jaegeuk Kim, 2015.
 15  *
 16  * This has not yet undergone a rigorous security audit.
 17  *
 18  * The usage of AES-XTS should conform to recommendations in NIST
 19  * Special Publication 800-38E and IEEE P1619/D16.
 20  */
 21 
 22 #include <linux/pagemap.h>
 23 #include <linux/mempool.h>
 24 #include <linux/module.h>
 25 #include <linux/scatterlist.h>
 26 #include <linux/ratelimit.h>
 27 #include <linux/dcache.h>
 28 #include <linux/namei.h>
 29 #include <crypto/aes.h>
 30 #include <crypto/skcipher.h>
 31 #include "fscrypt_private.h"
 32 
 33 static unsigned int num_prealloc_crypto_pages = 32;
 34 static unsigned int num_prealloc_crypto_ctxs = 128;
 35 
 36 module_param(num_prealloc_crypto_pages, uint, 0444);
 37 MODULE_PARM_DESC(num_prealloc_crypto_pages,
 38                 "Number of crypto pages to preallocate");
 39 module_param(num_prealloc_crypto_ctxs, uint, 0444);
 40 MODULE_PARM_DESC(num_prealloc_crypto_ctxs,
 41                 "Number of crypto contexts to preallocate");
 42 
 43 static mempool_t *fscrypt_bounce_page_pool = NULL;
 44 
 45 static LIST_HEAD(fscrypt_free_ctxs);
 46 static DEFINE_SPINLOCK(fscrypt_ctx_lock);
 47 
 48 static struct workqueue_struct *fscrypt_read_workqueue;
 49 static DEFINE_MUTEX(fscrypt_init_mutex);
 50 
 51 static struct kmem_cache *fscrypt_ctx_cachep;
 52 struct kmem_cache *fscrypt_info_cachep;
 53 
 54 void fscrypt_enqueue_decrypt_work(struct work_struct *work)
 55 {
 56         queue_work(fscrypt_read_workqueue, work);
 57 }
 58 EXPORT_SYMBOL(fscrypt_enqueue_decrypt_work);
 59 
 60 /**
 61  * fscrypt_release_ctx() - Releases an encryption context
 62  * @ctx: The encryption context to release.
 63  *
 64  * If the encryption context was allocated from the pre-allocated pool, returns
 65  * it to that pool. Else, frees it.
 66  *
 67  * If there's a bounce page in the context, this frees that.
 68  */
 69 void fscrypt_release_ctx(struct fscrypt_ctx *ctx)
 70 {
 71         unsigned long flags;
 72 
 73         if (ctx->flags & FS_CTX_HAS_BOUNCE_BUFFER_FL && ctx->w.bounce_page) {
 74                 mempool_free(ctx->w.bounce_page, fscrypt_bounce_page_pool);
 75                 ctx->w.bounce_page = NULL;
 76         }
 77         ctx->w.control_page = NULL;
 78         if (ctx->flags & FS_CTX_REQUIRES_FREE_ENCRYPT_FL) {
 79                 kmem_cache_free(fscrypt_ctx_cachep, ctx);
 80         } else {
 81                 spin_lock_irqsave(&fscrypt_ctx_lock, flags);
 82                 list_add(&ctx->free_list, &fscrypt_free_ctxs);
 83                 spin_unlock_irqrestore(&fscrypt_ctx_lock, flags);
 84         }
 85 }
 86 EXPORT_SYMBOL(fscrypt_release_ctx);
 87 
 88 /**
 89  * fscrypt_get_ctx() - Gets an encryption context
 90  * @inode:       The inode for which we are doing the crypto
 91  * @gfp_flags:   The gfp flag for memory allocation
 92  *
 93  * Allocates and initializes an encryption context.
 94  *
 95  * Return: An allocated and initialized encryption context on success; error
 96  * value or NULL otherwise.
 97  */
 98 struct fscrypt_ctx *fscrypt_get_ctx(const struct inode *inode, gfp_t gfp_flags)
 99 {
100         struct fscrypt_ctx *ctx = NULL;
101         struct fscrypt_info *ci = inode->i_crypt_info;
102         unsigned long flags;
103 
104         if (ci == NULL)
105                 return ERR_PTR(-ENOKEY);
106 
107         /*
108          * We first try getting the ctx from a free list because in
109          * the common case the ctx will have an allocated and
110          * initialized crypto tfm, so it's probably a worthwhile
111          * optimization. For the bounce page, we first try getting it
112          * from the kernel allocator because that's just about as fast
113          * as getting it from a list and because a cache of free pages
114          * should generally be a "last resort" option for a filesystem
115          * to be able to do its job.
116          */
117         spin_lock_irqsave(&fscrypt_ctx_lock, flags);
118         ctx = list_first_entry_or_null(&fscrypt_free_ctxs,
119                                         struct fscrypt_ctx, free_list);
120         if (ctx)
121                 list_del(&ctx->free_list);
122         spin_unlock_irqrestore(&fscrypt_ctx_lock, flags);
123         if (!ctx) {
124                 ctx = kmem_cache_zalloc(fscrypt_ctx_cachep, gfp_flags);
125                 if (!ctx)
126                         return ERR_PTR(-ENOMEM);
127                 ctx->flags |= FS_CTX_REQUIRES_FREE_ENCRYPT_FL;
128         } else {
129                 ctx->flags &= ~FS_CTX_REQUIRES_FREE_ENCRYPT_FL;
130         }
131         ctx->flags &= ~FS_CTX_HAS_BOUNCE_BUFFER_FL;
132         return ctx;
133 }
134 EXPORT_SYMBOL(fscrypt_get_ctx);
135 
136 void fscrypt_generate_iv(union fscrypt_iv *iv, u64 lblk_num,
137                          const struct fscrypt_info *ci)
138 {
139         memset(iv, 0, ci->ci_mode->ivsize);
140         iv->lblk_num = cpu_to_le64(lblk_num);
141 
142         if (ci->ci_flags & FS_POLICY_FLAG_DIRECT_KEY)
143                 memcpy(iv->nonce, ci->ci_nonce, FS_KEY_DERIVATION_NONCE_SIZE);
144 
145         if (ci->ci_essiv_tfm != NULL)
146                 crypto_cipher_encrypt_one(ci->ci_essiv_tfm, iv->raw, iv->raw);
147 }
148 
149 int fscrypt_do_page_crypto(const struct inode *inode, fscrypt_direction_t rw,
150                            u64 lblk_num, struct page *src_page,
151                            struct page *dest_page, unsigned int len,
152                            unsigned int offs, gfp_t gfp_flags)
153 {
154         union fscrypt_iv iv;
155         struct skcipher_request *req = NULL;
156         DECLARE_CRYPTO_WAIT(wait);
157         struct scatterlist dst, src;
158         struct fscrypt_info *ci = inode->i_crypt_info;
159         struct crypto_skcipher *tfm = ci->ci_ctfm;
160         int res = 0;
161 
162         BUG_ON(len == 0);
163 
164         fscrypt_generate_iv(&iv, lblk_num, ci);
165 
166         req = skcipher_request_alloc(tfm, gfp_flags);
167         if (!req)
168                 return -ENOMEM;
169 
170         skcipher_request_set_callback(
171                 req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
172                 crypto_req_done, &wait);
173 
174         sg_init_table(&dst, 1);
175         sg_set_page(&dst, dest_page, len, offs);
176         sg_init_table(&src, 1);
177         sg_set_page(&src, src_page, len, offs);
178         skcipher_request_set_crypt(req, &src, &dst, len, &iv);
179         if (rw == FS_DECRYPT)
180                 res = crypto_wait_req(crypto_skcipher_decrypt(req), &wait);
181         else
182                 res = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
183         skcipher_request_free(req);
184         if (res) {
185                 fscrypt_err(inode->i_sb,
186                             "%scryption failed for inode %lu, block %llu: %d",
187                             (rw == FS_DECRYPT ? "de" : "en"),
188                             inode->i_ino, lblk_num, res);
189                 return res;
190         }
191         return 0;
192 }
193 
194 struct page *fscrypt_alloc_bounce_page(struct fscrypt_ctx *ctx,
195                                        gfp_t gfp_flags)
196 {
197         ctx->w.bounce_page = mempool_alloc(fscrypt_bounce_page_pool, gfp_flags);
198         if (ctx->w.bounce_page == NULL)
199                 return ERR_PTR(-ENOMEM);
200         ctx->flags |= FS_CTX_HAS_BOUNCE_BUFFER_FL;
201         return ctx->w.bounce_page;
202 }
203 
204 /**
205  * fscypt_encrypt_page() - Encrypts a page
206  * @inode:     The inode for which the encryption should take place
207  * @page:      The page to encrypt. Must be locked for bounce-page
208  *             encryption.
209  * @len:       Length of data to encrypt in @page and encrypted
210  *             data in returned page.
211  * @offs:      Offset of data within @page and returned
212  *             page holding encrypted data.
213  * @lblk_num:  Logical block number. This must be unique for multiple
214  *             calls with same inode, except when overwriting
215  *             previously written data.
216  * @gfp_flags: The gfp flag for memory allocation
217  *
218  * Encrypts @page using the ctx encryption context. Performs encryption
219  * either in-place or into a newly allocated bounce page.
220  * Called on the page write path.
221  *
222  * Bounce page allocation is the default.
223  * In this case, the contents of @page are encrypted and stored in an
224  * allocated bounce page. @page has to be locked and the caller must call
225  * fscrypt_restore_control_page() on the returned ciphertext page to
226  * release the bounce buffer and the encryption context.
227  *
228  * In-place encryption is used by setting the FS_CFLG_OWN_PAGES flag in
229  * fscrypt_operations. Here, the input-page is returned with its content
230  * encrypted.
231  *
232  * Return: A page with the encrypted content on success. Else, an
233  * error value or NULL.
234  */
235 struct page *fscrypt_encrypt_page(const struct inode *inode,
236                                 struct page *page,
237                                 unsigned int len,
238                                 unsigned int offs,
239                                 u64 lblk_num, gfp_t gfp_flags)
240 
241 {
242         struct fscrypt_ctx *ctx;
243         struct page *ciphertext_page = page;
244         int err;
245 
246         BUG_ON(len % FS_CRYPTO_BLOCK_SIZE != 0);
247 
248         if (inode->i_sb->s_cop->flags & FS_CFLG_OWN_PAGES) {
249                 /* with inplace-encryption we just encrypt the page */
250                 err = fscrypt_do_page_crypto(inode, FS_ENCRYPT, lblk_num, page,
251                                              ciphertext_page, len, offs,
252                                              gfp_flags);
253                 if (err)
254                         return ERR_PTR(err);
255 
256                 return ciphertext_page;
257         }
258 
259         BUG_ON(!PageLocked(page));
260 
261         ctx = fscrypt_get_ctx(inode, gfp_flags);
262         if (IS_ERR(ctx))
263                 return (struct page *)ctx;
264 
265         /* The encryption operation will require a bounce page. */
266         ciphertext_page = fscrypt_alloc_bounce_page(ctx, gfp_flags);
267         if (IS_ERR(ciphertext_page))
268                 goto errout;
269 
270         ctx->w.control_page = page;
271         err = fscrypt_do_page_crypto(inode, FS_ENCRYPT, lblk_num,
272                                      page, ciphertext_page, len, offs,
273                                      gfp_flags);
274         if (err) {
275                 ciphertext_page = ERR_PTR(err);
276                 goto errout;
277         }
278         SetPagePrivate(ciphertext_page);
279         set_page_private(ciphertext_page, (unsigned long)ctx);
280         lock_page(ciphertext_page);
281         return ciphertext_page;
282 
283 errout:
284         fscrypt_release_ctx(ctx);
285         return ciphertext_page;
286 }
287 EXPORT_SYMBOL(fscrypt_encrypt_page);
288 
289 /**
290  * fscrypt_decrypt_page() - Decrypts a page in-place
291  * @inode:     The corresponding inode for the page to decrypt.
292  * @page:      The page to decrypt. Must be locked in case
293  *             it is a writeback page (FS_CFLG_OWN_PAGES unset).
294  * @len:       Number of bytes in @page to be decrypted.
295  * @offs:      Start of data in @page.
296  * @lblk_num:  Logical block number.
297  *
298  * Decrypts page in-place using the ctx encryption context.
299  *
300  * Called from the read completion callback.
301  *
302  * Return: Zero on success, non-zero otherwise.
303  */
304 int fscrypt_decrypt_page(const struct inode *inode, struct page *page,
305                         unsigned int len, unsigned int offs, u64 lblk_num)
306 {
307         if (!(inode->i_sb->s_cop->flags & FS_CFLG_OWN_PAGES))
308                 BUG_ON(!PageLocked(page));
309 
310         return fscrypt_do_page_crypto(inode, FS_DECRYPT, lblk_num, page, page,
311                                       len, offs, GFP_NOFS);
312 }
313 EXPORT_SYMBOL(fscrypt_decrypt_page);
314 
315 /*
316  * Validate dentries for encrypted directories to make sure we aren't
317  * potentially caching stale data after a key has been added or
318  * removed.
319  */
320 static int fscrypt_d_revalidate(struct dentry *dentry, unsigned int flags)
321 {
322         struct dentry *dir;
323         int dir_has_key, cached_with_key;
324 
325         if (flags & LOOKUP_RCU)
326                 return -ECHILD;
327 
328         dir = dget_parent(dentry);
329         if (!IS_ENCRYPTED(d_inode(dir))) {
330                 dput(dir);
331                 return 0;
332         }
333 
334         spin_lock(&dentry->d_lock);
335         cached_with_key = dentry->d_flags & DCACHE_ENCRYPTED_WITH_KEY;
336         spin_unlock(&dentry->d_lock);
337         dir_has_key = (d_inode(dir)->i_crypt_info != NULL);
338         dput(dir);
339 
340         /*
341          * If the dentry was cached without the key, and it is a
342          * negative dentry, it might be a valid name.  We can't check
343          * if the key has since been made available due to locking
344          * reasons, so we fail the validation so ext4_lookup() can do
345          * this check.
346          *
347          * We also fail the validation if the dentry was created with
348          * the key present, but we no longer have the key, or vice versa.
349          */
350         if ((!cached_with_key && d_is_negative(dentry)) ||
351                         (!cached_with_key && dir_has_key) ||
352                         (cached_with_key && !dir_has_key))
353                 return 0;
354         return 1;
355 }
356 
357 const struct dentry_operations fscrypt_d_ops = {
358         .d_revalidate = fscrypt_d_revalidate,
359 };
360 
361 void fscrypt_restore_control_page(struct page *page)
362 {
363         struct fscrypt_ctx *ctx;
364 
365         ctx = (struct fscrypt_ctx *)page_private(page);
366         set_page_private(page, (unsigned long)NULL);
367         ClearPagePrivate(page);
368         unlock_page(page);
369         fscrypt_release_ctx(ctx);
370 }
371 EXPORT_SYMBOL(fscrypt_restore_control_page);
372 
373 static void fscrypt_destroy(void)
374 {
375         struct fscrypt_ctx *pos, *n;
376 
377         list_for_each_entry_safe(pos, n, &fscrypt_free_ctxs, free_list)
378                 kmem_cache_free(fscrypt_ctx_cachep, pos);
379         INIT_LIST_HEAD(&fscrypt_free_ctxs);
380         mempool_destroy(fscrypt_bounce_page_pool);
381         fscrypt_bounce_page_pool = NULL;
382 }
383 
384 /**
385  * fscrypt_initialize() - allocate major buffers for fs encryption.
386  * @cop_flags:  fscrypt operations flags
387  *
388  * We only call this when we start accessing encrypted files, since it
389  * results in memory getting allocated that wouldn't otherwise be used.
390  *
391  * Return: Zero on success, non-zero otherwise.
392  */
393 int fscrypt_initialize(unsigned int cop_flags)
394 {
395         int i, res = -ENOMEM;
396 
397         /* No need to allocate a bounce page pool if this FS won't use it. */
398         if (cop_flags & FS_CFLG_OWN_PAGES)
399                 return 0;
400 
401         mutex_lock(&fscrypt_init_mutex);
402         if (fscrypt_bounce_page_pool)
403                 goto already_initialized;
404 
405         for (i = 0; i < num_prealloc_crypto_ctxs; i++) {
406                 struct fscrypt_ctx *ctx;
407 
408                 ctx = kmem_cache_zalloc(fscrypt_ctx_cachep, GFP_NOFS);
409                 if (!ctx)
410                         goto fail;
411                 list_add(&ctx->free_list, &fscrypt_free_ctxs);
412         }
413 
414         fscrypt_bounce_page_pool =
415                 mempool_create_page_pool(num_prealloc_crypto_pages, 0);
416         if (!fscrypt_bounce_page_pool)
417                 goto fail;
418 
419 already_initialized:
420         mutex_unlock(&fscrypt_init_mutex);
421         return 0;
422 fail:
423         fscrypt_destroy();
424         mutex_unlock(&fscrypt_init_mutex);
425         return res;
426 }
427 
428 void fscrypt_msg(struct super_block *sb, const char *level,
429                  const char *fmt, ...)
430 {
431         static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
432                                       DEFAULT_RATELIMIT_BURST);
433         struct va_format vaf;
434         va_list args;
435 
436         if (!__ratelimit(&rs))
437                 return;
438 
439         va_start(args, fmt);
440         vaf.fmt = fmt;
441         vaf.va = &args;
442         if (sb)
443                 printk("%sfscrypt (%s): %pV\n", level, sb->s_id, &vaf);
444         else
445                 printk("%sfscrypt: %pV\n", level, &vaf);
446         va_end(args);
447 }
448 
449 /**
450  * fscrypt_init() - Set up for fs encryption.
451  */
452 static int __init fscrypt_init(void)
453 {
454         /*
455          * Use an unbound workqueue to allow bios to be decrypted in parallel
456          * even when they happen to complete on the same CPU.  This sacrifices
457          * locality, but it's worthwhile since decryption is CPU-intensive.
458          *
459          * Also use a high-priority workqueue to prioritize decryption work,
460          * which blocks reads from completing, over regular application tasks.
461          */
462         fscrypt_read_workqueue = alloc_workqueue("fscrypt_read_queue",
463                                                  WQ_UNBOUND | WQ_HIGHPRI,
464                                                  num_online_cpus());
465         if (!fscrypt_read_workqueue)
466                 goto fail;
467 
468         fscrypt_ctx_cachep = KMEM_CACHE(fscrypt_ctx, SLAB_RECLAIM_ACCOUNT);
469         if (!fscrypt_ctx_cachep)
470                 goto fail_free_queue;
471 
472         fscrypt_info_cachep = KMEM_CACHE(fscrypt_info, SLAB_RECLAIM_ACCOUNT);
473         if (!fscrypt_info_cachep)
474                 goto fail_free_ctx;
475 
476         return 0;
477 
478 fail_free_ctx:
479         kmem_cache_destroy(fscrypt_ctx_cachep);
480 fail_free_queue:
481         destroy_workqueue(fscrypt_read_workqueue);
482 fail:
483         return -ENOMEM;
484 }
485 module_init(fscrypt_init)
486 
487 /**
488  * fscrypt_exit() - Shutdown the fs encryption system
489  */
490 static void __exit fscrypt_exit(void)
491 {
492         fscrypt_destroy();
493 
494         if (fscrypt_read_workqueue)
495                 destroy_workqueue(fscrypt_read_workqueue);
496         kmem_cache_destroy(fscrypt_ctx_cachep);
497         kmem_cache_destroy(fscrypt_info_cachep);
498 
499         fscrypt_essiv_cleanup();
500 }
501 module_exit(fscrypt_exit);
502 
503 MODULE_LICENSE("GPL");
504 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp