From: Ira Weiny ira.weiny@intel.com

Now that PKS can be enabled globaly (for all threads) add a test which spawns a thread and tests the same PKS functionality.

The test enables/disables PKS in 1 thread while attempting to access the page in another thread. We use the same test array as in the 'local' PKS testing.

Signed-off-by: Ira Weiny ira.weiny@intel.com --- arch/x86/mm/fault.c | 4 ++ lib/pks/pks_test.c | 128 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 124 insertions(+), 8 deletions(-)

diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 4b4ff9efa298..4c74f52fbc23 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1108,6 +1108,10 @@ static int spurious_kernel_fault_check(unsigned long error_code, pte_t *pte, if (global_pkey_is_enabled(pte, is_write, irq_state)) return 1;

+ /* + * NOTE: This must be after the global_pkey_is_enabled() call + * to allow the fixup code to be tested. + */ if (handle_pks_testing(error_code, irq_state)) return 1;

diff --git a/lib/pks/pks_test.c b/lib/pks/pks_test.c index 286c8b8457da..dfddccbe4cb6 100644 --- a/lib/pks/pks_test.c +++ b/lib/pks/pks_test.c @@ -154,7 +154,8 @@ static void check_exception(irqentry_state_t *irq_state) }

/* Check the exception state */ - if (!check_pkrs(test_armed_key, PKEY_DISABLE_ACCESS)) { + if (!check_pkrs(test_armed_key, + PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE)) { pr_err(" FAIL: PKRS cache and MSR\n"); test_exception_ctx->pass = false; } @@ -308,24 +309,29 @@ static int test_it(struct pks_test_ctx *ctx, struct pks_access_test *test, void return ret; }

-static int run_access_test(struct pks_test_ctx *ctx, - struct pks_access_test *test, - void *ptr) +static void set_protection(int pkey, enum pks_access_mode mode, bool global) { - switch (test->mode) { + switch (mode) { case PKS_TEST_NO_ACCESS: - pks_mknoaccess(ctx->pkey, false); + pks_mknoaccess(pkey, global); break; case PKS_TEST_RDWR: - pks_mkrdwr(ctx->pkey, false); + pks_mkrdwr(pkey, global); break; case PKS_TEST_RDONLY: - pks_mkread(ctx->pkey, false); + pks_mkread(pkey, global); break; default: pr_err("BUG in test invalid mode\n"); break; } +} + +static int run_access_test(struct pks_test_ctx *ctx, + struct pks_access_test *test, + void *ptr) +{ + set_protection(ctx->pkey, test->mode, false);

return test_it(ctx, test, ptr); } @@ -516,6 +522,110 @@ static void run_exception_test(void) pass ? "PASS" : "FAIL"); }

+struct shared_data { + struct mutex lock; + struct pks_test_ctx *ctx; + void *kmap_addr; + struct pks_access_test *test; +}; + +static int thread_main(void *d) +{ + struct shared_data *data = d; + struct pks_test_ctx *ctx = data->ctx; + + while (!kthread_should_stop()) { + mutex_lock(&data->lock); + /* + * wait for the main thread to hand us the page + * We should be spinning so hopefully we will not have gotten + * the global value from a schedule in. + */ + if (data->kmap_addr) { + if (test_it(ctx, data->test, data->kmap_addr)) + ctx->pass = false; + data->kmap_addr = NULL; + } + mutex_unlock(&data->lock); + } + + return 0; +} + +static void run_thread_access_test(struct shared_data *data, + struct pks_test_ctx *ctx, + struct pks_access_test *test, + void *ptr) +{ + set_protection(ctx->pkey, test->mode, true); + + pr_info("checking... mode %s; write %s\n", + get_mode_str(test->mode), test->write ? "TRUE" : "FALSE"); + + mutex_lock(&data->lock); + data->test = test; + data->kmap_addr = ptr; + mutex_unlock(&data->lock); + + while (data->kmap_addr) { + msleep(10); + } +} + +static void run_global_test(void) +{ + struct task_struct *other_task; + struct pks_test_ctx *ctx; + struct shared_data data; + bool pass = true; + void *ptr; + int i; + + pr_info(" ***** BEGIN: global pkey checking\n"); + + /* Set up context, data pgae, and thread */ + ctx = alloc_ctx("global pkey test"); + if (IS_ERR(ctx)) { + pr_err(" FAIL: no context\n"); + pass = false; + goto result; + } + ptr = alloc_test_page(ctx->pkey); + if (!ptr) { + pr_err(" FAIL: no vmalloc page\n"); + pass = false; + goto free_context; + } + other_task = kthread_run(thread_main, &data, "PKRS global test"); + if (IS_ERR(other_task)) { + pr_err(" FAIL: Failed to start thread\n"); + pass = false; + goto free_page; + } + + memset(&data, 0, sizeof(data)); + mutex_init(&data.lock); + data.ctx = ctx; + + /* Start testing */ + ctx->pass = true; + + for (i = 0; i < ARRAY_SIZE(pkey_test_ary); i++) { + run_thread_access_test(&data, ctx, &pkey_test_ary[i], ptr); + } + + kthread_stop(other_task); + pass = ctx->pass; + +free_page: + vfree(ptr); +free_context: + free_ctx(ctx); +result: + pr_info(" ***** END: global pkey checking : %s\n", + pass ? "PASS" : "FAIL"); +} + static void run_all(void) { struct pks_test_ctx *ctx[PKS_NUM_KEYS]; @@ -538,6 +648,8 @@ static void run_all(void) }

run_exception_test(); + + run_global_test(); }

static void crash_it(void)

From: Ira Weiny ira.weiny@intel.com

Device managed pages may have additional protections. These protections need to be removed prior to valid use by kernel users.

Check for special treatment of device managed pages in kmap and take action if needed. We use kmap as an interface for generic kernel code because under normal circumstances it would be a bug for general kernel code to not use kmap prior to accessing kernel memory. Therefore, this should allow any valid kernel users to seamlessly use these pages without issues.

Because of the critical nature of kmap it must be pointed out that the over head on regular DRAM is carefully implemented to be as fast as possible. Furthermore the underlying MSR write required on device pages when protected is better than a normal MSR write.

Specifically, WRMSR(MSR_IA32_PKRS) is not serializing but still maintains ordering properties similar to WRPKRU. The current SDM section on PKRS needs updating but should be the same as that of WRPKRU. So to quote from the WRPKRU text:

WRPKRU will never execute speculatively. Memory accesses affected by PKRU register will not execute (even speculatively) until all prior executions of WRPKRU have completed execution and updated the PKRU register.

Still this will make accessing pmem more expensive from the kernel but the overhead is minimized and many pmem users access this memory through user page mappings which are not affected at all.

Cc: Randy Dunlap rdunlap@infradead.org Signed-off-by: Ira Weiny ira.weiny@intel.com --- include/linux/highmem.h | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-)

diff --git a/include/linux/highmem.h b/include/linux/highmem.h index 14e6202ce47f..2a9806e3b8d2 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -8,6 +8,7 @@ #include <linux/mm.h> #include <linux/uaccess.h> #include <linux/hardirq.h> +#include <linux/memremap.h>

#include <asm/cacheflush.h>

@@ -31,6 +32,20 @@ static inline void invalidate_kernel_vmap_range(void *vaddr, int size)

#include <asm/kmap_types.h>

+static inline void dev_page_enable_access(struct page *page, bool global) +{ + if (!page_is_access_protected(page)) + return; + dev_access_enable(global); +} + +static inline void dev_page_disable_access(struct page *page, bool global) +{ + if (!page_is_access_protected(page)) + return; + dev_access_disable(global); +} + #ifdef CONFIG_HIGHMEM extern void *kmap_atomic_high_prot(struct page *page, pgprot_t prot); extern void kunmap_atomic_high(void *kvaddr); @@ -55,6 +70,11 @@ static inline void *kmap(struct page *page) else addr = kmap_high(page); kmap_flush_tlb((unsigned long)addr); + /* + * Even non-highmem pages may have additional access protections which + * need to be checked and potentially enabled. + */ + dev_page_enable_access(page, true); return addr; }

@@ -63,6 +83,11 @@ void kunmap_high(struct page *page); static inline void kunmap(struct page *page) { might_sleep(); + /* + * Even non-highmem pages may have additional access protections which + * need to be checked and potentially disabled. + */ + dev_page_disable_access(page, true); if (!PageHighMem(page)) return; kunmap_high(page); @@ -85,6 +110,7 @@ static inline void *kmap_atomic_prot(struct page *page, pgprot_t prot) { preempt_disable(); pagefault_disable(); + dev_page_enable_access(page, false); if (!PageHighMem(page)) return page_address(page); return kmap_atomic_high_prot(page, prot); @@ -137,6 +163,7 @@ static inline unsigned long totalhigh_pages(void) { return 0UL; } static inline void *kmap(struct page *page) { might_sleep(); + dev_page_enable_access(page, true); return page_address(page); }

@@ -146,6 +173,7 @@ static inline void kunmap_high(struct page *page)

static inline void kunmap(struct page *page) { + dev_page_disable_access(page, true); #ifdef ARCH_HAS_FLUSH_ON_KUNMAP kunmap_flush_on_unmap(page_address(page)); #endif @@ -155,6 +183,7 @@ static inline void *kmap_atomic(struct page *page) { preempt_disable(); pagefault_disable(); + dev_page_enable_access(page, false); return page_address(page); } #define kmap_atomic_prot(page, prot) kmap_atomic(page) @@ -216,7 +245,8 @@ static inline void kmap_atomic_idx_pop(void) #define kunmap_atomic(addr) \ do { \ BUILD_BUG_ON(__same_type((addr), struct page *)); \ - kunmap_atomic_high(addr); \ + dev_page_disable_access(kmap_to_page(addr), false); \ + kunmap_atomic_high(addr); \ pagefault_enable(); \ preempt_enable(); \ } while (0)

From: Ira Weiny ira.weiny@intel.com

The kmap() calls in this driver are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: Jens Axboe axboe@kernel.dk Signed-off-by: Ira Weiny ira.weiny@intel.com --- drivers/block/drbd/drbd_main.c | 4 ++-- drivers/block/drbd/drbd_receiver.c | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c index 573dbf6f0c31..f0d0c6b0745e 100644 --- a/drivers/block/drbd/drbd_main.c +++ b/drivers/block/drbd/drbd_main.c @@ -1532,9 +1532,9 @@ static int _drbd_no_send_page(struct drbd_peer_device *peer_device, struct page int err;

socket = peer_device->connection->data.socket; - addr = kmap(page) + offset; + addr = kmap_thread(page) + offset; err = drbd_send_all(peer_device->connection, socket, addr, size, msg_flags); - kunmap(page); + kunmap_thread(page); if (!err) peer_device->device->send_cnt += size >> 9; return err; diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c index 422363daa618..4704bc0564e2 100644 --- a/drivers/block/drbd/drbd_receiver.c +++ b/drivers/block/drbd/drbd_receiver.c @@ -1951,13 +1951,13 @@ read_in_block(struct drbd_peer_device *peer_device, u64 id, sector_t sector, page = peer_req->pages; page_chain_for_each(page) { unsigned len = min_t(int, ds, PAGE_SIZE); - data = kmap(page); + data = kmap_thread(page); err = drbd_recv_all_warn(peer_device->connection, data, len); if (drbd_insert_fault(device, DRBD_FAULT_RECEIVE)) { drbd_err(device, "Fault injection: Corrupting data on receive\n"); data[0] = data[0] ^ (unsigned long)-1; } - kunmap(page); + kunmap_thread(page); if (err) { drbd_free_peer_req(device, peer_req); return NULL; @@ -1992,7 +1992,7 @@ static int drbd_drain_block(struct drbd_peer_device *peer_device, int data_size)

page = drbd_alloc_pages(peer_device, 1, 1);

- data = kmap(page); + data = kmap_thread(page); while (data_size) { unsigned int len = min_t(int, data_size, PAGE_SIZE);

@@ -2001,7 +2001,7 @@ static int drbd_drain_block(struct drbd_peer_device *peer_device, int data_size) break; data_size -= len; } - kunmap(page); + kunmap_thread(page); drbd_free_pages(peer_device->device, page, 0); return err; } @@ -2033,10 +2033,10 @@ static int recv_dless_read(struct drbd_peer_device *peer_device, struct drbd_req D_ASSERT(peer_device->device, sector == bio->bi_iter.bi_sector);

bio_for_each_segment(bvec, bio, iter) { - void *mapped = kmap(bvec.bv_page) + bvec.bv_offset; + void *mapped = kmap_thread(bvec.bv_page) + bvec.bv_offset; expect = min_t(int, data_size, bvec.bv_len); err = drbd_recv_all_warn(peer_device->connection, mapped, expect); - kunmap(bvec.bv_page); + kunmap_thread(bvec.bv_page); if (err) return err; data_size -= expect;

From: Ira Weiny ira.weiny@intel.com

These kmap() calls in the gpu stack are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: David Airlie airlied@linux.ie Cc: Daniel Vetter daniel@ffwll.ch Cc: Patrik Jakobsson patrik.r.jakobsson@gmail.com Signed-off-by: Ira Weiny ira.weiny@intel.com --- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 12 ++++++------ drivers/gpu/drm/gma500/gma_display.c | 4 ++-- drivers/gpu/drm/gma500/mmu.c | 10 +++++----- drivers/gpu/drm/i915/gem/i915_gem_shmem.c | 4 ++-- .../gpu/drm/i915/gem/selftests/i915_gem_context.c | 4 ++-- drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c | 8 ++++---- drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c | 4 ++-- drivers/gpu/drm/i915/gt/intel_gtt.c | 4 ++-- drivers/gpu/drm/i915/gt/shmem_utils.c | 4 ++-- drivers/gpu/drm/i915/i915_gem.c | 8 ++++---- drivers/gpu/drm/i915/i915_gpu_error.c | 4 ++-- drivers/gpu/drm/i915/selftests/i915_perf.c | 4 ++-- drivers/gpu/drm/radeon/radeon_ttm.c | 4 ++-- 13 files changed, 37 insertions(+), 37 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 978bae731398..bd564bccb7a3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -2437,11 +2437,11 @@ static ssize_t amdgpu_ttm_gtt_read(struct file *f, char __user *buf,

page = adev->gart.pages[p]; if (page) { - ptr = kmap(page); + ptr = kmap_thread(page); ptr += off;

r = copy_to_user(buf, ptr, cur_size); - kunmap(adev->gart.pages[p]); + kunmap_thread(adev->gart.pages[p]); } else r = clear_user(buf, cur_size);

@@ -2507,9 +2507,9 @@ static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf, if (p->mapping != adev->mman.bdev.dev_mapping) return -EPERM;

- ptr = kmap(p); + ptr = kmap_thread(p); r = copy_to_user(buf, ptr + off, bytes); - kunmap(p); + kunmap_thread(p); if (r) return -EFAULT;

@@ -2558,9 +2558,9 @@ static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf, if (p->mapping != adev->mman.bdev.dev_mapping) return -EPERM;

- ptr = kmap(p); + ptr = kmap_thread(p); r = copy_from_user(ptr + off, buf, bytes); - kunmap(p); + kunmap_thread(p); if (r) return -EFAULT;

diff --git a/drivers/gpu/drm/gma500/gma_display.c b/drivers/gpu/drm/gma500/gma_display.c index 3df6d6e850f5..35f4e55c941f 100644 --- a/drivers/gpu/drm/gma500/gma_display.c +++ b/drivers/gpu/drm/gma500/gma_display.c @@ -400,9 +400,9 @@ int gma_crtc_cursor_set(struct drm_crtc *crtc, /* Copy the cursor to cursor mem */ tmp_dst = dev_priv->vram_addr + cursor_gt->offset; for (i = 0; i < cursor_pages; i++) { - tmp_src = kmap(gt->pages[i]); + tmp_src = kmap_thread(gt->pages[i]); memcpy(tmp_dst, tmp_src, PAGE_SIZE); - kunmap(gt->pages[i]); + kunmap_thread(gt->pages[i]); tmp_dst += PAGE_SIZE; }

diff --git a/drivers/gpu/drm/gma500/mmu.c b/drivers/gpu/drm/gma500/mmu.c index 505044c9a673..fba7a3a461fd 100644 --- a/drivers/gpu/drm/gma500/mmu.c +++ b/drivers/gpu/drm/gma500/mmu.c @@ -192,20 +192,20 @@ struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver, pd->invalid_pte = 0; }

- v = kmap(pd->dummy_pt); + v = kmap_thread(pd->dummy_pt); for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) v[i] = pd->invalid_pte;

- kunmap(pd->dummy_pt); + kunmap_thread(pd->dummy_pt);

- v = kmap(pd->p); + v = kmap_thread(pd->p); for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) v[i] = pd->invalid_pde;

- kunmap(pd->p); + kunmap_thread(pd->p);

clear_page(kmap(pd->dummy_page)); - kunmap(pd->dummy_page); + kunmap_thread(pd->dummy_page);

pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024); if (!pd->tables) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c index 38113d3c0138..274424795fb7 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c @@ -566,9 +566,9 @@ i915_gem_object_create_shmem_from_data(struct drm_i915_private *dev_priv, if (err < 0) goto fail;

- vaddr = kmap(page); + vaddr = kmap_thread(page); memcpy(vaddr, data, len); - kunmap(page); + kunmap_thread(page);

err = pagecache_write_end(file, file->f_mapping, offset, len, len, diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c index 7ffc3c751432..b466c677d007 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c @@ -1754,7 +1754,7 @@ static int check_scratch_page(struct i915_gem_context *ctx, u32 *out) return -EINVAL; }

- vaddr = kmap(page); + vaddr = kmap_thread(page); if (!vaddr) { pr_err("No (mappable) scratch page!\n"); return -EINVAL; @@ -1765,7 +1765,7 @@ static int check_scratch_page(struct i915_gem_context *ctx, u32 *out) pr_err("Inconsistent initial state of scratch page!\n"); err = -EINVAL; } - kunmap(page); + kunmap_thread(page);

return err; } diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c index 9c7402ce5bf9..447df22e2e06 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c @@ -143,7 +143,7 @@ static int check_partial_mapping(struct drm_i915_gem_object *obj, intel_gt_flush_ggtt_writes(&to_i915(obj->base.dev)->gt);

p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT); - cpu = kmap(p) + offset_in_page(offset); + cpu = kmap_thread(p) + offset_in_page(offset); drm_clflush_virt_range(cpu, sizeof(*cpu)); if (*cpu != (u32)page) { pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n", @@ -161,7 +161,7 @@ static int check_partial_mapping(struct drm_i915_gem_object *obj, } *cpu = 0; drm_clflush_virt_range(cpu, sizeof(*cpu)); - kunmap(p); + kunmap_thread(p);

out: __i915_vma_put(vma); @@ -236,7 +236,7 @@ static int check_partial_mappings(struct drm_i915_gem_object *obj, intel_gt_flush_ggtt_writes(&to_i915(obj->base.dev)->gt);

p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT); - cpu = kmap(p) + offset_in_page(offset); + cpu = kmap_thread(p) + offset_in_page(offset); drm_clflush_virt_range(cpu, sizeof(*cpu)); if (*cpu != (u32)page) { pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n", @@ -254,7 +254,7 @@ static int check_partial_mappings(struct drm_i915_gem_object *obj, } *cpu = 0; drm_clflush_virt_range(cpu, sizeof(*cpu)); - kunmap(p); + kunmap_thread(p); if (err) return err;

diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c index 7fb36b12fe7a..38da348282f1 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c @@ -731,7 +731,7 @@ static void swizzle_page(struct page *page) char *vaddr; int i;

- vaddr = kmap(page); + vaddr = kmap_thread(page);

for (i = 0; i < PAGE_SIZE; i += 128) { memcpy(temp, &vaddr[i], 64); @@ -739,7 +739,7 @@ static void swizzle_page(struct page *page) memcpy(&vaddr[i + 64], temp, 64); }

- kunmap(page); + kunmap_thread(page); }

/** diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.c b/drivers/gpu/drm/i915/gt/intel_gtt.c index 2a72cce63fd9..4cfb24e9ed62 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.c +++ b/drivers/gpu/drm/i915/gt/intel_gtt.c @@ -312,9 +312,9 @@ static void poison_scratch_page(struct page *page, unsigned long size) do { void *vaddr;

- vaddr = kmap(page); + vaddr = kmap_thread(page); memset(vaddr, POISON_FREE, PAGE_SIZE); - kunmap(page); + kunmap_thread(page);

page = pfn_to_page(page_to_pfn(page) + 1); size -= PAGE_SIZE; diff --git a/drivers/gpu/drm/i915/gt/shmem_utils.c b/drivers/gpu/drm/i915/gt/shmem_utils.c index 43c7acbdc79d..a40d3130cebf 100644 --- a/drivers/gpu/drm/i915/gt/shmem_utils.c +++ b/drivers/gpu/drm/i915/gt/shmem_utils.c @@ -142,12 +142,12 @@ static int __shmem_rw(struct file *file, loff_t off, if (IS_ERR(page)) return PTR_ERR(page);

- vaddr = kmap(page); + vaddr = kmap_thread(page); if (write) memcpy(vaddr + offset_in_page(off), ptr, this); else memcpy(ptr, vaddr + offset_in_page(off), this); - kunmap(page); + kunmap_thread(page); put_page(page);

len -= this; diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 9aa3066cb75d..cae8300fd224 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -312,14 +312,14 @@ shmem_pread(struct page *page, int offset, int len, char __user *user_data, char *vaddr; int ret;

- vaddr = kmap(page); + vaddr = kmap_thread(page);

if (needs_clflush) drm_clflush_virt_range(vaddr + offset, len);

ret = __copy_to_user(user_data, vaddr + offset, len);

- kunmap(page); + kunmap_thread(page);

return ret ? -EFAULT : 0; } @@ -708,7 +708,7 @@ shmem_pwrite(struct page *page, int offset, int len, char __user *user_data, char *vaddr; int ret;

- vaddr = kmap(page); + vaddr = kmap_thread(page);

if (needs_clflush_before) drm_clflush_virt_range(vaddr + offset, len); @@ -717,7 +717,7 @@ shmem_pwrite(struct page *page, int offset, int len, char __user *user_data, if (!ret && needs_clflush_after) drm_clflush_virt_range(vaddr + offset, len);

- kunmap(page); + kunmap_thread(page);

return ret ? -EFAULT : 0; } diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index 3e6cbb0d1150..aecd469b6b6e 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -1058,9 +1058,9 @@ i915_vma_coredump_create(const struct intel_gt *gt,

drm_clflush_pages(&page, 1);

- s = kmap(page); + s = kmap_thread(page); ret = compress_page(compress, s, dst, false); - kunmap(page); + kunmap_thread(page);

drm_clflush_pages(&page, 1);

diff --git a/drivers/gpu/drm/i915/selftests/i915_perf.c b/drivers/gpu/drm/i915/selftests/i915_perf.c index c2d001d9c0ec..7f7ef2d056f4 100644 --- a/drivers/gpu/drm/i915/selftests/i915_perf.c +++ b/drivers/gpu/drm/i915/selftests/i915_perf.c @@ -307,7 +307,7 @@ static int live_noa_gpr(void *arg) }

/* Poison the ce->vm so we detect writes not to the GGTT gt->scratch */ - scratch = kmap(ce->vm->scratch[0].base.page); + scratch = kmap_thread(ce->vm->scratch[0].base.page); memset(scratch, POISON_FREE, PAGE_SIZE);

rq = intel_context_create_request(ce); @@ -405,7 +405,7 @@ static int live_noa_gpr(void *arg) out_rq: i915_request_put(rq); out_ce: - kunmap(ce->vm->scratch[0].base.page); + kunmap_thread(ce->vm->scratch[0].base.page); intel_context_put(ce); out: stream_destroy(stream); diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c index 004344dce140..0aba0cac51e1 100644 --- a/drivers/gpu/drm/radeon/radeon_ttm.c +++ b/drivers/gpu/drm/radeon/radeon_ttm.c @@ -1013,11 +1013,11 @@ static ssize_t radeon_ttm_gtt_read(struct file *f, char __user *buf,

page = rdev->gart.pages[p]; if (page) { - ptr = kmap(page); + ptr = kmap_thread(page); ptr += off;

r = copy_to_user(buf, ptr, cur_size); - kunmap(rdev->gart.pages[p]); + kunmap_thread(rdev->gart.pages[p]); } else r = clear_user(buf, cur_size);

From: Ira Weiny ira.weiny@intel.com

The kmap() calls in these drivers are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: "David S. Miller" davem@davemloft.net Cc: Jakub Kicinski kuba@kernel.org Cc: Jesse Brandeburg jesse.brandeburg@intel.com Signed-off-by: Ira Weiny ira.weiny@intel.com --- drivers/net/ethernet/intel/igb/igb_ethtool.c | 4 ++-- drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c index 6e8231c1ddf0..ac9189752012 100644 --- a/drivers/net/ethernet/intel/igb/igb_ethtool.c +++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c @@ -1794,14 +1794,14 @@ static int igb_check_lbtest_frame(struct igb_rx_buffer *rx_buffer,

frame_size >>= 1;

- data = kmap(rx_buffer->page); + data = kmap_thread(rx_buffer->page);

if (data[3] != 0xFF || data[frame_size + 10] != 0xBE || data[frame_size + 12] != 0xAF) match = false;

- kunmap(rx_buffer->page); + kunmap_thread(rx_buffer->page);

return match; } diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index 71ec908266a6..7d469425f8b4 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -1963,14 +1963,14 @@ static bool ixgbe_check_lbtest_frame(struct ixgbe_rx_buffer *rx_buffer,

frame_size >>= 1;

- data = kmap(rx_buffer->page) + rx_buffer->page_offset; + data = kmap_thread(rx_buffer->page) + rx_buffer->page_offset;

if (data[3] != 0xFF || data[frame_size + 10] != 0xBE || data[frame_size + 12] != 0xAF) match = false;

- kunmap(rx_buffer->page); + kunmap_thread(rx_buffer->page);

return match; }

From: Ira Weiny ira.weiny@intel.com

The kmap() calls in this FS are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: Chris Mason clm@fb.com Cc: Josef Bacik josef@toxicpanda.com Cc: David Sterba dsterba@suse.com Signed-off-by: Ira Weiny ira.weiny@intel.com --- fs/btrfs/check-integrity.c | 4 ++-- fs/btrfs/compression.c | 4 ++-- fs/btrfs/inode.c | 16 ++++++++-------- fs/btrfs/lzo.c | 24 ++++++++++++------------ fs/btrfs/raid56.c | 34 +++++++++++++++++----------------- fs/btrfs/reflink.c | 8 ++++---- fs/btrfs/send.c | 4 ++-- fs/btrfs/zlib.c | 32 ++++++++++++++++---------------- fs/btrfs/zstd.c | 20 ++++++++++---------- 9 files changed, 73 insertions(+), 73 deletions(-)

diff --git a/fs/btrfs/check-integrity.c b/fs/btrfs/check-integrity.c index 81a8c87a5afb..9e5a02512ab5 100644 --- a/fs/btrfs/check-integrity.c +++ b/fs/btrfs/check-integrity.c @@ -2706,7 +2706,7 @@ static void __btrfsic_submit_bio(struct bio *bio)

bio_for_each_segment(bvec, bio, iter) { BUG_ON(bvec.bv_len != PAGE_SIZE); - mapped_datav[i] = kmap(bvec.bv_page); + mapped_datav[i] = kmap_thread(bvec.bv_page); i++;

if (dev_state->state->print_mask & @@ -2720,7 +2720,7 @@ static void __btrfsic_submit_bio(struct bio *bio) bio, &bio_is_patched, bio->bi_opf); bio_for_each_segment(bvec, bio, iter) - kunmap(bvec.bv_page); + kunmap_thread(bvec.bv_page); kfree(mapped_datav); } else if (NULL != dev_state && (bio->bi_opf & REQ_PREFLUSH)) { if (dev_state->state->print_mask & diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 1ab56a734e70..5944fb36d68a 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -1626,7 +1626,7 @@ static void heuristic_collect_sample(struct inode *inode, u64 start, u64 end, curr_sample_pos = 0; while (index < index_end) { page = find_get_page(inode->i_mapping, index); - in_data = kmap(page); + in_data = kmap_thread(page); /* Handle case where the start is not aligned to PAGE_SIZE */ i = start % PAGE_SIZE; while (i < PAGE_SIZE - SAMPLING_READ_SIZE) { @@ -1639,7 +1639,7 @@ static void heuristic_collect_sample(struct inode *inode, u64 start, u64 end, start += SAMPLING_INTERVAL; curr_sample_pos += SAMPLING_READ_SIZE; } - kunmap(page); + kunmap_thread(page); put_page(page);

index++; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 9570458aa847..9710a52c6c42 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -4603,7 +4603,7 @@ int btrfs_truncate_block(struct inode *inode, loff_t from, loff_t len, if (offset != blocksize) { if (!len) len = blocksize - offset; - kaddr = kmap(page); + kaddr = kmap_thread(page); if (front) memset(kaddr + (block_start - page_offset(page)), 0, offset); @@ -4611,7 +4611,7 @@ int btrfs_truncate_block(struct inode *inode, loff_t from, loff_t len, memset(kaddr + (block_start - page_offset(page)) + offset, 0, len); flush_dcache_page(page); - kunmap(page); + kunmap_thread(page); } ClearPageChecked(page); set_page_dirty(page); @@ -6509,9 +6509,9 @@ static noinline int uncompress_inline(struct btrfs_path *path, */

if (max_size + pg_offset < PAGE_SIZE) { - char *map = kmap(page); + char *map = kmap_thread(page); memset(map + pg_offset + max_size, 0, PAGE_SIZE - max_size - pg_offset); - kunmap(page); + kunmap_thread(page); } kfree(tmp); return ret; @@ -6704,7 +6704,7 @@ struct extent_map *btrfs_get_extent(struct btrfs_inode *inode, goto out; } } else { - map = kmap(page); + map = kmap_thread(page); read_extent_buffer(leaf, map + pg_offset, ptr, copy_size); if (pg_offset + copy_size < PAGE_SIZE) { @@ -6712,7 +6712,7 @@ struct extent_map *btrfs_get_extent(struct btrfs_inode *inode, PAGE_SIZE - pg_offset - copy_size); } - kunmap(page); + kunmap_thread(page); } flush_dcache_page(page); } @@ -8326,10 +8326,10 @@ vm_fault_t btrfs_page_mkwrite(struct vm_fault *vmf) zero_start = PAGE_SIZE;

if (zero_start != PAGE_SIZE) { - kaddr = kmap(page); + kaddr = kmap_thread(page); memset(kaddr + zero_start, 0, PAGE_SIZE - zero_start); flush_dcache_page(page); - kunmap(page); + kunmap_thread(page); } ClearPageChecked(page); set_page_dirty(page); diff --git a/fs/btrfs/lzo.c b/fs/btrfs/lzo.c index aa9cd11f4b78..f29dcc9ec573 100644 --- a/fs/btrfs/lzo.c +++ b/fs/btrfs/lzo.c @@ -140,7 +140,7 @@ int lzo_compress_pages(struct list_head *ws, struct address_space *mapping, *total_in = 0;

in_page = find_get_page(mapping, start >> PAGE_SHIFT); - data_in = kmap(in_page); + data_in = kmap_thread(in_page);

/* * store the size of all chunks of compressed data in @@ -151,7 +151,7 @@ int lzo_compress_pages(struct list_head *ws, struct address_space *mapping, ret = -ENOMEM; goto out; } - cpage_out = kmap(out_page); + cpage_out = kmap_thread(out_page); out_offset = LZO_LEN; tot_out = LZO_LEN; pages[0] = out_page; @@ -209,7 +209,7 @@ int lzo_compress_pages(struct list_head *ws, struct address_space *mapping, if (out_len == 0 && tot_in >= len) break;

- kunmap(out_page); + kunmap_thread(out_page); if (nr_pages == nr_dest_pages) { out_page = NULL; ret = -E2BIG; @@ -221,7 +221,7 @@ int lzo_compress_pages(struct list_head *ws, struct address_space *mapping, ret = -ENOMEM; goto out; } - cpage_out = kmap(out_page); + cpage_out = kmap_thread(out_page); pages[nr_pages++] = out_page;

pg_bytes_left = PAGE_SIZE; @@ -243,12 +243,12 @@ int lzo_compress_pages(struct list_head *ws, struct address_space *mapping, break;

bytes_left = len - tot_in; - kunmap(in_page); + kunmap_thread(in_page); put_page(in_page);

start += PAGE_SIZE; in_page = find_get_page(mapping, start >> PAGE_SHIFT); - data_in = kmap(in_page); + data_in = kmap_thread(in_page); in_len = min(bytes_left, PAGE_SIZE); }

@@ -258,10 +258,10 @@ int lzo_compress_pages(struct list_head *ws, struct address_space *mapping, }

/* store the size of all chunks of compressed data */ - cpage_out = kmap(pages[0]); + cpage_out = kmap_thread(pages[0]); write_compress_length(cpage_out, tot_out);

- kunmap(pages[0]); + kunmap_thread(pages[0]);

ret = 0; *total_out = tot_out; @@ -269,10 +269,10 @@ int lzo_compress_pages(struct list_head *ws, struct address_space *mapping, out: *out_pages = nr_pages; if (out_page) - kunmap(out_page); + kunmap_thread(out_page);

if (in_page) { - kunmap(in_page); + kunmap_thread(in_page); put_page(in_page); }

@@ -305,7 +305,7 @@ int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb) u64 disk_start = cb->start; struct bio *orig_bio = cb->orig_bio;

- data_in = kmap(pages_in[0]); + data_in = kmap_thread(pages_in[0]); tot_len = read_compress_length(data_in); /* * Compressed data header check. @@ -387,7 +387,7 @@ int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb) else kunmap(pages_in[page_in_index]);

- data_in = kmap(pages_in[++page_in_index]); + data_in = kmap_thread(pages_in[++page_in_index]);

in_page_bytes_left = PAGE_SIZE; in_offset = 0; diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index 255490f42b5d..34e646e4548c 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -262,13 +262,13 @@ static void cache_rbio_pages(struct btrfs_raid_bio *rbio) if (!rbio->bio_pages[i]) continue;

- s = kmap(rbio->bio_pages[i]); - d = kmap(rbio->stripe_pages[i]); + s = kmap_thread(rbio->bio_pages[i]); + d = kmap_thread(rbio->stripe_pages[i]);

copy_page(d, s);

- kunmap(rbio->bio_pages[i]); - kunmap(rbio->stripe_pages[i]); + kunmap_thread(rbio->bio_pages[i]); + kunmap_thread(rbio->stripe_pages[i]); SetPageUptodate(rbio->stripe_pages[i]); } set_bit(RBIO_CACHE_READY_BIT, &rbio->flags); @@ -1241,13 +1241,13 @@ static noinline void finish_rmw(struct btrfs_raid_bio *rbio) /* first collect one page from each data stripe */ for (stripe = 0; stripe < nr_data; stripe++) { p = page_in_rbio(rbio, stripe, pagenr, 0); - pointers[stripe] = kmap(p); + pointers[stripe] = kmap_thread(p); }

/* then add the parity stripe */ p = rbio_pstripe_page(rbio, pagenr); SetPageUptodate(p); - pointers[stripe++] = kmap(p); + pointers[stripe++] = kmap_thread(p);

if (has_qstripe) {

@@ -1257,7 +1257,7 @@ static noinline void finish_rmw(struct btrfs_raid_bio *rbio) */ p = rbio_qstripe_page(rbio, pagenr); SetPageUptodate(p); - pointers[stripe++] = kmap(p); + pointers[stripe++] = kmap_thread(p);

raid6_call.gen_syndrome(rbio->real_stripes, PAGE_SIZE, pointers); @@ -1269,7 +1269,7 @@ static noinline void finish_rmw(struct btrfs_raid_bio *rbio)

for (stripe = 0; stripe < rbio->real_stripes; stripe++) - kunmap(page_in_rbio(rbio, stripe, pagenr, 0)); + kunmap_thread(page_in_rbio(rbio, stripe, pagenr, 0)); }

/* @@ -1835,7 +1835,7 @@ static void __raid_recover_end_io(struct btrfs_raid_bio *rbio) } else { page = rbio_stripe_page(rbio, stripe, pagenr); } - pointers[stripe] = kmap(page); + pointers[stripe] = kmap_thread(page); }

/* all raid6 handling here */ @@ -1940,7 +1940,7 @@ static void __raid_recover_end_io(struct btrfs_raid_bio *rbio) } else { page = rbio_stripe_page(rbio, stripe, pagenr); } - kunmap(page); + kunmap_thread(page); } }

@@ -2379,18 +2379,18 @@ static noinline void finish_parity_scrub(struct btrfs_raid_bio *rbio, /* first collect one page from each data stripe */ for (stripe = 0; stripe < nr_data; stripe++) { p = page_in_rbio(rbio, stripe, pagenr, 0); - pointers[stripe] = kmap(p); + pointers[stripe] = kmap_thread(p); }

/* then add the parity stripe */ - pointers[stripe++] = kmap(p_page); + pointers[stripe++] = kmap_thread(p_page);

if (has_qstripe) { /* * raid6, add the qstripe and call the * library function to fill in our p/q */ - pointers[stripe++] = kmap(q_page); + pointers[stripe++] = kmap_thread(q_page);

raid6_call.gen_syndrome(rbio->real_stripes, PAGE_SIZE, pointers); @@ -2402,17 +2402,17 @@ static noinline void finish_parity_scrub(struct btrfs_raid_bio *rbio,

/* Check scrubbing parity and repair it */ p = rbio_stripe_page(rbio, rbio->scrubp, pagenr); - parity = kmap(p); + parity = kmap_thread(p); if (memcmp(parity, pointers[rbio->scrubp], PAGE_SIZE)) copy_page(parity, pointers[rbio->scrubp]); else /* Parity is right, needn't writeback */ bitmap_clear(rbio->dbitmap, pagenr, 1); - kunmap(p); + kunmap_thread(p);

for (stripe = 0; stripe < nr_data; stripe++) - kunmap(page_in_rbio(rbio, stripe, pagenr, 0)); - kunmap(p_page); + kunmap_thread(page_in_rbio(rbio, stripe, pagenr, 0)); + kunmap_thread(p_page); }

__free_page(p_page); diff --git a/fs/btrfs/reflink.c b/fs/btrfs/reflink.c index 5cd02514cf4d..10e53d7eba8c 100644 --- a/fs/btrfs/reflink.c +++ b/fs/btrfs/reflink.c @@ -92,10 +92,10 @@ static int copy_inline_to_page(struct inode *inode, if (comp_type == BTRFS_COMPRESS_NONE) { char *map;

- map = kmap(page); + map = kmap_thread(page); memcpy(map, data_start, datal); flush_dcache_page(page); - kunmap(page); + kunmap_thread(page); } else { ret = btrfs_decompress(comp_type, data_start, page, 0, inline_size, datal); @@ -119,10 +119,10 @@ static int copy_inline_to_page(struct inode *inode, if (datal < block_size) { char *map;

- map = kmap(page); + map = kmap_thread(page); memset(map + datal, 0, block_size - datal); flush_dcache_page(page); - kunmap(page); + kunmap_thread(page); }

SetPageUptodate(page); diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index d9813a5b075a..06c383d3dc43 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -4863,9 +4863,9 @@ static ssize_t fill_read_buf(struct send_ctx *sctx, u64 offset, u32 len) } }

- addr = kmap(page); + addr = kmap_thread(page); memcpy(sctx->read_buf + ret, addr + pg_offset, cur_len); - kunmap(page); + kunmap_thread(page); unlock_page(page); put_page(page); index++; diff --git a/fs/btrfs/zlib.c b/fs/btrfs/zlib.c index 05615a1099db..45b7a907bab3 100644 --- a/fs/btrfs/zlib.c +++ b/fs/btrfs/zlib.c @@ -126,7 +126,7 @@ int zlib_compress_pages(struct list_head *ws, struct address_space *mapping, ret = -ENOMEM; goto out; } - cpage_out = kmap(out_page); + cpage_out = kmap_thread(out_page); pages[0] = out_page; nr_pages = 1;

@@ -149,12 +149,12 @@ int zlib_compress_pages(struct list_head *ws, struct address_space *mapping,

for (i = 0; i < in_buf_pages; i++) { if (in_page) { - kunmap(in_page); + kunmap_thread(in_page); put_page(in_page); } in_page = find_get_page(mapping, start >> PAGE_SHIFT); - data_in = kmap(in_page); + data_in = kmap_thread(in_page); memcpy(workspace->buf + i * PAGE_SIZE, data_in, PAGE_SIZE); start += PAGE_SIZE; @@ -162,12 +162,12 @@ int zlib_compress_pages(struct list_head *ws, struct address_space *mapping, workspace->strm.next_in = workspace->buf; } else { if (in_page) { - kunmap(in_page); + kunmap_thread(in_page); put_page(in_page); } in_page = find_get_page(mapping, start >> PAGE_SHIFT); - data_in = kmap(in_page); + data_in = kmap_thread(in_page); start += PAGE_SIZE; workspace->strm.next_in = data_in; } @@ -196,7 +196,7 @@ int zlib_compress_pages(struct list_head *ws, struct address_space *mapping, * the stream end if required */ if (workspace->strm.avail_out == 0) { - kunmap(out_page); + kunmap_thread(out_page); if (nr_pages == nr_dest_pages) { out_page = NULL; ret = -E2BIG; @@ -207,7 +207,7 @@ int zlib_compress_pages(struct list_head *ws, struct address_space *mapping, ret = -ENOMEM; goto out; } - cpage_out = kmap(out_page); + cpage_out = kmap_thread(out_page); pages[nr_pages] = out_page; nr_pages++; workspace->strm.avail_out = PAGE_SIZE; @@ -234,7 +234,7 @@ int zlib_compress_pages(struct list_head *ws, struct address_space *mapping, goto out; } else if (workspace->strm.avail_out == 0) { /* get another page for the stream end */ - kunmap(out_page); + kunmap_thread(out_page); if (nr_pages == nr_dest_pages) { out_page = NULL; ret = -E2BIG; @@ -245,7 +245,7 @@ int zlib_compress_pages(struct list_head *ws, struct address_space *mapping, ret = -ENOMEM; goto out; } - cpage_out = kmap(out_page); + cpage_out = kmap_thread(out_page); pages[nr_pages] = out_page; nr_pages++; workspace->strm.avail_out = PAGE_SIZE; @@ -265,10 +265,10 @@ int zlib_compress_pages(struct list_head *ws, struct address_space *mapping, out: *out_pages = nr_pages; if (out_page) - kunmap(out_page); + kunmap_thread(out_page);

if (in_page) { - kunmap(in_page); + kunmap_thread(in_page); put_page(in_page); } return ret; @@ -289,7 +289,7 @@ int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb) u64 disk_start = cb->start; struct bio *orig_bio = cb->orig_bio;

- data_in = kmap(pages_in[page_in_index]); + data_in = kmap_thread(pages_in[page_in_index]); workspace->strm.next_in = data_in; workspace->strm.avail_in = min_t(size_t, srclen, PAGE_SIZE); workspace->strm.total_in = 0; @@ -311,7 +311,7 @@ int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb)

if (Z_OK != zlib_inflateInit2(&workspace->strm, wbits)) { pr_warn("BTRFS: inflateInit failed\n"); - kunmap(pages_in[page_in_index]); + kunmap_thread(pages_in[page_in_index]); return -EIO; } while (workspace->strm.total_in < srclen) { @@ -339,13 +339,13 @@ int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb)

if (workspace->strm.avail_in == 0) { unsigned long tmp; - kunmap(pages_in[page_in_index]); + kunmap_thread(pages_in[page_in_index]); page_in_index++; if (page_in_index >= total_pages_in) { data_in = NULL; break; } - data_in = kmap(pages_in[page_in_index]); + data_in = kmap_thread(pages_in[page_in_index]); workspace->strm.next_in = data_in; tmp = srclen - workspace->strm.total_in; workspace->strm.avail_in = min(tmp, @@ -359,7 +359,7 @@ int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb) done: zlib_inflateEnd(&workspace->strm); if (data_in) - kunmap(pages_in[page_in_index]); + kunmap_thread(pages_in[page_in_index]); if (!ret) zero_fill_bio(orig_bio); return ret; diff --git a/fs/btrfs/zstd.c b/fs/btrfs/zstd.c index 9a4871636c6c..48e03f6dcef7 100644 --- a/fs/btrfs/zstd.c +++ b/fs/btrfs/zstd.c @@ -399,7 +399,7 @@ int zstd_compress_pages(struct list_head *ws, struct address_space *mapping,

/* map in the first page of input data */ in_page = find_get_page(mapping, start >> PAGE_SHIFT); - workspace->in_buf.src = kmap(in_page); + workspace->in_buf.src = kmap_thread(in_page); workspace->in_buf.pos = 0; workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE);

@@ -411,7 +411,7 @@ int zstd_compress_pages(struct list_head *ws, struct address_space *mapping, goto out; } pages[nr_pages++] = out_page; - workspace->out_buf.dst = kmap(out_page); + workspace->out_buf.dst = kmap_thread(out_page); workspace->out_buf.pos = 0; workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE);

@@ -446,7 +446,7 @@ int zstd_compress_pages(struct list_head *ws, struct address_space *mapping, if (workspace->out_buf.pos == workspace->out_buf.size) { tot_out += PAGE_SIZE; max_out -= PAGE_SIZE; - kunmap(out_page); + kunmap_thread(out_page); if (nr_pages == nr_dest_pages) { out_page = NULL; ret = -E2BIG; @@ -458,7 +458,7 @@ int zstd_compress_pages(struct list_head *ws, struct address_space *mapping, goto out; } pages[nr_pages++] = out_page; - workspace->out_buf.dst = kmap(out_page); + workspace->out_buf.dst = kmap_thread(out_page); workspace->out_buf.pos = 0; workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE); @@ -479,7 +479,7 @@ int zstd_compress_pages(struct list_head *ws, struct address_space *mapping, start += PAGE_SIZE; len -= PAGE_SIZE; in_page = find_get_page(mapping, start >> PAGE_SHIFT); - workspace->in_buf.src = kmap(in_page); + workspace->in_buf.src = kmap_thread(in_page); workspace->in_buf.pos = 0; workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE); } @@ -518,7 +518,7 @@ int zstd_compress_pages(struct list_head *ws, struct address_space *mapping, goto out; } pages[nr_pages++] = out_page; - workspace->out_buf.dst = kmap(out_page); + workspace->out_buf.dst = kmap_thread(out_page); workspace->out_buf.pos = 0; workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE); } @@ -565,7 +565,7 @@ int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb) goto done; }

- workspace->in_buf.src = kmap(pages_in[page_in_index]); + workspace->in_buf.src = kmap_thread(pages_in[page_in_index]); workspace->in_buf.pos = 0; workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE);

@@ -601,14 +601,14 @@ int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb) break;

if (workspace->in_buf.pos == workspace->in_buf.size) { - kunmap(pages_in[page_in_index++]); + kunmap_thread(pages_in[page_in_index++]); if (page_in_index >= total_pages_in) { workspace->in_buf.src = NULL; ret = -EIO; goto done; } srclen -= PAGE_SIZE; - workspace->in_buf.src = kmap(pages_in[page_in_index]); + workspace->in_buf.src = kmap_thread(pages_in[page_in_index]); workspace->in_buf.pos = 0; workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE); } @@ -617,7 +617,7 @@ int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb) zero_fill_bio(orig_bio); done: if (workspace->in_buf.src) - kunmap(pages_in[page_in_index]); + kunmap_thread(pages_in[page_in_index]); return ret; }

From: Ira Weiny ira.weiny@intel.com

The kmap() calls in this FS are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: Herbert Xu herbert@gondor.apana.org.au Cc: Eric Biggers ebiggers@google.com Cc: Aditya Pakki pakki001@umn.edu Signed-off-by: Ira Weiny ira.weiny@intel.com --- fs/ecryptfs/crypto.c | 8 ++++---- fs/ecryptfs/read_write.c | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/fs/ecryptfs/crypto.c b/fs/ecryptfs/crypto.c index 0681540c48d9..e73e00994bee 100644 --- a/fs/ecryptfs/crypto.c +++ b/fs/ecryptfs/crypto.c @@ -469,10 +469,10 @@ int ecryptfs_encrypt_page(struct page *page) }

lower_offset = lower_offset_for_page(crypt_stat, page); - enc_extent_virt = kmap(enc_extent_page); + enc_extent_virt = kmap_thread(enc_extent_page); rc = ecryptfs_write_lower(ecryptfs_inode, enc_extent_virt, lower_offset, PAGE_SIZE); - kunmap(enc_extent_page); + kunmap_thread(enc_extent_page); if (rc < 0) { ecryptfs_printk(KERN_ERR, "Error attempting to write lower page; rc = [%d]\n", @@ -518,10 +518,10 @@ int ecryptfs_decrypt_page(struct page *page) BUG_ON(!(crypt_stat->flags & ECRYPTFS_ENCRYPTED));

lower_offset = lower_offset_for_page(crypt_stat, page); - page_virt = kmap(page); + page_virt = kmap_thread(page); rc = ecryptfs_read_lower(page_virt, lower_offset, PAGE_SIZE, ecryptfs_inode); - kunmap(page); + kunmap_thread(page); if (rc < 0) { ecryptfs_printk(KERN_ERR, "Error attempting to read lower page; rc = [%d]\n", diff --git a/fs/ecryptfs/read_write.c b/fs/ecryptfs/read_write.c index 0438997ac9d8..5eca4330c0c0 100644 --- a/fs/ecryptfs/read_write.c +++ b/fs/ecryptfs/read_write.c @@ -64,11 +64,11 @@ int ecryptfs_write_lower_page_segment(struct inode *ecryptfs_inode,

offset = ((((loff_t)page_for_lower->index) << PAGE_SHIFT) + offset_in_page); - virt = kmap(page_for_lower); + virt = kmap_thread(page_for_lower); rc = ecryptfs_write_lower(ecryptfs_inode, virt, offset, size); if (rc > 0) rc = 0; - kunmap(page_for_lower); + kunmap_thread(page_for_lower); return rc; }

@@ -251,11 +251,11 @@ int ecryptfs_read_lower_page_segment(struct page *page_for_ecryptfs, int rc;

offset = ((((loff_t)page_index) << PAGE_SHIFT) + offset_in_page); - virt = kmap(page_for_ecryptfs); + virt = kmap_thread(page_for_ecryptfs); rc = ecryptfs_read_lower(virt, offset, size, ecryptfs_inode); if (rc > 0) rc = 0; - kunmap(page_for_ecryptfs); + kunmap_thread(page_for_ecryptfs); flush_dcache_page(page_for_ecryptfs); return rc; }

From: Ira Weiny ira.weiny@intel.com

The kmap() calls in this FS are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: Ryusuke Konishi konishi.ryusuke@gmail.com Signed-off-by: Ira Weiny ira.weiny@intel.com --- fs/nilfs2/alloc.c | 34 +++++++++++++++++----------------- fs/nilfs2/cpfile.c | 4 ++-- 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/fs/nilfs2/alloc.c b/fs/nilfs2/alloc.c index adf3bb0a8048..2aa4c34094ef 100644 --- a/fs/nilfs2/alloc.c +++ b/fs/nilfs2/alloc.c @@ -524,7 +524,7 @@ int nilfs_palloc_prepare_alloc_entry(struct inode *inode, ret = nilfs_palloc_get_desc_block(inode, group, 1, &desc_bh); if (ret < 0) return ret; - desc_kaddr = kmap(desc_bh->b_page); + desc_kaddr = kmap_thread(desc_bh->b_page); desc = nilfs_palloc_block_get_group_desc( inode, group, desc_bh, desc_kaddr); n = nilfs_palloc_rest_groups_in_desc_block(inode, group, @@ -536,7 +536,7 @@ int nilfs_palloc_prepare_alloc_entry(struct inode *inode, inode, group, 1, &bitmap_bh); if (ret < 0) goto out_desc; - bitmap_kaddr = kmap(bitmap_bh->b_page); + bitmap_kaddr = kmap_thread(bitmap_bh->b_page); bitmap = bitmap_kaddr + bh_offset(bitmap_bh); pos = nilfs_palloc_find_available_slot( bitmap, group_offset, @@ -547,21 +547,21 @@ int nilfs_palloc_prepare_alloc_entry(struct inode *inode, desc, lock, -1); req->pr_entry_nr = entries_per_group * group + pos; - kunmap(desc_bh->b_page); - kunmap(bitmap_bh->b_page); + kunmap_thread(desc_bh->b_page); + kunmap_thread(bitmap_bh->b_page);

req->pr_desc_bh = desc_bh; req->pr_bitmap_bh = bitmap_bh; return 0; } - kunmap(bitmap_bh->b_page); + kunmap_thread(bitmap_bh->b_page); brelse(bitmap_bh); }

group_offset = 0; }

- kunmap(desc_bh->b_page); + kunmap_thread(desc_bh->b_page); brelse(desc_bh); }

@@ -569,7 +569,7 @@ int nilfs_palloc_prepare_alloc_entry(struct inode *inode, return -ENOSPC;

out_desc: - kunmap(desc_bh->b_page); + kunmap_thread(desc_bh->b_page); brelse(desc_bh); return ret; } @@ -605,10 +605,10 @@ void nilfs_palloc_commit_free_entry(struct inode *inode, spinlock_t *lock;

group = nilfs_palloc_group(inode, req->pr_entry_nr, &group_offset); - desc_kaddr = kmap(req->pr_desc_bh->b_page); + desc_kaddr = kmap_thread(req->pr_desc_bh->b_page); desc = nilfs_palloc_block_get_group_desc(inode, group, req->pr_desc_bh, desc_kaddr); - bitmap_kaddr = kmap(req->pr_bitmap_bh->b_page); + bitmap_kaddr = kmap_thread(req->pr_bitmap_bh->b_page); bitmap = bitmap_kaddr + bh_offset(req->pr_bitmap_bh); lock = nilfs_mdt_bgl_lock(inode, group);

@@ -620,8 +620,8 @@ void nilfs_palloc_commit_free_entry(struct inode *inode, else nilfs_palloc_group_desc_add_entries(desc, lock, 1);

- kunmap(req->pr_bitmap_bh->b_page); - kunmap(req->pr_desc_bh->b_page); + kunmap_thread(req->pr_bitmap_bh->b_page); + kunmap_thread(req->pr_desc_bh->b_page);

mark_buffer_dirty(req->pr_desc_bh); mark_buffer_dirty(req->pr_bitmap_bh); @@ -646,10 +646,10 @@ void nilfs_palloc_abort_alloc_entry(struct inode *inode, spinlock_t *lock;

group = nilfs_palloc_group(inode, req->pr_entry_nr, &group_offset); - desc_kaddr = kmap(req->pr_desc_bh->b_page); + desc_kaddr = kmap_thread(req->pr_desc_bh->b_page); desc = nilfs_palloc_block_get_group_desc(inode, group, req->pr_desc_bh, desc_kaddr); - bitmap_kaddr = kmap(req->pr_bitmap_bh->b_page); + bitmap_kaddr = kmap_thread(req->pr_bitmap_bh->b_page); bitmap = bitmap_kaddr + bh_offset(req->pr_bitmap_bh); lock = nilfs_mdt_bgl_lock(inode, group);

@@ -661,8 +661,8 @@ void nilfs_palloc_abort_alloc_entry(struct inode *inode, else nilfs_palloc_group_desc_add_entries(desc, lock, 1);

- kunmap(req->pr_bitmap_bh->b_page); - kunmap(req->pr_desc_bh->b_page); + kunmap_thread(req->pr_bitmap_bh->b_page); + kunmap_thread(req->pr_desc_bh->b_page);

brelse(req->pr_bitmap_bh); brelse(req->pr_desc_bh); @@ -754,7 +754,7 @@ int nilfs_palloc_freev(struct inode *inode, __u64 *entry_nrs, size_t nitems) /* Get the first entry number of the group */ group_min_nr = (__u64)group * epg;

- bitmap_kaddr = kmap(bitmap_bh->b_page); + bitmap_kaddr = kmap_thread(bitmap_bh->b_page); bitmap = bitmap_kaddr + bh_offset(bitmap_bh); lock = nilfs_mdt_bgl_lock(inode, group);

@@ -800,7 +800,7 @@ int nilfs_palloc_freev(struct inode *inode, __u64 *entry_nrs, size_t nitems) entry_start = rounddown(group_offset, epb); } while (true);

- kunmap(bitmap_bh->b_page); + kunmap_thread(bitmap_bh->b_page); mark_buffer_dirty(bitmap_bh); brelse(bitmap_bh);

diff --git a/fs/nilfs2/cpfile.c b/fs/nilfs2/cpfile.c index 86d4d850d130..402ab8bfce29 100644 --- a/fs/nilfs2/cpfile.c +++ b/fs/nilfs2/cpfile.c @@ -235,11 +235,11 @@ int nilfs_cpfile_get_checkpoint(struct inode *cpfile, ret = nilfs_cpfile_get_checkpoint_block(cpfile, cno, create, &cp_bh); if (ret < 0) goto out_header; - kaddr = kmap(cp_bh->b_page); + kaddr = kmap_thread(cp_bh->b_page); cp = nilfs_cpfile_block_get_checkpoint(cpfile, cno, cp_bh, kaddr); if (nilfs_checkpoint_invalid(cp)) { if (!create) { - kunmap(cp_bh->b_page); + kunmap_thread(cp_bh->b_page); brelse(cp_bh); ret = -ENOENT; goto out_header;

From: Ira Weiny ira.weiny@intel.com

The kmap() calls in this FS are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Signed-off-by: Ira Weiny ira.weiny@intel.com --- fs/hfsplus/bitmap.c | 20 ++++----- fs/hfsplus/bnode.c | 102 ++++++++++++++++++++++---------------------- fs/hfsplus/btree.c | 18 ++++---- 3 files changed, 70 insertions(+), 70 deletions(-)

diff --git a/fs/hfsplus/bitmap.c b/fs/hfsplus/bitmap.c index cebce0cfe340..9ec7c1559a0c 100644 --- a/fs/hfsplus/bitmap.c +++ b/fs/hfsplus/bitmap.c @@ -39,7 +39,7 @@ int hfsplus_block_allocate(struct super_block *sb, u32 size, start = size; goto out; } - pptr = kmap(page); + pptr = kmap_thread(page); curr = pptr + (offset & (PAGE_CACHE_BITS - 1)) / 32; i = offset % 32; offset &= ~(PAGE_CACHE_BITS - 1); @@ -74,7 +74,7 @@ int hfsplus_block_allocate(struct super_block *sb, u32 size, } curr++; } - kunmap(page); + kunmap_thread(page); offset += PAGE_CACHE_BITS; if (offset >= size) break; @@ -84,7 +84,7 @@ int hfsplus_block_allocate(struct super_block *sb, u32 size, start = size; goto out; } - curr = pptr = kmap(page); + curr = pptr = kmap_thread(page); if ((size ^ offset) / PAGE_CACHE_BITS) end = pptr + PAGE_CACHE_BITS / 32; else @@ -127,7 +127,7 @@ int hfsplus_block_allocate(struct super_block *sb, u32 size, len -= 32; } set_page_dirty(page); - kunmap(page); + kunmap_thread(page); offset += PAGE_CACHE_BITS; page = read_mapping_page(mapping, offset / PAGE_CACHE_BITS, NULL); @@ -135,7 +135,7 @@ int hfsplus_block_allocate(struct super_block *sb, u32 size, start = size; goto out; } - pptr = kmap(page); + pptr = kmap_thread(page); curr = pptr; end = pptr + PAGE_CACHE_BITS / 32; } @@ -151,7 +151,7 @@ int hfsplus_block_allocate(struct super_block *sb, u32 size, done: *curr = cpu_to_be32(n); set_page_dirty(page); - kunmap(page); + kunmap_thread(page); *max = offset + (curr - pptr) * 32 + i - start; sbi->free_blocks -= *max; hfsplus_mark_mdb_dirty(sb); @@ -185,7 +185,7 @@ int hfsplus_block_free(struct super_block *sb, u32 offset, u32 count) page = read_mapping_page(mapping, pnr, NULL); if (IS_ERR(page)) goto kaboom; - pptr = kmap(page); + pptr = kmap_thread(page); curr = pptr + (offset & (PAGE_CACHE_BITS - 1)) / 32; end = pptr + PAGE_CACHE_BITS / 32; len = count; @@ -215,11 +215,11 @@ int hfsplus_block_free(struct super_block *sb, u32 offset, u32 count) if (!count) break; set_page_dirty(page); - kunmap(page); + kunmap_thread(page); page = read_mapping_page(mapping, ++pnr, NULL); if (IS_ERR(page)) goto kaboom; - pptr = kmap(page); + pptr = kmap_thread(page); curr = pptr; end = pptr + PAGE_CACHE_BITS / 32; } @@ -231,7 +231,7 @@ int hfsplus_block_free(struct super_block *sb, u32 offset, u32 count) } out: set_page_dirty(page); - kunmap(page); + kunmap_thread(page); sbi->free_blocks += len; hfsplus_mark_mdb_dirty(sb); mutex_unlock(&sbi->alloc_mutex); diff --git a/fs/hfsplus/bnode.c b/fs/hfsplus/bnode.c index 177fae4e6581..62757d92fbbd 100644 --- a/fs/hfsplus/bnode.c +++ b/fs/hfsplus/bnode.c @@ -29,14 +29,14 @@ void hfs_bnode_read(struct hfs_bnode *node, void *buf, int off, int len) off &= ~PAGE_MASK;

l = min_t(int, len, PAGE_SIZE - off); - memcpy(buf, kmap(*pagep) + off, l); - kunmap(*pagep); + memcpy(buf, kmap_thread(*pagep) + off, l); + kunmap_thread(*pagep);

while ((len -= l) != 0) { buf += l; l = min_t(int, len, PAGE_SIZE); - memcpy(buf, kmap(*++pagep), l); - kunmap(*pagep); + memcpy(buf, kmap_thread(*++pagep), l); + kunmap_thread(*pagep); } }

@@ -82,16 +82,16 @@ void hfs_bnode_write(struct hfs_bnode *node, void *buf, int off, int len) off &= ~PAGE_MASK;

l = min_t(int, len, PAGE_SIZE - off); - memcpy(kmap(*pagep) + off, buf, l); + memcpy(kmap_thread(*pagep) + off, buf, l); set_page_dirty(*pagep); - kunmap(*pagep); + kunmap_thread(*pagep);

while ((len -= l) != 0) { buf += l; l = min_t(int, len, PAGE_SIZE); - memcpy(kmap(*++pagep), buf, l); + memcpy(kmap_thread(*++pagep), buf, l); set_page_dirty(*pagep); - kunmap(*pagep); + kunmap_thread(*pagep); } }

@@ -112,15 +112,15 @@ void hfs_bnode_clear(struct hfs_bnode *node, int off, int len) off &= ~PAGE_MASK;

l = min_t(int, len, PAGE_SIZE - off); - memset(kmap(*pagep) + off, 0, l); + memset(kmap_thread(*pagep) + off, 0, l); set_page_dirty(*pagep); - kunmap(*pagep); + kunmap_thread(*pagep);

while ((len -= l) != 0) { l = min_t(int, len, PAGE_SIZE); - memset(kmap(*++pagep), 0, l); + memset(kmap_thread(*++pagep), 0, l); set_page_dirty(*pagep); - kunmap(*pagep); + kunmap_thread(*pagep); } }

@@ -142,24 +142,24 @@ void hfs_bnode_copy(struct hfs_bnode *dst_node, int dst,

if (src == dst) { l = min_t(int, len, PAGE_SIZE - src); - memcpy(kmap(*dst_page) + src, kmap(*src_page) + src, l); - kunmap(*src_page); + memcpy(kmap_thread(*dst_page) + src, kmap_thread(*src_page) + src, l); + kunmap_thread(*src_page); set_page_dirty(*dst_page); - kunmap(*dst_page); + kunmap_thread(*dst_page);

while ((len -= l) != 0) { l = min_t(int, len, PAGE_SIZE); - memcpy(kmap(*++dst_page), kmap(*++src_page), l); - kunmap(*src_page); + memcpy(kmap_thread(*++dst_page), kmap_thread(*++src_page), l); + kunmap_thread(*src_page); set_page_dirty(*dst_page); - kunmap(*dst_page); + kunmap_thread(*dst_page); } } else { void *src_ptr, *dst_ptr;

do { - src_ptr = kmap(*src_page) + src; - dst_ptr = kmap(*dst_page) + dst; + src_ptr = kmap_thread(*src_page) + src; + dst_ptr = kmap_thread(*dst_page) + dst; if (PAGE_SIZE - src < PAGE_SIZE - dst) { l = PAGE_SIZE - src; src = 0; @@ -171,9 +171,9 @@ void hfs_bnode_copy(struct hfs_bnode *dst_node, int dst, } l = min(len, l); memcpy(dst_ptr, src_ptr, l); - kunmap(*src_page); + kunmap_thread(*src_page); set_page_dirty(*dst_page); - kunmap(*dst_page); + kunmap_thread(*dst_page); if (!dst) dst_page++; else @@ -202,27 +202,27 @@ void hfs_bnode_move(struct hfs_bnode *node, int dst, int src, int len)

if (src == dst) { while (src < len) { - memmove(kmap(*dst_page), kmap(*src_page), src); - kunmap(*src_page); + memmove(kmap_thread(*dst_page), kmap_thread(*src_page), src); + kunmap_thread(*src_page); set_page_dirty(*dst_page); - kunmap(*dst_page); + kunmap_thread(*dst_page); len -= src; src = PAGE_SIZE; src_page--; dst_page--; } src -= len; - memmove(kmap(*dst_page) + src, - kmap(*src_page) + src, len); - kunmap(*src_page); + memmove(kmap_thread(*dst_page) + src, + kmap_thread(*src_page) + src, len); + kunmap_thread(*src_page); set_page_dirty(*dst_page); - kunmap(*dst_page); + kunmap_thread(*dst_page); } else { void *src_ptr, *dst_ptr;

do { - src_ptr = kmap(*src_page) + src; - dst_ptr = kmap(*dst_page) + dst; + src_ptr = kmap_thread(*src_page) + src; + dst_ptr = kmap_thread(*dst_page) + dst; if (src < dst) { l = src; src = PAGE_SIZE; @@ -234,9 +234,9 @@ void hfs_bnode_move(struct hfs_bnode *node, int dst, int src, int len) } l = min(len, l); memmove(dst_ptr - l, src_ptr - l, l); - kunmap(*src_page); + kunmap_thread(*src_page); set_page_dirty(*dst_page); - kunmap(*dst_page); + kunmap_thread(*dst_page); if (dst == PAGE_SIZE) dst_page--; else @@ -251,26 +251,26 @@ void hfs_bnode_move(struct hfs_bnode *node, int dst, int src, int len)

if (src == dst) { l = min_t(int, len, PAGE_SIZE - src); - memmove(kmap(*dst_page) + src, - kmap(*src_page) + src, l); - kunmap(*src_page); + memmove(kmap_thread(*dst_page) + src, + kmap_thread(*src_page) + src, l); + kunmap_thread(*src_page); set_page_dirty(*dst_page); - kunmap(*dst_page); + kunmap_thread(*dst_page);

while ((len -= l) != 0) { l = min_t(int, len, PAGE_SIZE); - memmove(kmap(*++dst_page), - kmap(*++src_page), l); - kunmap(*src_page); + memmove(kmap_thread(*++dst_page), + kmap_thread(*++src_page), l); + kunmap_thread(*src_page); set_page_dirty(*dst_page); - kunmap(*dst_page); + kunmap_thread(*dst_page); } } else { void *src_ptr, *dst_ptr;

do { - src_ptr = kmap(*src_page) + src; - dst_ptr = kmap(*dst_page) + dst; + src_ptr = kmap_thread(*src_page) + src; + dst_ptr = kmap_thread(*dst_page) + dst; if (PAGE_SIZE - src < PAGE_SIZE - dst) { l = PAGE_SIZE - src; @@ -283,9 +283,9 @@ void hfs_bnode_move(struct hfs_bnode *node, int dst, int src, int len) } l = min(len, l); memmove(dst_ptr, src_ptr, l); - kunmap(*src_page); + kunmap_thread(*src_page); set_page_dirty(*dst_page); - kunmap(*dst_page); + kunmap_thread(*dst_page); if (!dst) dst_page++; else @@ -502,14 +502,14 @@ struct hfs_bnode *hfs_bnode_find(struct hfs_btree *tree, u32 num) if (!test_bit(HFS_BNODE_NEW, &node->flags)) return node;

- desc = (struct hfs_bnode_desc *)(kmap(node->page[0]) + + desc = (struct hfs_bnode_desc *)(kmap_thread(node->page[0]) + node->page_offset); node->prev = be32_to_cpu(desc->prev); node->next = be32_to_cpu(desc->next); node->num_recs = be16_to_cpu(desc->num_recs); node->type = desc->type; node->height = desc->height; - kunmap(node->page[0]); + kunmap_thread(node->page[0]);

switch (node->type) { case HFS_NODE_HEADER: @@ -593,14 +593,14 @@ struct hfs_bnode *hfs_bnode_create(struct hfs_btree *tree, u32 num) }

pagep = node->page; - memset(kmap(*pagep) + node->page_offset, 0, + memset(kmap_thread(*pagep) + node->page_offset, 0, min_t(int, PAGE_SIZE, tree->node_size)); set_page_dirty(*pagep); - kunmap(*pagep); + kunmap_thread(*pagep); for (i = 1; i < tree->pages_per_bnode; i++) { - memset(kmap(*++pagep), 0, PAGE_SIZE); + memset(kmap_thread(*++pagep), 0, PAGE_SIZE); set_page_dirty(*pagep); - kunmap(*pagep); + kunmap_thread(*pagep); } clear_bit(HFS_BNODE_NEW, &node->flags); wake_up(&node->lock_wq); diff --git a/fs/hfsplus/btree.c b/fs/hfsplus/btree.c index 66774f4cb4fd..74fcef3a1628 100644 --- a/fs/hfsplus/btree.c +++ b/fs/hfsplus/btree.c @@ -394,7 +394,7 @@ struct hfs_bnode *hfs_bmap_alloc(struct hfs_btree *tree)

off += node->page_offset; pagep = node->page + (off >> PAGE_SHIFT); - data = kmap(*pagep); + data = kmap_thread(*pagep); off &= ~PAGE_MASK; idx = 0;

@@ -407,7 +407,7 @@ struct hfs_bnode *hfs_bmap_alloc(struct hfs_btree *tree) idx += i; data[off] |= m; set_page_dirty(*pagep); - kunmap(*pagep); + kunmap_thread(*pagep); tree->free_nodes--; mark_inode_dirty(tree->inode); hfs_bnode_put(node); @@ -417,14 +417,14 @@ struct hfs_bnode *hfs_bmap_alloc(struct hfs_btree *tree) } } if (++off >= PAGE_SIZE) { - kunmap(*pagep); - data = kmap(*++pagep); + kunmap_thread(*pagep); + data = kmap_thread(*++pagep); off = 0; } idx += 8; len--; } - kunmap(*pagep); + kunmap_thread(*pagep); nidx = node->next; if (!nidx) { hfs_dbg(BNODE_MOD, "create new bmap node\n"); @@ -440,7 +440,7 @@ struct hfs_bnode *hfs_bmap_alloc(struct hfs_btree *tree) off = off16; off += node->page_offset; pagep = node->page + (off >> PAGE_SHIFT); - data = kmap(*pagep); + data = kmap_thread(*pagep); off &= ~PAGE_MASK; } } @@ -490,7 +490,7 @@ void hfs_bmap_free(struct hfs_bnode *node) } off += node->page_offset + nidx / 8; page = node->page[off >> PAGE_SHIFT]; - data = kmap(page); + data = kmap_thread(page); off &= ~PAGE_MASK; m = 1 << (~nidx & 7); byte = data[off]; @@ -498,13 +498,13 @@ void hfs_bmap_free(struct hfs_bnode *node) pr_crit("trying to free free bnode " "%u(%d)\n", node->this, node->type); - kunmap(page); + kunmap_thread(page); hfs_bnode_put(node); return; } data[off] = byte & ~m; set_page_dirty(page); - kunmap(page); + kunmap_thread(page); hfs_bnode_put(node); tree->free_nodes++; mark_inode_dirty(tree->inode);

From: Ira Weiny ira.weiny@intel.com

The kmap() calls in this FS are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: Trond Myklebust trond.myklebust@hammerspace.com Cc: Anna Schumaker anna.schumaker@netapp.com Signed-off-by: Ira Weiny ira.weiny@intel.com --- fs/nfs/dir.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index cb52db9a0cfb..fee321acccb4 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -213,7 +213,7 @@ int nfs_readdir_make_qstr(struct qstr *string, const char *name, unsigned int le static int nfs_readdir_add_to_array(struct nfs_entry *entry, struct page *page) { - struct nfs_cache_array *array = kmap(page); + struct nfs_cache_array *array = kmap_thread(page); struct nfs_cache_array_entry *cache_entry; int ret;

@@ -235,7 +235,7 @@ int nfs_readdir_add_to_array(struct nfs_entry *entry, struct page *page) if (entry->eof != 0) array->eof_index = array->size; out: - kunmap(page); + kunmap_thread(page); return ret; }

@@ -347,7 +347,7 @@ int nfs_readdir_search_array(nfs_readdir_descriptor_t *desc) struct nfs_cache_array *array; int status;

- array = kmap(desc->page); + array = kmap_thread(desc->page);

if (*desc->dir_cookie == 0) status = nfs_readdir_search_for_pos(array, desc); @@ -359,7 +359,7 @@ int nfs_readdir_search_array(nfs_readdir_descriptor_t *desc) desc->current_index += array->size; desc->page_index++; } - kunmap(desc->page); + kunmap_thread(desc->page); return status; }

@@ -602,10 +602,10 @@ int nfs_readdir_page_filler(nfs_readdir_descriptor_t *desc, struct nfs_entry *en

out_nopages: if (count == 0 || (status == -EBADCOOKIE && entry->eof != 0)) { - array = kmap(page); + array = kmap_thread(page); array->eof_index = array->size; status = 0; - kunmap(page); + kunmap_thread(page); }

put_page(scratch); @@ -669,7 +669,7 @@ int nfs_readdir_xdr_to_array(nfs_readdir_descriptor_t *desc, struct page *page, goto out; }

- array = kmap(page); + array = kmap_thread(page);

status = nfs_readdir_alloc_pages(pages, array_size); if (status < 0) @@ -691,7 +691,7 @@ int nfs_readdir_xdr_to_array(nfs_readdir_descriptor_t *desc, struct page *page,

nfs_readdir_free_pages(pages, array_size); out_release_array: - kunmap(page); + kunmap_thread(page); nfs4_label_free(entry.label); out: nfs_free_fattr(entry.fattr); @@ -803,7 +803,7 @@ int nfs_do_filldir(nfs_readdir_descriptor_t *desc) struct nfs_cache_array *array = NULL; struct nfs_open_dir_context *ctx = file->private_data;

- array = kmap(desc->page); + array = kmap_thread(desc->page); for (i = desc->cache_entry_index; i < array->size; i++) { struct nfs_cache_array_entry *ent;

@@ -827,7 +827,7 @@ int nfs_do_filldir(nfs_readdir_descriptor_t *desc) if (array->eof_index >= 0) desc->eof = true;

- kunmap(desc->page); + kunmap_thread(desc->page); dfprintk(DIRCACHE, "NFS: nfs_do_filldir() filling ended @ cookie %Lu; returning = %d\n", (unsigned long long)*desc->dir_cookie, res); return res;

From: Ira Weiny ira.weiny@intel.com

The kmap() calls in this FS are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: Christoph Hellwig hch@infradead.org Signed-off-by: Ira Weiny ira.weiny@intel.com --- fs/freevxfs/vxfs_immed.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fs/freevxfs/vxfs_immed.c b/fs/freevxfs/vxfs_immed.c index bfc780c682fb..9c42fec4cd85 100644 --- a/fs/freevxfs/vxfs_immed.c +++ b/fs/freevxfs/vxfs_immed.c @@ -69,9 +69,9 @@ vxfs_immed_readpage(struct file *fp, struct page *pp) u_int64_t offset = (u_int64_t)pp->index << PAGE_SHIFT; caddr_t kaddr;

- kaddr = kmap(pp); + kaddr = kmap_thread(pp); memcpy(kaddr, vip->vii_immed.vi_immed + offset, PAGE_SIZE); - kunmap(pp); + kunmap_thread(pp); flush_dcache_page(pp); SetPageUptodate(pp);

From: Ira Weiny ira.weiny@intel.com

The kmap() calls in this FS are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: Damien Le Moal damien.lemoal@wdc.com Cc: Naohiro Aota naohiro.aota@wdc.com Signed-off-by: Ira Weiny ira.weiny@intel.com --- fs/zonefs/super.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index 8ec7c8f109d7..2fd6c86beee1 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -1297,7 +1297,7 @@ static int zonefs_read_super(struct super_block *sb) if (ret) goto free_page;

- super = kmap(page); + super = kmap_thread(page);

ret = -EINVAL; if (le32_to_cpu(super->s_magic) != ZONEFS_MAGIC) @@ -1349,7 +1349,7 @@ static int zonefs_read_super(struct super_block *sb) ret = 0;

unmap: - kunmap(page); + kunmap_thread(page); free_page: __free_page(page);

From: Ira Weiny ira.weiny@intel.com

The kmap() calls in this FS are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: Richard Weinberger richard@nod.at Signed-off-by: Ira Weiny ira.weiny@intel.com --- fs/ubifs/file.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/fs/ubifs/file.c b/fs/ubifs/file.c index b77d1637bbbc..a3537447a885 100644 --- a/fs/ubifs/file.c +++ b/fs/ubifs/file.c @@ -111,7 +111,7 @@ static int do_readpage(struct page *page) ubifs_assert(c, !PageChecked(page)); ubifs_assert(c, !PagePrivate(page));

- addr = kmap(page); + addr = kmap_thread(page);

block = page->index << UBIFS_BLOCKS_PER_PAGE_SHIFT; beyond = (i_size + UBIFS_BLOCK_SIZE - 1) >> UBIFS_BLOCK_SHIFT; @@ -174,7 +174,7 @@ static int do_readpage(struct page *page) SetPageUptodate(page); ClearPageError(page); flush_dcache_page(page); - kunmap(page); + kunmap_thread(page); return 0;

error: @@ -182,7 +182,7 @@ static int do_readpage(struct page *page) ClearPageUptodate(page); SetPageError(page); flush_dcache_page(page); - kunmap(page); + kunmap_thread(page); return err; }

@@ -616,7 +616,7 @@ static int populate_page(struct ubifs_info *c, struct page *page, dbg_gen("ino %lu, pg %lu, i_size %lld, flags %#lx", inode->i_ino, page->index, i_size, page->flags);

- addr = zaddr = kmap(page); + addr = zaddr = kmap_thread(page);

end_index = (i_size - 1) >> PAGE_SHIFT; if (!i_size || page->index > end_index) { @@ -692,7 +692,7 @@ static int populate_page(struct ubifs_info *c, struct page *page, SetPageUptodate(page); ClearPageError(page); flush_dcache_page(page); - kunmap(page); + kunmap_thread(page); *n = nn; return 0;

@@ -700,7 +700,7 @@ static int populate_page(struct ubifs_info *c, struct page *page, ClearPageUptodate(page); SetPageError(page); flush_dcache_page(page); - kunmap(page); + kunmap_thread(page); ubifs_err(c, "bad data node (block %u, inode %lu)", page_block, inode->i_ino); return -EINVAL; @@ -918,7 +918,7 @@ static int do_writepage(struct page *page, int len) /* Update radix tree tags */ set_page_writeback(page);

- addr = kmap(page); + addr = kmap_thread(page); block = page->index << UBIFS_BLOCKS_PER_PAGE_SHIFT; i = 0; while (len) { @@ -950,7 +950,7 @@ static int do_writepage(struct page *page, int len) ClearPagePrivate(page); ClearPageChecked(page);

- kunmap(page); + kunmap_thread(page); unlock_page(page); end_page_writeback(page); return err;

From: Ira Weiny ira.weiny@intel.com

The kmap() calls in this FS are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: Anton Altaparmakov anton@tuxera.com Signed-off-by: Ira Weiny ira.weiny@intel.com --- fs/ntfs/aops.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fs/ntfs/aops.c b/fs/ntfs/aops.c index bb0a43860ad2..11633d732809 100644 --- a/fs/ntfs/aops.c +++ b/fs/ntfs/aops.c @@ -1099,7 +1099,7 @@ static int ntfs_write_mst_block(struct page *page, if (!nr_bhs) goto done; /* Map the page so we can access its contents. */ - kaddr = kmap(page); + kaddr = kmap_thread(page); /* Clear the page uptodate flag whilst the mst fixups are applied. */ BUG_ON(!PageUptodate(page)); ClearPageUptodate(page); @@ -1276,7 +1276,7 @@ static int ntfs_write_mst_block(struct page *page, iput(VFS_I(base_tni)); } SetPageUptodate(page); - kunmap(page); + kunmap_thread(page); done: if (unlikely(err && err != -ENOMEM)) { /*

From: Ira Weiny ira.weiny@intel.com

The kmap() calls in this FS are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: Hans de Goede hdegoede@redhat.com Signed-off-by: Ira Weiny ira.weiny@intel.com --- fs/vboxsf/file.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/fs/vboxsf/file.c b/fs/vboxsf/file.c index c4ab5996d97a..d9c7e6b7b4cc 100644 --- a/fs/vboxsf/file.c +++ b/fs/vboxsf/file.c @@ -216,7 +216,7 @@ static int vboxsf_readpage(struct file *file, struct page *page) u8 *buf; int err;

- buf = kmap(page); + buf = kmap_thread(page);

err = vboxsf_read(sf_handle->root, sf_handle->handle, off, &nread, buf); if (err == 0) { @@ -227,7 +227,7 @@ static int vboxsf_readpage(struct file *file, struct page *page) SetPageError(page); }

- kunmap(page); + kunmap_thread(page); unlock_page(page); return err; } @@ -268,10 +268,10 @@ static int vboxsf_writepage(struct page *page, struct writeback_control *wbc) if (!sf_handle) return -EBADF;

- buf = kmap(page); + buf = kmap_thread(page); err = vboxsf_write(sf_handle->root, sf_handle->handle, off, &nwrite, buf); - kunmap(page); + kunmap_thread(page);

kref_put(&sf_handle->refcount, vboxsf_handle_release);

@@ -302,10 +302,10 @@ static int vboxsf_write_end(struct file *file, struct address_space *mapping, if (!PageUptodate(page) && copied < len) zero_user(page, from + copied, len - copied);

- buf = kmap(page); + buf = kmap_thread(page); err = vboxsf_write(sf_handle->root, sf_handle->handle, pos, &nwritten, buf + from); - kunmap(page); + kunmap_thread(page);

if (err) { nwritten = 0;

From: Ira Weiny ira.weiny@intel.com

The kmap() calls in this FS are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: Nicolas Pitre nico@fluxnic.net Signed-off-by: Ira Weiny ira.weiny@intel.com --- fs/cramfs/inode.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/fs/cramfs/inode.c b/fs/cramfs/inode.c index 912308600d39..003c014a42ed 100644 --- a/fs/cramfs/inode.c +++ b/fs/cramfs/inode.c @@ -247,8 +247,8 @@ static void *cramfs_blkdev_read(struct super_block *sb, unsigned int offset, struct page *page = pages[i];

if (page) { - memcpy(data, kmap(page), PAGE_SIZE); - kunmap(page); + memcpy(data, kmap_thread(page), PAGE_SIZE); + kunmap_thread(page); put_page(page); } else memset(data, 0, PAGE_SIZE); @@ -826,7 +826,7 @@ static int cramfs_readpage(struct file *file, struct page *page)

maxblock = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT; bytes_filled = 0; - pgdata = kmap(page); + pgdata = kmap_thread(page);

if (page->index < maxblock) { struct super_block *sb = inode->i_sb; @@ -914,13 +914,13 @@ static int cramfs_readpage(struct file *file, struct page *page)

memset(pgdata + bytes_filled, 0, PAGE_SIZE - bytes_filled); flush_dcache_page(page); - kunmap(page); + kunmap_thread(page); SetPageUptodate(page); unlock_page(page); return 0;

err: - kunmap(page); + kunmap_thread(page); ClearPageUptodate(page); SetPageError(page); unlock_page(page);

From: Ira Weiny ira.weiny@intel.com

The kmap() calls in this FS are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: Gao Xiang xiang@kernel.org Cc: Chao Yu chao@kernel.org Signed-off-by: Ira Weiny ira.weiny@intel.com --- fs/erofs/super.c | 4 ++-- fs/erofs/xattr.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/fs/erofs/super.c b/fs/erofs/super.c index ddaa516c008a..41696b60f1b3 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -139,7 +139,7 @@ static int erofs_read_superblock(struct super_block *sb)

sbi = EROFS_SB(sb);

- data = kmap(page); + data = kmap_thread(page); dsb = (struct erofs_super_block *)(data + EROFS_SUPER_OFFSET);

ret = -EINVAL; @@ -189,7 +189,7 @@ static int erofs_read_superblock(struct super_block *sb) } ret = 0; out: - kunmap(page); + kunmap_thread(page); put_page(page); return ret; } diff --git a/fs/erofs/xattr.c b/fs/erofs/xattr.c index c8c381eadcd6..1771baa99d77 100644 --- a/fs/erofs/xattr.c +++ b/fs/erofs/xattr.c @@ -20,7 +20,7 @@ static inline void xattr_iter_end(struct xattr_iter *it, bool atomic) { /* the only user of kunmap() is 'init_inode_xattrs' */ if (!atomic) - kunmap(it->page); + kunmap_thread(it->page); else kunmap_atomic(it->kaddr);

@@ -96,7 +96,7 @@ static int init_inode_xattrs(struct inode *inode) }

/* read in shared xattr array (non-atomic, see kmalloc below) */ - it.kaddr = kmap(it.page); + it.kaddr = kmap_thread(it.page); atomic_map = false;

ih = (struct erofs_xattr_ibody_header *)(it.kaddr + it.ofs);

From: Ira Weiny ira.weiny@intel.com

There are 3 places in namei.c where the equivalent of ext2_put_page() is open coded. We want to use k[un]map_thread() instead of k[un]map() in ext2_[get|put]_page().

Move ext2_put_page() to ext2.h and use it in namei.c in prep for converting the k[un]map() code.

Cc: Jan Kara jack@suse.com Signed-off-by: Ira Weiny ira.weiny@intel.com --- fs/ext2/dir.c | 6 ------ fs/ext2/ext2.h | 8 ++++++++ fs/ext2/namei.c | 15 +++++---------- 3 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/fs/ext2/dir.c b/fs/ext2/dir.c index 70355ab6740e..f3194bf20733 100644 --- a/fs/ext2/dir.c +++ b/fs/ext2/dir.c @@ -66,12 +66,6 @@ static inline unsigned ext2_chunk_size(struct inode *inode) return inode->i_sb->s_blocksize; }

-static inline void ext2_put_page(struct page *page) -{ - kunmap(page); - put_page(page); -} - /* * Return the offset into page `page_nr' of the last valid * byte in that page, plus one. diff --git a/fs/ext2/ext2.h b/fs/ext2/ext2.h index 5136b7289e8d..021ec8b42ac3 100644 --- a/fs/ext2/ext2.h +++ b/fs/ext2/ext2.h @@ -16,6 +16,8 @@ #include <linux/blockgroup_lock.h> #include <linux/percpu_counter.h> #include <linux/rbtree.h> +#include <linux/mm.h> +#include <linux/highmem.h>

/* XXX Here for now... not interested in restructing headers JUST now */

@@ -745,6 +747,12 @@ extern int ext2_delete_entry (struct ext2_dir_entry_2 *, struct page *); extern int ext2_empty_dir (struct inode *); extern struct ext2_dir_entry_2 * ext2_dotdot (struct inode *, struct page **); extern void ext2_set_link(struct inode *, struct ext2_dir_entry_2 *, struct page *, struct inode *, int); +static inline void ext2_put_page(struct page *page) +{ + kunmap(page); + put_page(page); +} +

/* ialloc.c */ extern struct inode * ext2_new_inode (struct inode *, umode_t, const struct qstr *); diff --git a/fs/ext2/namei.c b/fs/ext2/namei.c index 5bf2c145643b..ea980f1e2e99 100644 --- a/fs/ext2/namei.c +++ b/fs/ext2/namei.c @@ -389,23 +389,18 @@ static int ext2_rename (struct inode * old_dir, struct dentry * old_dentry, if (dir_de) { if (old_dir != new_dir) ext2_set_link(old_inode, dir_de, dir_page, new_dir, 0); - else { - kunmap(dir_page); - put_page(dir_page); - } + else + ext2_put_page(dir_page); inode_dec_link_count(old_dir); } return 0;

out_dir: - if (dir_de) { - kunmap(dir_page); - put_page(dir_page); - } + if (dir_de) + ext2_put_page(dir_page); out_old: - kunmap(old_page); - put_page(old_page); + ext2_put_page(old_page); out: return err; }

From: Ira Weiny ira.weiny@intel.com

These kmap() calls are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Signed-off-by: Ira Weiny ira.weiny@intel.com --- fs/isofs/compress.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fs/isofs/compress.c b/fs/isofs/compress.c index bc12ac7e2312..ddd3fd99d2e1 100644 --- a/fs/isofs/compress.c +++ b/fs/isofs/compress.c @@ -344,7 +344,7 @@ static int zisofs_readpage(struct file *file, struct page *page) pages[i] = grab_cache_page_nowait(mapping, index); if (pages[i]) { ClearPageError(pages[i]); - kmap(pages[i]); + kmap_thread(pages[i]); } }

@@ -356,7 +356,7 @@ static int zisofs_readpage(struct file *file, struct page *page) flush_dcache_page(pages[i]); if (i == full_page && err) SetPageError(pages[i]); - kunmap(pages[i]); + kunmap_thread(pages[i]); unlock_page(pages[i]); if (i != full_page) put_page(pages[i]);

From: Ira Weiny ira.weiny@intel.com

These kmap() calls in these drivers are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: "David S. Miller" davem@davemloft.net Cc: Jakub Kicinski kuba@kernel.org Cc: Alexey Kuznetsov kuznet@ms2.inr.ac.ru Cc: Hideaki YOSHIFUJI yoshfuji@linux-ipv6.org Cc: Trond Myklebust trond.myklebust@hammerspace.com Cc: Anna Schumaker anna.schumaker@netapp.com Cc: Boris Pismenny borisp@nvidia.com Cc: Aviad Yehezkel aviadye@nvidia.com Cc: John Fastabend john.fastabend@gmail.com Cc: Daniel Borkmann daniel@iogearbox.net Signed-off-by: Ira Weiny ira.weiny@intel.com --- net/ceph/messenger.c | 4 ++-- net/core/datagram.c | 4 ++-- net/core/sock.c | 8 ++++---- net/ipv4/ip_output.c | 4 ++-- net/sunrpc/cache.c | 4 ++-- net/sunrpc/xdr.c | 8 ++++---- net/tls/tls_device.c | 4 ++-- 7 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index d4d7a0e52491..0c49b8e333da 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -1535,10 +1535,10 @@ static u32 ceph_crc32c_page(u32 crc, struct page *page, { char *kaddr;

- kaddr = kmap(page); + kaddr = kmap_thread(page); BUG_ON(kaddr == NULL); crc = crc32c(crc, kaddr + page_offset, length); - kunmap(page); + kunmap_thread(page);

return crc; } diff --git a/net/core/datagram.c b/net/core/datagram.c index 639745d4f3b9..cbd0a343074a 100644 --- a/net/core/datagram.c +++ b/net/core/datagram.c @@ -441,14 +441,14 @@ static int __skb_datagram_iter(const struct sk_buff *skb, int offset, end = start + skb_frag_size(frag); if ((copy = end - offset) > 0) { struct page *page = skb_frag_page(frag); - u8 *vaddr = kmap(page); + u8 *vaddr = kmap_thread(page);

if (copy > len) copy = len; n = INDIRECT_CALL_1(cb, simple_copy_to_iter, vaddr + skb_frag_off(frag) + offset - start, copy, data, to); - kunmap(page); + kunmap_thread(page); offset += n; if (n != copy) goto short_copy; diff --git a/net/core/sock.c b/net/core/sock.c index 6c5c6b18eff4..9b46a75cd8c1 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -2846,11 +2846,11 @@ ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, siz ssize_t res; struct msghdr msg = {.msg_flags = flags}; struct kvec iov; - char *kaddr = kmap(page); + char *kaddr = kmap_thread(page); iov.iov_base = kaddr + offset; iov.iov_len = size; res = kernel_sendmsg(sock, &msg, &iov, 1, size); - kunmap(page); + kunmap_thread(page); return res; } EXPORT_SYMBOL(sock_no_sendpage); @@ -2861,12 +2861,12 @@ ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page, ssize_t res; struct msghdr msg = {.msg_flags = flags}; struct kvec iov; - char *kaddr = kmap(page); + char *kaddr = kmap_thread(page);

iov.iov_base = kaddr + offset; iov.iov_len = size; res = kernel_sendmsg_locked(sk, &msg, &iov, 1, size); - kunmap(page); + kunmap_thread(page); return res; } EXPORT_SYMBOL(sock_no_sendpage_locked); diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index e6f2ada9e7d5..05304fb251a4 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -949,9 +949,9 @@ csum_page(struct page *page, int offset, int copy) { char *kaddr; __wsum csum; - kaddr = kmap(page); + kaddr = kmap_thread(page); csum = csum_partial(kaddr + offset, copy, 0); - kunmap(page); + kunmap_thread(page); return csum; }

diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c index baef5ee43dbb..88193f2a8e6f 100644 --- a/net/sunrpc/cache.c +++ b/net/sunrpc/cache.c @@ -935,9 +935,9 @@ static ssize_t cache_downcall(struct address_space *mapping, if (!page) goto out_slow;

- kaddr = kmap(page); + kaddr = kmap_thread(page); ret = cache_do_downcall(kaddr, buf, count, cd); - kunmap(page); + kunmap_thread(page); unlock_page(page); put_page(page); return ret; diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index be11d672b5b9..00afbb48fb0a 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -1353,7 +1353,7 @@ xdr_xcode_array2(struct xdr_buf *buf, unsigned int base, base &= ~PAGE_MASK; avail_page = min_t(unsigned int, PAGE_SIZE - base, avail_here); - c = kmap(*ppages) + base; + c = kmap_thread(*ppages) + base;

while (avail_here) { avail_here -= avail_page; @@ -1429,9 +1429,9 @@ xdr_xcode_array2(struct xdr_buf *buf, unsigned int base, } } if (avail_here) { - kunmap(*ppages); + kunmap_thread(*ppages); ppages++; - c = kmap(*ppages); + c = kmap_thread(*ppages); }

avail_page = min(avail_here, @@ -1471,7 +1471,7 @@ xdr_xcode_array2(struct xdr_buf *buf, unsigned int base, out: kfree(elem); if (ppages) - kunmap(*ppages); + kunmap_thread(*ppages); return err; }

diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index b74e2741f74f..ead5b1c485f8 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -576,13 +576,13 @@ int tls_device_sendpage(struct sock *sk, struct page *page, goto out; }

- kaddr = kmap(page); + kaddr = kmap_thread(page); iov.iov_base = kaddr + offset; iov.iov_len = size; iov_iter_kvec(&msg_iter, WRITE, &iov, 1, size); rc = tls_push_data(sk, &msg_iter, size, flags, TLS_RECORD_TYPE_DATA); - kunmap(page); + kunmap_thread(page);

out: release_sock(sk);

From: Ira Weiny ira.weiny@intel.com

These kmap() calls are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: "James E.J. Bottomley" jejb@linux.ibm.com Cc: "Martin K. Petersen" martin.petersen@oracle.com Signed-off-by: Ira Weiny ira.weiny@intel.com --- drivers/scsi/ipr.c | 8 ++++---- drivers/scsi/pmcraid.c | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c index b0aa58d117cc..a5a0b8feb661 100644 --- a/drivers/scsi/ipr.c +++ b/drivers/scsi/ipr.c @@ -3923,9 +3923,9 @@ static int ipr_copy_ucode_buffer(struct ipr_sglist *sglist, buffer += bsize_elem) { struct page *page = sg_page(sg);

- kaddr = kmap(page); + kaddr = kmap_thread(page); memcpy(kaddr, buffer, bsize_elem); - kunmap(page); + kunmap_thread(page);

sg->length = bsize_elem;

@@ -3938,9 +3938,9 @@ static int ipr_copy_ucode_buffer(struct ipr_sglist *sglist, if (len % bsize_elem) { struct page *page = sg_page(sg);

- kaddr = kmap(page); + kaddr = kmap_thread(page); memcpy(kaddr, buffer, len % bsize_elem); - kunmap(page); + kunmap_thread(page);

sg->length = len % bsize_elem; } diff --git a/drivers/scsi/pmcraid.c b/drivers/scsi/pmcraid.c index aa9ae2ae8579..4b05ba4b8a11 100644 --- a/drivers/scsi/pmcraid.c +++ b/drivers/scsi/pmcraid.c @@ -3269,13 +3269,13 @@ static int pmcraid_copy_sglist( for (i = 0; i < (len / bsize_elem); i++, sg = sg_next(sg), buffer += bsize_elem) { struct page *page = sg_page(sg);

- kaddr = kmap(page); + kaddr = kmap_thread(page); if (direction == DMA_TO_DEVICE) rc = copy_from_user(kaddr, buffer, bsize_elem); else rc = copy_to_user(buffer, kaddr, bsize_elem);

- kunmap(page); + kunmap_thread(page);

if (rc) { pmcraid_err("failed to copy user data into sg list\n"); @@ -3288,14 +3288,14 @@ static int pmcraid_copy_sglist( if (len % bsize_elem) { struct page *page = sg_page(sg);

- kaddr = kmap(page); + kaddr = kmap_thread(page);

if (direction == DMA_TO_DEVICE) rc = copy_from_user(kaddr, buffer, len % bsize_elem); else rc = copy_to_user(buffer, kaddr, len % bsize_elem);

- kunmap(page); + kunmap_thread(page);

sg->length = len % bsize_elem; }

From: Ira Weiny ira.weiny@intel.com

These kmap() calls are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: Stefano Stabellini sstabellini@kernel.org Signed-off-by: Ira Weiny ira.weiny@intel.com --- drivers/xen/gntalloc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/xen/gntalloc.c b/drivers/xen/gntalloc.c index 3fa40c723e8e..3b78e055feff 100644 --- a/drivers/xen/gntalloc.c +++ b/drivers/xen/gntalloc.c @@ -184,9 +184,9 @@ static int add_grefs(struct ioctl_gntalloc_alloc_gref *op, static void __del_gref(struct gntalloc_gref *gref) { if (gref->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) { - uint8_t *tmp = kmap(gref->page); + uint8_t *tmp = kmap_thread(gref->page); tmp[gref->notify.pgoff] = 0; - kunmap(gref->page); + kunmap_thread(gref->page); } if (gref->notify.flags & UNMAP_NOTIFY_SEND_EVENT) { notify_remote_via_evtchn(gref->notify.event);

From: Ira Weiny ira.weiny@intel.com

These kmap() calls are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: Greg Kroah-Hartman gregkh@linuxfoundation.org Signed-off-by: Ira Weiny ira.weiny@intel.com --- drivers/staging/rts5208/rtsx_transport.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/staging/rts5208/rtsx_transport.c b/drivers/staging/rts5208/rtsx_transport.c index 0027bcf638ad..f747cc23951b 100644 --- a/drivers/staging/rts5208/rtsx_transport.c +++ b/drivers/staging/rts5208/rtsx_transport.c @@ -92,13 +92,13 @@ unsigned int rtsx_stor_access_xfer_buf(unsigned char *buffer, while (sglen > 0) { unsigned int plen = min(sglen, (unsigned int) PAGE_SIZE - poff); - unsigned char *ptr = kmap(page); + unsigned char *ptr = kmap_thread(page);

if (dir == TO_XFER_BUF) memcpy(ptr + poff, buffer + cnt, plen); else memcpy(buffer + cnt, ptr + poff, plen); - kunmap(page); + kunmap_thread(page);

/* Start at the beginning of the next page */ poff = 0;

From: Ira Weiny ira.weiny@intel.com

These kmap() calls are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: Coly Li colyli@suse.de (maintainer:BCACHE (BLOCK LAYER CACHE)) Cc: Kent Overstreet kent.overstreet@gmail.com (maintainer:BCACHE (BLOCK LAYER CACHE)) Signed-off-by: Ira Weiny ira.weiny@intel.com --- drivers/md/bcache/request.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c index c7cadaafa947..a4571f6d09dd 100644 --- a/drivers/md/bcache/request.c +++ b/drivers/md/bcache/request.c @@ -44,10 +44,10 @@ static void bio_csum(struct bio *bio, struct bkey *k) uint64_t csum = 0;

bio_for_each_segment(bv, bio, iter) { - void *d = kmap(bv.bv_page) + bv.bv_offset; + void *d = kmap_thread(bv.bv_page) + bv.bv_offset;

csum = bch_crc64_update(csum, d, bv.bv_len); - kunmap(bv.bv_page); + kunmap_thread(bv.bv_page); }

k->ptr[KEY_PTRS(k)] = csum & (~0ULL >> 1);

From: Ira Weiny ira.weiny@intel.com

These kmap() calls are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: Greg Kroah-Hartman gregkh@linuxfoundation.org Signed-off-by: Ira Weiny ira.weiny@intel.com --- drivers/misc/vmw_vmci/vmci_queue_pair.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/drivers/misc/vmw_vmci/vmci_queue_pair.c b/drivers/misc/vmw_vmci/vmci_queue_pair.c index 8531ae781195..f308abb8ad03 100644 --- a/drivers/misc/vmw_vmci/vmci_queue_pair.c +++ b/drivers/misc/vmw_vmci/vmci_queue_pair.c @@ -343,7 +343,7 @@ static int qp_memcpy_to_queue_iter(struct vmci_queue *queue, size_t to_copy;

if (kernel_if->host) - va = kmap(kernel_if->u.h.page[page_index]); + va = kmap_thread(kernel_if->u.h.page[page_index]); else va = kernel_if->u.g.vas[page_index + 1]; /* Skip header. */ @@ -357,12 +357,12 @@ static int qp_memcpy_to_queue_iter(struct vmci_queue *queue, if (!copy_from_iter_full((u8 *)va + page_offset, to_copy, from)) { if (kernel_if->host) - kunmap(kernel_if->u.h.page[page_index]); + kunmap_thread(kernel_if->u.h.page[page_index]); return VMCI_ERROR_INVALID_ARGS; } bytes_copied += to_copy; if (kernel_if->host) - kunmap(kernel_if->u.h.page[page_index]); + kunmap_thread(kernel_if->u.h.page[page_index]); }

return VMCI_SUCCESS; @@ -391,7 +391,7 @@ static int qp_memcpy_from_queue_iter(struct iov_iter *to, int err;

if (kernel_if->host) - va = kmap(kernel_if->u.h.page[page_index]); + va = kmap_thread(kernel_if->u.h.page[page_index]); else va = kernel_if->u.g.vas[page_index + 1]; /* Skip header. */ @@ -405,12 +405,12 @@ static int qp_memcpy_from_queue_iter(struct iov_iter *to, err = copy_to_iter((u8 *)va + page_offset, to_copy, to); if (err != to_copy) { if (kernel_if->host) - kunmap(kernel_if->u.h.page[page_index]); + kunmap_thread(kernel_if->u.h.page[page_index]); return VMCI_ERROR_INVALID_ARGS; } bytes_copied += to_copy; if (kernel_if->host) - kunmap(kernel_if->u.h.page[page_index]); + kunmap_thread(kernel_if->u.h.page[page_index]); }

return VMCI_SUCCESS;

From: Ira Weiny ira.weiny@intel.com

This kmap() call is localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: Eric Biederman ebiederm@xmission.com Signed-off-by: Ira Weiny ira.weiny@intel.com --- kernel/kexec_core.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index c19c0dad1ebe..272a9920c0d6 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -815,7 +815,7 @@ static int kimage_load_normal_segment(struct kimage *image, if (result < 0) goto out;

- ptr = kmap(page); + ptr = kmap_thread(page); /* Start with a clear page */ clear_page(ptr); ptr += maddr & ~PAGE_MASK; @@ -828,7 +828,7 @@ static int kimage_load_normal_segment(struct kimage *image, memcpy(ptr, kbuf, uchunk); else result = copy_from_user(ptr, buf, uchunk); - kunmap(page); + kunmap_thread(page); if (result) { result = -EFAULT; goto out; @@ -879,7 +879,7 @@ static int kimage_load_crash_segment(struct kimage *image, goto out; } arch_kexec_post_alloc_pages(page_address(page), 1, 0); - ptr = kmap(page); + ptr = kmap_thread(page); ptr += maddr & ~PAGE_MASK; mchunk = min_t(size_t, mbytes, PAGE_SIZE - (maddr & ~PAGE_MASK)); @@ -895,7 +895,7 @@ static int kimage_load_crash_segment(struct kimage *image, else result = copy_from_user(ptr, buf, uchunk); kexec_flush_icache_page(page); - kunmap(page); + kunmap_thread(page); arch_kexec_pre_free_pages(page_address(page), 1); if (result) { result = -EFAULT;

From: Ira Weiny ira.weiny@intel.com

These kmap() calls are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Signed-off-by: Ira Weiny ira.weiny@intel.com --- mm/memory.c | 8 ++++---- mm/swapfile.c | 4 ++-- mm/userfaultfd.c | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/mm/memory.c b/mm/memory.c index fcfc4ca36eba..75a054882d7a 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4945,7 +4945,7 @@ int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, if (bytes > PAGE_SIZE-offset) bytes = PAGE_SIZE-offset;

- maddr = kmap(page); + maddr = kmap_thread(page); if (write) { copy_to_user_page(vma, page, addr, maddr + offset, buf, bytes); @@ -4954,7 +4954,7 @@ int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, copy_from_user_page(vma, page, addr, buf, maddr + offset, bytes); } - kunmap(page); + kunmap_thread(page); put_page(page); } len -= bytes; @@ -5216,14 +5216,14 @@ long copy_huge_page_from_user(struct page *dst_page,

for (i = 0; i < pages_per_huge_page; i++) { if (allow_pagefault) - page_kaddr = kmap(dst_page + i); + page_kaddr = kmap_thread(dst_page + i); else page_kaddr = kmap_atomic(dst_page + i); rc = copy_from_user(page_kaddr, (const void __user *)(src + i * PAGE_SIZE), PAGE_SIZE); if (allow_pagefault) - kunmap(dst_page + i); + kunmap_thread(dst_page + i); else kunmap_atomic(page_kaddr);

diff --git a/mm/swapfile.c b/mm/swapfile.c index debc94155f74..e3296ff95648 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -3219,7 +3219,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) error = PTR_ERR(page); goto bad_swap_unlock_inode; } - swap_header = kmap(page); + swap_header = kmap_thread(page);

maxpages = read_swap_header(p, swap_header, inode); if (unlikely(!maxpages)) { @@ -3395,7 +3395,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) filp_close(swap_file, NULL); out: if (page && !IS_ERR(page)) { - kunmap(page); + kunmap_thread(page); put_page(page); } if (name) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 9a3d451402d7..4d38c881bb2d 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -586,11 +586,11 @@ static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm, mmap_read_unlock(dst_mm); BUG_ON(!page);

- page_kaddr = kmap(page); + page_kaddr = kmap_thread(page); err = copy_from_user(page_kaddr, (const void __user *) src_addr, PAGE_SIZE); - kunmap(page); + kunmap_thread(page); if (unlikely(err)) { err = -EFAULT; goto out;

From: Ira Weiny ira.weiny@intel.com

These kmap() calls are localized to a single thread. To avoid the over head of global PKRS updates use the new kmap_thread() call.

Cc: Benjamin Herrenschmidt benh@kernel.crashing.org Cc: Paul Mackerras paulus@samba.org Signed-off-by: Ira Weiny ira.weiny@intel.com --- arch/powerpc/mm/mem.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c index 42e25874f5a8..6ef557b8dda6 100644 --- a/arch/powerpc/mm/mem.c +++ b/arch/powerpc/mm/mem.c @@ -573,9 +573,9 @@ void flush_icache_user_page(struct vm_area_struct *vma, struct page *page, { unsigned long maddr;

- maddr = (unsigned long) kmap(page) + (addr & ~PAGE_MASK); + maddr = (unsigned long) kmap_thread(page) + (addr & ~PAGE_MASK); flush_icache_range(maddr, maddr + len); - kunmap(page); + kunmap_thread(page); }

/*

From: Ira Weiny ira.weiny@intel.com

dax_direct_access() is a special case of accessing pmem via a page offset and without a struct page.

Because the dax driver is well aware of the special protections it has mapped memory with, call dev_access_[en|dis]able() directly instead of the unnecessary overhead of trying to get a page to kmap.

Similar to kmap, we leverage existing functions, dax_read_[un]lock(), because they are already required to surround the use of the memory returned from dax_direct_access().

Signed-off-by: Ira Weiny ira.weiny@intel.com --- drivers/dax/super.c | 2 ++ 1 file changed, 2 insertions(+)

diff --git a/drivers/dax/super.c b/drivers/dax/super.c index e84070b55463..0ddb3ee73e36 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -30,6 +30,7 @@ static DEFINE_SPINLOCK(dax_host_lock);

int dax_read_lock(void) { + dev_access_enable(false); return srcu_read_lock(&dax_srcu); } EXPORT_SYMBOL_GPL(dax_read_lock); @@ -37,6 +38,7 @@ EXPORT_SYMBOL_GPL(dax_read_lock); void dax_read_unlock(int id) { srcu_read_unlock(&dax_srcu, id); + dev_access_disable(false); } EXPORT_SYMBOL_GPL(dax_read_unlock);

From: Ira Weiny ira.weiny@intel.com

Protecting against stray writes is particularly important for PMEM because, unlike writes to anonymous memory, writes to PMEM persists across a reboot. Thus data corruption could result in permanent loss of data.

While stray writes are more serious than reads, protection is also enabled for reads. This helps to detect bugs in code which would incorrectly access device memory and prevents a more serious machine checks should those bug reads from a poison page.

Enable stray access protection by setting the flag in pgmap which requests it. There is no option presented to the user. If Zone Device Access Protection not be supported this flag will have no affect.

Signed-off-by: Ira Weiny ira.weiny@intel.com --- drivers/dax/device.c | 2 ++ drivers/nvdimm/pmem.c | 2 ++ 2 files changed, 4 insertions(+)

diff --git a/drivers/dax/device.c b/drivers/dax/device.c index 1e89513f3c59..e6fb35b4f0fb 100644 --- a/drivers/dax/device.c +++ b/drivers/dax/device.c @@ -430,6 +430,8 @@ int dev_dax_probe(struct device *dev) }

dev_dax->pgmap.type = MEMORY_DEVICE_GENERIC; + dev_dax->pgmap.flags |= PGMAP_PROT_ENABLED; + addr = devm_memremap_pages(dev, &dev_dax->pgmap); if (IS_ERR(addr)) return PTR_ERR(addr); diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index e4dc1ae990fc..9fcd8338e23f 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -426,6 +426,8 @@ static int pmem_attach_disk(struct device *dev, return -EBUSY; }

+ pmem->pgmap.flags |= PGMAP_PROT_ENABLED; + q = blk_alloc_queue(dev_to_node(dev)); if (!q) return -ENOMEM;