Some versions of iproute2 will output more than one line per entry, which
will cause the test to fail, like:
TEST: ipv6: list and flush cached exceptions [FAIL]
can't list cached exceptions
That happens, for example, with iproute2 4.15.0. When using the -oneline
option, this will work just fine:
TEST: ipv6: list and flush cached exceptions [ OK ]
This also works just fine with a more recent version of iproute2, like
5.4.0.
For some reason, two lines are printed for the IPv4 test no matter what
version of iproute2 is used. Use the same -oneline parameter there instead
of counting the lines twice.
Signed-off-by: Thadeu Lima de Souza Cascardo <cascardo(a)canonical.com>
---
tools/testing/selftests/net/pmtu.sh | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/tools/testing/selftests/net/pmtu.sh b/tools/testing/selftests/net/pmtu.sh
index ab367e75f095..d697815d2785 100755
--- a/tools/testing/selftests/net/pmtu.sh
+++ b/tools/testing/selftests/net/pmtu.sh
@@ -1249,8 +1249,7 @@ test_list_flush_ipv4_exception() {
done
run_cmd ${ns_a} ping -q -M want -i 0.1 -c 2 -s 1800 "${dst2}"
- # Each exception is printed as two lines
- if [ "$(${ns_a} ip route list cache | wc -l)" -ne 202 ]; then
+ if [ "$(${ns_a} ip -oneline route list cache | wc -l)" -ne 101 ]; then
err " can't list cached exceptions"
fail=1
fi
@@ -1300,7 +1299,7 @@ test_list_flush_ipv6_exception() {
run_cmd ${ns_a} ping -q -M want -i 0.1 -w 1 -s 1800 "${dst_prefix1}${i}"
done
run_cmd ${ns_a} ping -q -M want -i 0.1 -w 1 -s 1800 "${dst2}"
- if [ "$(${ns_a} ip -6 route list cache | wc -l)" -ne 101 ]; then
+ if [ "$(${ns_a} ip -oneline -6 route list cache | wc -l)" -ne 101 ]; then
err " can't list cached exceptions"
fail=1
fi
--
2.20.1
Hi
while testing on linux-next
I see that, when KBUILD_OUTPUT is set in the env, running something like (using TARGETS=exec as a random subsystem here...)
$ make TARGETS=exec INSTALL_PATH=/nfs/LTP-official-debian-aarch64-rootfs/opt/KSFT_next kselftest-install
works fine as usual, WHILE the alternative invocation (still documented in Documentation/dev-tools/kselftest.rst)
make -C tools/testing/selftests/ TARGETS=exec INSTALL_PATH=/nfs/LTP-official-debian-aarch64-rootfs/opt/KSFT_next install
fails miserably with:
...
...
REMOVE usr/include/rdma/cxgb3-abi.h usr/include/rdma/nes-abi.h
HDRINST usr/include/asm/kvm.h
INSTALL /kselftest/usr/include
mkdir: cannot create directory ‘/kselftest’: Permission denied
/home/crimar01/ARM/dev/src/pdsw/linux/Makefile:1187: recipe for target 'headers_install' failed
make[2]: *** [headers_install] Error 1
This is fixed by unsetting KBUILD_OUTPUT OR reverting:
303e6218ecec (ksft/fixes) selftests: Fix O= and KBUILD_OUTPUT handling for relative paths
since bypassing top makefile with -C, the definition of abs-objtree used by the above patch
is no more available.
As a side effect when KBUILD_OUTPUT is set, this breaks also the usage kselftest_install.sh.
$ ./kselftest_install.sh /home/crimar01/ARM/dev/nfs/LTP-official-debian-aarch64-rootfs/opt/KSFT_full_next
./kselftest_install.sh: Installing in specified location - /home/crimar01/ARM/dev/nfs/LTP-official-debian-aarch64-rootfs/opt/KSFT_full_next ...
make --no-builtin-rules INSTALL_HDR_PATH=$BUILD/usr \
ARCH=arm64 -C ../../.. headers_install
make[1]: Entering directory '/home/crimar01/ARM/dev/src/pdsw/linux'
make[2]: Entering directory '/home/crimar01/ARM/dev/src/pdsw/out_linux'
INSTALL /kselftest/usr/include
mkdir: cannot create directory ‘/kselftest’: Permission denied
/home/crimar01/ARM/dev/src/pdsw/linux/Makefile:1187: recipe for target 'headers_install' failed
make[2]: *** [headers_install] Error 1
make[2]: Leaving directory '/home/crimar01/ARM/dev/src/pdsw/out_linux'
Makefile:179: recipe for target 'sub-make' failed
make[1]: *** [sub-make] Error 2
make[1]: Leaving directory '/home/crimar01/ARM/dev/src/pdsw/linux'
Makefile:142: recipe for target 'khdr' failed
make: *** [khdr] Error 2
A possible fix would be (but duplicates in fact the main Makefile logic)
diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
index 319e094c3212..491d8b3ef1c7 100644
--- a/tools/testing/selftests/Makefile
+++ b/tools/testing/selftests/Makefile
@@ -88,6 +88,15 @@ ifdef building_out_of_srctree
override LDFLAGS =
endif
+ifeq ($(abs_objtree),)
+ifneq ($(KBUILD_OUTPUT),)
+abs_objtree := $(shell cd $(KBUILD_OUTPUT) && pwd)
+abs_objtree := $(realpath $(abs_objtree))
+else
+abs_objtree := $(shell pwd)
+endif
+endif #ifeq ($(abs_objtree),)
+
ifneq ($(O),)
BUILD := $(abs_objtree)
else
Any thoughts ? ... or am I missing something ?
(I think I'm starting to see this in latest CI linaro kselftest while they cross-compile for arm64)
Thanks
Cristian
On Wed, Nov 27, 2019 at 5:49 AM Jeffrin Jose
<jeffrin(a)rajagiritech.edu.in> wrote:
> Tested-by: Jeffrin Jose T <jeffrin(a)rajagiritech.edu.in>
> Signed-off-by: Jeffrin Jose T <jeffrin(a)rajagiritech.edu.in>
i do
--
software engineer
rajagiri school of engineering and technology
Fixes the issue caused by the fact that in C in the expression
of the form -1234L only 1234L is the actual literal, the unary
minus is an operation applied to the literal. Which means that
to express the lower bound for the type one has to negate the
upper bound and subtract 1.
Original error:
Expected test_data[i].expected.tv_sec == timestamp.tv_sec, but
test_data[i].expected.tv_sec == -2147483648
timestamp.tv_sec == 2147483648
1901-12-13 Lower bound of 32bit < 0 timestamp, no extra bits: msb:1
lower_bound:1 extra_bits: 0
Expected test_data[i].expected.tv_sec == timestamp.tv_sec, but
test_data[i].expected.tv_sec == 2147483648
timestamp.tv_sec == 6442450944
2038-01-19 Lower bound of 32bit <0 timestamp, lo extra sec bit on:
msb:1 lower_bound:1 extra_bits: 1
Expected test_data[i].expected.tv_sec == timestamp.tv_sec, but
test_data[i].expected.tv_sec == 6442450944
timestamp.tv_sec == 10737418240
2174-02-25 Lower bound of 32bit <0 timestamp, hi extra sec bit on:
msb:1 lower_bound:1 extra_bits: 2
not ok 1 - inode_test_xtimestamp_decoding
not ok 1 - ext4_inode_test
Reported-by: Geert Uytterhoeven geert(a)linux-m68k.org
Signed-off-by: Iurii Zaikin <yzaikin(a)google.com>
---
fs/ext4/inode-test.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/fs/ext4/inode-test.c b/fs/ext4/inode-test.c
index 92a9da1774aa..bbce1c328d85 100644
--- a/fs/ext4/inode-test.c
+++ b/fs/ext4/inode-test.c
@@ -25,7 +25,7 @@
* For constructing the negative timestamp lower bound value.
* binary: 10000000 00000000 00000000 00000000
*/
-#define LOWER_MSB_1 (-0x80000000L)
+#define LOWER_MSB_1 (-(UPPER_MSB_0) - 1L) /* avoid overflow */
/*
* For constructing the negative timestamp upper bound value.
* binary: 11111111 11111111 11111111 11111111
--
2.24.0.432.g9d3f5f5b63-goog
Hi,
Here is the 4th version of patches to fix some issues which happens on
the kernel with CONFIG_FUNCTION_TRACER=n or CONFIG_DYNAMIC_FTRACE=n.
In this version I fixed [1/4] to cleanup set_ftrace_notrace (Thanks Steve!)
Thank you,
---
Masami Hiramatsu (4):
selftests/ftrace: Fix to check the existence of set_ftrace_filter
selftests/ftrace: Fix ftrace test cases to check unsupported
selftests/ftrace: Do not to use absolute debugfs path
selftests/ftrace: Fix multiple kprobe testcase
.../ftrace/test.d/ftrace/func-filter-stacktrace.tc | 2 ++
.../selftests/ftrace/test.d/ftrace/func_cpumask.tc | 5 +++++
tools/testing/selftests/ftrace/test.d/functions | 5 ++++-
.../ftrace/test.d/kprobe/multiple_kprobes.tc | 6 +++---
.../inter-event/trigger-action-hist-xfail.tc | 4 ++--
.../inter-event/trigger-onchange-action-hist.tc | 2 +-
.../inter-event/trigger-snapshot-action-hist.tc | 4 ++--
7 files changed, 19 insertions(+), 9 deletions(-)
--
Masami Hiramatsu (Linaro) <mhiramat(a)kernel.org>
Fixes the issue caused by the fact that in C in the expression
of the form -1234L only 1234L is the actual literal, the unary
minus is an operation applied to the literal. Which means that
to express the lower bound for the type one has to negate the
upper bound and subtract 1.
Signed-off-by: Iurii Zaikin <yzaikin(a)google.com>
---
fs/ext4/inode-test.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/fs/ext4/inode-test.c b/fs/ext4/inode-test.c
index 92a9da1774aa..bbce1c328d85 100644
--- a/fs/ext4/inode-test.c
+++ b/fs/ext4/inode-test.c
@@ -25,7 +25,7 @@
* For constructing the negative timestamp lower bound value.
* binary: 10000000 00000000 00000000 00000000
*/
-#define LOWER_MSB_1 (-0x80000000L)
+#define LOWER_MSB_1 (-(UPPER_MSB_0) - 1L) /* avoid overflow */
/*
* For constructing the negative timestamp upper bound value.
* binary: 11111111 11111111 11111111 11111111
--
2.24.0.432.g9d3f5f5b63-goog
Hi Lurii,
On Tue, Nov 26, 2019 at 3:12 AM Linux Kernel Mailing List
<linux-kernel(a)vger.kernel.org> wrote:
> Commit: 1cbeab1b242d16fdb22dc3dab6a7d6afe746ae6d
> Parent: d460623c5fa126dc51bb2571dd7714ca75b0116c
> Refname: refs/heads/master
> Web: https://git.kernel.org/torvalds/c/1cbeab1b242d16fdb22dc3dab6a7d6afe746ae6d
> Author: Iurii Zaikin <yzaikin(a)google.com>
> AuthorDate: Thu Oct 17 15:12:33 2019 -0700
> Committer: Shuah Khan <skhan(a)linuxfoundation.org>
> CommitDate: Wed Oct 23 10:28:23 2019 -0600
>
> ext4: add kunit test for decoding extended timestamps
>
> KUnit tests for decoding extended 64 bit timestamps that verify the
> seconds part of [a/c/m] timestamps in ext4 inode structs are decoded
> correctly.
>
> Test data is derived from the table in the Inode Timestamps section of
> Documentation/filesystems/ext4/inodes.rst.
>
> KUnit tests run during boot and output the results to the debug log
> in TAP format (http://testanything.org/). Only useful for kernel devs
> running KUnit test harness and are not for inclusion into a production
> build.
>
> Signed-off-by: Iurii Zaikin <yzaikin(a)google.com>
> Reviewed-by: Theodore Ts'o <tytso(a)mit.edu>
> Reviewed-by: Brendan Higgins <brendanhiggins(a)google.com>
> Tested-by: Brendan Higgins <brendanhiggins(a)google.com>
> Reviewed-by: Shuah Khan <skhan(a)linuxfoundation.org>
> Signed-off-by: Shuah Khan <skhan(a)linuxfoundation.org>
While this test succeeds on arm64, it fails on m68k and arm32 (presumably
all 32-bit platforms?):
# Subtest: ext4_inode_test
1..1
# inode_test_xtimestamp_decoding: EXPECTATION FAILED at fs/ext4/inode-test.c:250
Expected test_data[i].expected.tv_sec == timestamp.tv_sec, but
test_data[i].expected.tv_sec == -2147483648
timestamp.tv_sec == 2147483648
1901-12-13 Lower bound of 32bit < 0 timestamp, no extra bits: msb:1
lower_bound:1 extra_bits: 0
# inode_test_xtimestamp_decoding: EXPECTATION FAILED at fs/ext4/inode-test.c:250
Expected test_data[i].expected.tv_sec == timestamp.tv_sec, but
test_data[i].expected.tv_sec == 2147483648
timestamp.tv_sec == 6442450944
2038-01-19 Lower bound of 32bit <0 timestamp, lo extra sec bit on:
msb:1 lower_bound:1 extra_bits: 1
# inode_test_xtimestamp_decoding: EXPECTATION FAILED at fs/ext4/inode-test.c:250
Expected test_data[i].expected.tv_sec == timestamp.tv_sec, but
test_data[i].expected.tv_sec == 6442450944
timestamp.tv_sec == 10737418240
2174-02-25 Lower bound of 32bit <0 timestamp, hi extra sec bit on:
msb:1 lower_bound:1 extra_bits: 2
not ok 1 - inode_test_xtimestamp_decoding
not ok 1 - ext4_inode_test
Gr{oetje,eeting}s,
Geert
--
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- geert(a)linux-m68k.org
In personal conversations with technical people, I call myself a hacker. But
when I'm talking to journalists I just say "programmer" or something like that.
-- Linus Torvalds
Previous error message for invalid kunitconfig was vague. Added to it so
that it lists invalid fields and prompts for them to be removed. Added
validate_config function returning whether or not this kconfig is valid.
Signed-off-by: Heidi Fahim <heidifahim(a)google.com>
---
tools/testing/kunit/kunit_kernel.py | 27 +++++++++++++++------------
1 file changed, 15 insertions(+), 12 deletions(-)
diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py
index bf3876835331..010d3f5030d2 100644
--- a/tools/testing/kunit/kunit_kernel.py
+++ b/tools/testing/kunit/kunit_kernel.py
@@ -93,6 +93,19 @@ class LinuxSourceTree(object):
return False
return True
+ def validate_config(self, build_dir):
+ kconfig_path = get_kconfig_path(build_dir)
+ validated_kconfig = kunit_config.Kconfig()
+ validated_kconfig.read_from_file(kconfig_path)
+ if not self._kconfig.is_subset_of(validated_kconfig):
+ invalid = self._kconfig.entries() - validated_kconfig.entries()
+ message = 'Provided Kconfig is not contained in validated .config. Invalid fields found in kunitconfig: %s' % (
+ ', '.join([str(e) for e in invalid])
+ )
+ logging.error(message)
+ return False
+ return True
+
def build_config(self, build_dir):
kconfig_path = get_kconfig_path(build_dir)
if build_dir and not os.path.exists(build_dir):
@@ -103,12 +116,7 @@ class LinuxSourceTree(object):
except ConfigError as e:
logging.error(e)
return False
- validated_kconfig = kunit_config.Kconfig()
- validated_kconfig.read_from_file(kconfig_path)
- if not self._kconfig.is_subset_of(validated_kconfig):
- logging.error('Provided Kconfig is not contained in validated .config!')
- return False
- return True
+ return self.validate_config(build_dir)
def build_reconfig(self, build_dir):
"""Creates a new .config if it is not a subset of the kunitconfig."""
@@ -133,12 +141,7 @@ class LinuxSourceTree(object):
except (ConfigError, BuildError) as e:
logging.error(e)
return False
- used_kconfig = kunit_config.Kconfig()
- used_kconfig.read_from_file(get_kconfig_path(build_dir))
- if not self._kconfig.is_subset_of(used_kconfig):
- logging.error('Provided Kconfig is not contained in final config!')
- return False
- return True
+ return self.validate_config(build_dir)
def run_kernel(self, args=[], timeout=None, build_dir=None):
args.extend(['mem=256M'])
--
2.24.0.432.g9d3f5f5b63-goog
Support for frequency limits in dev_pm_qos was removed when cpufreq was
switched to freq_qos, this series attempts to restore it by
reimplementing on top of freq_qos.
Discussion about removal is here:
https://lore.kernel.org/linux-pm/VI1PR04MB7023DF47D046AEADB4E051EBEE680@VI1…
The cpufreq core switched away because it needs contraints at the level
of a "cpufreq_policy" which cover multiple cpus so dev_pm_qos coupling
to struct device was not useful. Cpufreq could only use dev_pm_qos by
implementing an additional layer of aggregation anyway.
However in the devfreq subsystem scaling is always performed on a per-device
basis so dev_pm_qos is a very good match. Support for dev_pm_qos in devfreq
core is here:
https://patchwork.kernel.org/cover/11252409/
That series is RFC mostly because it needs these PM core patches.
Earlier versions got entangled in some locking cleanups but those are
not strictly necessary to get dev_pm_qos functionality.
In theory if freq_qos is extended to handle conflicting min/max values then
this sharing would be valuable. Right now freq_qos just ties two unrelated
pm_qos aggregations for min and max freq.
---
This is implemented by embeding a freq_qos_request inside dev_pm_qos_request:
the data field was already an union in order to deal with flag requests.
The internal freq_qos_apply is exported so that it can be called from
dev_pm_qos apply_constraints.
The dev_pm_qos_constraints_destroy function has no obvious equivalent in
freq_qos and the whole approach of "removing requests" is somewhat dubios:
request objects should be owned by consumers and the list of qos requests
should be empty when the target device is deleted.
Changes since v2:
* #define PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE FREQ_QOS_MAX_DEFAULT_VALUE
* #define FREQ_QOS_MAX_DEFAULT_VALUE S32_MAX (in new patch)
* Add initial kunit test for freq_qos, validating the MAX_DEFAULT_VALUE found
by Matthias.
Link to v2: https://patchwork.kernel.org/cover/11250413/
First two patches can be applied separated
Changes since v1:
* Don't rename or EXPORT_SYMBOL_GPL the freq_qos_apply function; just
drop the static marker.
Link to v1: https://patchwork.kernel.org/cover/11212887/
Leonard Crestez (4):
PM / QoS: Initial kunit test
PM / QOS: Redefine FREQ_QOS_MAX_DEFAULT_VALUE to S32_MAX
PM / QoS: Reorder pm_qos/freq_qos/dev_pm_qos structs
PM / QoS: Restore DEV_PM_QOS_MIN/MAX_FREQUENCY
drivers/base/Kconfig | 4 ++
drivers/base/power/Makefile | 1 +
drivers/base/power/qos-test.c | 116 ++++++++++++++++++++++++++++++++++
drivers/base/power/qos.c | 69 ++++++++++++++++++--
include/linux/pm_qos.h | 86 ++++++++++++++-----------
kernel/power/qos.c | 4 +-
6 files changed, 237 insertions(+), 43 deletions(-)
create mode 100644 drivers/base/power/qos-test.c
--
2.17.1
Hi,
Here is the 3rd version of patches to fix some issues which happens on
the kernel with CONFIG_FUNCTION_TRACER=n or CONFIG_DYNAMIC_FTRACE=n.
In this version and v2, I updated the descriptions of the first 2 patches
according to Steve's comment, added Steve's Reviewed-by to the 3rd patch,
and added the 4th patch which was newly found.
Thank you,
---
Masami Hiramatsu (4):
selftests/ftrace: Fix to check the existence of set_ftrace_filter
selftests/ftrace: Fix ftrace test cases to check unsupported
selftests/ftrace: Do not to use absolute debugfs path
selftests/ftrace: Fix multiple kprobe testcase
.../ftrace/test.d/ftrace/func-filter-stacktrace.tc | 2 ++
.../selftests/ftrace/test.d/ftrace/func_cpumask.tc | 5 +++++
tools/testing/selftests/ftrace/test.d/functions | 4 +++-
.../ftrace/test.d/kprobe/multiple_kprobes.tc | 6 +++---
.../inter-event/trigger-action-hist-xfail.tc | 4 ++--
.../inter-event/trigger-onchange-action-hist.tc | 2 +-
.../inter-event/trigger-snapshot-action-hist.tc | 4 ++--
7 files changed, 18 insertions(+), 9 deletions(-)
--
Masami Hiramatsu (Linaro) <mhiramat(a)kernel.org>
Hi Linus,
Please pull the following kselftest fixes update for Linux 5.5-rc1.
This kselftest fixes update for Linux 5.5-rc1 consists of several
fixes to tests and framework. Masami Hiramatsu fixed several tests
to build and run correctly on arm and other 32bit architectures.
diff is attached.
thanks,
-- Shuah
----------------------------------------------------------------
The following changes since commit ce3a677802121e038d2f062e90f96f84e7351da0:
selftests: watchdog: Add command line option to show watchdog_info
(2019-10-02 13:44:43 -0600)
are available in the Git repository at:
git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest
tags/linux-kselftest-5.5-rc1-fixes
for you to fetch changes up to ed2d8fa734e7759ac3788a19f308d3243d0eb164:
selftests: sync: Fix cast warnings on arm (2019-11-07 14:54:37 -0700)
----------------------------------------------------------------
linux-kselftest-5.5-rc1-fixes
This kselftest fixes update for Linux 5.5-rc1 consists of several
fixes to tests and framework. Masami Hiramatsu fixed several tests
to build and run correctly on arm and other 32bit architectures.
----------------------------------------------------------------
Kees Cook (2):
selftests: gen_kselftest_tar.sh: Do not clobber kselftest/
selftests: Move kselftest_module.sh into kselftest/
Masami Hiramatsu (6):
selftests: breakpoints: Fix a typo of function name
selftests: proc: Make va_max 1MB
selftests: vm: Build/Run 64bit tests only on 64bit arch
selftests: net: Use size_t and ssize_t for counting file size
selftests: net: Fix printf format warnings on arm
selftests: sync: Fix cast warnings on arm
Prabhakar Kushwaha (1):
kselftest: Fix NULL INSTALL_PATH for TARGETS runlist
Shuah Khan (1):
selftests: Fix O= and KBUILD_OUTPUT handling for relative paths
tools/testing/selftests/Makefile | 8 +++++---
.../selftests/breakpoints/breakpoint_test_arm64.c | 2 +-
tools/testing/selftests/gen_kselftest_tar.sh | 21
+++++++++++--------
.../{kselftest_module.sh => kselftest/module.sh} | 0
tools/testing/selftests/kselftest_install.sh | 24
+++++++++++-----------
tools/testing/selftests/lib/bitmap.sh | 2 +-
tools/testing/selftests/lib/prime_numbers.sh | 2 +-
tools/testing/selftests/lib/printf.sh | 2 +-
tools/testing/selftests/lib/strscpy.sh | 2 +-
tools/testing/selftests/net/so_txtime.c | 4 ++--
tools/testing/selftests/net/tcp_mmap.c | 8 ++++----
tools/testing/selftests/net/udpgso.c | 3 ++-
tools/testing/selftests/net/udpgso_bench_tx.c | 3 ++-
.../selftests/proc/proc-self-map-files-002.c | 6 +++++-
tools/testing/selftests/sync/sync.c | 6 +++---
tools/testing/selftests/vm/Makefile | 5 +++++
tools/testing/selftests/vm/run_vmtests | 10 +++++++++
17 files changed, 68 insertions(+), 40 deletions(-)
rename tools/testing/selftests/{kselftest_module.sh =>
kselftest/module.sh} (100%)
----------------------------------------------------------------
These counters will track hugetlb reservations rather than hugetlb
memory faulted in. This patch only adds the counter, following patches
add the charging and uncharging of the counter.
Problem:
Currently tasks attempting to allocate more hugetlb memory than is available get
a failure at mmap/shmget time. This is thanks to Hugetlbfs Reservations [1].
However, if a task attempts to allocate hugetlb memory only more than its
hugetlb_cgroup limit allows, the kernel will allow the mmap/shmget call,
but will SIGBUS the task when it attempts to fault the memory in.
We have developers interested in using hugetlb_cgroups, and they have expressed
dissatisfaction regarding this behavior. We'd like to improve this
behavior such that tasks violating the hugetlb_cgroup limits get an error on
mmap/shmget time, rather than getting SIGBUS'd when they try to fault
the excess memory in.
The underlying problem is that today's hugetlb_cgroup accounting happens
at hugetlb memory *fault* time, rather than at *reservation* time.
Thus, enforcing the hugetlb_cgroup limit only happens at fault time, and
the offending task gets SIGBUS'd.
Proposed Solution:
A new page counter named hugetlb.xMB.reservation_[limit|usage]_in_bytes. This
counter has slightly different semantics than
hugetlb.xMB.[limit|usage]_in_bytes:
- While usage_in_bytes tracks all *faulted* hugetlb memory,
reservation_usage_in_bytes tracks all *reserved* hugetlb memory and
hugetlb memory faulted in without a prior reservation.
- If a task attempts to reserve more memory than limit_in_bytes allows,
the kernel will allow it to do so. But if a task attempts to reserve
more memory than reservation_limit_in_bytes, the kernel will fail this
reservation.
This proposal is implemented in this patch series, with tests to verify
functionality and show the usage. We also added cgroup-v2 support to
hugetlb_cgroup so that the new use cases can be extended to v2.
Alternatives considered:
1. A new cgroup, instead of only a new page_counter attached to
the existing hugetlb_cgroup. Adding a new cgroup seemed like a lot of code
duplication with hugetlb_cgroup. Keeping hugetlb related page counters under
hugetlb_cgroup seemed cleaner as well.
2. Instead of adding a new counter, we considered adding a sysctl that modifies
the behavior of hugetlb.xMB.[limit|usage]_in_bytes, to do accounting at
reservation time rather than fault time. Adding a new page_counter seems
better as userspace could, if it wants, choose to enforce different cgroups
differently: one via limit_in_bytes, and another via
reservation_limit_in_bytes. This could be very useful if you're
transitioning how hugetlb memory is partitioned on your system one
cgroup at a time, for example. Also, someone may find usage for both
limit_in_bytes and reservation_limit_in_bytes concurrently, and this
approach gives them the option to do so.
Testing:
- Added tests passing.
- libhugetlbfs tests mostly passing, but some tests have trouble with and
without this patch series. Seems environment issue rather than code:
- Overall results:
********** TEST SUMMARY
* 2M
* 32-bit 64-bit
* Total testcases: 84 0
* Skipped: 0 0
* PASS: 66 0
* FAIL: 14 0
* Killed by signal: 0 0
* Bad configuration: 4 0
* Expected FAIL: 0 0
* Unexpected PASS: 0 0
* Test not present: 0 0
* Strange test result: 0 0
**********
- Failing tests:
- elflink_rw_and_share_test("linkhuge_rw") segfaults with and without this
patch series.
- LD_PRELOAD=libhugetlbfs.so HUGETLB_MORECORE=yes malloc (2M: 32):
FAIL Address is not hugepage
- LD_PRELOAD=libhugetlbfs.so HUGETLB_RESTRICT_EXE=unknown:malloc
HUGETLB_MORECORE=yes malloc (2M: 32):
FAIL Address is not hugepage
- LD_PRELOAD=libhugetlbfs.so HUGETLB_MORECORE=yes malloc_manysmall (2M: 32):
FAIL Address is not hugepage
- GLIBC_TUNABLES=glibc.malloc.tcache_count=0 LD_PRELOAD=libhugetlbfs.so
HUGETLB_MORECORE=yes heapshrink (2M: 32):
FAIL Heap not on hugepages
- GLIBC_TUNABLES=glibc.malloc.tcache_count=0 LD_PRELOAD=libhugetlbfs.so
libheapshrink.so HUGETLB_MORECORE=yes heapshrink (2M: 32):
FAIL Heap not on hugepages
- HUGETLB_ELFMAP=RW linkhuge_rw (2M: 32): FAIL small_data is not hugepage
- HUGETLB_ELFMAP=RW HUGETLB_MINIMAL_COPY=no linkhuge_rw (2M: 32):
FAIL small_data is not hugepage
- alloc-instantiate-race shared (2M: 32):
Bad configuration: sched_setaffinity(cpu1): Invalid argument -
FAIL Child 1 killed by signal Killed
- shmoverride_linked (2M: 32):
FAIL shmget failed size 2097152 from line 176: Invalid argument
- HUGETLB_SHM=yes shmoverride_linked (2M: 32):
FAIL shmget failed size 2097152 from line 176: Invalid argument
- shmoverride_linked_static (2M: 32):
FAIL shmget failed size 2097152 from line 176: Invalid argument
- HUGETLB_SHM=yes shmoverride_linked_static (2M: 32):
FAIL shmget failed size 2097152 from line 176: Invalid argument
- LD_PRELOAD=libhugetlbfs.so shmoverride_unlinked (2M: 32):
FAIL shmget failed size 2097152 from line 176: Invalid argument
- LD_PRELOAD=libhugetlbfs.so HUGETLB_SHM=yes shmoverride_unlinked (2M: 32):
FAIL shmget failed size 2097152 from line 176: Invalid argument
[1]: https://www.kernel.org/doc/html/latest/vm/hugetlbfs_reserv.html
Signed-off-by: Mina Almasry <almasrymina(a)google.com>
Acked-by: Hillf Danton <hdanton(a)sina.com>
---
include/linux/hugetlb.h | 23 ++++++++-
mm/hugetlb_cgroup.c | 111 ++++++++++++++++++++++++++++++----------
2 files changed, 107 insertions(+), 27 deletions(-)
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 53fc34f930d08..9c49a0ba894d3 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -320,6 +320,27 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
#ifdef CONFIG_HUGETLB_PAGE
+enum {
+ /* Tracks hugetlb memory faulted in. */
+ HUGETLB_RES_USAGE,
+ /* Tracks hugetlb memory reserved. */
+ HUGETLB_RES_RESERVATION_USAGE,
+ /* Limit for hugetlb memory faulted in. */
+ HUGETLB_RES_LIMIT,
+ /* Limit for hugetlb memory reserved. */
+ HUGETLB_RES_RESERVATION_LIMIT,
+ /* Max usage for hugetlb memory faulted in. */
+ HUGETLB_RES_MAX_USAGE,
+ /* Max usage for hugetlb memory reserved. */
+ HUGETLB_RES_RESERVATION_MAX_USAGE,
+ /* Faulted memory accounting fail count. */
+ HUGETLB_RES_FAILCNT,
+ /* Reserved memory accounting fail count. */
+ HUGETLB_RES_RESERVATION_FAILCNT,
+ HUGETLB_RES_NULL,
+ HUGETLB_RES_MAX,
+};
+
#define HSTATE_NAME_LEN 32
/* Defines one hugetlb page size */
struct hstate {
@@ -340,7 +361,7 @@ struct hstate {
unsigned int surplus_huge_pages_node[MAX_NUMNODES];
#ifdef CONFIG_CGROUP_HUGETLB
/* cgroup control files */
- struct cftype cgroup_files[5];
+ struct cftype cgroup_files[HUGETLB_RES_MAX];
#endif
char name[HSTATE_NAME_LEN];
};
diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c
index f1930fa0b445d..1ed4448ca41d3 100644
--- a/mm/hugetlb_cgroup.c
+++ b/mm/hugetlb_cgroup.c
@@ -25,6 +25,10 @@ struct hugetlb_cgroup {
* the counter to account for hugepages from hugetlb.
*/
struct page_counter hugepage[HUGE_MAX_HSTATE];
+ /*
+ * the counter to account for hugepage reservations from hugetlb.
+ */
+ struct page_counter reserved_hugepage[HUGE_MAX_HSTATE];
};
#define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
@@ -33,6 +37,14 @@ struct hugetlb_cgroup {
static struct hugetlb_cgroup *root_h_cgroup __read_mostly;
+static inline struct page_counter *
+hugetlb_cgroup_get_counter(struct hugetlb_cgroup *h_cg, int idx, bool reserved)
+{
+ if (reserved)
+ return &h_cg->reserved_hugepage[idx];
+ return &h_cg->hugepage[idx];
+}
+
static inline
struct hugetlb_cgroup *hugetlb_cgroup_from_css(struct cgroup_subsys_state *s)
{
@@ -254,30 +266,33 @@ void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages,
return;
}
-enum {
- RES_USAGE,
- RES_LIMIT,
- RES_MAX_USAGE,
- RES_FAILCNT,
-};
-
static u64 hugetlb_cgroup_read_u64(struct cgroup_subsys_state *css,
struct cftype *cft)
{
struct page_counter *counter;
+ struct page_counter *reserved_counter;
struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css);
counter = &h_cg->hugepage[MEMFILE_IDX(cft->private)];
+ reserved_counter = &h_cg->reserved_hugepage[MEMFILE_IDX(cft->private)];
switch (MEMFILE_ATTR(cft->private)) {
- case RES_USAGE:
+ case HUGETLB_RES_USAGE:
return (u64)page_counter_read(counter) * PAGE_SIZE;
- case RES_LIMIT:
+ case HUGETLB_RES_RESERVATION_USAGE:
+ return (u64)page_counter_read(reserved_counter) * PAGE_SIZE;
+ case HUGETLB_RES_LIMIT:
return (u64)counter->max * PAGE_SIZE;
- case RES_MAX_USAGE:
+ case HUGETLB_RES_RESERVATION_LIMIT:
+ return (u64)reserved_counter->max * PAGE_SIZE;
+ case HUGETLB_RES_MAX_USAGE:
return (u64)counter->watermark * PAGE_SIZE;
- case RES_FAILCNT:
+ case HUGETLB_RES_RESERVATION_MAX_USAGE:
+ return (u64)reserved_counter->watermark * PAGE_SIZE;
+ case HUGETLB_RES_FAILCNT:
return counter->failcnt;
+ case HUGETLB_RES_RESERVATION_FAILCNT:
+ return reserved_counter->failcnt;
default:
BUG();
}
@@ -291,6 +306,7 @@ static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of,
int ret, idx;
unsigned long nr_pages;
struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of));
+ bool reserved = false;
if (hugetlb_cgroup_is_root(h_cg)) /* Can't set limit on root */
return -EINVAL;
@@ -304,9 +320,14 @@ static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of,
nr_pages = round_down(nr_pages, 1 << huge_page_order(&hstates[idx]));
switch (MEMFILE_ATTR(of_cft(of)->private)) {
- case RES_LIMIT:
+ case HUGETLB_RES_RESERVATION_LIMIT:
+ reserved = true;
+ /* Fall through. */
+ case HUGETLB_RES_LIMIT:
mutex_lock(&hugetlb_limit_mutex);
- ret = page_counter_set_max(&h_cg->hugepage[idx], nr_pages);
+ ret = page_counter_set_max(hugetlb_cgroup_get_counter(h_cg, idx,
+ reserved),
+ nr_pages);
mutex_unlock(&hugetlb_limit_mutex);
break;
default:
@@ -320,18 +341,26 @@ static ssize_t hugetlb_cgroup_reset(struct kernfs_open_file *of,
char *buf, size_t nbytes, loff_t off)
{
int ret = 0;
- struct page_counter *counter;
+ struct page_counter *counter, *reserved_counter;
struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of));
counter = &h_cg->hugepage[MEMFILE_IDX(of_cft(of)->private)];
+ reserved_counter =
+ &h_cg->reserved_hugepage[MEMFILE_IDX(of_cft(of)->private)];
switch (MEMFILE_ATTR(of_cft(of)->private)) {
- case RES_MAX_USAGE:
+ case HUGETLB_RES_MAX_USAGE:
page_counter_reset_watermark(counter);
break;
- case RES_FAILCNT:
+ case HUGETLB_RES_RESERVATION_MAX_USAGE:
+ page_counter_reset_watermark(reserved_counter);
+ break;
+ case HUGETLB_RES_FAILCNT:
counter->failcnt = 0;
break;
+ case HUGETLB_RES_RESERVATION_FAILCNT:
+ reserved_counter->failcnt = 0;
+ break;
default:
ret = -EINVAL;
break;
@@ -357,37 +386,67 @@ static void __init __hugetlb_cgroup_file_init(int idx)
struct hstate *h = &hstates[idx];
/* format the size */
- mem_fmt(buf, 32, huge_page_size(h));
+ mem_fmt(buf, sizeof(buf), huge_page_size(h));
/* Add the limit file */
- cft = &h->cgroup_files[0];
+ cft = &h->cgroup_files[HUGETLB_RES_LIMIT];
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.limit_in_bytes", buf);
- cft->private = MEMFILE_PRIVATE(idx, RES_LIMIT);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_LIMIT);
+ cft->read_u64 = hugetlb_cgroup_read_u64;
+ cft->write = hugetlb_cgroup_write;
+
+ /* Add the reservation limit file */
+ cft = &h->cgroup_files[HUGETLB_RES_RESERVATION_LIMIT];
+ snprintf(cft->name, MAX_CFTYPE_NAME, "%s.reservation_limit_in_bytes",
+ buf);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_RESERVATION_LIMIT);
cft->read_u64 = hugetlb_cgroup_read_u64;
cft->write = hugetlb_cgroup_write;
/* Add the usage file */
- cft = &h->cgroup_files[1];
+ cft = &h->cgroup_files[HUGETLB_RES_USAGE];
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.usage_in_bytes", buf);
- cft->private = MEMFILE_PRIVATE(idx, RES_USAGE);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_USAGE);
+ cft->read_u64 = hugetlb_cgroup_read_u64;
+
+ /* Add the reservation usage file */
+ cft = &h->cgroup_files[HUGETLB_RES_RESERVATION_USAGE];
+ snprintf(cft->name, MAX_CFTYPE_NAME, "%s.reservation_usage_in_bytes",
+ buf);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_RESERVATION_USAGE);
cft->read_u64 = hugetlb_cgroup_read_u64;
/* Add the MAX usage file */
- cft = &h->cgroup_files[2];
+ cft = &h->cgroup_files[HUGETLB_RES_MAX_USAGE];
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.max_usage_in_bytes", buf);
- cft->private = MEMFILE_PRIVATE(idx, RES_MAX_USAGE);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_MAX_USAGE);
+ cft->write = hugetlb_cgroup_reset;
+ cft->read_u64 = hugetlb_cgroup_read_u64;
+
+ /* Add the MAX reservation usage file */
+ cft = &h->cgroup_files[HUGETLB_RES_RESERVATION_MAX_USAGE];
+ snprintf(cft->name, MAX_CFTYPE_NAME,
+ "%s.reservation_max_usage_in_bytes", buf);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_RESERVATION_MAX_USAGE);
cft->write = hugetlb_cgroup_reset;
cft->read_u64 = hugetlb_cgroup_read_u64;
/* Add the failcntfile */
- cft = &h->cgroup_files[3];
+ cft = &h->cgroup_files[HUGETLB_RES_FAILCNT];
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.failcnt", buf);
- cft->private = MEMFILE_PRIVATE(idx, RES_FAILCNT);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_FAILCNT);
+ cft->write = hugetlb_cgroup_reset;
+ cft->read_u64 = hugetlb_cgroup_read_u64;
+
+ /* Add the reservation failcntfile */
+ cft = &h->cgroup_files[HUGETLB_RES_RESERVATION_FAILCNT];
+ snprintf(cft->name, MAX_CFTYPE_NAME, "%s.reservation_failcnt", buf);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_RESERVATION_FAILCNT);
cft->write = hugetlb_cgroup_reset;
cft->read_u64 = hugetlb_cgroup_read_u64;
/* NULL terminate the last cft */
- cft = &h->cgroup_files[4];
+ cft = &h->cgroup_files[HUGETLB_RES_NULL];
memset(cft, 0, sizeof(*cft));
WARN_ON(cgroup_add_legacy_cftypes(&hugetlb_cgrp_subsys,
--
2.24.0.rc1.363.gb1bccd3e3d-goog
Hi,
Here is a set of well-reviewed (expect for one patch), lower-risk items
that can go into Linux 5.5. The one patch that wasn't reviewed is the
powerpc conversion, and it's still at this point a no-op, because
tracking isn't yet activated.
This is based on linux-next: b9d3d01405061bb42358fe53f824e894a1922ced
("Add linux-next specific files for 20191122").
This is essentially a cut-down v8 of "mm/gup: track dma-pinned pages:
FOLL_PIN" [1], and with one of the VFIO patches split into two patches.
The idea here is to get this long list of "noise" checked into 5.5, so
that the actual, higher-risk "track FOLL_PIN pages" (which is deferred:
not part of this series) will be a much shorter patchset to review.
For the v4l2-core changes, I've left those here (instead of sending
them separately to the -media tree), in order to get the name change
done now (put_user_page --> unpin_user_page). However, I've added a Cc
stable, as recommended during the last round of reviews.
Here are the relevant notes from the original cover letter, edited to
match the current situation:
This is a prerequisite to tracking dma-pinned pages. That in turn is a
prerequisite to solving the larger problem of proper interactions
between file-backed pages, and [R]DMA activities, as discussed in [1],
[2], [3], and in a remarkable number of email threads since about
2017. :)
A new internal gup flag, FOLL_PIN is introduced, and thoroughly
documented in the last patch's Documentation/vm/pin_user_pages.rst.
I believe that this will provide a good starting point for doing the
layout lease work that Ira Weiny has been working on. That's because
these new wrapper functions provide a clean, constrained, systematically
named set of functionality that, again, is required in order to even
know if a page is "dma-pinned".
In contrast to earlier approaches, the page tracking can be
incrementally applied to the kernel call sites that, until now, have
been simply calling get_user_pages() ("gup"). In other words, opt-in by
changing from this:
get_user_pages() (sets FOLL_GET)
put_page()
to this:
pin_user_pages() (sets FOLL_PIN)
put_user_page()
Because there are interdependencies with FOLL_LONGTERM, a similar
conversion as for FOLL_PIN, was applied. The change was from this:
get_user_pages(FOLL_LONGTERM) (also sets FOLL_GET)
put_page()
to this:
pin_longterm_pages() (sets FOLL_PIN | FOLL_LONGTERM)
put_user_page()
[1] https://lore.kernel.org/r/20191121071354.456618-1-jhubbard@nvidia.com
thanks,
John Hubbard
NVIDIA
Dan Williams (1):
mm: Cleanup __put_devmap_managed_page() vs ->page_free()
John Hubbard (18):
mm/gup: factor out duplicate code from four routines
mm/gup: move try_get_compound_head() to top, fix minor issues
goldish_pipe: rename local pin_user_pages() routine
mm: fix get_user_pages_remote()'s handling of FOLL_LONGTERM
vfio: fix FOLL_LONGTERM use, simplify get_user_pages_remote() call
mm/gup: introduce pin_user_pages*() and FOLL_PIN
goldish_pipe: convert to pin_user_pages() and put_user_page()
IB/{core,hw,umem}: set FOLL_PIN via pin_user_pages*(), fix up ODP
mm/process_vm_access: set FOLL_PIN via pin_user_pages_remote()
drm/via: set FOLL_PIN via pin_user_pages_fast()
fs/io_uring: set FOLL_PIN via pin_user_pages()
net/xdp: set FOLL_PIN via pin_user_pages()
media/v4l2-core: set pages dirty upon releasing DMA buffers
media/v4l2-core: pin_user_pages (FOLL_PIN) and put_user_page()
conversion
vfio, mm: pin_user_pages (FOLL_PIN) and put_user_page() conversion
powerpc: book3s64: convert to pin_user_pages() and put_user_page()
mm/gup_benchmark: use proper FOLL_WRITE flags instead of hard-coding
"1"
mm, tree-wide: rename put_user_page*() to unpin_user_page*()
Documentation/core-api/index.rst | 1 +
Documentation/core-api/pin_user_pages.rst | 233 ++++++++++++++
arch/powerpc/mm/book3s64/iommu_api.c | 12 +-
drivers/gpu/drm/via/via_dmablit.c | 6 +-
drivers/infiniband/core/umem.c | 4 +-
drivers/infiniband/core/umem_odp.c | 13 +-
drivers/infiniband/hw/hfi1/user_pages.c | 4 +-
drivers/infiniband/hw/mthca/mthca_memfree.c | 8 +-
drivers/infiniband/hw/qib/qib_user_pages.c | 4 +-
drivers/infiniband/hw/qib/qib_user_sdma.c | 8 +-
drivers/infiniband/hw/usnic/usnic_uiom.c | 4 +-
drivers/infiniband/sw/siw/siw_mem.c | 4 +-
drivers/media/v4l2-core/videobuf-dma-sg.c | 8 +-
drivers/nvdimm/pmem.c | 6 -
drivers/platform/goldfish/goldfish_pipe.c | 35 +--
drivers/vfio/vfio_iommu_type1.c | 35 +--
fs/io_uring.c | 6 +-
include/linux/mm.h | 77 +++--
mm/gup.c | 332 +++++++++++++-------
mm/gup_benchmark.c | 9 +-
mm/memremap.c | 80 ++---
mm/process_vm_access.c | 28 +-
net/xdp/xdp_umem.c | 4 +-
tools/testing/selftests/vm/gup_benchmark.c | 6 +-
24 files changed, 642 insertions(+), 285 deletions(-)
create mode 100644 Documentation/core-api/pin_user_pages.rst
--
2.24.0
Hi,
Here is the 2nd version of patches to fix some issues which happens on
the kernel with CONFIG_FUNCTION_TRACER=n or CONFIG_DYNAMIC_FTRACE=n.
In this version I just updated the descriptions of the first 2 patches
according to Steve's comment and add his Reviewed-by to the last patch.
Thank you,
---
Masami Hiramatsu (3):
selftests/ftrace: Fix to check the existence of set_ftrace_filter
selftests/ftrace: Fix ftrace test cases to check unsupported
selftests/ftrace: Do not to use absolute debugfs path
.../ftrace/test.d/ftrace/func-filter-stacktrace.tc | 2 ++
.../selftests/ftrace/test.d/ftrace/func_cpumask.tc | 5 +++++
tools/testing/selftests/ftrace/test.d/functions | 4 +++-
.../inter-event/trigger-action-hist-xfail.tc | 4 ++--
.../inter-event/trigger-onchange-action-hist.tc | 2 +-
.../inter-event/trigger-snapshot-action-hist.tc | 4 ++--
6 files changed, 15 insertions(+), 6 deletions(-)
--
Masami Hiramatsu (Linaro) <mhiramat(a)kernel.org>
Hi,
Here is a series of patches to fix some issues which happens on
the kernel with CONFIG_FUNCTION_TRACER=n but CONFIG_TRACER=y.
Thank you,
---
Masami Hiramatsu (3):
selftests/ftrace: Fix to check the existence of set_ftrace_filter
selftests/ftrace: Fix ftrace test cases to check unsupported
selftests/ftrace: Do not to use absolute debugfs path
.../ftrace/test.d/ftrace/func-filter-stacktrace.tc | 2 ++
.../selftests/ftrace/test.d/ftrace/func_cpumask.tc | 5 +++++
tools/testing/selftests/ftrace/test.d/functions | 4 +++-
.../inter-event/trigger-action-hist-xfail.tc | 4 ++--
.../inter-event/trigger-onchange-action-hist.tc | 2 +-
.../inter-event/trigger-snapshot-action-hist.tc | 4 ++--
6 files changed, 15 insertions(+), 6 deletions(-)
--
Masami Hiramatsu (Linaro) <mhiramat(a)kernel.org>
If settings files are present in upper directories of a test directory,
load those first before the local settings. This allows a top-level
directory to define settings for subdirectories while still allowing the
subdirectories to override them on a per-directory basis.
Signed-off-by: Kees Cook <keescook(a)chromium.org>
---
Note that this depends on Matt's patch:
https://lore.kernel.org/lkml/20191022171223.27934-1-matthieu.baerts@tessare…
---
tools/testing/selftests/kselftest/runner.sh | 30 +++++++++++++++------
1 file changed, 22 insertions(+), 8 deletions(-)
diff --git a/tools/testing/selftests/kselftest/runner.sh b/tools/testing/selftests/kselftest/runner.sh
index 84de7bc74f2c..666cfeaa8046 100644
--- a/tools/testing/selftests/kselftest/runner.sh
+++ b/tools/testing/selftests/kselftest/runner.sh
@@ -39,6 +39,27 @@ tap_timeout()
fi
}
+load_settings()
+{
+ local fullpath="$1"
+ local upperpath=${fullpath%/*}
+
+ # Load upper path settings first.
+ if [ "$fullpath" != "$upperpath" ] ; then
+ load_settings "$upperpath"
+ fi
+
+ # Load per-test-directory kselftest "settings" file.
+ local settings="$BASE_DIR/$fullpath/settings"
+ if [ -r "$settings" ] ; then
+ while read line ; do
+ local field=$(echo "$line" | cut -d= -f1)
+ local value=$(echo "$line" | cut -d= -f2-)
+ eval "kselftest_$field"="$value"
+ done < "$settings"
+ fi
+}
+
run_one()
{
DIR="$1"
@@ -50,14 +71,7 @@ run_one()
# Reset any "settings"-file variables.
export kselftest_timeout="$kselftest_default_timeout"
# Load per-test-directory kselftest "settings" file.
- settings="$BASE_DIR/$DIR/settings"
- if [ -r "$settings" ] ; then
- while read line ; do
- field=$(echo "$line" | cut -d= -f1)
- value=$(echo "$line" | cut -d= -f2-)
- eval "kselftest_$field"="$value"
- done < "$settings"
- fi
+ load_settings "$DIR"
TEST_HDR_MSG="selftests: $DIR: $BASENAME_TEST"
echo "# $TEST_HDR_MSG"
--
2.17.1
--
Kees Cook
Add documentation for the Python script used to build, run, and collect
results from the kernel known as kunit_tool. kunit_tool
(tools/testing/kunit/kunit.py) was already added in previous commits.
Signed-off-by: Brendan Higgins <brendanhiggins(a)google.com>
Reviewed-by: David Gow <davidgow(a)google.com>
---
Documentation/dev-tools/kunit/index.rst | 1 +
Documentation/dev-tools/kunit/kunit-tool.rst | 57 ++++++++++++++++++++
Documentation/dev-tools/kunit/start.rst | 5 +-
3 files changed, 62 insertions(+), 1 deletion(-)
create mode 100644 Documentation/dev-tools/kunit/kunit-tool.rst
diff --git a/Documentation/dev-tools/kunit/index.rst b/Documentation/dev-tools/kunit/index.rst
index 26ffb46bdf99d..c60d760a0eed1 100644
--- a/Documentation/dev-tools/kunit/index.rst
+++ b/Documentation/dev-tools/kunit/index.rst
@@ -9,6 +9,7 @@ KUnit - Unit Testing for the Linux Kernel
start
usage
+ kunit-tool
api/index
faq
diff --git a/Documentation/dev-tools/kunit/kunit-tool.rst b/Documentation/dev-tools/kunit/kunit-tool.rst
new file mode 100644
index 0000000000000..37509527c04e1
--- /dev/null
+++ b/Documentation/dev-tools/kunit/kunit-tool.rst
@@ -0,0 +1,57 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=================
+kunit_tool How-To
+=================
+
+What is kunit_tool?
+===================
+
+kunit_tool is a script (``tools/testing/kunit/kunit.py``) that aids in building
+the Linux kernel as UML (`User Mode Linux
+<http://user-mode-linux.sourceforge.net/>`_), running KUnit tests, parsing
+the test results and displaying them in a user friendly manner.
+
+What is a kunitconfig?
+======================
+
+It's just a defconfig that kunit_tool looks for in the base directory.
+kunit_tool uses it to generate a .config as you might expect. In addition, it
+verifies that the generated .config contains the CONFIG options in the
+kunitconfig; the reason it does this is so that it is easy to be sure that a
+CONFIG that enables a test actually ends up in the .config.
+
+How do I use kunit_tool?
+=================================
+
+If a kunitconfig is present at the root directory, all you have to do is:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py run
+
+However, you most likely want to use it with the following options:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py run --timeout=30 --jobs=`nproc --all`
+
+- ``--timeout`` sets a maximum amount of time to allow tests to run.
+- ``--jobs`` sets the number of threads to use to build the kernel.
+
+If you just want to use the defconfig that ships with the kernel, you can
+append the ``--defconfig`` flag as well:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py run --timeout=30 --jobs=`nproc --all` --defconfig
+
+.. note::
+ This command is particularly helpful for getting started because it
+ just works. No kunitconfig needs to be present.
+
+For a list of all the flags supported by kunit_tool, you can run:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py run --help
diff --git a/Documentation/dev-tools/kunit/start.rst b/Documentation/dev-tools/kunit/start.rst
index aeeddfafeea20..f4d9a4fa914f8 100644
--- a/Documentation/dev-tools/kunit/start.rst
+++ b/Documentation/dev-tools/kunit/start.rst
@@ -19,7 +19,10 @@ The wrapper can be run with:
.. code-block:: bash
- ./tools/testing/kunit/kunit.py run
+ ./tools/testing/kunit/kunit.py run --defconfig
+
+For more information on this wrapper (also called kunit_tool) checkout the
+:doc:`kunit-tool` page.
Creating a kunitconfig
======================
--
2.24.0.432.g9d3f5f5b63-goog
The current kunit execution model is to provide base kunit functionality
and tests built-in to the kernel. The aim of this series is to allow
building kunit itself and tests as modules. This in turn allows a
simple form of selective execution; load the module you wish to test.
In doing so, kunit itself (if also built as a module) will be loaded as
an implicit dependency.
Because this requires a core API modification - if a module delivers
multiple suites, they must be declared with the kunit_test_suites()
macro - we're proposing this patch set as a candidate to be applied to the
test tree before too many kunit consumers appear. We attempt to deal
with existing consumers in patch 3.
Changes since v3:
- removed symbol lookup patch for separate submission later
- removed use of sysctl_hung_task_timeout_seconds (patch 4, as discussed
with Brendan and Stephen)
- disabled build of string-stream-test when CONFIG_KUNIT_TEST=m; this
is to avoid having to deal with symbol lookup issues
- changed string-stream-impl.h back to string-stream.h (Brendan)
- added module build support to new list, ext4 tests
Changes since v2:
- moved string-stream.h header to lib/kunit/string-stream-impl.h (Brendan)
(patch 1)
- split out non-exported interfaces in try-catch-impl.h (Brendan)
(patch 2)
- added kunit_find_symbol() and KUNIT_INIT_SYMBOL to lookup non-exported
symbols (patches 3, 4)
- removed #ifdef MODULE around module licenses (Randy, Brendan, Andy)
(patch 4)
- replaced kunit_test_suite() with kunit_test_suites() rather than
supporting both (Brendan) (patch 4)
- lookup sysctl_hung_task_timeout_secs as kunit may be built as a module
and the symbol may not be available (patch 5)
Alan Maguire (6):
kunit: move string-stream.h to lib/kunit
kunit: hide unexported try-catch interface in try-catch-impl.h
kunit: allow kunit tests to be loaded as a module
kunit: remove timeout dependence on sysctl_hung_task_timeout_seconds
kunit: allow kunit to be loaded as a module
kunit: update documentation to describe module-based build
Documentation/dev-tools/kunit/faq.rst | 3 +-
Documentation/dev-tools/kunit/index.rst | 3 +
Documentation/dev-tools/kunit/usage.rst | 16 ++
fs/ext4/Kconfig | 2 +-
fs/ext4/Makefile | 5 +
fs/ext4/inode-test.c | 4 +-
include/kunit/assert.h | 3 +-
include/kunit/string-stream.h | 51 -----
include/kunit/test.h | 35 +++-
include/kunit/try-catch.h | 10 -
kernel/sysctl-test.c | 4 +-
lib/Kconfig.debug | 4 +-
lib/kunit/Kconfig | 6 +-
lib/kunit/Makefile | 14 +-
lib/kunit/assert.c | 10 +
lib/kunit/example-test.c | 88 ---------
lib/kunit/kunit-example-test.c | 90 +++++++++
lib/kunit/kunit-test.c | 334 ++++++++++++++++++++++++++++++++
lib/kunit/string-stream-test.c | 5 +-
lib/kunit/string-stream.c | 3 +-
lib/kunit/string-stream.h | 51 +++++
lib/kunit/test-test.c | 331 -------------------------------
lib/kunit/test.c | 25 ++-
lib/kunit/try-catch-impl.h | 28 +++
lib/kunit/try-catch.c | 37 +---
lib/list-test.c | 4 +-
26 files changed, 628 insertions(+), 538 deletions(-)
delete mode 100644 include/kunit/string-stream.h
delete mode 100644 lib/kunit/example-test.c
create mode 100644 lib/kunit/kunit-example-test.c
create mode 100644 lib/kunit/kunit-test.c
create mode 100644 lib/kunit/string-stream.h
delete mode 100644 lib/kunit/test-test.c
create mode 100644 lib/kunit/try-catch-impl.h
--
1.8.3.1
Summary
------------------------------------------------------------------------
kernel: 5.4.0-rc8
git repo: https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git
git branch: master
git commit: 1fef9976397fc9b951ee54467eccd65e0c508785
git describe: next-20191120
Test details: https://qa-reports.linaro.org/lkft/linux-next-oe/build/next-20191120
Regressions (compared to build next-20191119)
------------------------------------------------------------------------
No regressions
Fixes (compared to build next-20191119)
------------------------------------------------------------------------
No fixes
In total:
------------------------------------------------------------------------
Ran 0 total tests in the following environments and test suites.
pass 0
fail 0
xfail 0
skip 0
Environments
--------------
- x15 - arm
Test Suites
-----------
Failures
------------------------------------------------------------------------
x15:
Skips
------------------------------------------------------------------------
No skips
--
Linaro LKFT
https://lkft.linaro.org
Hi,
Christoph Hellwig has a preference to do things a little differently,
for the devmap cleanup in patch 5 ("mm: devmap: refactor 1-based
refcounting for ZONE_DEVICE pages"). That came up in a different
review thread, because the patch is out for review in two locations.
Here's that review thread:
https://lore.kernel.org/r/20191118070826.GB3099@infradead.org
...and I'm hoping that we can defer that request, because otherwise
it derails this series, which is starting to otherwise look like
it could be ready for 5.5.
There is a git repo and branch, for convenience:
git@github.com:johnhubbard/linux.git pin_user_pages_tracking_v6
Changes since v5:
* Fixed the refcounting for huge pages: in most cases, it was
only taking one GUP_PIN_COUNTING_BIAS's worth of refs, when it
should have been taking one GUP_PIN_COUNTING_BIAS for each subpage.
(Much thanks to Jan Kara for spotting that one!)
* Renamed user_page_ref_inc() to try_pin_page(), and added a new
try_pin_compound_head(). This definitely improves readability.
* Factored out some more duplication in the FOLL_PIN and FOLL_GET
cases, in gup.c.
* Fixed up some straggling "get_" --> "pin_" references in the comments.
* Added reviewed-by tags.
Changes since v4:
* Renamed put_user_page*() --> unpin_user_page().
* Removed all pin_longterm_pages*() calls. We will use FOLL_LONGTERM
at the call sites. (FOLL_PIN, however, remains an internal gup flag).
This is very nice: many patches just change three characters now:
get_user_pages --> pin_user_pages. I think we've found the right
balance of wrapper calls and gup flags, for the call sites.
* Updated a lot of documentation and commit logs to match the above
two large changes.
* Changed gup_benchmark tests and run_vmtests, to adapt to one less
use case: there is no pin_longterm_pages() call anymore.
* This includes a new devmap cleanup patch from Dan Williams, along
with a rebased follow-up: patches 4 and 5, already mentioned above.
* Fixed patch 10 ("mm/gup: introduce pin_user_pages*() and FOLL_PIN"),
so as to make pin_user_pages*() calls act as placeholders for the
corresponding get_user_pages*() calls, until a later patch fully
implements the DMA-pinning functionality.
Thanks to Jan Kara for noticing that.
* Fixed the implementation of pin_user_pages_remote().
* Further tweaked patch 2 ("mm/gup: factor out duplicate code from four
routines"), in response to Jan Kara's feedback.
* Dropped a few reviewed-by tags due to changes that invalidated
them.
Changes since v3:
* VFIO fix (patch 8): applied further cleanup: removed a pre-existing,
unnecessary release and reacquire of mmap_sem. Moved the DAX vma
checks from the vfio call site, to gup internals, and added comments
(and commit log) to clarify.
* Due to the above, made a corresponding fix to the
pin_longterm_pages_remote(), which was actually calling the wrong
gup internal function.
* Changed put_user_page() comments, to refer to pin*() APIs, rather than
get_user_pages*() APIs.
* Reverted an accidental whitespace-only change in the IB ODP code.
* Added a few more reviewed-by tags.
Changes since v2:
* Added a patch to convert IB/umem from normal gup, to gup_fast(). This
is also posted separately, in order to hopefully get some runtime
testing.
* Changed the page devmap code to be a little clearer,
thanks to Jerome for that.
* Split out the page devmap changes into a separate patch (and moved
Ira's Signed-off-by to that patch).
* Fixed my bug in IB: ODP code does not require pin_user_pages()
semantics. Therefore, revert the put_user_page() calls to put_page(),
and leave the get_user_pages() call as-is.
* As part of the revert, I am proposing here a change directly
from put_user_pages(), to release_pages(). I'd feel better if
someone agrees that this is the best way. It uses the more
efficient release_pages(), instead of put_page() in a loop,
and keep the change to just a few character on one line,
but OTOH it is not a pure revert.
* Loosened the FOLL_LONGTERM restrictions in the
__get_user_pages_locked() implementation, and used that in order
to fix up a VFIO bug. Thanks to Jason for that idea.
* Note the use of release_pages() in IB: is that OK?
* Added a few more WARN's and clarifying comments nearby.
* Many documentation improvements in various comments.
* Moved the new pin_user_pages.rst from Documentation/vm/ to
Documentation/core-api/ .
* Commit descriptions: added clarifying notes to the three patches
(drm/via, fs/io_uring, net/xdp) that already had put_user_page()
calls in place.
* Collected all pending Reviewed-by and Acked-by tags, from v1 and v2
email threads.
* Lot of churn from v2 --> v3, so it's possible that new bugs
sneaked in.
NOT DONE: separate patchset is required:
* __get_user_pages_locked(): stop compensating for
buggy callers who failed to set FOLL_GET. Instead, assert
that FOLL_GET is set (and fail if it's not).
======================================================================
Original cover letter (edited to fix up the patch description numbers)
This applies cleanly to linux-next and mmotm, and also to linux.git if
linux-next's commit 20cac10710c9 ("mm/gup_benchmark: fix MAP_HUGETLB
case") is first applied there.
This provides tracking of dma-pinned pages. This is a prerequisite to
solving the larger problem of proper interactions between file-backed
pages, and [R]DMA activities, as discussed in [1], [2], [3], and in
a remarkable number of email threads since about 2017. :)
A new internal gup flag, FOLL_PIN is introduced, and thoroughly
documented in the last patch's Documentation/vm/pin_user_pages.rst.
I believe that this will provide a good starting point for doing the
layout lease work that Ira Weiny has been working on. That's because
these new wrapper functions provide a clean, constrained, systematically
named set of functionality that, again, is required in order to even
know if a page is "dma-pinned".
In contrast to earlier approaches, the page tracking can be
incrementally applied to the kernel call sites that, until now, have
been simply calling get_user_pages() ("gup"). In other words, opt-in by
changing from this:
get_user_pages() (sets FOLL_GET)
put_page()
to this:
pin_user_pages() (sets FOLL_PIN)
put_user_page()
Because there are interdependencies with FOLL_LONGTERM, a similar
conversion as for FOLL_PIN, was applied. The change was from this:
get_user_pages(FOLL_LONGTERM) (also sets FOLL_GET)
put_page()
to this:
pin_longterm_pages() (sets FOLL_PIN | FOLL_LONGTERM)
put_user_page()
============================================================
Patch summary:
* Patches 1-9: refactoring and preparatory cleanup, independent fixes
* Patch 10: introduce pin_user_pages(), FOLL_PIN, but no functional
changes yet
* Patches 11-16: Convert existing put_user_page() callers, to use the
new pin*()
* Patch 17: Activate tracking of FOLL_PIN pages.
* Patches 18-20: convert various callers
* Patches: 21-23: gup_benchmark and run_vmtests support
* Patch 24: rename put_user_page*() --> unpin_user_page*()
============================================================
Testing:
* I've done some overall kernel testing (LTP, and a few other goodies),
and some directed testing to exercise some of the changes. And as you
can see, gup_benchmark is enhanced to exercise this. Basically, I've been
able to runtime test the core get_user_pages() and pin_user_pages() and
related routines, but not so much on several of the call sites--but those
are generally just a couple of lines changed, each.
Not much of the kernel is actually using this, which on one hand
reduces risk quite a lot. But on the other hand, testing coverage
is low. So I'd love it if, in particular, the Infiniband and PowerPC
folks could do a smoke test of this series for me.
Also, my runtime testing for the call sites so far is very weak:
* io_uring: Some directed tests from liburing exercise this, and they pass.
* process_vm_access.c: A small directed test passes.
* gup_benchmark: the enhanced version hits the new gup.c code, and passes.
* infiniband (still only have crude "IB pingpong" working, on a
good day: it's not exercising my conversions at runtime...)
* VFIO: compiles (I'm vowing to set up a run time test soon, but it's
not ready just yet)
* powerpc: it compiles...
* drm/via: compiles...
* goldfish: compiles...
* net/xdp: compiles...
* media/v4l2: compiles...
============================================================
Next:
* Get the block/bio_vec sites converted to use pin_user_pages().
* Work with Ira and Dave Chinner to weave this together with the
layout lease stuff.
============================================================
[1] Some slow progress on get_user_pages() (Apr 2, 2019): https://lwn.net/Articles/784574/
[2] DMA and get_user_pages() (LPC: Dec 12, 2018): https://lwn.net/Articles/774411/
[3] The trouble with get_user_pages() (Apr 30, 2018): https://lwn.net/Articles/753027/
Dan Williams (1):
mm: Cleanup __put_devmap_managed_page() vs ->page_free()
John Hubbard (23):
mm/gup: pass flags arg to __gup_device_* functions
mm/gup: factor out duplicate code from four routines
mm/gup: move try_get_compound_head() to top, fix minor issues
mm: devmap: refactor 1-based refcounting for ZONE_DEVICE pages
goldish_pipe: rename local pin_user_pages() routine
IB/umem: use get_user_pages_fast() to pin DMA pages
media/v4l2-core: set pages dirty upon releasing DMA buffers
vfio, mm: fix get_user_pages_remote() and FOLL_LONGTERM
mm/gup: introduce pin_user_pages*() and FOLL_PIN
goldish_pipe: convert to pin_user_pages() and put_user_page()
IB/{core,hw,umem}: set FOLL_PIN via pin_user_pages*(), fix up ODP
mm/process_vm_access: set FOLL_PIN via pin_user_pages_remote()
drm/via: set FOLL_PIN via pin_user_pages_fast()
fs/io_uring: set FOLL_PIN via pin_user_pages()
net/xdp: set FOLL_PIN via pin_user_pages()
mm/gup: track FOLL_PIN pages
media/v4l2-core: pin_user_pages (FOLL_PIN) and put_user_page()
conversion
vfio, mm: pin_user_pages (FOLL_PIN) and put_user_page() conversion
powerpc: book3s64: convert to pin_user_pages() and put_user_page()
mm/gup_benchmark: use proper FOLL_WRITE flags instead of hard-coding
"1"
mm/gup_benchmark: support pin_user_pages() and related calls
selftests/vm: run_vmtests: invoke gup_benchmark with basic FOLL_PIN
coverage
mm, tree-wide: rename put_user_page*() to unpin_user_page*()
Documentation/core-api/index.rst | 1 +
Documentation/core-api/pin_user_pages.rst | 233 +++++++++
arch/powerpc/mm/book3s64/iommu_api.c | 12 +-
drivers/gpu/drm/via/via_dmablit.c | 6 +-
drivers/infiniband/core/umem.c | 19 +-
drivers/infiniband/core/umem_odp.c | 13 +-
drivers/infiniband/hw/hfi1/user_pages.c | 4 +-
drivers/infiniband/hw/mthca/mthca_memfree.c | 8 +-
drivers/infiniband/hw/qib/qib_user_pages.c | 4 +-
drivers/infiniband/hw/qib/qib_user_sdma.c | 8 +-
drivers/infiniband/hw/usnic/usnic_uiom.c | 4 +-
drivers/infiniband/sw/siw/siw_mem.c | 4 +-
drivers/media/v4l2-core/videobuf-dma-sg.c | 8 +-
drivers/nvdimm/pmem.c | 6 -
drivers/platform/goldfish/goldfish_pipe.c | 35 +-
drivers/vfio/vfio_iommu_type1.c | 35 +-
fs/io_uring.c | 6 +-
include/linux/mm.h | 168 +++++-
include/linux/mmzone.h | 2 +
include/linux/page_ref.h | 10 +
mm/gup.c | 548 +++++++++++++++-----
mm/gup_benchmark.c | 74 ++-
mm/huge_memory.c | 54 +-
mm/hugetlb.c | 39 +-
mm/memremap.c | 76 ++-
mm/process_vm_access.c | 28 +-
mm/vmstat.c | 2 +
net/xdp/xdp_umem.c | 4 +-
tools/testing/selftests/vm/gup_benchmark.c | 21 +-
tools/testing/selftests/vm/run_vmtests | 22 +
30 files changed, 1104 insertions(+), 350 deletions(-)
create mode 100644 Documentation/core-api/pin_user_pages.rst
--
2.24.0
This patchset is being developed here:
<https://github.com/cyphar/linux/tree/openat2/master>
This is a re-send of
<https://lore.kernel.org/lkml/20191117011713.13032-1-cyphar@cyphar.com/>
but rebased on top of 5.4-rc8 (also my mails got duplicated the first
time I sent v17 -- hopefully that doesn't happen this time).
Patch changelog:
v17:
* Add a path_is_under() check for LOOKUP_IS_SCOPED in complete_walk(), as a
last line of defence to ensure that namei bugs will not break the contract
of LOOKUP_BENEATH or LOOKUP_IN_ROOT.
* Update based on feedback by Al Viro:
* Make nd_jump_link() free the passed path on error, so that callers don't
need to worry about it in the error path.
* Remove needless m_retry and r_retry variables in handle_dots().
* Always return -ECHILD from follow_dotdot_rcu().
v16: <https://lore.kernel.org/lkml/20191116002802.6663-1-cyphar@cyphar.com/>
v15: <https://lore.kernel.org/lkml/20191105090553.6350-1-cyphar@cyphar.com/>
v14: <https://lore.kernel.org/lkml/20191010054140.8483-1-cyphar@cyphar.com/>
<https://lore.kernel.org/lkml/20191026185700.10708-1-cyphar@cyphar.com>
v13: <https://lore.kernel.org/lkml/20190930183316.10190-1-cyphar@cyphar.com/>
v12: <https://lore.kernel.org/lkml/20190904201933.10736-1-cyphar@cyphar.com/>
v11: <https://lore.kernel.org/lkml/20190820033406.29796-1-cyphar@cyphar.com/>
<https://lore.kernel.org/lkml/20190728010207.9781-1-cyphar@cyphar.com/>
v10: <https://lore.kernel.org/lkml/20190719164225.27083-1-cyphar@cyphar.com/>
v09: <https://lore.kernel.org/lkml/20190706145737.5299-1-cyphar@cyphar.com/>
v08: <https://lore.kernel.org/lkml/20190520133305.11925-1-cyphar@cyphar.com/>
v07: <https://lore.kernel.org/lkml/20190507164317.13562-1-cyphar@cyphar.com/>
v06: <https://lore.kernel.org/lkml/20190506165439.9155-1-cyphar@cyphar.com/>
v05: <https://lore.kernel.org/lkml/20190320143717.2523-1-cyphar@cyphar.com/>
v04: <https://lore.kernel.org/lkml/20181112142654.341-1-cyphar@cyphar.com/>
v03: <https://lore.kernel.org/lkml/20181009070230.12884-1-cyphar@cyphar.com/>
v02: <https://lore.kernel.org/lkml/20181009065300.11053-1-cyphar@cyphar.com/>
v01: <https://lore.kernel.org/lkml/20180929103453.12025-1-cyphar@cyphar.com/>
For a very long time, extending openat(2) with new features has been
incredibly frustrating. This stems from the fact that openat(2) is
possibly the most famous counter-example to the mantra "don't silently
accept garbage from userspace" -- it doesn't check whether unknown flags
are present[1].
This means that (generally) the addition of new flags to openat(2) has
been fraught with backwards-compatibility issues (O_TMPFILE has to be
defined as __O_TMPFILE|O_DIRECTORY|[O_RDWR or O_WRONLY] to ensure old
kernels gave errors, since it's insecure to silently ignore the
flag[2]). All new security-related flags therefore have a tough road to
being added to openat(2).
Furthermore, the need for some sort of control over VFS's path resolution (to
avoid malicious paths resulting in inadvertent breakouts) has been a very
long-standing desire of many userspace applications. This patchset is a revival
of Al Viro's old AT_NO_JUMPS[3] patchset (which was a variant of David
Drysdale's O_BENEATH patchset[4] which was a spin-off of the Capsicum
project[5]) with a few additions and changes made based on the previous
discussion within [6] as well as others I felt were useful.
In line with the conclusions of the original discussion of AT_NO_JUMPS, the
flag has been split up into separate flags. However, instead of being an
openat(2) flag it is provided through a new syscall openat2(2) which provides
several other improvements to the openat(2) interface (see the patch
description for more details). The following new LOOKUP_* flags are added:
* LOOKUP_NO_XDEV blocks all mountpoint crossings (upwards, downwards,
or through absolute links). Absolute pathnames alone in openat(2) do not
trigger this. Magic-link traversal which implies a vfsmount jump is also
blocked (though magic-link jumps on the same vfsmount are permitted).
* LOOKUP_NO_MAGICLINKS blocks resolution through /proc/$pid/fd-style
links. This is done by blocking the usage of nd_jump_link() during
resolution in a filesystem. The term "magic-links" is used to match
with the only reference to these links in Documentation/, but I'm
happy to change the name.
It should be noted that this is different to the scope of
~LOOKUP_FOLLOW in that it applies to all path components. However,
you can do openat2(NO_FOLLOW|NO_MAGICLINKS) on a magic-link and it
will *not* fail (assuming that no parent component was a
magic-link), and you will have an fd for the magic-link.
In order to correctly detect magic-links, the introduction of a new
LOOKUP_MAGICLINK_JUMPED state flag was required.
* LOOKUP_BENEATH disallows escapes to outside the starting dirfd's
tree, using techniques such as ".." or absolute links. Absolute
paths in openat(2) are also disallowed. Conceptually this flag is to
ensure you "stay below" a certain point in the filesystem tree --
but this requires some additional to protect against various races
that would allow escape using "..".
Currently LOOKUP_BENEATH implies LOOKUP_NO_MAGICLINKS, because it
can trivially beam you around the filesystem (breaking the
protection). In future, there might be similar safety checks done as
in LOOKUP_IN_ROOT, but that requires more discussion.
In addition, two new flags are added that expand on the above ideas:
* LOOKUP_NO_SYMLINKS does what it says on the tin. No symlink
resolution is allowed at all, including magic-links. Just as with
LOOKUP_NO_MAGICLINKS this can still be used with NOFOLLOW to open an
fd for the symlink as long as no parent path had a symlink
component.
* LOOKUP_IN_ROOT is an extension of LOOKUP_BENEATH that, rather than
blocking attempts to move past the root, forces all such movements
to be scoped to the starting point. This provides chroot(2)-like
protection but without the cost of a chroot(2) for each filesystem
operation, as well as being safe against race attacks that chroot(2)
is not.
If a race is detected (as with LOOKUP_BENEATH) then an error is
generated, and similar to LOOKUP_BENEATH it is not permitted to cross
magic-links with LOOKUP_IN_ROOT.
The primary need for this is from container runtimes, which
currently need to do symlink scoping in userspace[7] when opening
paths in a potentially malicious container. There is a long list of
CVEs that could have bene mitigated by having RESOLVE_THIS_ROOT
(such as CVE-2017-1002101, CVE-2017-1002102, CVE-2018-15664, and
CVE-2019-5736, just to name a few).
In order to make all of the above more usable, I'm working on
libpathrs[8] which is a C-friendly library for safe path resolution. It
features a userspace-emulated backend if the kernel doesn't support
openat2(2). Hopefully we can get userspace to switch to using it, and
thus get openat2(2) support for free once it's ready.
Future work would include implementing things like RESOLVE_NO_AUTOMOUNT and
possibly a RESOLVE_NO_REMOTE (to allow programs to be sure they don't hit DoSes
though stale NFS handles).
[1]: https://lwn.net/Articles/588444/
[2]: https://lore.kernel.org/lkml/CA+55aFyyxJL1LyXZeBsf2ypriraj5ut1XkNDsunRBqgVj…
[3]: https://lore.kernel.org/lkml/20170429220414.GT29622@ZenIV.linux.org.uk
[4]: https://lore.kernel.org/lkml/1415094884-18349-1-git-send-email-drysdale@goo…
[5]: https://lore.kernel.org/lkml/1404124096-21445-1-git-send-email-drysdale@goo…
[6]: https://lwn.net/Articles/723057/
[7]: https://github.com/cyphar/filepath-securejoin
[8]: https://github.com/openSUSE/libpathrs
The current draft of the openat2(2) man-page is included below.
--8<---------------------------------------------------------------------------
OPENAT2(2) Linux Programmer's Manual OPENAT2(2)
NAME
openat2 - open and possibly create a file (extended)
SYNOPSIS
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
int openat2(int dirfd, const char *pathname, struct open_how *how, size_t size);
Note: There is no glibc wrapper for this system call; see NOTES.
DESCRIPTION
The openat2() system call opens the file specified by pathname. If the specified file
does not exist, it may optionally (if O_CREAT is specified in how.flags) be created by
openat2().
As with openat(2), if pathname is relative, then it is interpreted relative to the direc-
tory referred to by the file descriptor dirfd (or the current working directory of the
calling process, if dirfd is the special value AT_FDCWD.) If pathname is absolute, then
dirfd is ignored (unless how.resolve contains RESOLVE_IN_ROOT, in which case pathname is
resolved relative to dirfd.)
The openat2() system call is an extension of openat(2) and provides a superset of its
functionality. Rather than taking a single flag argument, an extensible structure (how)
is passed instead to allow for future extensions. size must be set to sizeof(struct
open_how), to facilitate future extensions (see the "Extensibility" section of the NOTES
for more detail on how extensions are handled.)
The open_how structure
The following structure indicates how pathname should be opened, and acts as a superset of
the flag and mode arguments to openat(2).
struct open_how {
__aligned_u64 flags; /* O_* flags. */
__u16 mode; /* Mode for O_{CREAT,TMPFILE}. */
__u16 __padding[3]; /* Must be zeroed. */
__aligned_u64 resolve; /* RESOLVE_* flags. */
};
Any future extensions to openat2() will be implemented as new fields appended to the above
structure (or through reuse of pre-existing padding space), with the zero value of the new
fields acting as though the extension were not present.
The meaning of each field is as follows:
flags
The file creation and status flags to use for this operation. All of the
O_* flags defined for openat(2) are valid openat2() flag values.
Unlike openat(2), it is an error to provide openat2() unknown or conflicting
flags in flags.
mode
File mode for the new file, with identical semantics to the mode argument to
openat(2). However, unlike openat(2), it is an error to provide openat2()
with a mode which contains bits other than 0777.
It is an error to provide openat2() a non-zero mode if flags does not con-
tain O_CREAT or O_TMPFILE.
resolve
Change how the components of pathname will be resolved (see path_resolu-
tion(7) for background information.) The primary use case for these flags
is to allow trusted programs to restrict how untrusted paths (or paths in-
side untrusted directories) are resolved. The full list of resolve flags is
given below.
RESOLVE_NO_XDEV
Disallow traversal of mount points during path resolution (including
all bind mounts).
Users of this flag are encouraged to make its use configurable (un-
less it is used for a specific security purpose), as bind mounts are
very widely used by end-users. Setting this flag indiscrimnately for
all uses of openat2() may result in spurious errors on previously-
functional systems.
RESOLVE_NO_SYMLINKS
Disallow resolution of symbolic links during path resolution. This
option implies RESOLVE_NO_MAGICLINKS.
If the trailing component is a symbolic link, and flags contains both
O_PATH and O_NOFOLLOW, then an O_PATH file descriptor referencing the
symbolic link will be returned.
Users of this flag are encouraged to make its use configurable (un-
less it is used for a specific security purpose), as symbolic links
are very widely used by end-users. Setting this flag indiscrimnately
for all uses of openat2() may result in spurious errors on previ-
ously-functional systems.
RESOLVE_NO_MAGICLINKS
Disallow all magic link resolution during path resolution.
If the trailing component is a magic link, and flags contains both
O_PATH and O_NOFOLLOW, then an O_PATH file descriptor referencing the
magic link will be returned.
Magic-links are symbolic link-like objects that are most notably
found in proc(5) (examples include /proc/[pid]/exe and
/proc/[pid]/fd/*.) Due to the potential danger of unknowingly open-
ing these magic links, it may be preferable for users to disable
their resolution entirely (see symboliclink(7) for more details.)
RESOLVE_BENEATH
Do not permit the path resolution to succeed if any component of the
resolution is not a descendant of the directory indicated by dirfd.
This results in absolute symbolic links (and absolute values of path-
name) to be rejected.
Currently, this flag also disables magic link resolution. However,
this may change in the future. The caller should explicitly specify
RESOLVE_NO_MAGICLINKS to ensure that magic links are not resolved.
RESOLVE_IN_ROOT
Treat dirfd as the root directory while resolving pathname (as though
the user called chroot(2) with dirfd as the argument.) Absolute sym-
bolic links and ".." path components will be scoped to dirfd. If
pathname is an absolute path, it is also treated relative to dirfd.
However, unlike chroot(2) (which changes the filesystem root perma-
nently for a process), RESOLVE_IN_ROOT allows a program to effi-
ciently restrict path resolution for only certain operations. It
also has several hardening features (such detecting escape attempts
during .. resolution) which chroot(2) does not.
Currently, this flag also disables magic link resolution. However,
this may change in the future. The caller should explicitly specify
RESOLVE_NO_MAGICLINKS to ensure that magic links are not resolved.
It is an error to provide openat2() unknown flags in resolve.
RETURN VALUE
On success, a new file descriptor is returned. On error, -1 is returned, and errno is set
appropriately.
ERRORS
The set of errors returned by openat2() includes all of the errors returned by openat(2),
as well as the following additional errors:
EINVAL An unknown flag or invalid value was specified in how.
EINVAL mode is non-zero, but flags does not contain O_CREAT or O_TMPFILE.
EINVAL size was smaller than any known version of struct open_how.
E2BIG An extension was specified in how, which the current kernel does not support (see
the "Extensibility" section of the NOTES for more detail on how extensions are han-
dled.)
EAGAIN resolve contains either RESOLVE_IN_ROOT or RESOLVE_BENEATH, and the kernel could
not ensure that a ".." component didn't escape (due to a race condition or poten-
tial attack.) Callers may choose to retry the openat2() call.
EXDEV resolve contains either RESOLVE_IN_ROOT or RESOLVE_BENEATH, and an escape from the
root during path resolution was detected.
EXDEV resolve contains RESOLVE_NO_XDEV, and a path component attempted to cross a mount
point.
ELOOP resolve contains RESOLVE_NO_SYMLINKS, and one of the path components was a symbolic
link (or magic link).
ELOOP resolve contains RESOLVE_NO_MAGICLINKS, and one of the path components was a magic
link.
VERSIONS
openat2() was added to Linux in kernel 5.FOO.
CONFORMING TO
This system call is Linux-specific.
The semantics of RESOLVE_BENEATH were modelled after FreeBSD's O_BENEATH.
NOTES
Glibc does not provide a wrapper for this system call; call it using systemcall(2).
Extensibility
In order to allow for struct open_how to be extended in future kernel revisions, openat2()
requires userspace to specify the size of struct open_how structure they are passing. By
providing this information, it is possible for openat2() to provide both forwards- and
backwards-compatibility — with size acting as an implicit version number (because new ex-
tension fields will always be appended, the size will always increase.) This extensibil-
ity design is very similar to other system calls such as perf_setattr(2),
perf_event_open(2), and clone(3).
If we let usize be the size of the structure according to userspace and ksize be the size
of the structure which the kernel supports, then there are only three cases to consider:
* If ksize equals usize, then there is no version mismatch and how can be used
verbatim.
* If ksize is larger than usize, then there are some extensions the kernel sup-
ports which the userspace program is unaware of. Because all extensions must
have their zero values be a no-op, the kernel treats all of the extension fields
not set by userspace to have zero values. This provides backwards-compatibil-
ity.
* If ksize is smaller than usize, then there are some extensions which the
userspace program is aware of but the kernel does not support. Because all ex-
tensions must have their zero values be a no-op, the kernel can safely ignore
the unsupported extension fields if they are all-zero. If any unsupported ex-
tension fields are non-zero, then -1 is returned and errno is set to E2BIG.
This provides forwards-compatibility.
Therefore, most userspace programs will not need to have any special handling of exten-
sions. However, if a userspace program wishes to determine what extensions the running
kernel supports, they may conduct a binary search on size (to find the largest value which
doesn't produce an error of E2BIG.)
SEE ALSO
openat(2), path_resolution(7), symlink(7)
Linux 2019-11-05 OPENAT2(2)
--8<---------------------------------------------------------------------------
Aleksa Sarai (13):
namei: only return -ECHILD from follow_dotdot_rcu()
nsfs: clean-up ns_get_path() signature to return int
namei: allow nd_jump_link() to produce errors
namei: allow set_root() to produce errors
namei: LOOKUP_NO_SYMLINKS: block symlink resolution
namei: LOOKUP_NO_MAGICLINKS: block magic-link resolution
namei: LOOKUP_NO_XDEV: block mountpoint crossing
namei: LOOKUP_BENEATH: O_BENEATH-like scoped resolution
namei: LOOKUP_IN_ROOT: chroot-like scoped resolution
namei: LOOKUP_{IN_ROOT,BENEATH}: permit limited ".." resolution
open: introduce openat2(2) syscall
selftests: add openat2(2) selftests
Documentation: path-lookup: include new LOOKUP flags
CREDITS | 4 +-
Documentation/filesystems/path-lookup.rst | 68 ++-
arch/alpha/kernel/syscalls/syscall.tbl | 1 +
arch/arm/tools/syscall.tbl | 1 +
arch/arm64/include/asm/unistd.h | 2 +-
arch/arm64/include/asm/unistd32.h | 2 +
arch/ia64/kernel/syscalls/syscall.tbl | 1 +
arch/m68k/kernel/syscalls/syscall.tbl | 1 +
arch/microblaze/kernel/syscalls/syscall.tbl | 1 +
arch/mips/kernel/syscalls/syscall_n32.tbl | 1 +
arch/mips/kernel/syscalls/syscall_n64.tbl | 1 +
arch/mips/kernel/syscalls/syscall_o32.tbl | 1 +
arch/parisc/kernel/syscalls/syscall.tbl | 1 +
arch/powerpc/kernel/syscalls/syscall.tbl | 1 +
arch/s390/kernel/syscalls/syscall.tbl | 1 +
arch/sh/kernel/syscalls/syscall.tbl | 1 +
arch/sparc/kernel/syscalls/syscall.tbl | 1 +
arch/x86/entry/syscalls/syscall_32.tbl | 1 +
arch/x86/entry/syscalls/syscall_64.tbl | 1 +
arch/xtensa/kernel/syscalls/syscall.tbl | 1 +
fs/namei.c | 185 +++++--
fs/nsfs.c | 29 +-
fs/open.c | 149 +++--
fs/proc/base.c | 3 +-
fs/proc/namespaces.c | 20 +-
include/linux/fcntl.h | 12 +-
include/linux/namei.h | 12 +-
include/linux/proc_ns.h | 4 +-
include/linux/syscalls.h | 3 +
include/uapi/asm-generic/unistd.h | 5 +-
include/uapi/linux/fcntl.h | 40 ++
kernel/bpf/offload.c | 12 +-
kernel/events/core.c | 2 +-
security/apparmor/apparmorfs.c | 6 +-
tools/testing/selftests/Makefile | 1 +
tools/testing/selftests/openat2/.gitignore | 1 +
tools/testing/selftests/openat2/Makefile | 8 +
tools/testing/selftests/openat2/helpers.c | 109 ++++
tools/testing/selftests/openat2/helpers.h | 107 ++++
.../testing/selftests/openat2/openat2_test.c | 316 +++++++++++
.../selftests/openat2/rename_attack_test.c | 160 ++++++
.../testing/selftests/openat2/resolve_test.c | 523 ++++++++++++++++++
42 files changed, 1686 insertions(+), 113 deletions(-)
create mode 100644 tools/testing/selftests/openat2/.gitignore
create mode 100644 tools/testing/selftests/openat2/Makefile
create mode 100644 tools/testing/selftests/openat2/helpers.c
create mode 100644 tools/testing/selftests/openat2/helpers.h
create mode 100644 tools/testing/selftests/openat2/openat2_test.c
create mode 100644 tools/testing/selftests/openat2/rename_attack_test.c
create mode 100644 tools/testing/selftests/openat2/resolve_test.c
base-commit: af42d3466bdc8f39806b26f593604fdc54140bcb
--
2.24.0
Add documentation for the Python script used to build, run, and collect
results from the kernel known as kunit_tool. kunit_tool
(tools/testing/kunit/kunit.py) was already added in previous commits.
Signed-off-by: Brendan Higgins <brendanhiggins(a)google.com>
Reviewed-by: David Gow <davidgow(a)google.com>
Cc: Randy Dunlap <rdunlap(a)infradead.org>
---
Documentation/dev-tools/kunit/index.rst | 1 +
Documentation/dev-tools/kunit/kunit-tool.rst | 57 ++++++++++++++++++++
Documentation/dev-tools/kunit/start.rst | 5 +-
3 files changed, 62 insertions(+), 1 deletion(-)
create mode 100644 Documentation/dev-tools/kunit/kunit-tool.rst
diff --git a/Documentation/dev-tools/kunit/index.rst b/Documentation/dev-tools/kunit/index.rst
index 26ffb46bdf99d..c60d760a0eed1 100644
--- a/Documentation/dev-tools/kunit/index.rst
+++ b/Documentation/dev-tools/kunit/index.rst
@@ -9,6 +9,7 @@ KUnit - Unit Testing for the Linux Kernel
start
usage
+ kunit-tool
api/index
faq
diff --git a/Documentation/dev-tools/kunit/kunit-tool.rst b/Documentation/dev-tools/kunit/kunit-tool.rst
new file mode 100644
index 0000000000000..50d46394e97e3
--- /dev/null
+++ b/Documentation/dev-tools/kunit/kunit-tool.rst
@@ -0,0 +1,57 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=================
+kunit_tool How-To
+=================
+
+What is kunit_tool?
+===================
+
+kunit_tool is a script (``tools/testing/kunit/kunit.py``) that aids in building
+the Linux kernel as UML (`User Mode Linux
+<http://user-mode-linux.sourceforge.net/>`_), running KUnit tests, parsing
+the test results and displaying them in a user friendly manner.
+
+What is a kunitconfig?
+======================
+
+It's just a defconfig that kunit_tool looks for in the base directory.
+kunit_tool uses it to generate a .config as you might expect. In addition, it
+verifies that the generated .config contains the CONFIG options in the
+kunitconfig; the reason it does this is so that it is easy to be sure that a
+CONFIG that enables a test actually ends up in the .config.
+
+How do I use kunit_tool?
+========================
+
+If a kunitconfig is present at the root directory, all you have to do is:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py run
+
+However, you most likely want to use it with the following options:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py run --timeout=30 --jobs=`nproc --all`
+
+- ``--timeout`` sets a maximum amount of time to allow tests to run.
+- ``--jobs`` sets the number of threads to use to build the kernel.
+
+If you just want to use the defconfig that ships with the kernel, you can
+append the ``--defconfig`` flag as well:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py run --timeout=30 --jobs=`nproc --all` --defconfig
+
+.. note::
+ This command is particularly helpful for getting started because it
+ just works. No kunitconfig needs to be present.
+
+For a list of all the flags supported by kunit_tool, you can run:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py run --help
diff --git a/Documentation/dev-tools/kunit/start.rst b/Documentation/dev-tools/kunit/start.rst
index aeeddfafeea20..f4d9a4fa914f8 100644
--- a/Documentation/dev-tools/kunit/start.rst
+++ b/Documentation/dev-tools/kunit/start.rst
@@ -19,7 +19,10 @@ The wrapper can be run with:
.. code-block:: bash
- ./tools/testing/kunit/kunit.py run
+ ./tools/testing/kunit/kunit.py run --defconfig
+
+For more information on this wrapper (also called kunit_tool) checkout the
+:doc:`kunit-tool` page.
Creating a kunitconfig
======================
--
2.24.0.432.g9d3f5f5b63-goog
Fix typos and gramatical errors in the Getting Started and Usage guide
for KUnit.
Reported-by: Randy Dunlap <rdunlap(a)infradead.org>
Link: https://patchwork.kernel.org/patch/11156481/
Reported-by: Rinat Ibragimov <ibragimovrinat(a)mail.ru>
Link: https://github.com/google/kunit-docs/issues/1
Signed-off-by: Brendan Higgins <brendanhiggins(a)google.com>
Reviewed-by: David Gow <davidgow(a)google.com>
---
Documentation/dev-tools/kunit/start.rst | 8 ++++----
Documentation/dev-tools/kunit/usage.rst | 24 ++++++++++++------------
2 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/Documentation/dev-tools/kunit/start.rst b/Documentation/dev-tools/kunit/start.rst
index f4d9a4fa914f8..9d6db892c41c0 100644
--- a/Documentation/dev-tools/kunit/start.rst
+++ b/Documentation/dev-tools/kunit/start.rst
@@ -26,7 +26,7 @@ For more information on this wrapper (also called kunit_tool) checkout the
Creating a kunitconfig
======================
-The Python script is a thin wrapper around Kbuild as such, it needs to be
+The Python script is a thin wrapper around Kbuild. As such, it needs to be
configured with a ``kunitconfig`` file. This file essentially contains the
regular Kernel config, with the specific test targets as well.
@@ -62,8 +62,8 @@ If everything worked correctly, you should see the following:
followed by a list of tests that are run. All of them should be passing.
.. note::
- Because it is building a lot of sources for the first time, the ``Building
- kunit kernel`` step may take a while.
+ Because it is building a lot of sources for the first time, the
+ ``Building KUnit kernel`` step may take a while.
Writing your first test
=======================
@@ -162,7 +162,7 @@ Now you can run the test:
.. code-block:: bash
- ./tools/testing/kunit/kunit.py
+ ./tools/testing/kunit/kunit.py run
You should see the following failure:
diff --git a/Documentation/dev-tools/kunit/usage.rst b/Documentation/dev-tools/kunit/usage.rst
index c6e69634e274b..b9a065ab681ee 100644
--- a/Documentation/dev-tools/kunit/usage.rst
+++ b/Documentation/dev-tools/kunit/usage.rst
@@ -16,7 +16,7 @@ Organization of this document
=============================
This document is organized into two main sections: Testing and Isolating
-Behavior. The first covers what a unit test is and how to use KUnit to write
+Behavior. The first covers what unit tests are and how to use KUnit to write
them. The second covers how to use KUnit to isolate code and make it possible
to unit test code that was otherwise un-unit-testable.
@@ -174,13 +174,13 @@ Test Suites
~~~~~~~~~~~
Now obviously one unit test isn't very helpful; the power comes from having
-many test cases covering all of your behaviors. Consequently it is common to
-have many *similar* tests; in order to reduce duplication in these closely
-related tests most unit testing frameworks provide the concept of a *test
-suite*, in KUnit we call it a *test suite*; all it is is just a collection of
-test cases for a unit of code with a set up function that gets invoked before
-every test cases and then a tear down function that gets invoked after every
-test case completes.
+many test cases covering all of a unit's behaviors. Consequently it is common
+to have many *similar* tests; in order to reduce duplication in these closely
+related tests most unit testing frameworks - including KUnit - provide the
+concept of a *test suite*. A *test suite* is just a collection of test cases
+for a unit of code with a set up function that gets invoked before every test
+case and then a tear down function that gets invoked after every test case
+completes.
Example:
@@ -211,7 +211,7 @@ KUnit test framework.
.. note::
A test case will only be run if it is associated with a test suite.
-For a more information on these types of things see the :doc:`api/test`.
+For more information on these types of things see the :doc:`api/test`.
Isolating Behavior
==================
@@ -338,7 +338,7 @@ We can easily test this code by *faking out* the underlying EEPROM:
return count;
}
- ssize_t fake_eeprom_write(struct eeprom *this, size_t offset, const char *buffer, size_t count)
+ ssize_t fake_eeprom_write(struct eeprom *parent, size_t offset, const char *buffer, size_t count)
{
struct fake_eeprom *this = container_of(parent, struct fake_eeprom, parent);
@@ -454,7 +454,7 @@ KUnit on non-UML architectures
By default KUnit uses UML as a way to provide dependencies for code under test.
Under most circumstances KUnit's usage of UML should be treated as an
implementation detail of how KUnit works under the hood. Nevertheless, there
-are instances where being able to run architecture specific code, or test
+are instances where being able to run architecture specific code or test
against real hardware is desirable. For these reasons KUnit supports running on
other architectures.
@@ -557,7 +557,7 @@ run your tests on your hardware setup just by compiling for your architecture.
.. important::
Always prefer tests that run on UML to tests that only run under a particular
architecture, and always prefer tests that run under QEMU or another easy
- (and monitarily free) to obtain software environment to a specific piece of
+ (and monetarily free) to obtain software environment to a specific piece of
hardware.
Nevertheless, there are still valid reasons to write an architecture or hardware
--
2.24.0.432.g9d3f5f5b63-goog
Fix typos and gramatical errors in the Getting Started and Usage guide
for KUnit.
Reported-by: Randy Dunlap <rdunlap(a)infradead.org>
Link: https://patchwork.kernel.org/patch/11156481/
Signed-off-by: Brendan Higgins <brendanhiggins(a)google.com>
---
Documentation/dev-tools/kunit/start.rst | 6 +++---
Documentation/dev-tools/kunit/usage.rst | 22 +++++++++++-----------
2 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/Documentation/dev-tools/kunit/start.rst b/Documentation/dev-tools/kunit/start.rst
index f4d9a4fa914f8..db146c7d77490 100644
--- a/Documentation/dev-tools/kunit/start.rst
+++ b/Documentation/dev-tools/kunit/start.rst
@@ -26,7 +26,7 @@ For more information on this wrapper (also called kunit_tool) checkout the
Creating a kunitconfig
======================
-The Python script is a thin wrapper around Kbuild as such, it needs to be
+The Python script is a thin wrapper around Kbuild. As such, it needs to be
configured with a ``kunitconfig`` file. This file essentially contains the
regular Kernel config, with the specific test targets as well.
@@ -62,8 +62,8 @@ If everything worked correctly, you should see the following:
followed by a list of tests that are run. All of them should be passing.
.. note::
- Because it is building a lot of sources for the first time, the ``Building
- kunit kernel`` step may take a while.
+ Because it is building a lot of sources for the first time, the
+ ``Building KUnit kernel`` step may take a while.
Writing your first test
=======================
diff --git a/Documentation/dev-tools/kunit/usage.rst b/Documentation/dev-tools/kunit/usage.rst
index c6e69634e274b..ae42a0d128c27 100644
--- a/Documentation/dev-tools/kunit/usage.rst
+++ b/Documentation/dev-tools/kunit/usage.rst
@@ -16,7 +16,7 @@ Organization of this document
=============================
This document is organized into two main sections: Testing and Isolating
-Behavior. The first covers what a unit test is and how to use KUnit to write
+Behavior. The first covers what unit tests are and how to use KUnit to write
them. The second covers how to use KUnit to isolate code and make it possible
to unit test code that was otherwise un-unit-testable.
@@ -174,13 +174,13 @@ Test Suites
~~~~~~~~~~~
Now obviously one unit test isn't very helpful; the power comes from having
-many test cases covering all of your behaviors. Consequently it is common to
-have many *similar* tests; in order to reduce duplication in these closely
-related tests most unit testing frameworks provide the concept of a *test
-suite*, in KUnit we call it a *test suite*; all it is is just a collection of
-test cases for a unit of code with a set up function that gets invoked before
-every test cases and then a tear down function that gets invoked after every
-test case completes.
+many test cases covering all of a unit's behaviors. Consequently it is common
+to have many *similar* tests; in order to reduce duplication in these closely
+related tests most unit testing frameworks - including KUnit - provide the
+concept of a *test suite*. A *test suite* is just a collection of test cases
+for a unit of code with a set up function that gets invoked before every test
+case and then a tear down function that gets invoked after every test case
+completes.
Example:
@@ -211,7 +211,7 @@ KUnit test framework.
.. note::
A test case will only be run if it is associated with a test suite.
-For a more information on these types of things see the :doc:`api/test`.
+For more information on these types of things see the :doc:`api/test`.
Isolating Behavior
==================
@@ -454,7 +454,7 @@ KUnit on non-UML architectures
By default KUnit uses UML as a way to provide dependencies for code under test.
Under most circumstances KUnit's usage of UML should be treated as an
implementation detail of how KUnit works under the hood. Nevertheless, there
-are instances where being able to run architecture specific code, or test
+are instances where being able to run architecture specific code or test
against real hardware is desirable. For these reasons KUnit supports running on
other architectures.
@@ -557,7 +557,7 @@ run your tests on your hardware setup just by compiling for your architecture.
.. important::
Always prefer tests that run on UML to tests that only run under a particular
architecture, and always prefer tests that run under QEMU or another easy
- (and monitarily free) to obtain software environment to a specific piece of
+ (and monetarily free) to obtain software environment to a specific piece of
hardware.
Nevertheless, there are still valid reasons to write an architecture or hardware
--
2.24.0.432.g9d3f5f5b63-goog
Hi,
Please note that two of these patches are also out for review
separately, and may go in earlier than this series. This series
requires those, so to ease review and testing, they are also
included here: patches 4 and 5, the devmap cleanups.
There is a git repo and branch, for convenience:
git@github.com:johnhubbard/linux.git pin_user_pages_tracking_v5
Despite the large number of changes, it does feel like the review
comments are converging, btw.
Changes since v4:
* Renamed put_user_page*() --> unpin_user_page().
* Removed all pin_longterm_pages*() calls. We will use FOLL_LONGTERM
at the call sites. (FOLL_PIN, however, remains an internal gup flag).
This is very nice: many patches just change three characters now:
get_user_pages --> pin_user_pages. I think we've found the right
balance of wrapper calls and gup flags, for the call sites.
* Updated a lot of documentation and commit logs to match the above
two large changes.
* Changed gup_benchmark tests and run_vmtests, to adapt to one less
use case: there is no pin_longterm_pages() call anymore.
* This includes a new devmap cleanup patch from Dan Williams, along
with a rebased follow-up: patches 4 and 5, already mentioned above.
* Fixed patch 10 ("mm/gup: introduce pin_user_pages*() and FOLL_PIN"),
so as to make pin_user_pages*() calls act as placeholders for the
corresponding get_user_pages*() calls, until a later patch fully
implements the DMA-pinning functionality.
Thanks to Jan Kara for noticing that.
* Fixed the implementation of pin_user_pages_remote().
* Further tweaked patch 2 ("mm/gup: factor out duplicate code from four
routines"), in response to Jan Kara's feedback.
* Dropped a few reviewed-by tags due to changes that invalidated
them.
Changes since v3:
* VFIO fix (patch 8): applied further cleanup: removed a pre-existing,
unnecessary release and reacquire of mmap_sem. Moved the DAX vma
checks from the vfio call site, to gup internals, and added comments
(and commit log) to clarify.
* Due to the above, made a corresponding fix to the
pin_longterm_pages_remote(), which was actually calling the wrong
gup internal function.
* Changed put_user_page() comments, to refer to pin*() APIs, rather than
get_user_pages*() APIs.
* Reverted an accidental whitespace-only change in the IB ODP code.
* Added a few more reviewed-by tags.
Changes since v2:
* Added a patch to convert IB/umem from normal gup, to gup_fast(). This
is also posted separately, in order to hopefully get some runtime
testing.
* Changed the page devmap code to be a little clearer,
thanks to Jerome for that.
* Split out the page devmap changes into a separate patch (and moved
Ira's Signed-off-by to that patch).
* Fixed my bug in IB: ODP code does not require pin_user_pages()
semantics. Therefore, revert the put_user_page() calls to put_page(),
and leave the get_user_pages() call as-is.
* As part of the revert, I am proposing here a change directly
from put_user_pages(), to release_pages(). I'd feel better if
someone agrees that this is the best way. It uses the more
efficient release_pages(), instead of put_page() in a loop,
and keep the change to just a few character on one line,
but OTOH it is not a pure revert.
* Loosened the FOLL_LONGTERM restrictions in the
__get_user_pages_locked() implementation, and used that in order
to fix up a VFIO bug. Thanks to Jason for that idea.
* Note the use of release_pages() in IB: is that OK?
* Added a few more WARN's and clarifying comments nearby.
* Many documentation improvements in various comments.
* Moved the new pin_user_pages.rst from Documentation/vm/ to
Documentation/core-api/ .
* Commit descriptions: added clarifying notes to the three patches
(drm/via, fs/io_uring, net/xdp) that already had put_user_page()
calls in place.
* Collected all pending Reviewed-by and Acked-by tags, from v1 and v2
email threads.
* Lot of churn from v2 --> v3, so it's possible that new bugs
sneaked in.
NOT DONE: separate patchset is required:
* __get_user_pages_locked(): stop compensating for
buggy callers who failed to set FOLL_GET. Instead, assert
that FOLL_GET is set (and fail if it's not).
======================================================================
Original cover letter (edited to fix up the patch description numbers)
This applies cleanly to linux-next and mmotm, and also to linux.git if
linux-next's commit 20cac10710c9 ("mm/gup_benchmark: fix MAP_HUGETLB
case") is first applied there.
This provides tracking of dma-pinned pages. This is a prerequisite to
solving the larger problem of proper interactions between file-backed
pages, and [R]DMA activities, as discussed in [1], [2], [3], and in
a remarkable number of email threads since about 2017. :)
A new internal gup flag, FOLL_PIN is introduced, and thoroughly
documented in the last patch's Documentation/vm/pin_user_pages.rst.
I believe that this will provide a good starting point for doing the
layout lease work that Ira Weiny has been working on. That's because
these new wrapper functions provide a clean, constrained, systematically
named set of functionality that, again, is required in order to even
know if a page is "dma-pinned".
In contrast to earlier approaches, the page tracking can be
incrementally applied to the kernel call sites that, until now, have
been simply calling get_user_pages() ("gup"). In other words, opt-in by
changing from this:
get_user_pages() (sets FOLL_GET)
put_page()
to this:
pin_user_pages() (sets FOLL_PIN)
put_user_page()
Because there are interdependencies with FOLL_LONGTERM, a similar
conversion as for FOLL_PIN, was applied. The change was from this:
get_user_pages(FOLL_LONGTERM) (also sets FOLL_GET)
put_page()
to this:
pin_longterm_pages() (sets FOLL_PIN | FOLL_LONGTERM)
put_user_page()
============================================================
Patch summary:
* Patches 1-9: refactoring and preparatory cleanup, independent fixes
* Patch 10: introduce pin_user_pages(), FOLL_PIN, but no functional
changes yet
* Patches 11-16: Convert existing put_user_page() callers, to use the
new pin*()
* Patch 17: Activate tracking of FOLL_PIN pages.
* Patches 18-20: convert various callers
* Patches: 21-23: gup_benchmark and run_vmtests support
* Patch 24: rename put_user_page*() --> unpin_user_page*()
============================================================
Testing:
* I've done some overall kernel testing (LTP, and a few other goodies),
and some directed testing to exercise some of the changes. And as you
can see, gup_benchmark is enhanced to exercise this. Basically, I've been
able to runtime test the core get_user_pages() and pin_user_pages() and
related routines, but not so much on several of the call sites--but those
are generally just a couple of lines changed, each.
Not much of the kernel is actually using this, which on one hand
reduces risk quite a lot. But on the other hand, testing coverage
is low. So I'd love it if, in particular, the Infiniband and PowerPC
folks could do a smoke test of this series for me.
Also, my runtime testing for the call sites so far is very weak:
* io_uring: Some directed tests from liburing exercise this, and they pass.
* process_vm_access.c: A small directed test passes.
* gup_benchmark: the enhanced version hits the new gup.c code, and passes.
* infiniband (still only have crude "IB pingpong" working, on a
good day: it's not exercising my conversions at runtime...)
* VFIO: compiles (I'm vowing to set up a run time test soon, but it's
not ready just yet)
* powerpc: it compiles...
* drm/via: compiles...
* goldfish: compiles...
* net/xdp: compiles...
* media/v4l2: compiles...
============================================================
Next:
* Get the block/bio_vec sites converted to use pin_user_pages().
* Work with Ira and Dave Chinner to weave this together with the
layout lease stuff.
============================================================
[1] Some slow progress on get_user_pages() (Apr 2, 2019): https://lwn.net/Articles/784574/
[2] DMA and get_user_pages() (LPC: Dec 12, 2018): https://lwn.net/Articles/774411/
[3] The trouble with get_user_pages() (Apr 30, 2018): https://lwn.net/Articles/753027/
Dan Williams (1):
mm: Cleanup __put_devmap_managed_page() vs ->page_free()
John Hubbard (23):
mm/gup: pass flags arg to __gup_device_* functions
mm/gup: factor out duplicate code from four routines
mm/gup: move try_get_compound_head() to top, fix minor issues
mm: devmap: refactor 1-based refcounting for ZONE_DEVICE pages
goldish_pipe: rename local pin_user_pages() routine
IB/umem: use get_user_pages_fast() to pin DMA pages
media/v4l2-core: set pages dirty upon releasing DMA buffers
vfio, mm: fix get_user_pages_remote() and FOLL_LONGTERM
mm/gup: introduce pin_user_pages*() and FOLL_PIN
goldish_pipe: convert to pin_user_pages() and put_user_page()
IB/{core,hw,umem}: set FOLL_PIN via pin_user_pages*(), fix up ODP
mm/process_vm_access: set FOLL_PIN via pin_user_pages_remote()
drm/via: set FOLL_PIN via pin_user_pages_fast()
fs/io_uring: set FOLL_PIN via pin_user_pages()
net/xdp: set FOLL_PIN via pin_user_pages()
mm/gup: track FOLL_PIN pages
media/v4l2-core: pin_user_pages (FOLL_PIN) and put_user_page()
conversion
vfio, mm: pin_user_pages (FOLL_PIN) and put_user_page() conversion
powerpc: book3s64: convert to pin_user_pages() and put_user_page()
mm/gup_benchmark: use proper FOLL_WRITE flags instead of hard-coding
"1"
mm/gup_benchmark: support pin_user_pages() and related calls
selftests/vm: run_vmtests: invoke gup_benchmark with basic FOLL_PIN
coverage
mm, tree-wide: rename put_user_page*() to unpin_user_page*()
Documentation/core-api/index.rst | 1 +
Documentation/core-api/pin_user_pages.rst | 233 ++++++++
arch/powerpc/mm/book3s64/iommu_api.c | 12 +-
drivers/gpu/drm/via/via_dmablit.c | 6 +-
drivers/infiniband/core/umem.c | 19 +-
drivers/infiniband/core/umem_odp.c | 13 +-
drivers/infiniband/hw/hfi1/user_pages.c | 4 +-
drivers/infiniband/hw/mthca/mthca_memfree.c | 8 +-
drivers/infiniband/hw/qib/qib_user_pages.c | 4 +-
drivers/infiniband/hw/qib/qib_user_sdma.c | 8 +-
drivers/infiniband/hw/usnic/usnic_uiom.c | 4 +-
drivers/infiniband/sw/siw/siw_mem.c | 4 +-
drivers/media/v4l2-core/videobuf-dma-sg.c | 8 +-
drivers/nvdimm/pmem.c | 6 -
drivers/platform/goldfish/goldfish_pipe.c | 35 +-
drivers/vfio/vfio_iommu_type1.c | 35 +-
fs/io_uring.c | 6 +-
include/linux/mm.h | 155 +++++-
include/linux/mmzone.h | 2 +
include/linux/page_ref.h | 10 +
mm/gup.c | 561 +++++++++++++++-----
mm/gup_benchmark.c | 74 ++-
mm/huge_memory.c | 54 +-
mm/hugetlb.c | 39 +-
mm/memremap.c | 76 ++-
mm/process_vm_access.c | 28 +-
mm/vmstat.c | 2 +
net/xdp/xdp_umem.c | 4 +-
tools/testing/selftests/vm/gup_benchmark.c | 21 +-
tools/testing/selftests/vm/run_vmtests | 22 +
30 files changed, 1104 insertions(+), 350 deletions(-)
create mode 100644 Documentation/core-api/pin_user_pages.rst
--
2.24.0
These changes are based on Jason's rdma/hmm branch (5.4.0-rc5).
Patch 1 was previously posted here [1] but was dropped from that orginal
series. Hopefully, the tests will reduce concerns about edge conditions.
I'm sure more tests could be usefully added but I thought this was a good
starting point.
Changes since v3:
patch 1:
Unchanged except rebased on Jason's latest hmm (bbe3329e354d3ab5dc18).
patch 2:
Is now part of Jason's tree.
patch 3 (now 2):
Major changes to incorporate Jason's review feedback.
* drivers/char/hmm_dmirror.c driver moved to lib/test_hmm.c
* XArray used instead of "page tables".
* platform device driver removed.
* remove redundant copyright.
Changes since v2:
patch 1:
Removed hmm_range_needs_fault() and just use hmm_range_need_fault().
Updated the change log to include that it fixes a bug where
hmm_range_fault() incorrectly returned an error when no fault is requested.
patch 2:
Removed the confusing change log wording about DMA.
Changed hmm_range_fault() to return the PFN of the zero page like any other
page.
patch 3:
Adjusted the test code to match the new zero page behavior.
Changes since v1:
Rebased to Jason's rdma/hmm branch (5.4.0-rc1).
Cleaned up locking for the test driver's page tables.
Incorporated Christoph Hellwig's comments.
[1] https://lore.kernel.org/linux-mm/20190726005650.2566-6-rcampbell@nvidia.com/
Ralph Campbell (2):
mm/hmm: make full use of walk_page_range()
mm/hmm/test: add self tests for HMM
MAINTAINERS | 3 +
include/uapi/linux/test_hmm.h | 59 ++
lib/Kconfig.debug | 11 +
lib/Makefile | 1 +
lib/test_hmm.c | 1306 ++++++++++++++++++++++++
mm/hmm.c | 121 ++-
tools/testing/selftests/vm/.gitignore | 1 +
tools/testing/selftests/vm/Makefile | 3 +
tools/testing/selftests/vm/config | 2 +
tools/testing/selftests/vm/hmm-tests.c | 1295 +++++++++++++++++++++++
tools/testing/selftests/vm/run_vmtests | 16 +
tools/testing/selftests/vm/test_hmm.sh | 97 ++
12 files changed, 2852 insertions(+), 63 deletions(-)
create mode 100644 include/uapi/linux/test_hmm.h
create mode 100644 lib/test_hmm.c
create mode 100644 tools/testing/selftests/vm/hmm-tests.c
create mode 100755 tools/testing/selftests/vm/test_hmm.sh
--
2.20.1
This patchset is being developed here:
<https://github.com/cyphar/linux/tree/openat2/master>
Patch changelog:
v17:
* Add a path_is_under() check for LOOKUP_IS_SCOPED in complete_walk(), as a
last line of defence to ensure that namei bugs will not break the contract
of LOOKUP_BENEATH or LOOKUP_IN_ROOT.
* Update based on feedback by Al Viro:
* Make nd_jump_link() free the passed path on error, so that callers don't
need to worry about it in the error path.
* Remove needless m_retry and r_retry variables in handle_dots().
* Always return -ECHILD from follow_dotdot_rcu().
v16: <https://lore.kernel.org/lkml/20191116002802.6663-1-cyphar@cyphar.com/>
v15: <https://lore.kernel.org/lkml/20191105090553.6350-1-cyphar@cyphar.com/>
v14: <https://lore.kernel.org/lkml/20191010054140.8483-1-cyphar@cyphar.com/>
<https://lore.kernel.org/lkml/20191026185700.10708-1-cyphar@cyphar.com>
v13: <https://lore.kernel.org/lkml/20190930183316.10190-1-cyphar@cyphar.com/>
v12: <https://lore.kernel.org/lkml/20190904201933.10736-1-cyphar@cyphar.com/>
v11: <https://lore.kernel.org/lkml/20190820033406.29796-1-cyphar@cyphar.com/>
<https://lore.kernel.org/lkml/20190728010207.9781-1-cyphar@cyphar.com/>
v10: <https://lore.kernel.org/lkml/20190719164225.27083-1-cyphar@cyphar.com/>
v09: <https://lore.kernel.org/lkml/20190706145737.5299-1-cyphar@cyphar.com/>
v08: <https://lore.kernel.org/lkml/20190520133305.11925-1-cyphar@cyphar.com/>
v07: <https://lore.kernel.org/lkml/20190507164317.13562-1-cyphar@cyphar.com/>
v06: <https://lore.kernel.org/lkml/20190506165439.9155-1-cyphar@cyphar.com/>
v05: <https://lore.kernel.org/lkml/20190320143717.2523-1-cyphar@cyphar.com/>
v04: <https://lore.kernel.org/lkml/20181112142654.341-1-cyphar@cyphar.com/>
v03: <https://lore.kernel.org/lkml/20181009070230.12884-1-cyphar@cyphar.com/>
v02: <https://lore.kernel.org/lkml/20181009065300.11053-1-cyphar@cyphar.com/>
v01: <https://lore.kernel.org/lkml/20180929103453.12025-1-cyphar@cyphar.com/>
For a very long time, extending openat(2) with new features has been
incredibly frustrating. This stems from the fact that openat(2) is
possibly the most famous counter-example to the mantra "don't silently
accept garbage from userspace" -- it doesn't check whether unknown flags
are present[1].
This means that (generally) the addition of new flags to openat(2) has
been fraught with backwards-compatibility issues (O_TMPFILE has to be
defined as __O_TMPFILE|O_DIRECTORY|[O_RDWR or O_WRONLY] to ensure old
kernels gave errors, since it's insecure to silently ignore the
flag[2]). All new security-related flags therefore have a tough road to
being added to openat(2).
Furthermore, the need for some sort of control over VFS's path resolution (to
avoid malicious paths resulting in inadvertent breakouts) has been a very
long-standing desire of many userspace applications. This patchset is a revival
of Al Viro's old AT_NO_JUMPS[3] patchset (which was a variant of David
Drysdale's O_BENEATH patchset[4] which was a spin-off of the Capsicum
project[5]) with a few additions and changes made based on the previous
discussion within [6] as well as others I felt were useful.
In line with the conclusions of the original discussion of AT_NO_JUMPS, the
flag has been split up into separate flags. However, instead of being an
openat(2) flag it is provided through a new syscall openat2(2) which provides
several other improvements to the openat(2) interface (see the patch
description for more details). The following new LOOKUP_* flags are added:
* LOOKUP_NO_XDEV blocks all mountpoint crossings (upwards, downwards,
or through absolute links). Absolute pathnames alone in openat(2) do not
trigger this. Magic-link traversal which implies a vfsmount jump is also
blocked (though magic-link jumps on the same vfsmount are permitted).
* LOOKUP_NO_MAGICLINKS blocks resolution through /proc/$pid/fd-style
links. This is done by blocking the usage of nd_jump_link() during
resolution in a filesystem. The term "magic-links" is used to match
with the only reference to these links in Documentation/, but I'm
happy to change the name.
It should be noted that this is different to the scope of
~LOOKUP_FOLLOW in that it applies to all path components. However,
you can do openat2(NO_FOLLOW|NO_MAGICLINKS) on a magic-link and it
will *not* fail (assuming that no parent component was a
magic-link), and you will have an fd for the magic-link.
In order to correctly detect magic-links, the introduction of a new
LOOKUP_MAGICLINK_JUMPED state flag was required.
* LOOKUP_BENEATH disallows escapes to outside the starting dirfd's
tree, using techniques such as ".." or absolute links. Absolute
paths in openat(2) are also disallowed. Conceptually this flag is to
ensure you "stay below" a certain point in the filesystem tree --
but this requires some additional to protect against various races
that would allow escape using "..".
Currently LOOKUP_BENEATH implies LOOKUP_NO_MAGICLINKS, because it
can trivially beam you around the filesystem (breaking the
protection). In future, there might be similar safety checks done as
in LOOKUP_IN_ROOT, but that requires more discussion.
In addition, two new flags are added that expand on the above ideas:
* LOOKUP_NO_SYMLINKS does what it says on the tin. No symlink
resolution is allowed at all, including magic-links. Just as with
LOOKUP_NO_MAGICLINKS this can still be used with NOFOLLOW to open an
fd for the symlink as long as no parent path had a symlink
component.
* LOOKUP_IN_ROOT is an extension of LOOKUP_BENEATH that, rather than
blocking attempts to move past the root, forces all such movements
to be scoped to the starting point. This provides chroot(2)-like
protection but without the cost of a chroot(2) for each filesystem
operation, as well as being safe against race attacks that chroot(2)
is not.
If a race is detected (as with LOOKUP_BENEATH) then an error is
generated, and similar to LOOKUP_BENEATH it is not permitted to cross
magic-links with LOOKUP_IN_ROOT.
The primary need for this is from container runtimes, which
currently need to do symlink scoping in userspace[7] when opening
paths in a potentially malicious container. There is a long list of
CVEs that could have bene mitigated by having RESOLVE_THIS_ROOT
(such as CVE-2017-1002101, CVE-2017-1002102, CVE-2018-15664, and
CVE-2019-5736, just to name a few).
In order to make all of the above more usable, I'm working on
libpathrs[8] which is a C-friendly library for safe path resolution. It
features a userspace-emulated backend if the kernel doesn't support
openat2(2). Hopefully we can get userspace to switch to using it, and
thus get openat2(2) support for free once it's ready.
Future work would include implementing things like RESOLVE_NO_AUTOMOUNT and
possibly a RESOLVE_NO_REMOTE (to allow programs to be sure they don't hit DoSes
though stale NFS handles).
[1]: https://lwn.net/Articles/588444/
[2]: https://lore.kernel.org/lkml/CA+55aFyyxJL1LyXZeBsf2ypriraj5ut1XkNDsunRBqgVj…
[3]: https://lore.kernel.org/lkml/20170429220414.GT29622@ZenIV.linux.org.uk
[4]: https://lore.kernel.org/lkml/1415094884-18349-1-git-send-email-drysdale@goo…
[5]: https://lore.kernel.org/lkml/1404124096-21445-1-git-send-email-drysdale@goo…
[6]: https://lwn.net/Articles/723057/
[7]: https://github.com/cyphar/filepath-securejoin
[8]: https://github.com/openSUSE/libpathrs
The current draft of the openat2(2) man-page is included below.
--8<---------------------------------------------------------------------------
OPENAT2(2) Linux Programmer's Manual OPENAT2(2)
NAME
openat2 - open and possibly create a file (extended)
SYNOPSIS
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
int openat2(int dirfd, const char *pathname, struct open_how *how, size_t size);
Note: There is no glibc wrapper for this system call; see NOTES.
DESCRIPTION
The openat2() system call opens the file specified by pathname. If the specified file
does not exist, it may optionally (if O_CREAT is specified in how.flags) be created by
openat2().
As with openat(2), if pathname is relative, then it is interpreted relative to the direc-
tory referred to by the file descriptor dirfd (or the current working directory of the
calling process, if dirfd is the special value AT_FDCWD.) If pathname is absolute, then
dirfd is ignored (unless how.resolve contains RESOLVE_IN_ROOT, in which case pathname is
resolved relative to dirfd.)
The openat2() system call is an extension of openat(2) and provides a superset of its
functionality. Rather than taking a single flag argument, an extensible structure (how)
is passed instead to allow for future extensions. size must be set to sizeof(struct
open_how), to facilitate future extensions (see the "Extensibility" section of the NOTES
for more detail on how extensions are handled.)
The open_how structure
The following structure indicates how pathname should be opened, and acts as a superset of
the flag and mode arguments to openat(2).
struct open_how {
__aligned_u64 flags; /* O_* flags. */
__u16 mode; /* Mode for O_{CREAT,TMPFILE}. */
__u16 __padding[3]; /* Must be zeroed. */
__aligned_u64 resolve; /* RESOLVE_* flags. */
};
Any future extensions to openat2() will be implemented as new fields appended to the above
structure (or through reuse of pre-existing padding space), with the zero value of the new
fields acting as though the extension were not present.
The meaning of each field is as follows:
flags
The file creation and status flags to use for this operation. All of the
O_* flags defined for openat(2) are valid openat2() flag values.
Unlike openat(2), it is an error to provide openat2() unknown or conflicting
flags in flags.
mode
File mode for the new file, with identical semantics to the mode argument to
openat(2). However, unlike openat(2), it is an error to provide openat2()
with a mode which contains bits other than 0777.
It is an error to provide openat2() a non-zero mode if flags does not con-
tain O_CREAT or O_TMPFILE.
resolve
Change how the components of pathname will be resolved (see path_resolu-
tion(7) for background information.) The primary use case for these flags
is to allow trusted programs to restrict how untrusted paths (or paths in-
side untrusted directories) are resolved. The full list of resolve flags is
given below.
RESOLVE_NO_XDEV
Disallow traversal of mount points during path resolution (including
all bind mounts).
Users of this flag are encouraged to make its use configurable (un-
less it is used for a specific security purpose), as bind mounts are
very widely used by end-users. Setting this flag indiscrimnately for
all uses of openat2() may result in spurious errors on previously-
functional systems.
RESOLVE_NO_SYMLINKS
Disallow resolution of symbolic links during path resolution. This
option implies RESOLVE_NO_MAGICLINKS.
If the trailing component is a symbolic link, and flags contains both
O_PATH and O_NOFOLLOW, then an O_PATH file descriptor referencing the
symbolic link will be returned.
Users of this flag are encouraged to make its use configurable (un-
less it is used for a specific security purpose), as symbolic links
are very widely used by end-users. Setting this flag indiscrimnately
for all uses of openat2() may result in spurious errors on previ-
ously-functional systems.
RESOLVE_NO_MAGICLINKS
Disallow all magic link resolution during path resolution.
If the trailing component is a magic link, and flags contains both
O_PATH and O_NOFOLLOW, then an O_PATH file descriptor referencing the
magic link will be returned.
Magic-links are symbolic link-like objects that are most notably
found in proc(5) (examples include /proc/[pid]/exe and
/proc/[pid]/fd/*.) Due to the potential danger of unknowingly open-
ing these magic links, it may be preferable for users to disable
their resolution entirely (see symboliclink(7) for more details.)
RESOLVE_BENEATH
Do not permit the path resolution to succeed if any component of the
resolution is not a descendant of the directory indicated by dirfd.
This results in absolute symbolic links (and absolute values of path-
name) to be rejected.
Currently, this flag also disables magic link resolution. However,
this may change in the future. The caller should explicitly specify
RESOLVE_NO_MAGICLINKS to ensure that magic links are not resolved.
RESOLVE_IN_ROOT
Treat dirfd as the root directory while resolving pathname (as though
the user called chroot(2) with dirfd as the argument.) Absolute sym-
bolic links and ".." path components will be scoped to dirfd. If
pathname is an absolute path, it is also treated relative to dirfd.
However, unlike chroot(2) (which changes the filesystem root perma-
nently for a process), RESOLVE_IN_ROOT allows a program to effi-
ciently restrict path resolution for only certain operations. It
also has several hardening features (such detecting escape attempts
during .. resolution) which chroot(2) does not.
Currently, this flag also disables magic link resolution. However,
this may change in the future. The caller should explicitly specify
RESOLVE_NO_MAGICLINKS to ensure that magic links are not resolved.
It is an error to provide openat2() unknown flags in resolve.
RETURN VALUE
On success, a new file descriptor is returned. On error, -1 is returned, and errno is set
appropriately.
ERRORS
The set of errors returned by openat2() includes all of the errors returned by openat(2),
as well as the following additional errors:
EINVAL An unknown flag or invalid value was specified in how.
EINVAL mode is non-zero, but flags does not contain O_CREAT or O_TMPFILE.
EINVAL size was smaller than any known version of struct open_how.
E2BIG An extension was specified in how, which the current kernel does not support (see
the "Extensibility" section of the NOTES for more detail on how extensions are han-
dled.)
EAGAIN resolve contains either RESOLVE_IN_ROOT or RESOLVE_BENEATH, and the kernel could
not ensure that a ".." component didn't escape (due to a race condition or poten-
tial attack.) Callers may choose to retry the openat2() call.
EXDEV resolve contains either RESOLVE_IN_ROOT or RESOLVE_BENEATH, and an escape from the
root during path resolution was detected.
EXDEV resolve contains RESOLVE_NO_XDEV, and a path component attempted to cross a mount
point.
ELOOP resolve contains RESOLVE_NO_SYMLINKS, and one of the path components was a symbolic
link (or magic link).
ELOOP resolve contains RESOLVE_NO_MAGICLINKS, and one of the path components was a magic
link.
VERSIONS
openat2() was added to Linux in kernel 5.FOO.
CONFORMING TO
This system call is Linux-specific.
The semantics of RESOLVE_BENEATH were modelled after FreeBSD's O_BENEATH.
NOTES
Glibc does not provide a wrapper for this system call; call it using systemcall(2).
Extensibility
In order to allow for struct open_how to be extended in future kernel revisions, openat2()
requires userspace to specify the size of struct open_how structure they are passing. By
providing this information, it is possible for openat2() to provide both forwards- and
backwards-compatibility — with size acting as an implicit version number (because new ex-
tension fields will always be appended, the size will always increase.) This extensibil-
ity design is very similar to other system calls such as perf_setattr(2),
perf_event_open(2), and clone(3).
If we let usize be the size of the structure according to userspace and ksize be the size
of the structure which the kernel supports, then there are only three cases to consider:
* If ksize equals usize, then there is no version mismatch and how can be used
verbatim.
* If ksize is larger than usize, then there are some extensions the kernel sup-
ports which the userspace program is unaware of. Because all extensions must
have their zero values be a no-op, the kernel treats all of the extension fields
not set by userspace to have zero values. This provides backwards-compatibil-
ity.
* If ksize is smaller than usize, then there are some extensions which the
userspace program is aware of but the kernel does not support. Because all ex-
tensions must have their zero values be a no-op, the kernel can safely ignore
the unsupported extension fields if they are all-zero. If any unsupported ex-
tension fields are non-zero, then -1 is returned and errno is set to E2BIG.
This provides forwards-compatibility.
Therefore, most userspace programs will not need to have any special handling of exten-
sions. However, if a userspace program wishes to determine what extensions the running
kernel supports, they may conduct a binary search on size (to find the largest value which
doesn't produce an error of E2BIG.)
SEE ALSO
openat(2), path_resolution(7), symlink(7)
Linux 2019-11-05 OPENAT2(2)
--8<---------------------------------------------------------------------------
Aleksa Sarai (13):
namei: only return -ECHILD from follow_dotdot_rcu()
nsfs: clean-up ns_get_path() signature to return int
namei: allow nd_jump_link() to produce errors
namei: allow set_root() to produce errors
namei: LOOKUP_NO_SYMLINKS: block symlink resolution
namei: LOOKUP_NO_MAGICLINKS: block magic-link resolution
namei: LOOKUP_NO_XDEV: block mountpoint crossing
namei: LOOKUP_BENEATH: O_BENEATH-like scoped resolution
namei: LOOKUP_IN_ROOT: chroot-like scoped resolution
namei: LOOKUP_{IN_ROOT,BENEATH}: permit limited ".." resolution
open: introduce openat2(2) syscall
selftests: add openat2(2) selftests
Documentation: path-lookup: include new LOOKUP flags
CREDITS | 4 +-
Documentation/filesystems/path-lookup.rst | 68 ++-
arch/alpha/kernel/syscalls/syscall.tbl | 1 +
arch/arm/tools/syscall.tbl | 1 +
arch/arm64/include/asm/unistd.h | 2 +-
arch/arm64/include/asm/unistd32.h | 2 +
arch/ia64/kernel/syscalls/syscall.tbl | 1 +
arch/m68k/kernel/syscalls/syscall.tbl | 1 +
arch/microblaze/kernel/syscalls/syscall.tbl | 1 +
arch/mips/kernel/syscalls/syscall_n32.tbl | 1 +
arch/mips/kernel/syscalls/syscall_n64.tbl | 1 +
arch/mips/kernel/syscalls/syscall_o32.tbl | 1 +
arch/parisc/kernel/syscalls/syscall.tbl | 1 +
arch/powerpc/kernel/syscalls/syscall.tbl | 1 +
arch/s390/kernel/syscalls/syscall.tbl | 1 +
arch/sh/kernel/syscalls/syscall.tbl | 1 +
arch/sparc/kernel/syscalls/syscall.tbl | 1 +
arch/x86/entry/syscalls/syscall_32.tbl | 1 +
arch/x86/entry/syscalls/syscall_64.tbl | 1 +
arch/xtensa/kernel/syscalls/syscall.tbl | 1 +
fs/namei.c | 185 +++++--
fs/nsfs.c | 29 +-
fs/open.c | 149 +++--
fs/proc/base.c | 3 +-
fs/proc/namespaces.c | 20 +-
include/linux/fcntl.h | 12 +-
include/linux/namei.h | 12 +-
include/linux/proc_ns.h | 4 +-
include/linux/syscalls.h | 3 +
include/uapi/asm-generic/unistd.h | 5 +-
include/uapi/linux/fcntl.h | 40 ++
kernel/bpf/offload.c | 12 +-
kernel/events/core.c | 2 +-
security/apparmor/apparmorfs.c | 6 +-
tools/testing/selftests/Makefile | 1 +
tools/testing/selftests/openat2/.gitignore | 1 +
tools/testing/selftests/openat2/Makefile | 8 +
tools/testing/selftests/openat2/helpers.c | 109 ++++
tools/testing/selftests/openat2/helpers.h | 107 ++++
.../testing/selftests/openat2/openat2_test.c | 316 +++++++++++
.../selftests/openat2/rename_attack_test.c | 160 ++++++
.../testing/selftests/openat2/resolve_test.c | 523 ++++++++++++++++++
42 files changed, 1686 insertions(+), 113 deletions(-)
create mode 100644 tools/testing/selftests/openat2/.gitignore
create mode 100644 tools/testing/selftests/openat2/Makefile
create mode 100644 tools/testing/selftests/openat2/helpers.c
create mode 100644 tools/testing/selftests/openat2/helpers.h
create mode 100644 tools/testing/selftests/openat2/openat2_test.c
create mode 100644 tools/testing/selftests/openat2/rename_attack_test.c
create mode 100644 tools/testing/selftests/openat2/resolve_test.c
base-commit: 31f4f5b495a62c9a8b15b1c3581acd5efeb9af8c
--
2.24.0
This patchset is being developed here:
<https://github.com/cyphar/linux/tree/openat2/master>
Patch changelog:
v16:
* Update based on review by Al Viro:
* Handle magic-link related errors from with nd_jump_link() and drop
LOOKUP_MAGICLINK_JUMPED.
* Drop the slash-skipping logic for LOOKUP_IN_ROOT since it's not actually
necessary (link_path_walk() already does it, and it's possible that doing
it slightly breaks downstream code).
* Update outdated open_how documentation to match new semantics.
* Update commit message to further explain why -EAGAIN is preferable for
path_is_under() when checking whether ".." is safe.
* Split out the set_root() and nd_jump_root() errors from the
LOOKUP_BENEATH patch.
* Expand path-lookup documentation changes to describe all new LOOKUP flags.
* Cleanup the signature of ns_get_path() such that it returns an int
(previously it returned a void * -- even though the only two
possible return values were ERR_PTR or NULL).
v15: <https://lore.kernel.org/lkml/20191105090553.6350-1-cyphar@cyphar.com/>
v14: <https://lore.kernel.org/lkml/20191010054140.8483-1-cyphar@cyphar.com/>
<https://lore.kernel.org/lkml/20191026185700.10708-1-cyphar@cyphar.com>
v13: <https://lore.kernel.org/lkml/20190930183316.10190-1-cyphar@cyphar.com/>
v12: <https://lore.kernel.org/lkml/20190904201933.10736-1-cyphar@cyphar.com/>
v11: <https://lore.kernel.org/lkml/20190820033406.29796-1-cyphar@cyphar.com/>
<https://lore.kernel.org/lkml/20190728010207.9781-1-cyphar@cyphar.com/>
v10: <https://lore.kernel.org/lkml/20190719164225.27083-1-cyphar@cyphar.com/>
v09: <https://lore.kernel.org/lkml/20190706145737.5299-1-cyphar@cyphar.com/>
v08: <https://lore.kernel.org/lkml/20190520133305.11925-1-cyphar@cyphar.com/>
v07: <https://lore.kernel.org/lkml/20190507164317.13562-1-cyphar@cyphar.com/>
v06: <https://lore.kernel.org/lkml/20190506165439.9155-1-cyphar@cyphar.com/>
v05: <https://lore.kernel.org/lkml/20190320143717.2523-1-cyphar@cyphar.com/>
v04: <https://lore.kernel.org/lkml/20181112142654.341-1-cyphar@cyphar.com/>
v03: <https://lore.kernel.org/lkml/20181009070230.12884-1-cyphar@cyphar.com/>
v02: <https://lore.kernel.org/lkml/20181009065300.11053-1-cyphar@cyphar.com/>
v01: <https://lore.kernel.org/lkml/20180929103453.12025-1-cyphar@cyphar.com/>
For a very long time, extending openat(2) with new features has been
incredibly frustrating. This stems from the fact that openat(2) is
possibly the most famous counter-example to the mantra "don't silently
accept garbage from userspace" -- it doesn't check whether unknown flags
are present[1].
This means that (generally) the addition of new flags to openat(2) has
been fraught with backwards-compatibility issues (O_TMPFILE has to be
defined as __O_TMPFILE|O_DIRECTORY|[O_RDWR or O_WRONLY] to ensure old
kernels gave errors, since it's insecure to silently ignore the
flag[2]). All new security-related flags therefore have a tough road to
being added to openat(2).
Furthermore, the need for some sort of control over VFS's path resolution (to
avoid malicious paths resulting in inadvertent breakouts) has been a very
long-standing desire of many userspace applications. This patchset is a revival
of Al Viro's old AT_NO_JUMPS[3] patchset (which was a variant of David
Drysdale's O_BENEATH patchset[4] which was a spin-off of the Capsicum
project[5]) with a few additions and changes made based on the previous
discussion within [6] as well as others I felt were useful.
In line with the conclusions of the original discussion of AT_NO_JUMPS, the
flag has been split up into separate flags. However, instead of being an
openat(2) flag it is provided through a new syscall openat2(2) which provides
several other improvements to the openat(2) interface (see the patch
description for more details). The following new LOOKUP_* flags are added:
* LOOKUP_NO_XDEV blocks all mountpoint crossings (upwards, downwards,
or through absolute links). Absolute pathnames alone in openat(2) do not
trigger this. Magic-link traversal which implies a vfsmount jump is also
blocked (though magic-link jumps on the same vfsmount are permitted).
* LOOKUP_NO_MAGICLINKS blocks resolution through /proc/$pid/fd-style
links. This is done by blocking the usage of nd_jump_link() during
resolution in a filesystem. The term "magic-links" is used to match
with the only reference to these links in Documentation/, but I'm
happy to change the name.
It should be noted that this is different to the scope of
~LOOKUP_FOLLOW in that it applies to all path components. However,
you can do openat2(NO_FOLLOW|NO_MAGICLINKS) on a magic-link and it
will *not* fail (assuming that no parent component was a
magic-link), and you will have an fd for the magic-link.
In order to correctly detect magic-links, the introduction of a new
LOOKUP_MAGICLINK_JUMPED state flag was required.
* LOOKUP_BENEATH disallows escapes to outside the starting dirfd's
tree, using techniques such as ".." or absolute links. Absolute
paths in openat(2) are also disallowed. Conceptually this flag is to
ensure you "stay below" a certain point in the filesystem tree --
but this requires some additional to protect against various races
that would allow escape using "..".
Currently LOOKUP_BENEATH implies LOOKUP_NO_MAGICLINKS, because it
can trivially beam you around the filesystem (breaking the
protection). In future, there might be similar safety checks done as
in LOOKUP_IN_ROOT, but that requires more discussion.
In addition, two new flags are added that expand on the above ideas:
* LOOKUP_NO_SYMLINKS does what it says on the tin. No symlink
resolution is allowed at all, including magic-links. Just as with
LOOKUP_NO_MAGICLINKS this can still be used with NOFOLLOW to open an
fd for the symlink as long as no parent path had a symlink
component.
* LOOKUP_IN_ROOT is an extension of LOOKUP_BENEATH that, rather than
blocking attempts to move past the root, forces all such movements
to be scoped to the starting point. This provides chroot(2)-like
protection but without the cost of a chroot(2) for each filesystem
operation, as well as being safe against race attacks that chroot(2)
is not.
If a race is detected (as with LOOKUP_BENEATH) then an error is
generated, and similar to LOOKUP_BENEATH it is not permitted to cross
magic-links with LOOKUP_IN_ROOT.
The primary need for this is from container runtimes, which
currently need to do symlink scoping in userspace[7] when opening
paths in a potentially malicious container. There is a long list of
CVEs that could have bene mitigated by having RESOLVE_THIS_ROOT
(such as CVE-2017-1002101, CVE-2017-1002102, CVE-2018-15664, and
CVE-2019-5736, just to name a few).
In order to make all of the above more usable, I'm working on
libpathrs[8] which is a C-friendly library for safe path resolution. It
features a userspace-emulated backend if the kernel doesn't support
openat2(2). Hopefully we can get userspace to switch to using it, and
thus get openat2(2) support for free once it's ready.
Future work would include implementing things like RESOLVE_NO_AUTOMOUNT and
possibly a RESOLVE_NO_REMOTE (to allow programs to be sure they don't hit DoSes
though stale NFS handles).
[1]: https://lwn.net/Articles/588444/
[2]: https://lore.kernel.org/lkml/CA+55aFyyxJL1LyXZeBsf2ypriraj5ut1XkNDsunRBqgVj…
[3]: https://lore.kernel.org/lkml/20170429220414.GT29622@ZenIV.linux.org.uk
[4]: https://lore.kernel.org/lkml/1415094884-18349-1-git-send-email-drysdale@goo…
[5]: https://lore.kernel.org/lkml/1404124096-21445-1-git-send-email-drysdale@goo…
[6]: https://lwn.net/Articles/723057/
[7]: https://github.com/cyphar/filepath-securejoin
[8]: https://github.com/openSUSE/libpathrs
The current draft of the openat2(2) man-page is included below.
--8<---------------------------------------------------------------------------
OPENAT2(2) Linux Programmer's Manual OPENAT2(2)
NAME
openat2 - open and possibly create a file (extended)
SYNOPSIS
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
int openat2(int dirfd, const char *pathname, struct open_how *how, size_t size);
Note: There is no glibc wrapper for this system call; see NOTES.
DESCRIPTION
The openat2() system call opens the file specified by pathname. If the specified file
does not exist, it may optionally (if O_CREAT is specified in how.flags) be created by
openat2().
As with openat(2), if pathname is relative, then it is interpreted relative to the direc-
tory referred to by the file descriptor dirfd (or the current working directory of the
calling process, if dirfd is the special value AT_FDCWD.) If pathname is absolute, then
dirfd is ignored (unless how.resolve contains RESOLVE_IN_ROOT, in which case pathname is
resolved relative to dirfd.)
The openat2() system call is an extension of openat(2) and provides a superset of its
functionality. Rather than taking a single flag argument, an extensible structure (how)
is passed instead to allow for future extensions. size must be set to sizeof(struct
open_how), to facilitate future extensions (see the "Extensibility" section of the NOTES
for more detail on how extensions are handled.)
The open_how structure
The following structure indicates how pathname should be opened, and acts as a superset of
the flag and mode arguments to openat(2).
struct open_how {
__aligned_u64 flags; /* O_* flags. */
__u16 mode; /* Mode for O_{CREAT,TMPFILE}. */
__u16 __padding[3]; /* Must be zeroed. */
__aligned_u64 resolve; /* RESOLVE_* flags. */
};
Any future extensions to openat2() will be implemented as new fields appended to the above
structure (or through reuse of pre-existing padding space), with the zero value of the new
fields acting as though the extension were not present.
The meaning of each field is as follows:
flags
The file creation and status flags to use for this operation. All of the
O_* flags defined for openat(2) are valid openat2() flag values.
Unlike openat(2), it is an error to provide openat2() unknown or conflicting
flags in flags.
mode
File mode for the new file, with identical semantics to the mode argument to
openat(2). However, unlike openat(2), it is an error to provide openat2()
with a mode which contains bits other than 0777.
It is an error to provide openat2() a non-zero mode if flags does not con-
tain O_CREAT or O_TMPFILE.
resolve
Change how the components of pathname will be resolved (see path_resolu-
tion(7) for background information.) The primary use case for these flags
is to allow trusted programs to restrict how untrusted paths (or paths in-
side untrusted directories) are resolved. The full list of resolve flags is
given below.
RESOLVE_NO_XDEV
Disallow traversal of mount points during path resolution (including
all bind mounts).
Users of this flag are encouraged to make its use configurable (un-
less it is used for a specific security purpose), as bind mounts are
very widely used by end-users. Setting this flag indiscrimnately for
all uses of openat2() may result in spurious errors on previously-
functional systems.
RESOLVE_NO_SYMLINKS
Disallow resolution of symbolic links during path resolution. This
option implies RESOLVE_NO_MAGICLINKS.
If the trailing component is a symbolic link, and flags contains both
O_PATH and O_NOFOLLOW, then an O_PATH file descriptor referencing the
symbolic link will be returned.
Users of this flag are encouraged to make its use configurable (un-
less it is used for a specific security purpose), as symbolic links
are very widely used by end-users. Setting this flag indiscrimnately
for all uses of openat2() may result in spurious errors on previ-
ously-functional systems.
RESOLVE_NO_MAGICLINKS
Disallow all magic link resolution during path resolution.
If the trailing component is a magic link, and flags contains both
O_PATH and O_NOFOLLOW, then an O_PATH file descriptor referencing the
magic link will be returned.
Magic-links are symbolic link-like objects that are most notably
found in proc(5) (examples include /proc/[pid]/exe and
/proc/[pid]/fd/*.) Due to the potential danger of unknowingly open-
ing these magic links, it may be preferable for users to disable
their resolution entirely (see symboliclink(7) for more details.)
RESOLVE_BENEATH
Do not permit the path resolution to succeed if any component of the
resolution is not a descendant of the directory indicated by dirfd.
This results in absolute symbolic links (and absolute values of path-
name) to be rejected.
Currently, this flag also disables magic link resolution. However,
this may change in the future. The caller should explicitly specify
RESOLVE_NO_MAGICLINKS to ensure that magic links are not resolved.
RESOLVE_IN_ROOT
Treat dirfd as the root directory while resolving pathname (as though
the user called chroot(2) with dirfd as the argument.) Absolute sym-
bolic links and ".." path components will be scoped to dirfd. If
pathname is an absolute path, it is also treated relative to dirfd.
However, unlike chroot(2) (which changes the filesystem root perma-
nently for a process), RESOLVE_IN_ROOT allows a program to effi-
ciently restrict path resolution for only certain operations. It
also has several hardening features (such detecting escape attempts
during .. resolution) which chroot(2) does not.
Currently, this flag also disables magic link resolution. However,
this may change in the future. The caller should explicitly specify
RESOLVE_NO_MAGICLINKS to ensure that magic links are not resolved.
It is an error to provide openat2() unknown flags in resolve.
RETURN VALUE
On success, a new file descriptor is returned. On error, -1 is returned, and errno is set
appropriately.
ERRORS
The set of errors returned by openat2() includes all of the errors returned by openat(2),
as well as the following additional errors:
EINVAL An unknown flag or invalid value was specified in how.
EINVAL mode is non-zero, but flags does not contain O_CREAT or O_TMPFILE.
EINVAL size was smaller than any known version of struct open_how.
E2BIG An extension was specified in how, which the current kernel does not support (see
the "Extensibility" section of the NOTES for more detail on how extensions are han-
dled.)
EAGAIN resolve contains either RESOLVE_IN_ROOT or RESOLVE_BENEATH, and the kernel could
not ensure that a ".." component didn't escape (due to a race condition or poten-
tial attack.) Callers may choose to retry the openat2() call.
EXDEV resolve contains either RESOLVE_IN_ROOT or RESOLVE_BENEATH, and an escape from the
root during path resolution was detected.
EXDEV resolve contains RESOLVE_NO_XDEV, and a path component attempted to cross a mount
point.
ELOOP resolve contains RESOLVE_NO_SYMLINKS, and one of the path components was a symbolic
link (or magic link).
ELOOP resolve contains RESOLVE_NO_MAGICLINKS, and one of the path components was a magic
link.
VERSIONS
openat2() was added to Linux in kernel 5.FOO.
CONFORMING TO
This system call is Linux-specific.
The semantics of RESOLVE_BENEATH were modelled after FreeBSD's O_BENEATH.
NOTES
Glibc does not provide a wrapper for this system call; call it using systemcall(2).
Extensibility
In order to allow for struct open_how to be extended in future kernel revisions, openat2()
requires userspace to specify the size of struct open_how structure they are passing. By
providing this information, it is possible for openat2() to provide both forwards- and
backwards-compatibility — with size acting as an implicit version number (because new ex-
tension fields will always be appended, the size will always increase.) This extensibil-
ity design is very similar to other system calls such as perf_setattr(2),
perf_event_open(2), and clone(3).
If we let usize be the size of the structure according to userspace and ksize be the size
of the structure which the kernel supports, then there are only three cases to consider:
* If ksize equals usize, then there is no version mismatch and how can be used
verbatim.
* If ksize is larger than usize, then there are some extensions the kernel sup-
ports which the userspace program is unaware of. Because all extensions must
have their zero values be a no-op, the kernel treats all of the extension fields
not set by userspace to have zero values. This provides backwards-compatibil-
ity.
* If ksize is smaller than usize, then there are some extensions which the
userspace program is aware of but the kernel does not support. Because all ex-
tensions must have their zero values be a no-op, the kernel can safely ignore
the unsupported extension fields if they are all-zero. If any unsupported ex-
tension fields are non-zero, then -1 is returned and errno is set to E2BIG.
This provides forwards-compatibility.
Therefore, most userspace programs will not need to have any special handling of exten-
sions. However, if a userspace program wishes to determine what extensions the running
kernel supports, they may conduct a binary search on size (to find the largest value which
doesn't produce an error of E2BIG.)
SEE ALSO
openat(2), path_resolution(7), symlink(7)
Linux 2019-11-05 OPENAT2(2)
--8<---------------------------------------------------------------------------
Aleksa Sarai (12):
nsfs: clean-up ns_get_path() signature to return int
namei: allow nd_jump_link() to produce errors
namei: allow set_root() to produce errors
namei: LOOKUP_NO_SYMLINKS: block symlink resolution
namei: LOOKUP_NO_MAGICLINKS: block magic-link resolution
namei: LOOKUP_NO_XDEV: block mountpoint crossing
namei: LOOKUP_BENEATH: O_BENEATH-like scoped resolution
namei: LOOKUP_IN_ROOT: chroot-like scoped resolution
namei: LOOKUP_{IN_ROOT,BENEATH}: permit limited ".." resolution
open: introduce openat2(2) syscall
selftests: add openat2(2) selftests
Documentation: path-lookup: include new LOOKUP flags
CREDITS | 4 +-
Documentation/filesystems/path-lookup.rst | 68 ++-
arch/alpha/kernel/syscalls/syscall.tbl | 1 +
arch/arm/tools/syscall.tbl | 1 +
arch/arm64/include/asm/unistd.h | 2 +-
arch/arm64/include/asm/unistd32.h | 2 +
arch/ia64/kernel/syscalls/syscall.tbl | 1 +
arch/m68k/kernel/syscalls/syscall.tbl | 1 +
arch/microblaze/kernel/syscalls/syscall.tbl | 1 +
arch/mips/kernel/syscalls/syscall_n32.tbl | 1 +
arch/mips/kernel/syscalls/syscall_n64.tbl | 1 +
arch/mips/kernel/syscalls/syscall_o32.tbl | 1 +
arch/parisc/kernel/syscalls/syscall.tbl | 1 +
arch/powerpc/kernel/syscalls/syscall.tbl | 1 +
arch/s390/kernel/syscalls/syscall.tbl | 1 +
arch/sh/kernel/syscalls/syscall.tbl | 1 +
arch/sparc/kernel/syscalls/syscall.tbl | 1 +
arch/x86/entry/syscalls/syscall_32.tbl | 1 +
arch/x86/entry/syscalls/syscall_64.tbl | 1 +
arch/xtensa/kernel/syscalls/syscall.tbl | 1 +
fs/namei.c | 166 ++++--
fs/nsfs.c | 29 +-
fs/open.c | 149 +++--
fs/proc/base.c | 5 +-
fs/proc/namespaces.c | 23 +-
include/linux/fcntl.h | 12 +-
include/linux/namei.h | 12 +-
include/linux/proc_ns.h | 4 +-
include/linux/syscalls.h | 3 +
include/uapi/asm-generic/unistd.h | 5 +-
include/uapi/linux/fcntl.h | 40 ++
kernel/bpf/offload.c | 12 +-
kernel/events/core.c | 2 +-
security/apparmor/apparmorfs.c | 8 +-
tools/testing/selftests/Makefile | 1 +
tools/testing/selftests/openat2/.gitignore | 1 +
tools/testing/selftests/openat2/Makefile | 8 +
tools/testing/selftests/openat2/helpers.c | 109 ++++
tools/testing/selftests/openat2/helpers.h | 107 ++++
.../testing/selftests/openat2/openat2_test.c | 316 +++++++++++
.../selftests/openat2/rename_attack_test.c | 160 ++++++
.../testing/selftests/openat2/resolve_test.c | 523 ++++++++++++++++++
42 files changed, 1675 insertions(+), 112 deletions(-)
create mode 100644 tools/testing/selftests/openat2/.gitignore
create mode 100644 tools/testing/selftests/openat2/Makefile
create mode 100644 tools/testing/selftests/openat2/helpers.c
create mode 100644 tools/testing/selftests/openat2/helpers.h
create mode 100644 tools/testing/selftests/openat2/openat2_test.c
create mode 100644 tools/testing/selftests/openat2/rename_attack_test.c
create mode 100644 tools/testing/selftests/openat2/resolve_test.c
base-commit: 31f4f5b495a62c9a8b15b1c3581acd5efeb9af8c
--
2.24.0
From: Michael Ellerman <mpe(a)ellerman.id.au>
[ Upstream commit 69f8117f17b332a68cd8f4bf8c2d0d3d5b84efc5 ]
Use TEST_GEN_PROGS and don't redefine all, this makes the out-of-tree
build work. We need to move the extra dependencies below the include
of lib.mk, because it adds the $(OUTPUT) prefix if it's defined.
We can also drop the clean rule, lib.mk does it for us.
Signed-off-by: Michael Ellerman <mpe(a)ellerman.id.au>
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
---
tools/testing/selftests/powerpc/cache_shape/Makefile | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/tools/testing/selftests/powerpc/cache_shape/Makefile b/tools/testing/selftests/powerpc/cache_shape/Makefile
index 1be547434a49c..7e0c175b82978 100644
--- a/tools/testing/selftests/powerpc/cache_shape/Makefile
+++ b/tools/testing/selftests/powerpc/cache_shape/Makefile
@@ -1,11 +1,6 @@
# SPDX-License-Identifier: GPL-2.0
-TEST_PROGS := cache_shape
-
-all: $(TEST_PROGS)
-
-$(TEST_PROGS): ../harness.c ../utils.c
+TEST_GEN_PROGS := cache_shape
include ../../lib.mk
-clean:
- rm -f $(TEST_PROGS) *.o
+$(TEST_GEN_PROGS): ../harness.c ../utils.c
--
2.20.1
From: Joel Stanley <joel(a)jms.id.au>
[ Upstream commit 27825349d7b238533a47e3d98b8bb0efd886b752 ]
We should use TEST_GEN_PROGS, not TEST_PROGS. That tells the selftests
makefile (lib.mk) that those tests are generated (built), and so it
adds the $(OUTPUT) prefix for us, making the out-of-tree build work
correctly.
It also means we don't need our own clean rule, lib.mk does it.
We also have to update the signal_tm rule to use $(OUTPUT).
Signed-off-by: Joel Stanley <joel(a)jms.id.au>
Signed-off-by: Michael Ellerman <mpe(a)ellerman.id.au>
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
---
tools/testing/selftests/powerpc/signal/Makefile | 11 +++--------
1 file changed, 3 insertions(+), 8 deletions(-)
diff --git a/tools/testing/selftests/powerpc/signal/Makefile b/tools/testing/selftests/powerpc/signal/Makefile
index a7cbd5082e271..4213978f3ee2c 100644
--- a/tools/testing/selftests/powerpc/signal/Makefile
+++ b/tools/testing/selftests/powerpc/signal/Makefile
@@ -1,14 +1,9 @@
# SPDX-License-Identifier: GPL-2.0
-TEST_PROGS := signal signal_tm
-
-all: $(TEST_PROGS)
-
-$(TEST_PROGS): ../harness.c ../utils.c signal.S
+TEST_GEN_PROGS := signal signal_tm
CFLAGS += -maltivec
-signal_tm: CFLAGS += -mhtm
+$(OUTPUT)/signal_tm: CFLAGS += -mhtm
include ../../lib.mk
-clean:
- rm -f $(TEST_PROGS) *.o
+$(TEST_GEN_PROGS): ../harness.c ../utils.c signal.S
--
2.20.1
From: "Shuah Khan (Samsung OSG)" <shuah(a)kernel.org>
[ Upstream commit 9a244229a4b850b11952a0df79607c69b18fd8df ]
When /dev/watchdog open fails, watchdog exits with "watchdog not enabled"
message. This is incorrect when open fails due to insufficient privilege.
Fix message to clearly state the reason when open fails with EACCESS when
a non-root user runs it.
Signed-off-by: Shuah Khan (Samsung OSG) <shuah(a)kernel.org>
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
---
tools/testing/selftests/watchdog/watchdog-test.c | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/tools/testing/selftests/watchdog/watchdog-test.c b/tools/testing/selftests/watchdog/watchdog-test.c
index 6e290874b70e2..e029e2017280f 100644
--- a/tools/testing/selftests/watchdog/watchdog-test.c
+++ b/tools/testing/selftests/watchdog/watchdog-test.c
@@ -89,7 +89,13 @@ int main(int argc, char *argv[])
fd = open("/dev/watchdog", O_WRONLY);
if (fd == -1) {
- printf("Watchdog device not enabled.\n");
+ if (errno == ENOENT)
+ printf("Watchdog device not enabled.\n");
+ else if (errno == EACCES)
+ printf("Run watchdog as root.\n");
+ else
+ printf("Watchdog device open failed %s\n",
+ strerror(errno));
exit(-1);
}
--
2.20.1
From: Michael Ellerman <mpe(a)ellerman.id.au>
[ Upstream commit 69f8117f17b332a68cd8f4bf8c2d0d3d5b84efc5 ]
Use TEST_GEN_PROGS and don't redefine all, this makes the out-of-tree
build work. We need to move the extra dependencies below the include
of lib.mk, because it adds the $(OUTPUT) prefix if it's defined.
We can also drop the clean rule, lib.mk does it for us.
Signed-off-by: Michael Ellerman <mpe(a)ellerman.id.au>
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
---
tools/testing/selftests/powerpc/cache_shape/Makefile | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/tools/testing/selftests/powerpc/cache_shape/Makefile b/tools/testing/selftests/powerpc/cache_shape/Makefile
index ede4d3dae7505..689f6c8ebcd8d 100644
--- a/tools/testing/selftests/powerpc/cache_shape/Makefile
+++ b/tools/testing/selftests/powerpc/cache_shape/Makefile
@@ -1,12 +1,7 @@
# SPDX-License-Identifier: GPL-2.0
-TEST_PROGS := cache_shape
-
-all: $(TEST_PROGS)
-
-$(TEST_PROGS): ../harness.c ../utils.c
+TEST_GEN_PROGS := cache_shape
top_srcdir = ../../../../..
include ../../lib.mk
-clean:
- rm -f $(TEST_PROGS) *.o
+$(TEST_GEN_PROGS): ../harness.c ../utils.c
--
2.20.1
From: Joel Stanley <joel(a)jms.id.au>
[ Upstream commit 27825349d7b238533a47e3d98b8bb0efd886b752 ]
We should use TEST_GEN_PROGS, not TEST_PROGS. That tells the selftests
makefile (lib.mk) that those tests are generated (built), and so it
adds the $(OUTPUT) prefix for us, making the out-of-tree build work
correctly.
It also means we don't need our own clean rule, lib.mk does it.
We also have to update the signal_tm rule to use $(OUTPUT).
Signed-off-by: Joel Stanley <joel(a)jms.id.au>
Signed-off-by: Michael Ellerman <mpe(a)ellerman.id.au>
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
---
tools/testing/selftests/powerpc/signal/Makefile | 11 +++--------
1 file changed, 3 insertions(+), 8 deletions(-)
diff --git a/tools/testing/selftests/powerpc/signal/Makefile b/tools/testing/selftests/powerpc/signal/Makefile
index 1fca25c6ace06..209a958dca127 100644
--- a/tools/testing/selftests/powerpc/signal/Makefile
+++ b/tools/testing/selftests/powerpc/signal/Makefile
@@ -1,15 +1,10 @@
# SPDX-License-Identifier: GPL-2.0
-TEST_PROGS := signal signal_tm
-
-all: $(TEST_PROGS)
-
-$(TEST_PROGS): ../harness.c ../utils.c signal.S
+TEST_GEN_PROGS := signal signal_tm
CFLAGS += -maltivec
-signal_tm: CFLAGS += -mhtm
+$(OUTPUT)/signal_tm: CFLAGS += -mhtm
top_srcdir = ../../../../..
include ../../lib.mk
-clean:
- rm -f $(TEST_PROGS) *.o
+$(TEST_GEN_PROGS): ../harness.c ../utils.c signal.S
--
2.20.1
From: Joel Stanley <joel(a)jms.id.au>
[ Upstream commit c39b79082a38a4f8c801790edecbbb4d62ed2992 ]
We should use TEST_GEN_PROGS, not TEST_PROGS. That tells the selftests
makefile (lib.mk) that those tests are generated (built), and so it
adds the $(OUTPUT) prefix for us, making the out-of-tree build work
correctly.
It also means we don't need our own clean rule, lib.mk does it.
We also have to update the ptrace-pkey and core-pkey rules to use
$(OUTPUT).
Signed-off-by: Joel Stanley <joel(a)jms.id.au>
Signed-off-by: Michael Ellerman <mpe(a)ellerman.id.au>
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
---
tools/testing/selftests/powerpc/ptrace/Makefile | 13 ++++---------
1 file changed, 4 insertions(+), 9 deletions(-)
diff --git a/tools/testing/selftests/powerpc/ptrace/Makefile b/tools/testing/selftests/powerpc/ptrace/Makefile
index 923d531265f8c..9f9423430059e 100644
--- a/tools/testing/selftests/powerpc/ptrace/Makefile
+++ b/tools/testing/selftests/powerpc/ptrace/Makefile
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: GPL-2.0
-TEST_PROGS := ptrace-gpr ptrace-tm-gpr ptrace-tm-spd-gpr \
+TEST_GEN_PROGS := ptrace-gpr ptrace-tm-gpr ptrace-tm-spd-gpr \
ptrace-tar ptrace-tm-tar ptrace-tm-spd-tar ptrace-vsx ptrace-tm-vsx \
ptrace-tm-spd-vsx ptrace-tm-spr ptrace-hwbreak ptrace-pkey core-pkey \
perf-hwbreak
@@ -7,14 +7,9 @@ TEST_PROGS := ptrace-gpr ptrace-tm-gpr ptrace-tm-spd-gpr \
top_srcdir = ../../../../..
include ../../lib.mk
-all: $(TEST_PROGS)
-
CFLAGS += -m64 -I../../../../../usr/include -I../tm -mhtm -fno-pie
-ptrace-pkey core-pkey: child.h
-ptrace-pkey core-pkey: LDLIBS += -pthread
-
-$(TEST_PROGS): ../harness.c ../utils.c ../lib/reg.S ptrace.h
+$(OUTPUT)/ptrace-pkey $(OUTPUT)/core-pkey: child.h
+$(OUTPUT)/ptrace-pkey $(OUTPUT)/core-pkey: LDLIBS += -pthread
-clean:
- rm -f $(TEST_PROGS) *.o
+$(TEST_GEN_PROGS): ../harness.c ../utils.c ../lib/reg.S ptrace.h
--
2.20.1
From: Andrea Parri <andrea.parri(a)amarulasolutions.com>
[ Upstream commit fb363e2d20351e1d16629df19e7bce1a31b3227a ]
Fixes the following warnings:
dirty_log_test.c: In function ‘help’:
dirty_log_test.c:216:9: warning: format ‘%lu’ expects argument of type ‘long unsigned int’, but argument 2 has type ‘int’ [-Wformat=]
printf(" -i: specify iteration counts (default: %"PRIu64")\n",
^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
In file included from include/test_util.h:18:0,
from dirty_log_test.c:16:
/usr/include/inttypes.h:105:34: note: format string is defined here
# define PRIu64 __PRI64_PREFIX "u"
dirty_log_test.c:218:9: warning: format ‘%lu’ expects argument of type ‘long unsigned int’, but argument 2 has type ‘int’ [-Wformat=]
printf(" -I: specify interval in ms (default: %"PRIu64" ms)\n",
^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
In file included from include/test_util.h:18:0,
from dirty_log_test.c:16:
/usr/include/inttypes.h:105:34: note: format string is defined here
# define PRIu64 __PRI64_PREFIX "u"
Signed-off-by: Andrea Parri <andrea.parri(a)amarulasolutions.com>
Signed-off-by: Shuah Khan (Samsung OSG) <shuah(a)kernel.org>
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
---
tools/testing/selftests/kvm/dirty_log_test.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/tools/testing/selftests/kvm/dirty_log_test.c b/tools/testing/selftests/kvm/dirty_log_test.c
index 0c2cdc105f968..a9c4b5e21d7e7 100644
--- a/tools/testing/selftests/kvm/dirty_log_test.c
+++ b/tools/testing/selftests/kvm/dirty_log_test.c
@@ -31,9 +31,9 @@
/* How many pages to dirty for each guest loop */
#define TEST_PAGES_PER_LOOP 1024
/* How many host loops to run (one KVM_GET_DIRTY_LOG for each loop) */
-#define TEST_HOST_LOOP_N 32
+#define TEST_HOST_LOOP_N 32UL
/* Interval for each host loop (ms) */
-#define TEST_HOST_LOOP_INTERVAL 10
+#define TEST_HOST_LOOP_INTERVAL 10UL
/*
* Guest variables. We use these variables to share data between host
--
2.20.1
From: "Shuah Khan (Samsung OSG)" <shuah(a)kernel.org>
[ Upstream commit 9a244229a4b850b11952a0df79607c69b18fd8df ]
When /dev/watchdog open fails, watchdog exits with "watchdog not enabled"
message. This is incorrect when open fails due to insufficient privilege.
Fix message to clearly state the reason when open fails with EACCESS when
a non-root user runs it.
Signed-off-by: Shuah Khan (Samsung OSG) <shuah(a)kernel.org>
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
---
tools/testing/selftests/watchdog/watchdog-test.c | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/tools/testing/selftests/watchdog/watchdog-test.c b/tools/testing/selftests/watchdog/watchdog-test.c
index 6e290874b70e2..e029e2017280f 100644
--- a/tools/testing/selftests/watchdog/watchdog-test.c
+++ b/tools/testing/selftests/watchdog/watchdog-test.c
@@ -89,7 +89,13 @@ int main(int argc, char *argv[])
fd = open("/dev/watchdog", O_WRONLY);
if (fd == -1) {
- printf("Watchdog device not enabled.\n");
+ if (errno == ENOENT)
+ printf("Watchdog device not enabled.\n");
+ else if (errno == EACCES)
+ printf("Run watchdog as root.\n");
+ else
+ printf("Watchdog device open failed %s\n",
+ strerror(errno));
exit(-1);
}
--
2.20.1
From: Quentin Monnet <quentin.monnet(a)netronome.com>
[ Upstream commit c5fa5d602221362f8341ecd9e32d83194abf5bd9 ]
The return value for each test in test_libbpf.sh is compared with
if (( $? == 0 )) ; then ...
This works well with bash, but not with dash, that /bin/sh is aliased to
on some systems (such as Ubuntu).
Let's replace this comparison by something that works on both shells.
Signed-off-by: Quentin Monnet <quentin.monnet(a)netronome.com>
Reviewed-by: Jakub Kicinski <jakub.kicinski(a)netronome.com>
Signed-off-by: Alexei Starovoitov <ast(a)kernel.org>
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
---
tools/testing/selftests/bpf/test_libbpf.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tools/testing/selftests/bpf/test_libbpf.sh b/tools/testing/selftests/bpf/test_libbpf.sh
index 8b1bc96d8e0cc..2989b2e2d856d 100755
--- a/tools/testing/selftests/bpf/test_libbpf.sh
+++ b/tools/testing/selftests/bpf/test_libbpf.sh
@@ -6,7 +6,7 @@ export TESTNAME=test_libbpf
# Determine selftest success via shell exit code
exit_handler()
{
- if (( $? == 0 )); then
+ if [ $? -eq 0 ]; then
echo "selftests: $TESTNAME [PASS]";
else
echo "$TESTNAME: failed at file $LAST_LOADED" 1>&2
--
2.20.1
Add documentation for the Python script used to build, run, and collect
results from the kernel known as kunit_tool. kunit_tool
(tools/testing/kunit/kunit.py) was already added in previous commits.
Signed-off-by: Brendan Higgins <brendanhiggins(a)google.com>
---
Documentation/dev-tools/kunit/index.rst | 1 +
Documentation/dev-tools/kunit/kunit-tool.rst | 57 ++++++++++++++++++++
Documentation/dev-tools/kunit/start.rst | 5 +-
3 files changed, 62 insertions(+), 1 deletion(-)
create mode 100644 Documentation/dev-tools/kunit/kunit-tool.rst
diff --git a/Documentation/dev-tools/kunit/index.rst b/Documentation/dev-tools/kunit/index.rst
index 26ffb46bdf99d..c60d760a0eed1 100644
--- a/Documentation/dev-tools/kunit/index.rst
+++ b/Documentation/dev-tools/kunit/index.rst
@@ -9,6 +9,7 @@ KUnit - Unit Testing for the Linux Kernel
start
usage
+ kunit-tool
api/index
faq
diff --git a/Documentation/dev-tools/kunit/kunit-tool.rst b/Documentation/dev-tools/kunit/kunit-tool.rst
new file mode 100644
index 0000000000000..37509527c04e1
--- /dev/null
+++ b/Documentation/dev-tools/kunit/kunit-tool.rst
@@ -0,0 +1,57 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=================
+kunit_tool How-To
+=================
+
+What is kunit_tool?
+===================
+
+kunit_tool is a script (``tools/testing/kunit/kunit.py``) that aids in building
+the Linux kernel as UML (`User Mode Linux
+<http://user-mode-linux.sourceforge.net/>`_), running KUnit tests, parsing
+the test results and displaying them in a user friendly manner.
+
+What is a kunitconfig?
+======================
+
+It's just a defconfig that kunit_tool looks for in the base directory.
+kunit_tool uses it to generate a .config as you might expect. In addition, it
+verifies that the generated .config contains the CONFIG options in the
+kunitconfig; the reason it does this is so that it is easy to be sure that a
+CONFIG that enables a test actually ends up in the .config.
+
+How do I use kunit_tool?
+=================================
+
+If a kunitconfig is present at the root directory, all you have to do is:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py run
+
+However, you most likely want to use it with the following options:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py run --timeout=30 --jobs=`nproc --all`
+
+- ``--timeout`` sets a maximum amount of time to allow tests to run.
+- ``--jobs`` sets the number of threads to use to build the kernel.
+
+If you just want to use the defconfig that ships with the kernel, you can
+append the ``--defconfig`` flag as well:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py run --timeout=30 --jobs=`nproc --all` --defconfig
+
+.. note::
+ This command is particularly helpful for getting started because it
+ just works. No kunitconfig needs to be present.
+
+For a list of all the flags supported by kunit_tool, you can run:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py run --help
diff --git a/Documentation/dev-tools/kunit/start.rst b/Documentation/dev-tools/kunit/start.rst
index aeeddfafeea20..4248a6f9038b8 100644
--- a/Documentation/dev-tools/kunit/start.rst
+++ b/Documentation/dev-tools/kunit/start.rst
@@ -19,7 +19,10 @@ The wrapper can be run with:
.. code-block:: bash
- ./tools/testing/kunit/kunit.py run
+ ./tools/testing/kunit/kunit.py run --timeout=30 --jobs=`nproc --all` --defconfig
+
+For more information on this wrapper (also called kunit_tool) checkout the
+:doc:`kunit-tool` page.
Creating a kunitconfig
======================
--
2.24.0.432.g9d3f5f5b63-goog
Currently the following command produces an error message:
linux# make kselftest TARGETS=bpf O=/mnt/linux-build
# selftests: bpf: test_libbpf.sh
# ./test_libbpf.sh: line 23: ./test_libbpf_open: No such file or directory
# test_libbpf: failed at file test_l4lb.o
# selftests: test_libbpf [FAILED]
The error message might not affect make return code, therefore one might
need to grep make output in order to detect it.
The current logic prepends $(OUTPUT) only to the first member of
$(TEST_PROGS). After that, run_one() does
cd `dirname $TEST`
For all tests except the first one, `dirname $TEST` is ., which means
they cannot access the files generated in $(OUTPUT).
Fix by using $(addprefix) to prepend $(OUTPUT)/ to each member of
$(TEST_PROGS).
Fixes: 1a940687e424 ("selftests: lib.mk: copy test scripts and test files for make O=dir run")
Signed-off-by: Ilya Leoshkevich <iii(a)linux.ibm.com>
---
v1->v2:
- Append / to $(OUTPUT).
- Use $(addprefix) instead of $(foreach).
v2->v3:
- Split the patch in two.
- Improve the commit message.
v3->v4:
- Drop the first patch.
- Add a note regarding make return code to the commit message.
tools/testing/selftests/lib.mk | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/tools/testing/selftests/lib.mk b/tools/testing/selftests/lib.mk
index 1c8a1963d03f..0cf510df1ee2 100644
--- a/tools/testing/selftests/lib.mk
+++ b/tools/testing/selftests/lib.mk
@@ -75,7 +75,8 @@ ifdef building_out_of_srctree
@rsync -aq $(TEST_PROGS) $(TEST_PROGS_EXTENDED) $(TEST_FILES) $(OUTPUT)
fi
@if [ "X$(TEST_PROGS)" != "X" ]; then
- $(call RUN_TESTS, $(TEST_GEN_PROGS) $(TEST_CUSTOM_PROGS) $(OUTPUT)/$(TEST_PROGS))
+ $(call RUN_TESTS, $(TEST_GEN_PROGS) $(TEST_CUSTOM_PROGS) \
+ $(addprefix $(OUTPUT)/,$(TEST_PROGS)))
else
$(call RUN_TESTS, $(TEST_GEN_PROGS) $(TEST_CUSTOM_PROGS))
fi
--
2.23.0
Greetings,
Find attached email very confidential. reply for more details
Thanks.
Peter Wong
----------------------------------------------------
This email was sent by the shareware version of Postman Professional.
Add documentation for the Python script used to build, run, and collect
results from the kernel known as kunit_tool. kunit_tool
(tools/testing/kunit/kunit.py) was already added in previous commits.
Signed-off-by: Brendan Higgins <brendanhiggins(a)google.com>
---
Documentation/dev-tools/kunit/index.rst | 1 +
Documentation/dev-tools/kunit/kunit-tool.rst | 57 ++++++++++++++++++++
Documentation/dev-tools/kunit/start.rst | 3 ++
3 files changed, 61 insertions(+)
create mode 100644 Documentation/dev-tools/kunit/kunit-tool.rst
diff --git a/Documentation/dev-tools/kunit/index.rst b/Documentation/dev-tools/kunit/index.rst
index 26ffb46bdf99d..c60d760a0eed1 100644
--- a/Documentation/dev-tools/kunit/index.rst
+++ b/Documentation/dev-tools/kunit/index.rst
@@ -9,6 +9,7 @@ KUnit - Unit Testing for the Linux Kernel
start
usage
+ kunit-tool
api/index
faq
diff --git a/Documentation/dev-tools/kunit/kunit-tool.rst b/Documentation/dev-tools/kunit/kunit-tool.rst
new file mode 100644
index 0000000000000..aa1a93649a45a
--- /dev/null
+++ b/Documentation/dev-tools/kunit/kunit-tool.rst
@@ -0,0 +1,57 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=================
+kunit_tool How-To
+=================
+
+What is kunit_tool?
+===================
+
+kunit_tool is a set of scripts that aid in building the Linux kernel as UML
+(`User Mode Linux <http://user-mode-linux.sourceforge.net/old/>`_), running
+KUnit tests, parsing the test results and displaying them in a user friendly
+manner.
+
+What is a kunitconfig?
+======================
+
+It's just a defconfig that kunit_tool looks for in the base directory.
+kunit_tool uses it to generate a .config as you might expect. In addition, it
+verifies that the generated .config contains the CONFIG options in the
+kunitconfig; the reason it does this is so that it is easy to be sure that a
+CONFIG that enables a test actually ends up in the .config.
+
+How do I use kunit_tool?
+=================================
+
+If a kunitconfig is present at the root directory, all you have to do is:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py run
+
+However, you most likely want to use it with the following options:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py run --timeout=30 --jobs=8
+
+- ``--timeout`` sets a maximum amount of time to allow tests to run.
+- ``--jobs`` sets the number of threads to use to build the kernel.
+
+If you just want to use the defconfig that ships with the kernel, you can
+append the ``--defconfig`` flag as well:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py run --timeout=30 --jobs=8 --defconfig
+
+.. note::
+ This command is particularly helpful for getting started because it
+ just works. No kunitconfig needs to be present.
+
+For a list of all the flags supported by kunit_tool, you can run:
+
+.. code-block:: bash
+
+ ./tools/testing/kunit/kunit.py run --help
diff --git a/Documentation/dev-tools/kunit/start.rst b/Documentation/dev-tools/kunit/start.rst
index aeeddfafeea20..1535c4394cfa2 100644
--- a/Documentation/dev-tools/kunit/start.rst
+++ b/Documentation/dev-tools/kunit/start.rst
@@ -21,6 +21,9 @@ The wrapper can be run with:
./tools/testing/kunit/kunit.py run
+For more information on this wrapper (also called kunit_tool) checkout the
+:doc:`kunit-tool` page.
+
Creating a kunitconfig
======================
The Python script is a thin wrapper around Kbuild as such, it needs to be
--
2.24.0.432.g9d3f5f5b63-goog
On Thu, Nov 14, 2019 at 1:31 PM Brendan Higgins
<brendanhiggins(a)google.com> wrote:
>
> +kselftest and kunit lists to document this decision.
Sorry for the spam. I accidentally CC'ed the doc list instead of the
kselftest list in my previous email.
> On Wed, Nov 13, 2019 at 11:54 PM Alan Maguire <alan.maguire(a)oracle.com> wrote:
> >
> > On Wed, 13 Nov 2019, Stephen Boyd wrote:
> >
> > > Quoting Brendan Higgins (2019-11-11 13:41:38)
> > > > +Stephen Boyd - since he is more of an expert on the hung task timer than I am.
> > > >
> > > > On Fri, Nov 8, 2019 at 7:30 AM Alan Maguire <alan.maguire(a)oracle.com> wrote:
> > > > >
> > > > > On Thu, 7 Nov 2019, Brendan Higgins wrote:
> > > > >
> > > > > > On Thu, Oct 17, 2019 at 11:09 AM Alan Maguire <alan.maguire(a)oracle.com> wrote:
> > > > > > > +MODULE_LICENSE("GPL");
> > > > > > > diff --git a/lib/kunit/try-catch.c b/lib/kunit/try-catch.c
> > > > > > > index 1c1e9af..72fc8ed 100644
> > > > > > > --- a/lib/kunit/try-catch.c
> > > > > > > +++ b/lib/kunit/try-catch.c
> > > > > > > @@ -31,6 +31,8 @@ static int kunit_generic_run_threadfn_adapter(void *data)
> > > > > > > complete_and_exit(try_catch->try_completion, 0);
> > > > > > > }
> > > > > > >
> > > > > > > +KUNIT_VAR_SYMBOL(sysctl_hung_task_timeout_secs, unsigned long);
> > > > > >
> > > > > > Can you just export sysctl_hung_task_timeout_secs?
> > > > > >
> > > > > > I don't mean to make you redo all this work for one symbol twice, but
> > > > > > I thought we agreed on just exposing this symbol, but in a namespace.
> > > > > > It seemed like a good use case for that namespaced exporting thing
> > > > > > that Luis was talking about. As I understood it, you would have to
> > > > > > export it in the module that defines it, and then use the new
> > > > > > MODULE_IMPORT_NS() macro here.
> > > > > >
> > > > >
> > > > > Sure, I can certainly look into that, though I wonder if we should
> > > > > consider another possibility - should kunit have its own sysctl table for
> > > > > things like configuring timeouts? I can look at adding a patch for that
> > > >
> > > > So on the one hand, yes, I would like to have configurable test
> > > > timeouts for KUnit, but that is not what the parameter check is for
> > > > here. This is to make sure KUnit times a test case out before the hung
> > > > task timer does.
> > > >
> > > > > prior to the module patch so the issues with exporting the hung task
> > > > > timeout would go away. Now the reason I suggest this isn't as much a hack
> > > > > to solve this specific problem, rather it seems to fit better with the
> > > > > longer-term intent expressed by the comment around use of the field (at
> > > > > least as I read it, I may be wrong).
> > > >
> > > > Not really. Although I do agree that adding configurability here might
> > > > be a good idea, I believe we would need to clamp such a value by
> > > > sysctl_hung_task_timeout_secs regardless since we don't want to be
> > > > killed by the hung task timer; thus, we still need access to
> > > > sysctl_hung_task_timeout_secs either way, and so doing what you are
> > > > proposing would be off topic.
> > > >
> > > > > Exporting the symbol does allow us to piggy-back on an existing value, but
> > > > > maybe we should support out our own tunable "kunit_timeout_secs" here?
> > > > > Doing so would also lay the groundwork for supporting other kunit
> > > > > tunables in the future if needed. What do you think?
> > > >
> > > > The goal is not to piggy back on the value as I mentioned above.
> > > > Stephen, do you have any thoughts on this? Do you see any other
> > > > preferable solution to what Alan is trying to do?
> > >
> > > One idea would be to make some sort of process flag that says "this is a
> > > kunit task, ignore me with regards to the hung task timeout". Then we
> > > can hardcode the 5 minute kunit timeout. I'm not sure we have any more
> > > flags though.
> > >
> > > Or drop the whole timeout clamping logic, let the hung task timeout kick
> > > in and potentially oops the kernel, but then continue to let the test
> > > run and maybe sometimes get the kunit timeout here. This last option
> > > doesn't sound so bad to me given that this is all a corner case anyway
> > > where we don't expect to actually ever hit this problem so letting the
> > > hung task detector do its job is probably fine. This nicely avoids
> > > having to export this symbol to modules too.
> > >
> >
> > Thanks for suggestions! This latter approach seems fine to me; presumably
> > something has gone wrong if we are tripping the hung task timeout anyway,
> > so having an oops to document that seems fine. Brendan, what do you think?
>
> If Stephen thinks it is fine to drop the clamping logic, I think it is
> fine too. I think it would probably be good to replace it with a
> comment under the TODO that explains that a hung test *can* cause an
> oops if the hung task timeout is less than the kunit timeout value. It
> would probably be good to also select a timeout value that is less
> than the default hung task timeout. We might also want to link to this
> discussion. I fully expect that the timeout logic will get more
> attention at some point in the future.
>
> One more thing: Alan, can you submit the commit that drops the
> clamping logic in its own commit? I would prefer to make sure that it
> is easy to spot in the commit history.
>
> Cheers!
This patchset is being developed here:
<https://github.com/cyphar/linux/tree/openat2/master>
Patch changelog:
v15:
* Fix code style for LOOKUP_IN_ROOT handling in path_init(). [Linus Torvalds]
* Split out patches for each individual LOOKUP flag.
* Reword commit messages to give more background information about the
series, as well as mention the semantics of each flag in more detail.
v14: <https://lore.kernel.org/lkml/20191010054140.8483-1-cyphar@cyphar.com/>
<https://lore.kernel.org/lkml/20191026185700.10708-1-cyphar@cyphar.com>
v13: <https://lore.kernel.org/lkml/20190930183316.10190-1-cyphar@cyphar.com/>
v12: <https://lore.kernel.org/lkml/20190904201933.10736-1-cyphar@cyphar.com/>
v11: <https://lore.kernel.org/lkml/20190820033406.29796-1-cyphar@cyphar.com/>
<https://lore.kernel.org/lkml/20190728010207.9781-1-cyphar@cyphar.com/>
v10: <https://lore.kernel.org/lkml/20190719164225.27083-1-cyphar@cyphar.com/>
v09: <https://lore.kernel.org/lkml/20190706145737.5299-1-cyphar@cyphar.com/>
v08: <https://lore.kernel.org/lkml/20190520133305.11925-1-cyphar@cyphar.com/>
v07: <https://lore.kernel.org/lkml/20190507164317.13562-1-cyphar@cyphar.com/>
v06: <https://lore.kernel.org/lkml/20190506165439.9155-1-cyphar@cyphar.com/>
v05: <https://lore.kernel.org/lkml/20190320143717.2523-1-cyphar@cyphar.com/>
v04: <https://lore.kernel.org/lkml/20181112142654.341-1-cyphar@cyphar.com/>
v03: <https://lore.kernel.org/lkml/20181009070230.12884-1-cyphar@cyphar.com/>
v02: <https://lore.kernel.org/lkml/20181009065300.11053-1-cyphar@cyphar.com/>
v01: <https://lore.kernel.org/lkml/20180929103453.12025-1-cyphar@cyphar.com/>
For a very long time, extending openat(2) with new features has been
incredibly frustrating. This stems from the fact that openat(2) is
possibly the most famous counter-example to the mantra "don't silently
accept garbage from userspace" -- it doesn't check whether unknown flags
are present[1].
This means that (generally) the addition of new flags to openat(2) has
been fraught with backwards-compatibility issues (O_TMPFILE has to be
defined as __O_TMPFILE|O_DIRECTORY|[O_RDWR or O_WRONLY] to ensure old
kernels gave errors, since it's insecure to silently ignore the
flag[2]). All new security-related flags therefore have a tough road to
being added to openat(2).
Furthermore, the need for some sort of control over VFS's path resolution (to
avoid malicious paths resulting in inadvertent breakouts) has been a very
long-standing desire of many userspace applications. This patchset is a revival
of Al Viro's old AT_NO_JUMPS[3] patchset (which was a variant of David
Drysdale's O_BENEATH patchset[4] which was a spin-off of the Capsicum
project[5]) with a few additions and changes made based on the previous
discussion within [6] as well as others I felt were useful.
In line with the conclusions of the original discussion of AT_NO_JUMPS, the
flag has been split up into separate flags. However, instead of being an
openat(2) flag it is provided through a new syscall openat2(2) which provides
several other improvements to the openat(2) interface (see the patch
description for more details). The following new LOOKUP_* flags are added:
* LOOKUP_NO_XDEV blocks all mountpoint crossings (upwards, downwards,
or through absolute links). Absolute pathnames alone in openat(2) do not
trigger this. Magic-link traversal which implies a vfsmount jump is also
blocked (though magic-link jumps on the same vfsmount are permitted).
* LOOKUP_NO_MAGICLINKS blocks resolution through /proc/$pid/fd-style
links. This is done by blocking the usage of nd_jump_link() during
resolution in a filesystem. The term "magic-links" is used to match
with the only reference to these links in Documentation/, but I'm
happy to change the name.
It should be noted that this is different to the scope of
~LOOKUP_FOLLOW in that it applies to all path components. However,
you can do openat2(NO_FOLLOW|NO_MAGICLINKS) on a magic-link and it
will *not* fail (assuming that no parent component was a
magic-link), and you will have an fd for the magic-link.
In order to correctly detect magic-links, the introduction of a new
LOOKUP_MAGICLINK_JUMPED state flag was required.
* LOOKUP_BENEATH disallows escapes to outside the starting dirfd's
tree, using techniques such as ".." or absolute links. Absolute
paths in openat(2) are also disallowed. Conceptually this flag is to
ensure you "stay below" a certain point in the filesystem tree --
but this requires some additional to protect against various races
that would allow escape using "..".
Currently LOOKUP_BENEATH implies LOOKUP_NO_MAGICLINKS, because it
can trivially beam you around the filesystem (breaking the
protection). In future, there might be similar safety checks done as
in LOOKUP_IN_ROOT, but that requires more discussion.
In addition, two new flags are added that expand on the above ideas:
* LOOKUP_NO_SYMLINKS does what it says on the tin. No symlink
resolution is allowed at all, including magic-links. Just as with
LOOKUP_NO_MAGICLINKS this can still be used with NOFOLLOW to open an
fd for the symlink as long as no parent path had a symlink
component.
* LOOKUP_IN_ROOT is an extension of LOOKUP_BENEATH that, rather than
blocking attempts to move past the root, forces all such movements
to be scoped to the starting point. This provides chroot(2)-like
protection but without the cost of a chroot(2) for each filesystem
operation, as well as being safe against race attacks that chroot(2)
is not.
If a race is detected (as with LOOKUP_BENEATH) then an error is
generated, and similar to LOOKUP_BENEATH it is not permitted to cross
magic-links with LOOKUP_IN_ROOT.
The primary need for this is from container runtimes, which
currently need to do symlink scoping in userspace[7] when opening
paths in a potentially malicious container. There is a long list of
CVEs that could have bene mitigated by having RESOLVE_THIS_ROOT
(such as CVE-2017-1002101, CVE-2017-1002102, CVE-2018-15664, and
CVE-2019-5736, just to name a few).
In order to make all of the above more usable, I'm working on
libpathrs[8] which is a C-friendly library for safe path resolution. It
features a userspace-emulated backend if the kernel doesn't support
openat2(2). Hopefully we can get userspace to switch to using it, and
thus get openat2(2) support for free once it's ready.
Future work would include implementing things like RESOLVE_NO_AUTOMOUNT and
possibly a RESOLVE_NO_REMOTE (to allow programs to be sure they don't hit DoSes
though stale NFS handles).
[1]: https://lwn.net/Articles/588444/
[2]: https://lore.kernel.org/lkml/CA+55aFyyxJL1LyXZeBsf2ypriraj5ut1XkNDsunRBqgVj…
[3]: https://lore.kernel.org/lkml/20170429220414.GT29622@ZenIV.linux.org.uk
[4]: https://lore.kernel.org/lkml/1415094884-18349-1-git-send-email-drysdale@goo…
[5]: https://lore.kernel.org/lkml/1404124096-21445-1-git-send-email-drysdale@goo…
[6]: https://lwn.net/Articles/723057/
[7]: https://github.com/cyphar/filepath-securejoin
[8]: https://github.com/openSUSE/libpathrs
The current draft of the openat2(2) man-page is included below.
--8<---------------------------------------------------------------------------
OPENAT2(2) Linux Programmer's Manual OPENAT2(2)
NAME
openat2 - open and possibly create a file (extended)
SYNOPSIS
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
int openat2(int dirfd, const char *pathname, struct open_how *how, size_t size);
Note: There is no glibc wrapper for this system call; see NOTES.
DESCRIPTION
The openat2() system call opens the file specified by pathname. If the specified file
does not exist, it may optionally (if O_CREAT is specified in how.flags) be created by
openat2().
As with openat(2), if pathname is relative, then it is interpreted relative to the direc-
tory referred to by the file descriptor dirfd (or the current working directory of the
calling process, if dirfd is the special value AT_FDCWD.) If pathname is absolute, then
dirfd is ignored (unless how.resolve contains RESOLVE_IN_ROOT, in which case pathname is
resolved relative to dirfd.)
The openat2() system call is an extension of openat(2) and provides a superset of its
functionality. Rather than taking a single flag argument, an extensible structure (how)
is passed instead to allow for future extensions. size must be set to sizeof(struct
open_how), to facilitate future extensions (see the "Extensibility" section of the NOTES
for more detail on how extensions are handled.)
The open_how structure
The following structure indicates how pathname should be opened, and acts as a superset of
the flag and mode arguments to openat(2).
struct open_how {
__aligned_u64 flags; /* O_* flags. */
__u16 mode; /* Mode for O_{CREAT,TMPFILE}. */
__u16 __padding[3]; /* Must be zeroed. */
__aligned_u64 resolve; /* RESOLVE_* flags. */
};
Any future extensions to openat2() will be implemented as new fields appended to the above
structure (or through reuse of pre-existing padding space), with the zero value of the new
fields acting as though the extension were not present.
The meaning of each field is as follows:
flags
The file creation and status flags to use for this operation. All of the
O_* flags defined for openat(2) are valid openat2() flag values.
Unlike openat(2), it is an error to provide openat2() unknown or conflicting
flags in flags.
mode
File mode for the new file, with identical semantics to the mode argument to
openat(2). However, unlike openat(2), it is an error to provide openat2()
with a mode which contains bits other than 0777.
It is an error to provide openat2() a non-zero mode if flags does not con-
tain O_CREAT or O_TMPFILE.
resolve
Change how the components of pathname will be resolved (see path_resolu-
tion(7) for background information.) The primary use case for these flags
is to allow trusted programs to restrict how untrusted paths (or paths in-
side untrusted directories) are resolved. The full list of resolve flags is
given below.
RESOLVE_NO_XDEV
Disallow traversal of mount points during path resolution (including
all bind mounts).
Users of this flag are encouraged to make its use configurable (un-
less it is used for a specific security purpose), as bind mounts are
very widely used by end-users. Setting this flag indiscrimnately for
all uses of openat2() may result in spurious errors on previously-
functional systems.
RESOLVE_NO_SYMLINKS
Disallow resolution of symbolic links during path resolution. This
option implies RESOLVE_NO_MAGICLINKS.
If the trailing component is a symbolic link, and flags contains both
O_PATH and O_NOFOLLOW, then an O_PATH file descriptor referencing the
symbolic link will be returned.
Users of this flag are encouraged to make its use configurable (un-
less it is used for a specific security purpose), as symbolic links
are very widely used by end-users. Setting this flag indiscrimnately
for all uses of openat2() may result in spurious errors on previ-
ously-functional systems.
RESOLVE_NO_MAGICLINKS
Disallow all magic link resolution during path resolution.
If the trailing component is a magic link, and flags contains both
O_PATH and O_NOFOLLOW, then an O_PATH file descriptor referencing the
magic link will be returned.
Magic-links are symbolic link-like objects that are most notably
found in proc(5) (examples include /proc/[pid]/exe and
/proc/[pid]/fd/*.) Due to the potential danger of unknowingly open-
ing these magic links, it may be preferable for users to disable
their resolution entirely (see symboliclink(7) for more details.)
RESOLVE_BENEATH
Do not permit the path resolution to succeed if any component of the
resolution is not a descendant of the directory indicated by dirfd.
This results in absolute symbolic links (and absolute values of path-
name) to be rejected.
Currently, this flag also disables magic link resolution. However,
this may change in the future. The caller should explicitly specify
RESOLVE_NO_MAGICLINKS to ensure that magic links are not resolved.
RESOLVE_IN_ROOT
Treat dirfd as the root directory while resolving pathname (as though
the user called chroot(2) with dirfd as the argument.) Absolute sym-
bolic links and ".." path components will be scoped to dirfd. If
pathname is an absolute path, it is also treated relative to dirfd.
However, unlike chroot(2) (which changes the filesystem root perma-
nently for a process), RESOLVE_IN_ROOT allows a program to effi-
ciently restrict path resolution for only certain operations. It
also has several hardening features (such detecting escape attempts
during .. resolution) which chroot(2) does not.
Currently, this flag also disables magic link resolution. However,
this may change in the future. The caller should explicitly specify
RESOLVE_NO_MAGICLINKS to ensure that magic links are not resolved.
It is an error to provide openat2() unknown flags in resolve.
RETURN VALUE
On success, a new file descriptor is returned. On error, -1 is returned, and errno is set
appropriately.
ERRORS
The set of errors returned by openat2() includes all of the errors returned by openat(2),
as well as the following additional errors:
EINVAL An unknown flag or invalid value was specified in how.
EINVAL mode is non-zero, but flags does not contain O_CREAT or O_TMPFILE.
EINVAL size was smaller than any known version of struct open_how.
E2BIG An extension was specified in how, which the current kernel does not support (see
the "Extensibility" section of the NOTES for more detail on how extensions are han-
dled.)
EAGAIN resolve contains either RESOLVE_IN_ROOT or RESOLVE_BENEATH, and the kernel could
not ensure that a ".." component didn't escape (due to a race condition or poten-
tial attack.) Callers may choose to retry the openat2() call.
EXDEV resolve contains either RESOLVE_IN_ROOT or RESOLVE_BENEATH, and an escape from the
root during path resolution was detected.
EXDEV resolve contains RESOLVE_NO_XDEV, and a path component attempted to cross a mount
point.
ELOOP resolve contains RESOLVE_NO_SYMLINKS, and one of the path components was a symbolic
link (or magic link).
ELOOP resolve contains RESOLVE_NO_MAGICLINKS, and one of the path components was a magic
link.
VERSIONS
openat2() was added to Linux in kernel 5.FOO.
CONFORMING TO
This system call is Linux-specific.
The semantics of RESOLVE_BENEATH were modelled after FreeBSD's O_BENEATH.
NOTES
Glibc does not provide a wrapper for this system call; call it using systemcall(2).
Extensibility
In order to allow for struct open_how to be extended in future kernel revisions, openat2()
requires userspace to specify the size of struct open_how structure they are passing. By
providing this information, it is possible for openat2() to provide both forwards- and
backwards-compatibility — with size acting as an implicit version number (because new ex-
tension fields will always be appended, the size will always increase.) This extensibil-
ity design is very similar to other system calls such as perf_setattr(2),
perf_event_open(2), and clone(3).
If we let usize be the size of the structure according to userspace and ksize be the size
of the structure which the kernel supports, then there are only three cases to consider:
* If ksize equals usize, then there is no version mismatch and how can be used
verbatim.
* If ksize is larger than usize, then there are some extensions the kernel sup-
ports which the userspace program is unaware of. Because all extensions must
have their zero values be a no-op, the kernel treats all of the extension fields
not set by userspace to have zero values. This provides backwards-compatibil-
ity.
* If ksize is smaller than usize, then there are some extensions which the
userspace program is aware of but the kernel does not support. Because all ex-
tensions must have their zero values be a no-op, the kernel can safely ignore
the unsupported extension fields if they are all-zero. If any unsupported ex-
tension fields are non-zero, then -1 is returned and errno is set to E2BIG.
This provides forwards-compatibility.
Therefore, most userspace programs will not need to have any special handling of exten-
sions. However, if a userspace program wishes to determine what extensions the running
kernel supports, they may conduct a binary search on size (to find the largest value which
doesn't produce an error of E2BIG.)
SEE ALSO
openat(2), path_resolution(7), symlink(7)
Linux 2019-11-05 OPENAT2(2)
--8<---------------------------------------------------------------------------
Aleksa Sarai (9):
namei: LOOKUP_NO_SYMLINKS: block symlink resolution
namei: LOOKUP_NO_MAGICLINKS: block magic-link resolution
namei: LOOKUP_NO_XDEV: block mountpoint crossing
namei: LOOKUP_BENEATH: O_BENEATH-like scoped resolution
namei: LOOKUP_IN_ROOT: chroot-like scoped resolution
namei: LOOKUP_{IN_ROOT,BENEATH}: permit limited ".." resolution
open: introduce openat2(2) syscall
selftests: add openat2(2) selftests
Documentation: path-lookup: mention LOOKUP_MAGICLINK_JUMPED
CREDITS | 4 +-
Documentation/filesystems/path-lookup.rst | 18 +-
arch/alpha/kernel/syscalls/syscall.tbl | 1 +
arch/arm/tools/syscall.tbl | 1 +
arch/arm64/include/asm/unistd.h | 2 +-
arch/arm64/include/asm/unistd32.h | 2 +
arch/ia64/kernel/syscalls/syscall.tbl | 1 +
arch/m68k/kernel/syscalls/syscall.tbl | 1 +
arch/microblaze/kernel/syscalls/syscall.tbl | 1 +
arch/mips/kernel/syscalls/syscall_n32.tbl | 1 +
arch/mips/kernel/syscalls/syscall_n64.tbl | 1 +
arch/mips/kernel/syscalls/syscall_o32.tbl | 1 +
arch/parisc/kernel/syscalls/syscall.tbl | 1 +
arch/powerpc/kernel/syscalls/syscall.tbl | 1 +
arch/s390/kernel/syscalls/syscall.tbl | 1 +
arch/sh/kernel/syscalls/syscall.tbl | 1 +
arch/sparc/kernel/syscalls/syscall.tbl | 1 +
arch/x86/entry/syscalls/syscall_32.tbl | 1 +
arch/x86/entry/syscalls/syscall_64.tbl | 1 +
arch/xtensa/kernel/syscalls/syscall.tbl | 1 +
fs/namei.c | 176 +++++-
fs/open.c | 149 +++--
include/linux/fcntl.h | 12 +-
include/linux/namei.h | 11 +
include/linux/syscalls.h | 3 +
include/uapi/asm-generic/unistd.h | 5 +-
include/uapi/linux/fcntl.h | 41 ++
tools/testing/selftests/Makefile | 1 +
tools/testing/selftests/openat2/.gitignore | 1 +
tools/testing/selftests/openat2/Makefile | 8 +
tools/testing/selftests/openat2/helpers.c | 109 ++++
tools/testing/selftests/openat2/helpers.h | 107 ++++
.../testing/selftests/openat2/openat2_test.c | 316 +++++++++++
.../selftests/openat2/rename_attack_test.c | 160 ++++++
.../testing/selftests/openat2/resolve_test.c | 523 ++++++++++++++++++
35 files changed, 1591 insertions(+), 73 deletions(-)
create mode 100644 tools/testing/selftests/openat2/.gitignore
create mode 100644 tools/testing/selftests/openat2/Makefile
create mode 100644 tools/testing/selftests/openat2/helpers.c
create mode 100644 tools/testing/selftests/openat2/helpers.h
create mode 100644 tools/testing/selftests/openat2/openat2_test.c
create mode 100644 tools/testing/selftests/openat2/rename_attack_test.c
create mode 100644 tools/testing/selftests/openat2/resolve_test.c
base-commit: a99d8080aaf358d5d23581244e5da23b35e340b9
--
2.23.0
On kvm_create_max_vcpus test remove unneeded local
variable in the loop that add vcpus to the VM.
Signed-off-by: Wainer dos Santos Moschetta <wainersm(a)redhat.com>
---
tools/testing/selftests/kvm/kvm_create_max_vcpus.c | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/tools/testing/selftests/kvm/kvm_create_max_vcpus.c b/tools/testing/selftests/kvm/kvm_create_max_vcpus.c
index 231d79e57774..6f38c3dc0d56 100644
--- a/tools/testing/selftests/kvm/kvm_create_max_vcpus.c
+++ b/tools/testing/selftests/kvm/kvm_create_max_vcpus.c
@@ -29,12 +29,9 @@ void test_vcpu_creation(int first_vcpu_id, int num_vcpus)
vm = vm_create(VM_MODE_DEFAULT, DEFAULT_GUEST_PHY_PAGES, O_RDWR);
- for (i = 0; i < num_vcpus; i++) {
- int vcpu_id = first_vcpu_id + i;
-
+ for (i = first_vcpu_id; i < first_vcpu_id + num_vcpus; i++)
/* This asserts that the vCPU was created. */
- vm_vcpu_add(vm, vcpu_id);
- }
+ vm_vcpu_add(vm, i);
kvm_vm_free(vm);
}
--
2.18.1
OK, here we go. Any VFIO and Infiniband runtime testing from anyone, is
especially welcome here.
Changes since v3:
* VFIO fix (patch 8): applied further cleanup: removed a pre-existing,
unnecessary release and reacquire of mmap_sem. Moved the DAX vma
checks from the vfio call site, to gup internals, and added comments
(and commit log) to clarify.
* Due to the above, made a corresponding fix to the
pin_longterm_pages_remote(), which was actually calling the wrong
gup internal function.
* Changed put_user_page() comments, to refer to pin*() APIs, rather than
get_user_pages*() APIs.
* Reverted an accidental whitespace-only change in the IB ODP code.
* Added a few more reviewed-by tags.
Changes since v2:
* Added a patch to convert IB/umem from normal gup, to gup_fast(). This
is also posted separately, in order to hopefully get some runtime
testing.
* Changed the page devmap code to be a little clearer,
thanks to Jerome for that.
* Split out the page devmap changes into a separate patch (and moved
Ira's Signed-off-by to that patch).
* Fixed my bug in IB: ODP code does not require pin_user_pages()
semantics. Therefore, revert the put_user_page() calls to put_page(),
and leave the get_user_pages() call as-is.
* As part of the revert, I am proposing here a change directly
from put_user_pages(), to release_pages(). I'd feel better if
someone agrees that this is the best way. It uses the more
efficient release_pages(), instead of put_page() in a loop,
and keep the change to just a few character on one line,
but OTOH it is not a pure revert.
* Loosened the FOLL_LONGTERM restrictions in the
__get_user_pages_locked() implementation, and used that in order
to fix up a VFIO bug. Thanks to Jason for that idea.
* Note the use of release_pages() in IB: is that OK?
* Added a few more WARN's and clarifying comments nearby.
* Many documentation improvements in various comments.
* Moved the new pin_user_pages.rst from Documentation/vm/ to
Documentation/core-api/ .
* Commit descriptions: added clarifying notes to the three patches
(drm/via, fs/io_uring, net/xdp) that already had put_user_page()
calls in place.
* Collected all pending Reviewed-by and Acked-by tags, from v1 and v2
email threads.
* Lot of churn from v2 --> v3, so it's possible that new bugs
sneaked in.
NOT DONE: separate patchset is required:
* __get_user_pages_locked(): stop compensating for
buggy callers who failed to set FOLL_GET. Instead, assert
that FOLL_GET is set (and fail if it's not).
======================================================================
Original cover letter (edited to fix up the patch description numbers)
This applies cleanly to linux-next and mmotm, and also to linux.git if
linux-next's commit 20cac10710c9 ("mm/gup_benchmark: fix MAP_HUGETLB
case") is first applied there.
This provides tracking of dma-pinned pages. This is a prerequisite to
solving the larger problem of proper interactions between file-backed
pages, and [R]DMA activities, as discussed in [1], [2], [3], and in
a remarkable number of email threads since about 2017. :)
A new internal gup flag, FOLL_PIN is introduced, and thoroughly
documented in the last patch's Documentation/vm/pin_user_pages.rst.
I believe that this will provide a good starting point for doing the
layout lease work that Ira Weiny has been working on. That's because
these new wrapper functions provide a clean, constrained, systematically
named set of functionality that, again, is required in order to even
know if a page is "dma-pinned".
In contrast to earlier approaches, the page tracking can be
incrementally applied to the kernel call sites that, until now, have
been simply calling get_user_pages() ("gup"). In other words, opt-in by
changing from this:
get_user_pages() (sets FOLL_GET)
put_page()
to this:
pin_user_pages() (sets FOLL_PIN)
put_user_page()
Because there are interdependencies with FOLL_LONGTERM, a similar
conversion as for FOLL_PIN, was applied. The change was from this:
get_user_pages(FOLL_LONGTERM) (also sets FOLL_GET)
put_page()
to this:
pin_longterm_pages() (sets FOLL_PIN | FOLL_LONGTERM)
put_user_page()
============================================================
Patch summary:
* Patches 1-8: refactoring and preparatory cleanup, independent fixes
* Patch 9: introduce pin_user_pages(), FOLL_PIN, but no functional
changes yet
* Patches 10-15: Convert existing put_user_page() callers, to use the
new pin*()
* Patch 16: Activate tracking of FOLL_PIN pages.
* Patches 17-19: convert FOLL_LONGTERM callers
* Patches: 20-22: gup_benchmark and run_vmtests support
* Patch 23: enforce FOLL_LONGTERM as a gup-internal (only) flag
============================================================
Testing:
* I've done some overall kernel testing (LTP, and a few other goodies),
and some directed testing to exercise some of the changes. And as you
can see, gup_benchmark is enhanced to exercise this. Basically, I've been
able to runtime test the core get_user_pages() and pin_user_pages() and
related routines, but not so much on several of the call sites--but those
are generally just a couple of lines changed, each.
Not much of the kernel is actually using this, which on one hand
reduces risk quite a lot. But on the other hand, testing coverage
is low. So I'd love it if, in particular, the Infiniband and PowerPC
folks could do a smoke test of this series for me.
Also, my runtime testing for the call sites so far is very weak:
* io_uring: Some directed tests from liburing exercise this, and they pass.
* process_vm_access.c: A small directed test passes.
* gup_benchmark: the enhanced version hits the new gup.c code, and passes.
* infiniband (still only have crude "IB pingpong" working, on a
good day: it's not exercising my conversions at runtime...)
* VFIO: compiles (I'm vowing to set up a run time test soon, but it's
not ready just yet)
* powerpc: it compiles...
* drm/via: compiles...
* goldfish: compiles...
* net/xdp: compiles...
* media/v4l2: compiles...
============================================================
Next:
* Get the block/bio_vec sites converted to use pin_user_pages().
* Work with Ira and Dave Chinner to weave this together with the
layout lease stuff.
============================================================
[1] Some slow progress on get_user_pages() (Apr 2, 2019): https://lwn.net/Articles/784574/
[2] DMA and get_user_pages() (LPC: Dec 12, 2018): https://lwn.net/Articles/774411/
[3] The trouble with get_user_pages() (Apr 30, 2018): https://lwn.net/Articles/753027/
John Hubbard (23):
mm/gup: pass flags arg to __gup_device_* functions
mm/gup: factor out duplicate code from four routines
mm/gup: move try_get_compound_head() to top, fix minor issues
mm: devmap: refactor 1-based refcounting for ZONE_DEVICE pages
goldish_pipe: rename local pin_user_pages() routine
IB/umem: use get_user_pages_fast() to pin DMA pages
media/v4l2-core: set pages dirty upon releasing DMA buffers
vfio, mm: fix get_user_pages_remote() and FOLL_LONGTERM
mm/gup: introduce pin_user_pages*() and FOLL_PIN
goldish_pipe: convert to pin_user_pages() and put_user_page()
IB/{core,hw,umem}: set FOLL_PIN, FOLL_LONGTERM via
pin_longterm_pages*()
mm/process_vm_access: set FOLL_PIN via pin_user_pages_remote()
drm/via: set FOLL_PIN via pin_user_pages_fast()
fs/io_uring: set FOLL_PIN via pin_user_pages()
net/xdp: set FOLL_PIN via pin_user_pages()
mm/gup: track FOLL_PIN pages
media/v4l2-core: pin_longterm_pages (FOLL_PIN) and put_user_page()
conversion
vfio, mm: pin_longterm_pages (FOLL_PIN) and put_user_page() conversion
powerpc: book3s64: convert to pin_longterm_pages() and put_user_page()
mm/gup_benchmark: use proper FOLL_WRITE flags instead of hard-coding
"1"
mm/gup_benchmark: support pin_user_pages() and related calls
selftests/vm: run_vmtests: invoke gup_benchmark with basic FOLL_PIN
coverage
mm/gup: remove support for gup(FOLL_LONGTERM)
Documentation/core-api/index.rst | 1 +
Documentation/core-api/pin_user_pages.rst | 218 +++++++
arch/powerpc/mm/book3s64/iommu_api.c | 15 +-
drivers/gpu/drm/via/via_dmablit.c | 2 +-
drivers/infiniband/core/umem.c | 17 +-
drivers/infiniband/core/umem_odp.c | 13 +-
drivers/infiniband/hw/hfi1/user_pages.c | 4 +-
drivers/infiniband/hw/mthca/mthca_memfree.c | 3 +-
drivers/infiniband/hw/qib/qib_user_pages.c | 8 +-
drivers/infiniband/hw/qib/qib_user_sdma.c | 2 +-
drivers/infiniband/hw/usnic/usnic_uiom.c | 9 +-
drivers/infiniband/sw/siw/siw_mem.c | 5 +-
drivers/media/v4l2-core/videobuf-dma-sg.c | 10 +-
drivers/platform/goldfish/goldfish_pipe.c | 35 +-
drivers/vfio/vfio_iommu_type1.c | 30 +-
fs/io_uring.c | 5 +-
include/linux/mm.h | 164 ++++-
include/linux/mmzone.h | 2 +
include/linux/page_ref.h | 10 +
mm/gup.c | 636 ++++++++++++++++----
mm/gup_benchmark.c | 87 ++-
mm/huge_memory.c | 54 +-
mm/hugetlb.c | 39 +-
mm/memremap.c | 67 +--
mm/process_vm_access.c | 28 +-
mm/vmstat.c | 2 +
net/xdp/xdp_umem.c | 4 +-
tools/testing/selftests/vm/gup_benchmark.c | 34 +-
tools/testing/selftests/vm/run_vmtests | 22 +
29 files changed, 1191 insertions(+), 335 deletions(-)
create mode 100644 Documentation/core-api/pin_user_pages.rst
--
2.24.0
Hi,
The cover letter is long, so the more important stuff is first:
* Jason, if you or someone could look at the the VFIO cleanup (patch 8)
and conversion to FOLL_PIN (patch 18), to make sure it's use of
remote and longterm gup matches what we discussed during the review
of v2, I'd appreciate it.
* Also for Jason and IB: as noted below, in patch 11, I am (too?) boldly
converting from put_user_pages() to release_pages().
* Jerome, I am going to take a look at doing your FOLL_GET change idea
(some callers should set FOLL_GET) separately, because it blew up "a
little bit" in my face. It's definitely a separate--tiny, but risky--project.
It also looks more reasonable when applied on top of this series here
(and it conflicts a lot), so I'm going to send it as a follow-up.
Changes since v2:
* Added a patch to convert IB/umem from normal gup, to gup_fast(). This
is also posted separately, in order to hopefully get some runtime
testing.
* Changed the page devmap code to be a little clearer,
thanks to Jerome for that.
* Split out the page devmap changes into a separate patch (and moved
Ira's Signed-off-by to that patch).
* Fixed my bug in IB: ODP code does not require pin_user_pages()
semantics. Therefore, revert the put_user_page() calls to put_page(),
and leave the get_user_pages() call as-is.
* As part of the revert, I am proposing here a change directly
from put_user_pages(), to release_pages(). I'd feel better if
someone agrees that this is the best way. It uses the more
efficient release_pages(), instead of put_page() in a loop,
and keep the change to just a few character on one line,
but OTOH it is not a pure revert.
* Loosened the FOLL_LONGTERM restrictions in the
__get_user_pages_locked() implementation, and used that in order
to fix up a VFIO bug. Thanks to Jason for that idea.
* Note the use of release_pages() in IB: is that OK?
* Added a few more WARN's and clarifying comments nearby.
* Many documentation improvements in various comments.
* Moved the new pin_user_pages.rst from Documentation/vm/ to
Documentation/core-api/ .
* Commit descriptions: added clarifying notes to the three patches
(drm/via, fs/io_uring, net/xdp) that already had put_user_page()
calls in place.
* Collected all pending Reviewed-by and Acked-by tags, from v1 and v2
email threads.
* Lot of churn from v2 --> v3, so it's possible that new bugs
sneaked in.
NOT DONE: separate patchset is required:
* __get_user_pages_locked(): stop compensating for
buggy callers who failed to set FOLL_GET. Instead, assert
that FOLL_GET is set (and fail if it's not).
======================================================================
Original cover letter (edited to fix up the patch description numbers)
This applies cleanly to linux-next and mmotm, and also to linux.git if
linux-next's commit 20cac10710c9 ("mm/gup_benchmark: fix MAP_HUGETLB
case") is first applied there.
This provides tracking of dma-pinned pages. This is a prerequisite to
solving the larger problem of proper interactions between file-backed
pages, and [R]DMA activities, as discussed in [1], [2], [3], and in
a remarkable number of email threads since about 2017. :)
A new internal gup flag, FOLL_PIN is introduced, and thoroughly
documented in the last patch's Documentation/vm/pin_user_pages.rst.
I believe that this will provide a good starting point for doing the
layout lease work that Ira Weiny has been working on. That's because
these new wrapper functions provide a clean, constrained, systematically
named set of functionality that, again, is required in order to even
know if a page is "dma-pinned".
In contrast to earlier approaches, the page tracking can be
incrementally applied to the kernel call sites that, until now, have
been simply calling get_user_pages() ("gup"). In other words, opt-in by
changing from this:
get_user_pages() (sets FOLL_GET)
put_page()
to this:
pin_user_pages() (sets FOLL_PIN)
put_user_page()
Because there are interdependencies with FOLL_LONGTERM, a similar
conversion as for FOLL_PIN, was applied. The change was from this:
get_user_pages(FOLL_LONGTERM) (also sets FOLL_GET)
put_page()
to this:
pin_longterm_pages() (sets FOLL_PIN | FOLL_LONGTERM)
put_user_page()
============================================================
Patch summary:
* Patches 1-8: refactoring and preparatory cleanup, independent fixes
* Patch 9: introduce pin_user_pages(), FOLL_PIN, but no functional
changes yet
* Patches 10-15: Convert existing put_user_page() callers, to use the
new pin*()
* Patch 16: Activate tracking of FOLL_PIN pages.
* Patches 17-19: convert FOLL_LONGTERM callers
* Patches: 20-22: gup_benchmark and run_vmtests support
* Patch 23: enforce FOLL_LONGTERM as a gup-internal (only) flag
============================================================
Testing:
* I've done some overall kernel testing (LTP, and a few other goodies),
and some directed testing to exercise some of the changes. And as you
can see, gup_benchmark is enhanced to exercise this. Basically, I've been
able to runtime test the core get_user_pages() and pin_user_pages() and
related routines, but not so much on several of the call sites--but those
are generally just a couple of lines changed, each.
Not much of the kernel is actually using this, which on one hand
reduces risk quite a lot. But on the other hand, testing coverage
is low. So I'd love it if, in particular, the Infiniband and PowerPC
folks could do a smoke test of this series for me.
Also, my runtime testing for the call sites so far is very weak:
* io_uring: Some directed tests from liburing exercise this, and they pass.
* process_vm_access.c: A small directed test passes.
* gup_benchmark: the enhanced version hits the new gup.c code, and passes.
* infiniband (still only have crude "IB pingpong" working, on a
good day: it's not exercising my conversions at runtime...)
* VFIO: compiles (I'm vowing to set up a run time test soon, but it's
not ready just yet)
* powerpc: it compiles...
* drm/via: compiles...
* goldfish: compiles...
* net/xdp: compiles...
* media/v4l2: compiles...
============================================================
Next:
* Get the block/bio_vec sites converted to use pin_user_pages().
* Work with Ira and Dave Chinner to weave this together with the
layout lease stuff.
============================================================
[1] Some slow progress on get_user_pages() (Apr 2, 2019): https://lwn.net/Articles/784574/
[2] DMA and get_user_pages() (LPC: Dec 12, 2018): https://lwn.net/Articles/774411/
[3] The trouble with get_user_pages() (Apr 30, 2018): https://lwn.net/Articles/753027/
John Hubbard (23):
mm/gup: pass flags arg to __gup_device_* functions
mm/gup: factor out duplicate code from four routines
mm/gup: move try_get_compound_head() to top, fix minor issues
mm: devmap: refactor 1-based refcounting for ZONE_DEVICE pages
goldish_pipe: rename local pin_user_pages() routine
IB/umem: use get_user_pages_fast() to pin DMA pages
media/v4l2-core: set pages dirty upon releasing DMA buffers
vfio, mm: fix get_user_pages_remote() and FOLL_LONGTERM
mm/gup: introduce pin_user_pages*() and FOLL_PIN
goldish_pipe: convert to pin_user_pages() and put_user_page()
IB/{core,hw,umem}: set FOLL_PIN, FOLL_LONGTERM via
pin_longterm_pages*()
mm/process_vm_access: set FOLL_PIN via pin_user_pages_remote()
drm/via: set FOLL_PIN via pin_user_pages_fast()
fs/io_uring: set FOLL_PIN via pin_user_pages()
net/xdp: set FOLL_PIN via pin_user_pages()
mm/gup: track FOLL_PIN pages
media/v4l2-core: pin_longterm_pages (FOLL_PIN) and put_user_page()
conversion
vfio, mm: pin_longterm_pages (FOLL_PIN) and put_user_page() conversion
powerpc: book3s64: convert to pin_longterm_pages() and put_user_page()
mm/gup_benchmark: use proper FOLL_WRITE flags instead of hard-coding
"1"
mm/gup_benchmark: support pin_user_pages() and related calls
selftests/vm: run_vmtests: invoke gup_benchmark with basic FOLL_PIN
coverage
mm/gup: remove support for gup(FOLL_LONGTERM)
Documentation/core-api/index.rst | 1 +
Documentation/core-api/pin_user_pages.rst | 218 +++++++
arch/powerpc/mm/book3s64/iommu_api.c | 15 +-
drivers/gpu/drm/via/via_dmablit.c | 2 +-
drivers/infiniband/core/umem.c | 17 +-
drivers/infiniband/core/umem_odp.c | 24 +-
drivers/infiniband/hw/hfi1/user_pages.c | 4 +-
drivers/infiniband/hw/mthca/mthca_memfree.c | 3 +-
drivers/infiniband/hw/qib/qib_user_pages.c | 8 +-
drivers/infiniband/hw/qib/qib_user_sdma.c | 2 +-
drivers/infiniband/hw/usnic/usnic_uiom.c | 9 +-
drivers/infiniband/sw/siw/siw_mem.c | 5 +-
drivers/media/v4l2-core/videobuf-dma-sg.c | 10 +-
drivers/platform/goldfish/goldfish_pipe.c | 35 +-
drivers/vfio/vfio_iommu_type1.c | 35 +-
fs/io_uring.c | 5 +-
include/linux/mm.h | 164 +++++-
include/linux/mmzone.h | 2 +
include/linux/page_ref.h | 10 +
mm/gup.c | 608 ++++++++++++++++----
mm/gup_benchmark.c | 87 ++-
mm/huge_memory.c | 54 +-
mm/hugetlb.c | 39 +-
mm/memremap.c | 67 +--
mm/process_vm_access.c | 28 +-
mm/vmstat.c | 2 +
net/xdp/xdp_umem.c | 4 +-
tools/testing/selftests/vm/gup_benchmark.c | 34 +-
tools/testing/selftests/vm/run_vmtests | 22 +
29 files changed, 1180 insertions(+), 334 deletions(-)
create mode 100644 Documentation/core-api/pin_user_pages.rst
--
2.24.0
From: Petr Machata <petrm(a)mellanox.com>
[ Upstream commit 372809055f6c830ff978564e09f58bcb9e9b937c ]
Immediately after mlxsw module is probed and lldpad started, added APP
entries are briefly in "unknown" state before becoming "pending". That's
the state that lldpad_app_wait_set() typically sees, and since there are
no pending entries at that time, it bails out. However the entries have
not been pushed to the kernel yet at that point, and thus the test case
fails.
Fix by waiting for both unknown and pending entries to disappear before
proceeding.
Fixes: d159261f3662 ("selftests: mlxsw: Add test for trust-DSCP")
Signed-off-by: Petr Machata <petrm(a)mellanox.com>
Signed-off-by: Ido Schimmel <idosch(a)mellanox.com>
Signed-off-by: David S. Miller <davem(a)davemloft.net>
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
---
tools/testing/selftests/net/forwarding/lib.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tools/testing/selftests/net/forwarding/lib.sh b/tools/testing/selftests/net/forwarding/lib.sh
index ca53b539aa2d1..08bac6cf1bb3a 100644
--- a/tools/testing/selftests/net/forwarding/lib.sh
+++ b/tools/testing/selftests/net/forwarding/lib.sh
@@ -251,7 +251,7 @@ lldpad_app_wait_set()
{
local dev=$1; shift
- while lldptool -t -i $dev -V APP -c app | grep -q pending; do
+ while lldptool -t -i $dev -V APP -c app | grep -Eq "pending|unknown"; do
echo "$dev: waiting for lldpad to push pending APP updates"
sleep 5
done
--
2.20.1
On Tue, Nov 12, 2019 at 3:08 PM John Hubbard <jhubbard(a)nvidia.com> wrote:
>
> On 11/12/19 2:43 PM, Dan Williams wrote:
> ...
> > Ah, sorry. This was the first time I had looked at this series and
> > jumped in without reading the background.
> >
> > Your patch as is looks ok, I assume you've removed the FOLL_LONGTERM
> > warning in get_user_pages_remote in another patch?
> >
>
> Actually, I haven't gone quite that far. Actually this patch is the last
> change to that function. Therefore, at the end of this patchset,
> get_user_pages_remote() ends up with this check in it which
> is a less-restrictive version of the warning:
>
> /*
> * Current FOLL_LONGTERM behavior is incompatible with
> * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
> * vmas. However, this only comes up if locked is set, and there are
> * callers that do request FOLL_LONGTERM, but do not set locked. So,
> * allow what we can.
> */
> if (gup_flags & FOLL_LONGTERM) {
> if (WARN_ON_ONCE(locked))
> return -EINVAL;
> }
>
> Is that OK, or did you want to go further (possibly in a follow-up
> patchset, as I'm hoping to get this one in soon)?
That looks ok. Something to maybe push down into the core in a future
cleanup, but not something that needs to be done now.
> ...
> >>> I think check_vma_flags() should do the ((FOLL_LONGTERM | FOLL_GET) &&
> >>> vma_is_fsdax()) check and that would also remove the need for
> >>> __gup_longterm_locked.
> >>>
> >>
> >> Good idea, but there is still the call to check_and_migrate_cma_pages(),
> >> inside __gup_longterm_locked(). So it's a little more involved and
> >> we can't trivially delete __gup_longterm_locked() yet, right?
> >
> > [ add Aneesh ]
> >
> > Yes, you're right. I had overlooked that had snuck in there. That to
> > me similarly needs to be pushed down into the core with its own FOLL
> > flag, or it needs to be an explicit fixup that each caller does after
> > get_user_pages. The fact that migration silently happens as a side
> > effect of gup is too magical for my taste.
> >
>
> Yes. It's an intrusive side effect that is surprising, and not in a
> "happy surprise" way. :) . Fixing up the CMA pages by splitting that
> functionality into separate function calls sounds like an improvement
> worth exploring.
Right, future work.
Greetings,
Find attached email very confidential. reply for more details
Thanks.
Peter Wong
----------------------------------------------------
This email was sent by the shareware version of Postman Professional.
It is necessary to set fd to -1 when inotify_add_watch() fails in
cg_prepare_for_wait. Otherwise the fd which has been closed in
cg_prepare_for_wait may be misused in other functions such as
cg_enter_and_wait_for_frozen and cg_freeze_wait.
Fixes: 5313bfe425c8 ("selftests: cgroup: add freezer controller self-tests")
Signed-off-by: Hewenliang <hewenliang4(a)huawei.com>
---
tools/testing/selftests/cgroup/test_freezer.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/tools/testing/selftests/cgroup/test_freezer.c b/tools/testing/selftests/cgroup/test_freezer.c
index 0fc1b6d4b0f9..62a27ab3c2f3 100644
--- a/tools/testing/selftests/cgroup/test_freezer.c
+++ b/tools/testing/selftests/cgroup/test_freezer.c
@@ -72,6 +72,7 @@ static int cg_prepare_for_wait(const char *cgroup)
if (ret == -1) {
debug("Error: inotify_add_watch() failed\n");
close(fd);
+ fd = -1;
}
return fd;
--
2.19.1
Hi,
Changes since v1:
* Changed the function signature of __huge_pt_done() from int to void.
* Renamed __remove_refs_from_head() to put_compound_head().
* Improved the comment documentation in mm.h and gup.c
* Merged Documentation/vm/pin_user_pages.rst into the "introduce
FOLL_PIN" patch.
* Fixed Documentation/vm/pin_user_pages.rst:
* Fixed up a TODO about DAX.
* 31, not 32 bits total are available for counting
* Deleted some stale comments from the commit description of the
VFIO patch.
* Added Reviewed-by tags from Ira Weiny and Jens Axboe, and Acked-by
from Björn Töpel.
======================================================================
Original cover letter (edited to fix up the patch description numbers)
This applies cleanly to linux-next and mmotm, and also to linux.git if
linux-next's commit 20cac10710c9 ("mm/gup_benchmark: fix MAP_HUGETLB
case") is first applied there.
This provides tracking of dma-pinned pages. This is a prerequisite to
solving the larger problem of proper interactions between file-backed
pages, and [R]DMA activities, as discussed in [1], [2], [3], and in
a remarkable number of email threads since about 2017. :)
A new internal gup flag, FOLL_PIN is introduced, and thoroughly
documented in the last patch's Documentation/vm/pin_user_pages.rst.
I believe that this will provide a good starting point for doing the
layout lease work that Ira Weiny has been working on. That's because
these new wrapper functions provide a clean, constrained, systematically
named set of functionality that, again, is required in order to even
know if a page is "dma-pinned".
In contrast to earlier approaches, the page tracking can be
incrementally applied to the kernel call sites that, until now, have
been simply calling get_user_pages() ("gup"). In other words, opt-in by
changing from this:
get_user_pages() (sets FOLL_GET)
put_page()
to this:
pin_user_pages() (sets FOLL_PIN)
put_user_page()
Because there are interdependencies with FOLL_LONGTERM, a similar
conversion as for FOLL_PIN, was applied. The change was from this:
get_user_pages(FOLL_LONGTERM) (also sets FOLL_GET)
put_page()
to this:
pin_longterm_pages() (sets FOLL_PIN | FOLL_LONGTERM)
put_user_page()
============================================================
Patch summary:
* Patches 1-4: refactoring and preparatory cleanup, independent fixes
(Patch 4: V4L2-core bug fix (can be separately applied))
* Patch 5: introduce pin_user_pages(), FOLL_PIN, but no functional
changes yet
* Patches 6-11: Convert existing put_user_page() callers, to use the
new pin*()
* Patch 12: Activate tracking of FOLL_PIN pages.
* Patches 13-15: convert FOLL_LONGTERM callers
* Patches: 16-17: gup_benchmark and run_vmtests support
* Patch 18: enforce FOLL_LONGTERM as a gup-internal (only) flag
============================================================
Testing:
* I've done some overall kernel testing (LTP, and a few other goodies),
and some directed testing to exercise some of the changes. And as you
can see, gup_benchmark is enhanced to exercise this. Basically, I've been
able to runtime test the core get_user_pages() and pin_user_pages() and
related routines, but not so much on several of the call sites--but those
are generally just a couple of lines changed, each.
Not much of the kernel is actually using this, which on one hand
reduces risk quite a lot. But on the other hand, testing coverage
is low. So I'd love it if, in particular, the Infiniband and PowerPC
folks could do a smoke test of this series for me.
Also, my runtime testing for the call sites so far is very weak:
* io_uring: Some directed tests from liburing exercise this, and they pass.
* process_vm_access.c: A small directed test passes.
* gup_benchmark: the enhanced version hits the new gup.c code, and passes.
* infiniband (still only have crude "IB pingpong" working, on a
good day: it's not exercising my conversions at runtime...)
* VFIO: compiles (I'm vowing to set up a run time test soon, but it's
not ready just yet)
* powerpc: it compiles...
* drm/via: compiles...
* goldfish: compiles...
* net/xdp: compiles...
* media/v4l2: compiles...
============================================================
Next:
* Get the block/bio_vec sites converted to use pin_user_pages().
* Work with Ira and Dave Chinner to weave this together with the
layout lease stuff.
============================================================
[1] Some slow progress on get_user_pages() (Apr 2, 2019): https://lwn.net/Articles/784574/
[2] DMA and get_user_pages() (LPC: Dec 12, 2018): https://lwn.net/Articles/774411/
[3] The trouble with get_user_pages() (Apr 30, 2018): https://lwn.net/Articles/753027/
John Hubbard (18):
mm/gup: pass flags arg to __gup_device_* functions
mm/gup: factor out duplicate code from four routines
goldish_pipe: rename local pin_user_pages() routine
media/v4l2-core: set pages dirty upon releasing DMA buffers
mm/gup: introduce pin_user_pages*() and FOLL_PIN
goldish_pipe: convert to pin_user_pages() and put_user_page()
infiniband: set FOLL_PIN, FOLL_LONGTERM via pin_longterm_pages*()
mm/process_vm_access: set FOLL_PIN via pin_user_pages_remote()
drm/via: set FOLL_PIN via pin_user_pages_fast()
fs/io_uring: set FOLL_PIN via pin_user_pages()
net/xdp: set FOLL_PIN via pin_user_pages()
mm/gup: track FOLL_PIN pages
media/v4l2-core: pin_longterm_pages (FOLL_PIN) and put_user_page()
conversion
vfio, mm: pin_longterm_pages (FOLL_PIN) and put_user_page() conversion
powerpc: book3s64: convert to pin_longterm_pages() and put_user_page()
mm/gup_benchmark: support pin_user_pages() and related calls
selftests/vm: run_vmtests: invoke gup_benchmark with basic FOLL_PIN
coverage
mm/gup: remove support for gup(FOLL_LONGTERM)
Documentation/vm/index.rst | 1 +
Documentation/vm/pin_user_pages.rst | 212 +++++++
arch/powerpc/mm/book3s64/iommu_api.c | 15 +-
drivers/gpu/drm/via/via_dmablit.c | 2 +-
drivers/infiniband/core/umem.c | 5 +-
drivers/infiniband/core/umem_odp.c | 10 +-
drivers/infiniband/hw/hfi1/user_pages.c | 4 +-
drivers/infiniband/hw/mthca/mthca_memfree.c | 3 +-
drivers/infiniband/hw/qib/qib_user_pages.c | 8 +-
drivers/infiniband/hw/qib/qib_user_sdma.c | 2 +-
drivers/infiniband/hw/usnic/usnic_uiom.c | 9 +-
drivers/infiniband/sw/siw/siw_mem.c | 5 +-
drivers/media/v4l2-core/videobuf-dma-sg.c | 10 +-
drivers/platform/goldfish/goldfish_pipe.c | 35 +-
drivers/vfio/vfio_iommu_type1.c | 15 +-
fs/io_uring.c | 5 +-
include/linux/mm.h | 142 ++++-
include/linux/mmzone.h | 2 +
include/linux/page_ref.h | 10 +
mm/gup.c | 594 ++++++++++++++++----
mm/gup_benchmark.c | 81 ++-
mm/huge_memory.c | 32 +-
mm/hugetlb.c | 28 +-
mm/memremap.c | 4 +-
mm/process_vm_access.c | 28 +-
mm/vmstat.c | 2 +
net/xdp/xdp_umem.c | 4 +-
tools/testing/selftests/vm/gup_benchmark.c | 28 +-
tools/testing/selftests/vm/run_vmtests | 22 +
29 files changed, 1054 insertions(+), 264 deletions(-)
create mode 100644 Documentation/vm/pin_user_pages.rst
--
2.23.0
The current kunit execution model is to provide base kunit functionality
and tests built-in to the kernel. The aim of this series is to allow
building kunit itself and tests as modules. This in turn allows a
simple form of selective execution; load the module you wish to test.
In doing so, kunit itself (if also built as a module) will be loaded as
an implicit dependency.
Because this requires a core API modification - if a module delivers
multiple suites, they must be declared with the kunit_test_suites()
macro - we're proposing this patch set as a candidate to be applied to the
test tree before too many kunit consumers appear. We attempt to deal
with existing consumers in patch 4.
Changes since v2:
- moved string-stream.h header to lib/kunit/string-stream-impl.h (Brendan)
(patch 1)
- split out non-exported interfaces in try-catch-impl.h (Brendan)
(patch 2)
- added kunit_find_symbol() and KUNIT_INIT_*SYMBOL to lookup non-exported
symbols. KUNIT_INIT_*SYMBOL() is defined so that a mismatch between
local symbol definition and definition of symbol in target will trigger
a compilation error when the object is compiled built-in (Brendan)
(patches 3, 4)
- removed #ifdef MODULE around module licenses (Randy, Brendan, Andy)
(patch 4)
- replaced kunit_test_suite() with kunit_test_suites() rather than
supporting both (Brendan) (patch 4)
- lookup sysctl_hung_task_timeout_secs as kunit may be built as a module
and the symbol may not be available (patch 5)
- fixed whitespace issues in doc (patch 6)
Alan Maguire (6):
kunit: move string-stream.h to lib/kunit/string-stream-impl.h
kunit: hide unexported try-catch interface in try-catch-impl.h
kunit: add kunit_find_symbol() function for symbol lookup
kunit: allow kunit tests to be loaded as a module
kunit: allow kunit to be loaded as a module
kunit: update documentation to describe module-based build
Documentation/dev-tools/kunit/faq.rst | 3 +-
Documentation/dev-tools/kunit/index.rst | 3 +
Documentation/dev-tools/kunit/usage.rst | 16 +++++
include/kunit/assert.h | 3 +-
include/kunit/string-stream.h | 51 -------------
include/kunit/test.h | 123 +++++++++++++++++++++++++++++---
include/kunit/try-catch.h | 10 ---
kernel/sysctl-test.c | 4 +-
lib/Kconfig.debug | 2 +-
lib/kunit/Kconfig | 6 +-
lib/kunit/Makefile | 4 +-
lib/kunit/assert.c | 9 +++
lib/kunit/example-test.c | 4 +-
lib/kunit/string-stream-impl.h | 51 +++++++++++++
lib/kunit/string-stream-test.c | 46 ++++++++----
lib/kunit/string-stream.c | 3 +-
lib/kunit/test-test.c | 50 ++++++++++---
lib/kunit/test.c | 49 +++++++++++++
lib/kunit/try-catch-impl.h | 23 ++++++
lib/kunit/try-catch.c | 6 ++
20 files changed, 363 insertions(+), 103 deletions(-)
delete mode 100644 include/kunit/string-stream.h
create mode 100644 lib/kunit/string-stream-impl.h
create mode 100644 lib/kunit/try-catch-impl.h
--
1.8.3.1
When installing kselftests to its own directory and run the
test_lwt_ip_encap.sh it will complain that test_lwt_ip_encap.o can't be
found. Same with the test_tc_edt.sh test it will complain that
test_tc_edt.o can't be found.
$ ./test_lwt_ip_encap.sh
starting egress IPv4 encap test
Error opening object test_lwt_ip_encap.o: No such file or directory
Object hashing failed!
Cannot initialize ELF context!
Failed to parse eBPF program: Invalid argument
Rework to add test_lwt_ip_encap.o and test_tc_edt.o to TEST_FILES so the
object file gets installed when installing kselftest.
Fixes: 74b5a5968fe8 ("selftests/bpf: Replace test_progs and test_maps w/ general rule")
Signed-off-by: Anders Roxell <anders.roxell(a)linaro.org>
Acked-by: Song Liu <songliubraving(a)fb.com>
---
tools/testing/selftests/bpf/Makefile | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile
index b334a6db15c1..b03dc2298fea 100644
--- a/tools/testing/selftests/bpf/Makefile
+++ b/tools/testing/selftests/bpf/Makefile
@@ -38,7 +38,8 @@ TEST_GEN_PROGS += test_progs-bpf_gcc
endif
TEST_GEN_FILES =
-TEST_FILES =
+TEST_FILES = test_lwt_ip_encap.o \
+ test_tc_edt.o
# Order correspond to 'make run_tests' order
TEST_PROGS := test_kmod.sh \
--
2.20.1
When installing kselftests to its own directory and running the
test_lwt_ip_encap.sh it will complain that test_lwt_ip_encap.o can't be
find.
$ ./test_lwt_ip_encap.sh
starting egress IPv4 encap test
Error opening object test_lwt_ip_encap.o: No such file or directory
Object hashing failed!
Cannot initialize ELF context!
Failed to parse eBPF program: Invalid argument
Rework to add test_lwt_ip_encap.o to TEST_FILES so the object file gets
installed when installing kselftest.
Fixes: 74b5a5968fe8 ("selftests/bpf: Replace test_progs and test_maps w/ general rule")
Signed-off-by: Anders Roxell <anders.roxell(a)linaro.org>
Acked-by: Song Liu <songliubraving(a)fb.com>
---
tools/testing/selftests/bpf/Makefile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile
index b334a6db15c1..cc09b5df9403 100644
--- a/tools/testing/selftests/bpf/Makefile
+++ b/tools/testing/selftests/bpf/Makefile
@@ -38,7 +38,7 @@ TEST_GEN_PROGS += test_progs-bpf_gcc
endif
TEST_GEN_FILES =
-TEST_FILES =
+TEST_FILES = test_lwt_ip_encap.o
# Order correspond to 'make run_tests' order
TEST_PROGS := test_kmod.sh \
--
2.20.1
When installing kselftests to its own directory and running the
test_lwt_ip_encap.sh it will complain that test_lwt_ip_encap.o can't be
find.
$ ./test_lwt_ip_encap.sh
starting egress IPv4 encap test
Error opening object test_lwt_ip_encap.o: No such file or directory
Object hashing failed!
Cannot initialize ELF context!
Failed to parse eBPF program: Invalid argument
Rework to add test_lwt_ip_encap.o to TEST_FILES so the object file gets
installed when installing kselftest.
Fixes: 74b5a5968fe8 ("selftests/bpf: Replace test_progs and test_maps w/ general rule")
Signed-off-by: Anders Roxell <anders.roxell(a)linaro.org>
---
tools/testing/selftests/bpf/Makefile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile
index b334a6db15c1..cc09b5df9403 100644
--- a/tools/testing/selftests/bpf/Makefile
+++ b/tools/testing/selftests/bpf/Makefile
@@ -38,7 +38,7 @@ TEST_GEN_PROGS += test_progs-bpf_gcc
endif
TEST_GEN_FILES =
-TEST_FILES =
+TEST_FILES = test_lwt_ip_encap.o
# Order correspond to 'make run_tests' order
TEST_PROGS := test_kmod.sh \
--
2.20.1
From: Breno Leitao <leitao(a)debian.org>
[ Upstream commit 44d947eff19d64384efc06069509db7a0a1103b0 ]
There are cases where the test is not expecting to have the transaction
aborted, but, the test process might have been rescheduled, either in the
OS level or by KVM (if it is running on a KVM guest machine). The process
reschedule will cause a treclaim/recheckpoint which will cause the
transaction to doom, aborting the transaction as soon as the process is
rescheduled back to the CPU. This might cause the test to fail, but this is
not a failure in essence.
If that is the case, TEXASR[FC] is indicated with either
TM_CAUSE_RESCHEDULE or TM_CAUSE_KVM_RESCHEDULE for KVM interruptions.
In this scenario, ignore these two failures and avoid the whole test to
return failure.
Signed-off-by: Breno Leitao <leitao(a)debian.org>
Reviewed-by: Gustavo Romero <gromero(a)linux.ibm.com>
Signed-off-by: Michael Ellerman <mpe(a)ellerman.id.au>
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
---
tools/testing/selftests/powerpc/tm/tm-unavailable.c | 9 ++++++---
tools/testing/selftests/powerpc/tm/tm.h | 9 +++++++++
2 files changed, 15 insertions(+), 3 deletions(-)
diff --git a/tools/testing/selftests/powerpc/tm/tm-unavailable.c b/tools/testing/selftests/powerpc/tm/tm-unavailable.c
index 156c8e750259b..09894f4ff62e6 100644
--- a/tools/testing/selftests/powerpc/tm/tm-unavailable.c
+++ b/tools/testing/selftests/powerpc/tm/tm-unavailable.c
@@ -236,7 +236,8 @@ void *tm_una_ping(void *input)
}
/* Check if we were not expecting a failure and a it occurred. */
- if (!expecting_failure() && is_failure(cr_)) {
+ if (!expecting_failure() && is_failure(cr_) &&
+ !failure_is_reschedule()) {
printf("\n\tUnexpected transaction failure 0x%02lx\n\t",
failure_code());
return (void *) -1;
@@ -244,9 +245,11 @@ void *tm_una_ping(void *input)
/*
* Check if TM failed due to the cause we were expecting. 0xda is a
- * TM_CAUSE_FAC_UNAV cause, otherwise it's an unexpected cause.
+ * TM_CAUSE_FAC_UNAV cause, otherwise it's an unexpected cause, unless
+ * it was caused by a reschedule.
*/
- if (is_failure(cr_) && !failure_is_unavailable()) {
+ if (is_failure(cr_) && !failure_is_unavailable() &&
+ !failure_is_reschedule()) {
printf("\n\tUnexpected failure cause 0x%02lx\n\t",
failure_code());
return (void *) -1;
diff --git a/tools/testing/selftests/powerpc/tm/tm.h b/tools/testing/selftests/powerpc/tm/tm.h
index df4204247d45c..5518b1d4ef8b2 100644
--- a/tools/testing/selftests/powerpc/tm/tm.h
+++ b/tools/testing/selftests/powerpc/tm/tm.h
@@ -52,6 +52,15 @@ static inline bool failure_is_unavailable(void)
return (failure_code() & TM_CAUSE_FAC_UNAV) == TM_CAUSE_FAC_UNAV;
}
+static inline bool failure_is_reschedule(void)
+{
+ if ((failure_code() & TM_CAUSE_RESCHED) == TM_CAUSE_RESCHED ||
+ (failure_code() & TM_CAUSE_KVM_RESCHED) == TM_CAUSE_KVM_RESCHED)
+ return true;
+
+ return false;
+}
+
static inline bool failure_is_nesting(void)
{
return (__builtin_get_texasru() & 0x400000);
--
2.20.1
hello,
i got a warning during kernel compilation.
net/core/skbuff.o: warning: objtool: skb_push.cold()+0x15: unreachable instruction
related clips...
--------------------x----------------x---------------------
(gdb) l skb_push.cold
1880 void *skb_push(struct sk_buff *skb, unsigned int len)
1881 {
1882 skb->data -= len;
1883 skb->len += len;
1884 if (unlikely(skb->data < skb->head))
1885 skb_under_panic(skb, len, __builtin_return_address(0));
1886 return skb->data;
1887 }
1888 EXPORT_SYMBOL(skb_push);
1889
(gdb) l *0xffffffff815ffc8e
0xffffffff815ffc8e is in skb_push (net/core/skbuff.c:1885).
1880 void *skb_push(struct sk_buff *skb, unsigned int len)
1881 {
1882 skb->data -= len;
1883 skb->len += len;
1884 if (unlikely(skb->data < skb->head))
1885 skb_under_panic(skb, len, __builtin_return_address(0));
1886 return skb->data;
1887 }
1888 EXPORT_SYMBOL(skb_push);
1889
(gdb)
------------------x-----------------------x------------------------------------
$uname -a
Linux debian 5.4.0-rc1+ #1 SMP Sat Nov 9 21:29:48 IST 2019 x86_64 GNU/Linux
$
this kernel is from linux-kselftest tree
---------------------------x-------------x------------------------------
$gcc --version
gcc (Debian 9.2.1-14) 9.2.1 20191025
Copyright (C) 2019 Free Software Foundation, Inc.
This is free software; see the source for copying conditions. There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-------------------------x-------------------x-----------------------------
Linux debian 5.4.0-rc1+ #1 SMP Sat Nov 9 21:29:48 IST 2019 x86_64 GNU/Linux
GNU Make 4.2.1
Binutils 2.33.1
Util-linux 2.33.1
Mount 2.33.1
Linux C Library 2.29
Dynamic linker (ldd) 2.29
Procps 3.3.15
Kbd 2.0.4
Console-tools 2.0.4
Sh-utils 8.30
Udev 241
-------------------------------x-----------------x----------------------------
--
software engineer
rajagiri school of engineering and technology
hello,
i just got some mail deleted. so incase some of you have
send mail to me after https://lkml.org/lkml/2019/11/4/824
please kindly resend it to me.
I have also noticed the error in 5.4.0-rc1+ of linux-kselftest tree.
iam attaching two related files again.
--
software engineer
rajagiri school of engineering and technology
From: Nicolas Geoffray <ngeoffray(a)google.com>
F_SEAL_FUTURE_WRITE has unexpected behavior when used with MAP_PRIVATE:
A private mapping created after the memfd file that gets sealed with
F_SEAL_FUTURE_WRITE loses the copy-on-write at fork behavior, meaning
children and parent share the same memory, even though the mapping is
private.
The reason for this is due to the code below:
static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
{
struct shmem_inode_info *info = SHMEM_I(file_inode(file));
if (info->seals & F_SEAL_FUTURE_WRITE) {
/*
* New PROT_WRITE and MAP_SHARED mmaps are not allowed when
* "future write" seal active.
*/
if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE))
return -EPERM;
/*
* Since the F_SEAL_FUTURE_WRITE seals allow for a MAP_SHARED
* read-only mapping, take care to not allow mprotect to revert
* protections.
*/
vma->vm_flags &= ~(VM_MAYWRITE);
}
...
}
And for the mm to know if a mapping is copy-on-write:
static inline bool is_cow_mapping(vm_flags_t flags)
{
return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
}
The patch fixes the issue by making the mprotect revert protection
happen only for shared mappings. For private mappings, using mprotect
will have no effect on the seal behavior.
Cc: kernel-team(a)android.com
Signed-off-by: Nicolas Geoffray <ngeoffray(a)google.com>
Signed-off-by: Joel Fernandes (Google) <joel(a)joelfernandes.org>
---
Google bug: 143833776
mm/shmem.c | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/mm/shmem.c b/mm/shmem.c
index 447fd575587c..6ac5e867ef13 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -2214,11 +2214,14 @@ static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
return -EPERM;
/*
- * Since the F_SEAL_FUTURE_WRITE seals allow for a MAP_SHARED
- * read-only mapping, take care to not allow mprotect to revert
- * protections.
+ * Since an F_SEAL_FUTURE_WRITE sealed memfd can be mapped as
+ * MAP_SHARED and read-only, take care to not allow mprotect to
+ * revert protections on such mappings. Do this only for shared
+ * mappings. For private mappings, don't need to mask VM_MAYWRITE
+ * as we still want them to be COW-writable.
*/
- vma->vm_flags &= ~(VM_MAYWRITE);
+ if (vma->vm_flags & VM_SHARED)
+ vma->vm_flags &= ~(VM_MAYWRITE);
}
file_accessed(file);
--
2.24.0.rc1.363.gb1bccd3e3d-goog
Hi
this patchset aims to add the initial arch-specific arm64 support to
kselftest starting with signals-related test-cases.
This series is based on v5.4-rc4.
A common internal test-case layout is proposed for signal tests and it is
wired-up to the toplevel kselftest Makefile, so that it should be possible
at the end to run it on an arm64 target in the usual way with KSFT.
~/linux# make TARGETS=arm64 kselftest
New KSFT arm64 testcases live inside tools/testing/selftests/arm64 grouped
by family inside subdirectories: arm64/signal is the first family proposed
with this series.
This series converts also to this subdirectory scheme the pre-existing
KSFT arm64 tags tests (already merged in v5.3), moving them into their own
arm64/tags subdirectory.
Thanks
Cristian
Notes:
-----
- further details in the included READMEs
- more tests still to be written (current strategy is going through the
related Kernel signal-handling code and write a test for each possible
and sensible code-path)
A few ideas for more TODO testcases:
- mangle_pstate_invalid_ssbs_regs: mess with SSBS bits on every
possible configured behavior
- fake_sigreturn_unmapped_sp: SP into unmapped addrs
- fake_sigreturn_kernelspace_sp: SP into kernel addrs
- fake_sigreturn_sve_bad_extra_context: SVE extra context badly formed
- fake_sigreturn_misaligned_sp_4: misaligned SP by 4
(i.e., __alignof__(struct _aarch64_ctx))
- fake_sigreturn_misaligned_sp_8: misaligned SP by 8
(i.e., sizeof(struct _aarch64_ctx))
- fake_sigreturn_bad_size_non_aligned: a size that doesn't overflow
__reserved[], but is not a multiple of 16
- fake_sigreturn_bad_size_tiny: a size that is less than 16
- fake_sigreturn_bad_size_overflow_tiny: a size that does overflow
__reserved[], but by less than 16 bytes?
- mangle_sve_invalid_extra_context: SVE extra_context invalid
- SVE signal testcases and special handling will be part of an additional
patch still to be released
- KSFT arm64 tags test patch
https://lore.kernel.org/linux-arm-kernel/c1e6aad230658bc175b42d92daeff2e300…
is relocated into its own directory under tools/testing/selftests/arm64/tags
Changes:
--------
v9 --> v10:
- rebased on v5.4-rc4
- removed some test_init stale code related to PAN/UAO
(not used nor needed and wrong)
v8-->v9:
- fixed a couple of misplaced .gitignore
v7-->v8:
- removed SSBS test case
- split remnants of SSBS patch (v7 05/11), containing some helpers,
into two distinct patches
v6-->v7:
- rebased on v5.4-rc2
- renamed SUBTARGETS arm64/ toplevel Makefile ENV to ARM64_SUBTARGETS
- fixed fake_sigreturn alignment routines (off by one)
- fixed SSBS test: avoid using MRS/MSR as whole and SKIP when SSBS not
supported
- reporting KSFT_SKIP when needed (usually if test_init(0 fails)
- using ID_AA64PFR1_EL1.SSBS to check SSBS support instead of HWCAP_SSBS
v5-->v6:
- added arm64 toplevel Makefile SUBTARGETS env var to be able to selectively
build only some arm64/ tests subdirectories
- removed unneed toplevel Makefile exports and fixed Copyright
- better checks for supported features and features names helpers
- converted some run-time critical assert() to abort() to avoid
issues when -NDEBUG is set
- default_handler() signal handler refactored and split
- using SIGTRAP for get_current_context()
- use volatile where proper
- refactor and relocate test_init() invocation
- review usage of MRS SSBS instructions depending on HW_SSBS
- cleanup fake_sigreturn trampoline
- cleanup get_starting_header helper
- avoiding timeout test failures wherever possible (fail immediately
if possible)
v4-->v5:
- rebased on arm64/for-next-core merging 01/11 with KSFT tags tests:
commit 9ce1263033cd ("selftests, arm64: add a selftest for passing tagged pointers to kernel")
- moved .gitignore up on elevel
- moved kernel header search mechanism into KSFT arm64 toplevel Makefile
so that it can be used easily also by each arm64 KSFT subsystem inside
subdirs of arm64
v3-->v4:
- rebased on v5.3-rc6
- added test descriptions
- fixed commit messages (imperative mood)
- added missing includes and removed unneeded ones
- added/used new get_starting_head() helper
- fixed/simplified signal.S::fakke_sigreturn()
- added set_regval() macro and .init initialization func
- better synchonization in get_current_context()
- macroization of mangle_pstate_invalid_mode_el
- split mangle_pstate_invalid_mode_el h/t
- removed standalone mode
- simplified CPU features checks
- fixed/refactored get_header() and validation routines
- simplfied docs
v2-->v3:
- rebased on v5.3-rc2
- better test result characterization looking for
SEGV_ACCERR in si_code on SIGSEGV
- using KSFT Framework macros for retvalues
- removed SAFE_WRITE()/dump_uc: buggy, un-needed and unused
- reviewed generation process of test_arm64_signals.sh runner script
- re-added a fixed fake_sigreturn_misaligned_sp testcase and a properly
extended fake_sigreturn() helper
- added tests' TODO notes
v1-->v2:
- rebased on 5.2-rc7
- various makefile's cleanups
- mixed READMEs fixes
- fixed test_arm64_signals.sh runner script
- cleaned up assembly code in signal.S
- improved get_current_context() logic
- fixed SAFE_WRITE()
- common support code split into more chunks, each one introduced when
needed by some new testcases
- fixed some headers validation routines in testcases.c
- removed some still broken/immature tests:
+ fake_sigreturn_misaligned
+ fake_sigreturn_overflow_reserved
+ mangle_pc_invalid
+ mangle_sp_misaligned
- fixed some other testcases:
+ mangle_pstate_ssbs_regs: better checks of SSBS bit when feature unsupported
+ mangle_pstate_invalid_compat_toggle: name fix
+ mangle_pstate_invalid_mode_el[1-3]: precautionary zeroing PSTATE.MODE
+ fake_sigreturn_bad_magic, fake_sigreturn_bad_size,
fake_sigreturn_bad_size_for_magic0:
- accounting for available space...dropping extra when needed
- keeping alignent
- new testcases on FPSMID context:
+ fake_sigreturn_missing_fpsimd
+ fake_sigreturn_duplicated_fpsimd
Cristian Marussi (12):
kselftest: arm64: extend toplevel skeleton Makefile
kselftest: arm64: mangle_pstate_invalid_compat_toggle and common utils
kselftest: arm64: mangle_pstate_invalid_daif_bits
kselftest: arm64: mangle_pstate_invalid_mode_el[123][ht]
kselftest: arm64: extend test_init functionalities
kselftest: arm64: add helper get_current_context
kselftest: arm64: fake_sigreturn_bad_magic
kselftest: arm64: fake_sigreturn_bad_size_for_magic0
kselftest: arm64: fake_sigreturn_missing_fpsimd
kselftest: arm64: fake_sigreturn_duplicated_fpsimd
kselftest: arm64: fake_sigreturn_bad_size
kselftest: arm64: fake_sigreturn_misaligned_sp
tools/testing/selftests/Makefile | 1 +
tools/testing/selftests/arm64/Makefile | 64 +++-
tools/testing/selftests/arm64/README | 25 ++
.../testing/selftests/arm64/signal/.gitignore | 3 +
tools/testing/selftests/arm64/signal/Makefile | 32 ++
tools/testing/selftests/arm64/signal/README | 59 ++++
.../testing/selftests/arm64/signal/signals.S | 64 ++++
.../selftests/arm64/signal/test_signals.c | 29 ++
.../selftests/arm64/signal/test_signals.h | 100 ++++++
.../arm64/signal/test_signals_utils.c | 328 ++++++++++++++++++
.../arm64/signal/test_signals_utils.h | 120 +++++++
.../testcases/fake_sigreturn_bad_magic.c | 52 +++
.../testcases/fake_sigreturn_bad_size.c | 77 ++++
.../fake_sigreturn_bad_size_for_magic0.c | 46 +++
.../fake_sigreturn_duplicated_fpsimd.c | 50 +++
.../testcases/fake_sigreturn_misaligned_sp.c | 37 ++
.../testcases/fake_sigreturn_missing_fpsimd.c | 50 +++
.../mangle_pstate_invalid_compat_toggle.c | 31 ++
.../mangle_pstate_invalid_daif_bits.c | 35 ++
.../mangle_pstate_invalid_mode_el1h.c | 15 +
.../mangle_pstate_invalid_mode_el1t.c | 15 +
.../mangle_pstate_invalid_mode_el2h.c | 15 +
.../mangle_pstate_invalid_mode_el2t.c | 15 +
.../mangle_pstate_invalid_mode_el3h.c | 15 +
.../mangle_pstate_invalid_mode_el3t.c | 15 +
.../mangle_pstate_invalid_mode_template.h | 28 ++
.../arm64/signal/testcases/testcases.c | 196 +++++++++++
.../arm64/signal/testcases/testcases.h | 104 ++++++
.../selftests/arm64/{ => tags}/.gitignore | 0
tools/testing/selftests/arm64/tags/Makefile | 7 +
.../arm64/{ => tags}/run_tags_test.sh | 0
.../selftests/arm64/{ => tags}/tags_test.c | 0
32 files changed, 1623 insertions(+), 5 deletions(-)
create mode 100644 tools/testing/selftests/arm64/README
create mode 100644 tools/testing/selftests/arm64/signal/.gitignore
create mode 100644 tools/testing/selftests/arm64/signal/Makefile
create mode 100644 tools/testing/selftests/arm64/signal/README
create mode 100644 tools/testing/selftests/arm64/signal/signals.S
create mode 100644 tools/testing/selftests/arm64/signal/test_signals.c
create mode 100644 tools/testing/selftests/arm64/signal/test_signals.h
create mode 100644 tools/testing/selftests/arm64/signal/test_signals_utils.c
create mode 100644 tools/testing/selftests/arm64/signal/test_signals_utils.h
create mode 100644 tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_magic.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size_for_magic0.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_duplicated_fpsimd.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_misaligned_sp.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_missing_fpsimd.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/mangle_pstate_invalid_compat_toggle.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/mangle_pstate_invalid_daif_bits.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/mangle_pstate_invalid_mode_el1h.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/mangle_pstate_invalid_mode_el1t.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/mangle_pstate_invalid_mode_el2h.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/mangle_pstate_invalid_mode_el2t.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/mangle_pstate_invalid_mode_el3h.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/mangle_pstate_invalid_mode_el3t.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/mangle_pstate_invalid_mode_template.h
create mode 100644 tools/testing/selftests/arm64/signal/testcases/testcases.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/testcases.h
rename tools/testing/selftests/arm64/{ => tags}/.gitignore (100%)
create mode 100644 tools/testing/selftests/arm64/tags/Makefile
rename tools/testing/selftests/arm64/{ => tags}/run_tags_test.sh (100%)
rename tools/testing/selftests/arm64/{ => tags}/tags_test.c (100%)
--
2.17.1
Hello,
Is there anything blocking this from getting merged?
This patch series fixes the following problem:
linux# make kselftest TARGETS=bpf O=/mnt/linux-build
# selftests: bpf: test_libbpf.sh
# ./test_libbpf.sh: line 23: ./test_libbpf_open: No such file or directory
# test_libbpf: failed at file test_l4lb.o
# selftests: test_libbpf [FAILED]
Patch 1 appends / to $(OUTPUT) in order to make it more uniform with the
rest of the tree.
Patch 2 fixes the problem by prepending $(OUTPUT) to all members of
$(TEST_PROGS).
v1->v2:
- Append / to $(OUTPUT).
- Use $(addprefix) instead of $(foreach).
v2->v3:
- Split the patch in two.
- Improve the commit message.
Ilya Leoshkevich (2):
selftests: append / to $(OUTPUT)
selftests: fix prepending $(OUTPUT) to $(TEST_PROGS)
tools/testing/selftests/Makefile | 16 ++++++++--------
tools/testing/selftests/lib.mk | 3 ++-
2 files changed, 10 insertions(+), 9 deletions(-)
--
2.23.0
The livepatch selftests compare expected dmesg output to verify kernel
behavior. They currently filter out "tainting kernel with
TAINT_LIVEPATCH" messages which may be logged when loading livepatch
modules.
Further filter the log to also drop "loading out-of-tree module taints
kernel" messages in case the klp_test modules have been build without
the in-tree module flag.
Signed-off-by: Joe Lawrence <joe.lawrence(a)redhat.com>
---
Note: I stumbled across this in a testing scenario and thought it might
be generally useful to extend this admittedly fragile mechanism. Since
there are no related livepatch-core changes, this can go through Shuah's
kselftest tree if she prefers. -- Joe
tools/testing/selftests/livepatch/functions.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tools/testing/selftests/livepatch/functions.sh b/tools/testing/selftests/livepatch/functions.sh
index 79b0affd21fb..57975c323542 100644
--- a/tools/testing/selftests/livepatch/functions.sh
+++ b/tools/testing/selftests/livepatch/functions.sh
@@ -221,7 +221,7 @@ function check_result {
local expect="$*"
local result
- result=$(dmesg | grep -v 'tainting' | grep -e 'livepatch:' -e 'test_klp' | sed 's/^\[[ 0-9.]*\] //')
+ result=$(dmesg | grep -ve '\<taints\>' -ve '\<tainting\>' | grep -e 'livepatch:' -e 'test_klp' | sed 's/^\[[ 0-9.]*\] //')
if [[ "$expect" == "$result" ]] ; then
echo "ok"
--
2.21.0
Hi,
This applies cleanly to linux-next and mmotm, and also to linux.git if
linux-next's commit 20cac10710c9 ("mm/gup_benchmark: fix MAP_HUGETLB
case") is first applied there.
This provides tracking of dma-pinned pages. This is a prerequisite to
solving the larger problem of proper interactions between file-backed
pages, and [R]DMA activities, as discussed in [1], [2], [3], and in
a remarkable number of email threads since about 2017. :)
A new internal gup flag, FOLL_PIN is introduced, and thoroughly
documented in the last patch's Documentation/vm/pin_user_pages.rst.
I believe that this will provide a good starting point for doing the
layout lease work that Ira Weiny has been working on. That's because
these new wrapper functions provide a clean, constrained, systematically
named set of functionality that, again, is required in order to even
know if a page is "dma-pinned".
In contrast to earlier approaches, the page tracking can be
incrementally applied to the kernel call sites that, until now, have
been simply calling get_user_pages() ("gup"). In other words, opt-in by
changing from this:
get_user_pages() (sets FOLL_GET)
put_page()
to this:
pin_user_pages() (sets FOLL_PIN)
put_user_page()
Because there are interdependencies with FOLL_LONGTERM, a similar
conversion as for FOLL_PIN, was applied. The change was from this:
get_user_pages(FOLL_LONGTERM) (also sets FOLL_GET)
put_page()
to this:
pin_longterm_pages() (sets FOLL_PIN | FOLL_LONGTERM)
put_user_page()
============================================================
Patch summary:
* Patches 1-4: refactoring and preparatory cleanup, independent fixes
(Patch 4: V4L2-core bug fix (can be separately applied))
* Patch 5: introduce pin_user_pages(), FOLL_PIN, but no functional
changes yet
* Patches 6-11: Convert existing put_user_page() callers, to use the
new pin*()
* Patch 12: Activate tracking of FOLL_PIN pages.
* Patches 13-15: convert FOLL_LONGTERM callers
* Patches: 16-17: gup_benchmark and run_vmtests support
* Patch 18: enforce FOLL_LONGTERM as a gup-internal (only) flag
* Patch 19: Documentation/vm/pin_user_pages.rst
============================================================
Testing:
* I've done some overall kernel testing (LTP, and a few other goodies),
and some directed testing to exercise some of the changes. And as you
can see, gup_benchmark is enhanced to exercise this. Basically, I've been
able to runtime test the core get_user_pages() and pin_user_pages() and
related routines, but not so much on several of the call sites--but those
are generally just a couple of lines changed, each.
Not much of the kernel is actually using this, which on one hand
reduces risk quite a lot. But on the other hand, testing coverage
is low. So I'd love it if, in particular, the Infiniband and PowerPC
folks could do a smoke test of this series for me.
Also, my runtime testing for the call sites so far is very weak:
* io_uring: Some directed tests from liburing exercise this, and they pass.
* process_vm_access.c: A small directed test passes.
* gup_benchmark: the enhanced version hits the new gup.c code, and passes.
* infiniband (still only have crude "IB pingpong" working, on a
good day: it's not exercising my conversions at runtime...)
* VFIO: compiles (I'm vowing to set up a run time test soon, but it's
not ready just yet)
* powerpc: it compiles...
* drm/via: compiles...
* goldfish: compiles...
* net/xdp: compiles...
* media/v4l2: compiles...
============================================================
Next:
* Get the block/bio_vec sites converted to use pin_user_pages().
* Work with Ira and Dave Chinner to weave this together with the
layout lease stuff.
============================================================
[1] Some slow progress on get_user_pages() (Apr 2, 2019): https://lwn.net/Articles/784574/
[2] DMA and get_user_pages() (LPC: Dec 12, 2018): https://lwn.net/Articles/774411/
[3] The trouble with get_user_pages() (Apr 30, 2018): https://lwn.net/Articles/753027/
John Hubbard (19):
mm/gup: pass flags arg to __gup_device_* functions
mm/gup: factor out duplicate code from four routines
goldish_pipe: rename local pin_user_pages() routine
media/v4l2-core: set pages dirty upon releasing DMA buffers
mm/gup: introduce pin_user_pages*() and FOLL_PIN
goldish_pipe: convert to pin_user_pages() and put_user_page()
infiniband: set FOLL_PIN, FOLL_LONGTERM via pin_longterm_pages*()
mm/process_vm_access: set FOLL_PIN via pin_user_pages_remote()
drm/via: set FOLL_PIN via pin_user_pages_fast()
fs/io_uring: set FOLL_PIN via pin_user_pages()
net/xdp: set FOLL_PIN via pin_user_pages()
mm/gup: track FOLL_PIN pages
media/v4l2-core: pin_longterm_pages (FOLL_PIN) and put_user_page()
conversion
vfio, mm: pin_longterm_pages (FOLL_PIN) and put_user_page() conversion
powerpc: book3s64: convert to pin_longterm_pages() and put_user_page()
mm/gup_benchmark: support pin_user_pages() and related calls
selftests/vm: run_vmtests: invoke gup_benchmark with basic FOLL_PIN
coverage
mm/gup: remove support for gup(FOLL_LONGTERM)
Documentation/vm: add pin_user_pages.rst
Documentation/vm/index.rst | 1 +
Documentation/vm/pin_user_pages.rst | 213 +++++++
arch/powerpc/mm/book3s64/iommu_api.c | 15 +-
drivers/gpu/drm/via/via_dmablit.c | 2 +-
drivers/infiniband/core/umem.c | 5 +-
drivers/infiniband/core/umem_odp.c | 10 +-
drivers/infiniband/hw/hfi1/user_pages.c | 4 +-
drivers/infiniband/hw/mthca/mthca_memfree.c | 3 +-
drivers/infiniband/hw/qib/qib_user_pages.c | 8 +-
drivers/infiniband/hw/qib/qib_user_sdma.c | 2 +-
drivers/infiniband/hw/usnic/usnic_uiom.c | 9 +-
drivers/infiniband/sw/siw/siw_mem.c | 5 +-
drivers/media/v4l2-core/videobuf-dma-sg.c | 10 +-
drivers/platform/goldfish/goldfish_pipe.c | 35 +-
drivers/vfio/vfio_iommu_type1.c | 15 +-
fs/io_uring.c | 5 +-
include/linux/mm.h | 133 ++++-
include/linux/mmzone.h | 2 +
include/linux/page_ref.h | 10 +
mm/gup.c | 622 ++++++++++++++++----
mm/gup_benchmark.c | 81 ++-
mm/huge_memory.c | 32 +-
mm/hugetlb.c | 28 +-
mm/memremap.c | 4 +-
mm/process_vm_access.c | 28 +-
mm/vmstat.c | 2 +
net/xdp/xdp_umem.c | 4 +-
tools/testing/selftests/vm/gup_benchmark.c | 28 +-
tools/testing/selftests/vm/run_vmtests | 22 +
29 files changed, 1066 insertions(+), 272 deletions(-)
create mode 100644 Documentation/vm/pin_user_pages.rst
--
2.23.0
Greetings,
Find the attached mail very confidential. reply for more details
Thanks.
Peter Wong
----------------------------------------------------
This email was sent by the shareware version of Postman Professional.
Verify that in this scenario
------------------------ N2
| |
------ ------ N3 ----
| R1 | | R2 |------|H2|
------ ------ ----
| |
------------------------ N1
|
----
|H1|
----
where H1's default route goes through R1 and R1's default route goes
through R2 over N2, traceroute6 from H1 to H2 reports R2's address
on N2 and not N1.
Signed-off-by: Francesco Ruggeri <fruggeri(a)arista.com>
---
tools/testing/selftests/net/Makefile | 1 +
.../testing/selftests/net/icmp6_reply_addr.sh | 159 ++++++++++++++++++
2 files changed, 160 insertions(+)
create mode 100755 tools/testing/selftests/net/icmp6_reply_addr.sh
diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile
index 0bd6b23c97ef..daeaeb59d5ca 100644
--- a/tools/testing/selftests/net/Makefile
+++ b/tools/testing/selftests/net/Makefile
@@ -11,6 +11,7 @@ TEST_PROGS += udpgso_bench.sh fib_rule_tests.sh msg_zerocopy.sh psock_snd.sh
TEST_PROGS += udpgro_bench.sh udpgro.sh test_vxlan_under_vrf.sh reuseport_addr_any.sh
TEST_PROGS += test_vxlan_fdb_changelink.sh so_txtime.sh ipv6_flowlabel.sh
TEST_PROGS += tcp_fastopen_backup_key.sh fcnal-test.sh l2tp.sh
+TEST_PROGS += icmp6_reply_addr.sh
TEST_PROGS_EXTENDED := in_netns.sh
TEST_GEN_FILES = socket nettest
TEST_GEN_FILES += psock_fanout psock_tpacket msg_zerocopy reuseport_addr_any
diff --git a/tools/testing/selftests/net/icmp6_reply_addr.sh b/tools/testing/selftests/net/icmp6_reply_addr.sh
new file mode 100755
index 000000000000..551834cb9272
--- /dev/null
+++ b/tools/testing/selftests/net/icmp6_reply_addr.sh
@@ -0,0 +1,159 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Verify that in this scenario
+#
+# ------------------------ N2
+# | |
+# ------ ------ N3 ----
+# | R1 | | R2 |------|H2|
+# ------ ------ ----
+# | |
+# ------------------------ N1
+# |
+# ----
+# |H1|
+# ----
+#
+# where H1's default route goes through R1 and R1's default route goes
+# through R2 over N2, traceroute6 from H1 to H2 reports R2's address
+# on N2 and not N1.
+#
+# Addresses are assigned as follows:
+#
+# N1: 2000:101::/64
+# N2: 2000:102::/64
+# N3: 2000:103::/64
+#
+# R1's host part of address: 1
+# R2's host part of address: 2
+# H1's host part of address: 3
+# H2's host part of address: 4
+#
+# For example:
+# the IPv6 address of R1's interface on N2 is 2000:102::1/64
+
+####################################################################
+# helpers
+#
+# Interface on network <net> in node <node> is called <node><net>
+#
+
+node()
+{
+ host=$1
+ shift
+ ip netns exec ${host} $*
+}
+
+create_nodes()
+{
+ for n in $*; do
+ ip netns add $n
+ node $n ip link set lo up
+ done
+}
+
+delete_nodes()
+{
+ for n in $*; do
+ ip netns del $n
+ done
+}
+
+create_veth_net()
+{
+ net=$1
+ h1=$2
+ h2=$3
+
+ ip link add ${h1}${net} type veth peer name ${h2}${net}
+ ip link set ${h1}${net} netns ${h1}
+ node ${h1} ip link set ${h1}${net} up
+ ip link set ${h2}${net} netns ${h2}
+ node ${h2} ip link set ${h2}${net} up
+}
+
+create_macvlan_net()
+{
+ net=$1
+ shift
+ nodes=$*
+
+ ip link add ${net} type dummy
+ ip link set ${net} up
+
+ for n in ${nodes}; do
+ ip link add link ${net} dev ${n}${net} type macvlan mode bridge
+ ip link set ${n}${net} netns $n
+ node ${n} ip link set ${n}${net} up
+ done
+}
+
+delete_macvlan_nets()
+{
+ nets=$*
+
+ for n in ${nets}; do
+ ip link del ${n}
+ done
+}
+
+# end helpers
+####################################################################
+
+if [ "$(id -u)" -ne 0 ]; then
+ echo "SKIP: Need root privileges"
+ exit 0
+fi
+
+if [ ! -x "$(command -v traceroute6)" ]; then
+ echo "SKIP: Could not run test without traceroute6"
+ exit 0
+fi
+
+create_nodes host1 host2 rtr1 rtr2
+
+create_macvlan_net net1 host1 rtr1 rtr2
+create_veth_net net2 rtr1 rtr2
+create_veth_net net3 rtr2 host2
+
+# Configure interfaces and routes in host1
+node host1 ip -6 addr add 2000:101::3/64 dev host1net1
+node host1 ip -6 route add default via 2000:101::1
+
+# Configure interfaces and routes in rtr1
+node rtr1 ip -6 addr add 2000:101::1/64 dev rtr1net1
+node rtr1 ip -6 addr add 2000:102::1/64 dev rtr1net2
+node rtr1 ip -6 route add default via 2000:102::2
+node rtr1 sysctl net.ipv6.conf.all.forwarding=1 >/dev/null
+
+# Configure interfaces and routes in rtr2
+node rtr2 ip -6 addr add 2000:101::2/64 dev rtr2net1
+node rtr2 ip -6 addr add 2000:102::2/64 dev rtr2net2
+node rtr2 ip -6 addr add 2000:103::2/64 dev rtr2net3
+node rtr2 sysctl net.ipv6.conf.all.forwarding=1 >/dev/null
+
+# Configure interfaces and routes in host2
+node host2 ip -6 addr add 2000:103::4/64 dev host2net3
+node host2 ip -6 route add default via 2000:103::2
+
+# Ping host2 from host1
+echo "Priming the network"
+node host1 ping6 -c5 2000:103::4 >/dev/null
+
+# Traceroute host2 from host1
+echo "Running traceroute6"
+if node host1 traceroute6 2000:103::4 | grep -q 2000:102::2; then
+ ret=0
+ echo "Found 2000:102::2. Test passed."
+else
+ ret=1
+ echo "Did not find 2000:102::2. Test failed."
+fi
+
+delete_macvlan_nets net1
+delete_nodes host1 host2 rtr1 rtr2
+
+exit ${ret}
+
--
2.19.1
I'm planning to add some kernel self tests which use a user level program
in tools/testing/selftests/vm/ and a kernel module. See:
https://lore.kernel.org/linux-mm/20191023195515.13168-1-rcampbell@nvidia.co…
The question is where to put the kernel module source code.
I see some test modules that are in lib/test_*.ko and my patch
initially placed the hmm-dmirror module in drivers/char/ since
it creates a character device.
Any advice?
These counters will track hugetlb reservations rather than hugetlb
memory faulted in. This patch only adds the counter, following patches
add the charging and uncharging of the counter.
Problem:
Currently tasks attempting to allocate more hugetlb memory than is available get
a failure at mmap/shmget time. This is thanks to Hugetlbfs Reservations [1].
However, if a task attempts to allocate hugetlb memory only more than its
hugetlb_cgroup limit allows, the kernel will allow the mmap/shmget call,
but will SIGBUS the task when it attempts to fault the memory in.
We have developers interested in using hugetlb_cgroups, and they have expressed
dissatisfaction regarding this behavior. We'd like to improve this
behavior such that tasks violating the hugetlb_cgroup limits get an error on
mmap/shmget time, rather than getting SIGBUS'd when they try to fault
the excess memory in.
The underlying problem is that today's hugetlb_cgroup accounting happens
at hugetlb memory *fault* time, rather than at *reservation* time.
Thus, enforcing the hugetlb_cgroup limit only happens at fault time, and
the offending task gets SIGBUS'd.
Proposed Solution:
A new page counter named hugetlb.xMB.reservation_[limit|usage]_in_bytes. This
counter has slightly different semantics than
hugetlb.xMB.[limit|usage]_in_bytes:
- While usage_in_bytes tracks all *faulted* hugetlb memory,
reservation_usage_in_bytes tracks all *reserved* hugetlb memory and
hugetlb memory faulted in without a prior reservation.
- If a task attempts to reserve more memory than limit_in_bytes allows,
the kernel will allow it to do so. But if a task attempts to reserve
more memory than reservation_limit_in_bytes, the kernel will fail this
reservation.
This proposal is implemented in this patch series, with tests to verify
functionality and show the usage. We also added cgroup-v2 support to
hugetlb_cgroup so that the new use cases can be extended to v2.
Alternatives considered:
1. A new cgroup, instead of only a new page_counter attached to
the existing hugetlb_cgroup. Adding a new cgroup seemed like a lot of code
duplication with hugetlb_cgroup. Keeping hugetlb related page counters under
hugetlb_cgroup seemed cleaner as well.
2. Instead of adding a new counter, we considered adding a sysctl that modifies
the behavior of hugetlb.xMB.[limit|usage]_in_bytes, to do accounting at
reservation time rather than fault time. Adding a new page_counter seems
better as userspace could, if it wants, choose to enforce different cgroups
differently: one via limit_in_bytes, and another via
reservation_limit_in_bytes. This could be very useful if you're
transitioning how hugetlb memory is partitioned on your system one
cgroup at a time, for example. Also, someone may find usage for both
limit_in_bytes and reservation_limit_in_bytes concurrently, and this
approach gives them the option to do so.
Testing:
- Added tests passing.
- libhugetlbfs tests mostly passing, but some tests have trouble with and
without this patch series. Seems environment issue rather than code:
- Overall results:
********** TEST SUMMARY
* 2M
* 32-bit 64-bit
* Total testcases: 84 0
* Skipped: 0 0
* PASS: 66 0
* FAIL: 14 0
* Killed by signal: 0 0
* Bad configuration: 4 0
* Expected FAIL: 0 0
* Unexpected PASS: 0 0
* Test not present: 0 0
* Strange test result: 0 0
**********
- Failing tests:
- elflink_rw_and_share_test("linkhuge_rw") segfaults with and without this
patch series.
- LD_PRELOAD=libhugetlbfs.so HUGETLB_MORECORE=yes malloc (2M: 32):
FAIL Address is not hugepage
- LD_PRELOAD=libhugetlbfs.so HUGETLB_RESTRICT_EXE=unknown:malloc
HUGETLB_MORECORE=yes malloc (2M: 32):
FAIL Address is not hugepage
- LD_PRELOAD=libhugetlbfs.so HUGETLB_MORECORE=yes malloc_manysmall (2M: 32):
FAIL Address is not hugepage
- GLIBC_TUNABLES=glibc.malloc.tcache_count=0 LD_PRELOAD=libhugetlbfs.so
HUGETLB_MORECORE=yes heapshrink (2M: 32):
FAIL Heap not on hugepages
- GLIBC_TUNABLES=glibc.malloc.tcache_count=0 LD_PRELOAD=libhugetlbfs.so
libheapshrink.so HUGETLB_MORECORE=yes heapshrink (2M: 32):
FAIL Heap not on hugepages
- HUGETLB_ELFMAP=RW linkhuge_rw (2M: 32): FAIL small_data is not hugepage
- HUGETLB_ELFMAP=RW HUGETLB_MINIMAL_COPY=no linkhuge_rw (2M: 32):
FAIL small_data is not hugepage
- alloc-instantiate-race shared (2M: 32):
Bad configuration: sched_setaffinity(cpu1): Invalid argument -
FAIL Child 1 killed by signal Killed
- shmoverride_linked (2M: 32):
FAIL shmget failed size 2097152 from line 176: Invalid argument
- HUGETLB_SHM=yes shmoverride_linked (2M: 32):
FAIL shmget failed size 2097152 from line 176: Invalid argument
- shmoverride_linked_static (2M: 32):
FAIL shmget failed size 2097152 from line 176: Invalid argument
- HUGETLB_SHM=yes shmoverride_linked_static (2M: 32):
FAIL shmget failed size 2097152 from line 176: Invalid argument
- LD_PRELOAD=libhugetlbfs.so shmoverride_unlinked (2M: 32):
FAIL shmget failed size 2097152 from line 176: Invalid argument
- LD_PRELOAD=libhugetlbfs.so HUGETLB_SHM=yes shmoverride_unlinked (2M: 32):
FAIL shmget failed size 2097152 from line 176: Invalid argument
[1]: https://www.kernel.org/doc/html/latest/vm/hugetlbfs_reserv.html
Signed-off-by: Mina Almasry <almasrymina(a)google.com>
Acked-by: Hillf Danton <hdanton(a)sina.com>
---
include/linux/hugetlb.h | 23 ++++++++-
mm/hugetlb_cgroup.c | 111 ++++++++++++++++++++++++++++++----------
2 files changed, 107 insertions(+), 27 deletions(-)
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 53fc34f930d08..9c49a0ba894d3 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -320,6 +320,27 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
#ifdef CONFIG_HUGETLB_PAGE
+enum {
+ /* Tracks hugetlb memory faulted in. */
+ HUGETLB_RES_USAGE,
+ /* Tracks hugetlb memory reserved. */
+ HUGETLB_RES_RESERVATION_USAGE,
+ /* Limit for hugetlb memory faulted in. */
+ HUGETLB_RES_LIMIT,
+ /* Limit for hugetlb memory reserved. */
+ HUGETLB_RES_RESERVATION_LIMIT,
+ /* Max usage for hugetlb memory faulted in. */
+ HUGETLB_RES_MAX_USAGE,
+ /* Max usage for hugetlb memory reserved. */
+ HUGETLB_RES_RESERVATION_MAX_USAGE,
+ /* Faulted memory accounting fail count. */
+ HUGETLB_RES_FAILCNT,
+ /* Reserved memory accounting fail count. */
+ HUGETLB_RES_RESERVATION_FAILCNT,
+ HUGETLB_RES_NULL,
+ HUGETLB_RES_MAX,
+};
+
#define HSTATE_NAME_LEN 32
/* Defines one hugetlb page size */
struct hstate {
@@ -340,7 +361,7 @@ struct hstate {
unsigned int surplus_huge_pages_node[MAX_NUMNODES];
#ifdef CONFIG_CGROUP_HUGETLB
/* cgroup control files */
- struct cftype cgroup_files[5];
+ struct cftype cgroup_files[HUGETLB_RES_MAX];
#endif
char name[HSTATE_NAME_LEN];
};
diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c
index f1930fa0b445d..1ed4448ca41d3 100644
--- a/mm/hugetlb_cgroup.c
+++ b/mm/hugetlb_cgroup.c
@@ -25,6 +25,10 @@ struct hugetlb_cgroup {
* the counter to account for hugepages from hugetlb.
*/
struct page_counter hugepage[HUGE_MAX_HSTATE];
+ /*
+ * the counter to account for hugepage reservations from hugetlb.
+ */
+ struct page_counter reserved_hugepage[HUGE_MAX_HSTATE];
};
#define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
@@ -33,6 +37,14 @@ struct hugetlb_cgroup {
static struct hugetlb_cgroup *root_h_cgroup __read_mostly;
+static inline struct page_counter *
+hugetlb_cgroup_get_counter(struct hugetlb_cgroup *h_cg, int idx, bool reserved)
+{
+ if (reserved)
+ return &h_cg->reserved_hugepage[idx];
+ return &h_cg->hugepage[idx];
+}
+
static inline
struct hugetlb_cgroup *hugetlb_cgroup_from_css(struct cgroup_subsys_state *s)
{
@@ -254,30 +266,33 @@ void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages,
return;
}
-enum {
- RES_USAGE,
- RES_LIMIT,
- RES_MAX_USAGE,
- RES_FAILCNT,
-};
-
static u64 hugetlb_cgroup_read_u64(struct cgroup_subsys_state *css,
struct cftype *cft)
{
struct page_counter *counter;
+ struct page_counter *reserved_counter;
struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css);
counter = &h_cg->hugepage[MEMFILE_IDX(cft->private)];
+ reserved_counter = &h_cg->reserved_hugepage[MEMFILE_IDX(cft->private)];
switch (MEMFILE_ATTR(cft->private)) {
- case RES_USAGE:
+ case HUGETLB_RES_USAGE:
return (u64)page_counter_read(counter) * PAGE_SIZE;
- case RES_LIMIT:
+ case HUGETLB_RES_RESERVATION_USAGE:
+ return (u64)page_counter_read(reserved_counter) * PAGE_SIZE;
+ case HUGETLB_RES_LIMIT:
return (u64)counter->max * PAGE_SIZE;
- case RES_MAX_USAGE:
+ case HUGETLB_RES_RESERVATION_LIMIT:
+ return (u64)reserved_counter->max * PAGE_SIZE;
+ case HUGETLB_RES_MAX_USAGE:
return (u64)counter->watermark * PAGE_SIZE;
- case RES_FAILCNT:
+ case HUGETLB_RES_RESERVATION_MAX_USAGE:
+ return (u64)reserved_counter->watermark * PAGE_SIZE;
+ case HUGETLB_RES_FAILCNT:
return counter->failcnt;
+ case HUGETLB_RES_RESERVATION_FAILCNT:
+ return reserved_counter->failcnt;
default:
BUG();
}
@@ -291,6 +306,7 @@ static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of,
int ret, idx;
unsigned long nr_pages;
struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of));
+ bool reserved = false;
if (hugetlb_cgroup_is_root(h_cg)) /* Can't set limit on root */
return -EINVAL;
@@ -304,9 +320,14 @@ static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of,
nr_pages = round_down(nr_pages, 1 << huge_page_order(&hstates[idx]));
switch (MEMFILE_ATTR(of_cft(of)->private)) {
- case RES_LIMIT:
+ case HUGETLB_RES_RESERVATION_LIMIT:
+ reserved = true;
+ /* Fall through. */
+ case HUGETLB_RES_LIMIT:
mutex_lock(&hugetlb_limit_mutex);
- ret = page_counter_set_max(&h_cg->hugepage[idx], nr_pages);
+ ret = page_counter_set_max(hugetlb_cgroup_get_counter(h_cg, idx,
+ reserved),
+ nr_pages);
mutex_unlock(&hugetlb_limit_mutex);
break;
default:
@@ -320,18 +341,26 @@ static ssize_t hugetlb_cgroup_reset(struct kernfs_open_file *of,
char *buf, size_t nbytes, loff_t off)
{
int ret = 0;
- struct page_counter *counter;
+ struct page_counter *counter, *reserved_counter;
struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of));
counter = &h_cg->hugepage[MEMFILE_IDX(of_cft(of)->private)];
+ reserved_counter =
+ &h_cg->reserved_hugepage[MEMFILE_IDX(of_cft(of)->private)];
switch (MEMFILE_ATTR(of_cft(of)->private)) {
- case RES_MAX_USAGE:
+ case HUGETLB_RES_MAX_USAGE:
page_counter_reset_watermark(counter);
break;
- case RES_FAILCNT:
+ case HUGETLB_RES_RESERVATION_MAX_USAGE:
+ page_counter_reset_watermark(reserved_counter);
+ break;
+ case HUGETLB_RES_FAILCNT:
counter->failcnt = 0;
break;
+ case HUGETLB_RES_RESERVATION_FAILCNT:
+ reserved_counter->failcnt = 0;
+ break;
default:
ret = -EINVAL;
break;
@@ -357,37 +386,67 @@ static void __init __hugetlb_cgroup_file_init(int idx)
struct hstate *h = &hstates[idx];
/* format the size */
- mem_fmt(buf, 32, huge_page_size(h));
+ mem_fmt(buf, sizeof(buf), huge_page_size(h));
/* Add the limit file */
- cft = &h->cgroup_files[0];
+ cft = &h->cgroup_files[HUGETLB_RES_LIMIT];
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.limit_in_bytes", buf);
- cft->private = MEMFILE_PRIVATE(idx, RES_LIMIT);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_LIMIT);
+ cft->read_u64 = hugetlb_cgroup_read_u64;
+ cft->write = hugetlb_cgroup_write;
+
+ /* Add the reservation limit file */
+ cft = &h->cgroup_files[HUGETLB_RES_RESERVATION_LIMIT];
+ snprintf(cft->name, MAX_CFTYPE_NAME, "%s.reservation_limit_in_bytes",
+ buf);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_RESERVATION_LIMIT);
cft->read_u64 = hugetlb_cgroup_read_u64;
cft->write = hugetlb_cgroup_write;
/* Add the usage file */
- cft = &h->cgroup_files[1];
+ cft = &h->cgroup_files[HUGETLB_RES_USAGE];
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.usage_in_bytes", buf);
- cft->private = MEMFILE_PRIVATE(idx, RES_USAGE);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_USAGE);
+ cft->read_u64 = hugetlb_cgroup_read_u64;
+
+ /* Add the reservation usage file */
+ cft = &h->cgroup_files[HUGETLB_RES_RESERVATION_USAGE];
+ snprintf(cft->name, MAX_CFTYPE_NAME, "%s.reservation_usage_in_bytes",
+ buf);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_RESERVATION_USAGE);
cft->read_u64 = hugetlb_cgroup_read_u64;
/* Add the MAX usage file */
- cft = &h->cgroup_files[2];
+ cft = &h->cgroup_files[HUGETLB_RES_MAX_USAGE];
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.max_usage_in_bytes", buf);
- cft->private = MEMFILE_PRIVATE(idx, RES_MAX_USAGE);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_MAX_USAGE);
+ cft->write = hugetlb_cgroup_reset;
+ cft->read_u64 = hugetlb_cgroup_read_u64;
+
+ /* Add the MAX reservation usage file */
+ cft = &h->cgroup_files[HUGETLB_RES_RESERVATION_MAX_USAGE];
+ snprintf(cft->name, MAX_CFTYPE_NAME,
+ "%s.reservation_max_usage_in_bytes", buf);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_RESERVATION_MAX_USAGE);
cft->write = hugetlb_cgroup_reset;
cft->read_u64 = hugetlb_cgroup_read_u64;
/* Add the failcntfile */
- cft = &h->cgroup_files[3];
+ cft = &h->cgroup_files[HUGETLB_RES_FAILCNT];
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.failcnt", buf);
- cft->private = MEMFILE_PRIVATE(idx, RES_FAILCNT);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_FAILCNT);
+ cft->write = hugetlb_cgroup_reset;
+ cft->read_u64 = hugetlb_cgroup_read_u64;
+
+ /* Add the reservation failcntfile */
+ cft = &h->cgroup_files[HUGETLB_RES_RESERVATION_FAILCNT];
+ snprintf(cft->name, MAX_CFTYPE_NAME, "%s.reservation_failcnt", buf);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_RESERVATION_FAILCNT);
cft->write = hugetlb_cgroup_reset;
cft->read_u64 = hugetlb_cgroup_read_u64;
/* NULL terminate the last cft */
- cft = &h->cgroup_files[4];
+ cft = &h->cgroup_files[HUGETLB_RES_NULL];
memset(cft, 0, sizeof(*cft));
WARN_ON(cgroup_add_legacy_cftypes(&hugetlb_cgrp_subsys,
--
2.24.0.rc0.303.g954a862665-goog
Since commit 5821ba969511 ("selftests: Add test plan API to kselftest.h
and adjust callers") accidentally introduced 'a' typo in the front of
run_test() function, breakpoint_test_arm64.c became not able to be
compiled.
Remove the 'a' from arun_test().
Fixes: 5821ba969511 ("selftests: Add test plan API to kselftest.h and adjust callers")
Reported-by: Jun Takahashi <takahashi.jun_s(a)aa.socionext.com>
Signed-off-by: Masami Hiramatsu <mhiramat(a)kernel.org>
Cc: Kees Cook <keescook(a)chromium.org>
---
.../selftests/breakpoints/breakpoint_test_arm64.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c b/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c
index 58ed5eeab709..ad41ea69001b 100644
--- a/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c
+++ b/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c
@@ -109,7 +109,7 @@ static bool set_watchpoint(pid_t pid, int size, int wp)
return false;
}
-static bool arun_test(int wr_size, int wp_size, int wr, int wp)
+static bool run_test(int wr_size, int wp_size, int wr, int wp)
{
int status;
siginfo_t siginfo;
The 'functions' directive is not only for functions, but also works for
structs/unions. So the name is misleading. This patch renames it to
'identifiers', which specific the functions/types to be included in
documentation. We keep the old name as an alias of the new one before
all documentation are updated.
Signed-off-by: Changbin Du <changbin.du(a)gmail.com>
---
v2:
o use 'identifiers' as the new directive name.
---
Documentation/doc-guide/kernel-doc.rst | 29 ++++++++++++++------------
Documentation/sphinx/kerneldoc.py | 19 ++++++++++-------
2 files changed, 28 insertions(+), 20 deletions(-)
diff --git a/Documentation/doc-guide/kernel-doc.rst b/Documentation/doc-guide/kernel-doc.rst
index 192c36af39e2..fff6604631ea 100644
--- a/Documentation/doc-guide/kernel-doc.rst
+++ b/Documentation/doc-guide/kernel-doc.rst
@@ -476,6 +476,22 @@ internal: *[source-pattern ...]*
.. kernel-doc:: drivers/gpu/drm/i915/intel_audio.c
:internal:
+identifiers: *[ function/type ...]*
+ Include documentation for each *function* and *type* in *source*.
+ If no *function* is specified, the documentation for all functions
+ and types in the *source* will be included.
+
+ Examples::
+
+ .. kernel-doc:: lib/bitmap.c
+ :identifiers: bitmap_parselist bitmap_parselist_user
+
+ .. kernel-doc:: lib/idr.c
+ :identifiers:
+
+functions: *[ function/type ...]*
+ This is an alias of the 'identifiers' directive and deprecated.
+
doc: *title*
Include documentation for the ``DOC:`` paragraph identified by *title* in
*source*. Spaces are allowed in *title*; do not quote the *title*. The *title*
@@ -488,19 +504,6 @@ doc: *title*
.. kernel-doc:: drivers/gpu/drm/i915/intel_audio.c
:doc: High Definition Audio over HDMI and Display Port
-functions: *[ function ...]*
- Include documentation for each *function* in *source*.
- If no *function* is specified, the documentation for all functions
- and types in the *source* will be included.
-
- Examples::
-
- .. kernel-doc:: lib/bitmap.c
- :functions: bitmap_parselist bitmap_parselist_user
-
- .. kernel-doc:: lib/idr.c
- :functions:
-
Without options, the kernel-doc directive includes all documentation comments
from the source file.
diff --git a/Documentation/sphinx/kerneldoc.py b/Documentation/sphinx/kerneldoc.py
index 1159405cb920..0689f9c37f1e 100644
--- a/Documentation/sphinx/kerneldoc.py
+++ b/Documentation/sphinx/kerneldoc.py
@@ -59,9 +59,10 @@ class KernelDocDirective(Directive):
optional_arguments = 4
option_spec = {
'doc': directives.unchanged_required,
- 'functions': directives.unchanged,
'export': directives.unchanged,
'internal': directives.unchanged,
+ 'identifiers': directives.unchanged,
+ 'functions': directives.unchanged, # alias of 'identifiers'
}
has_content = False
@@ -71,6 +72,7 @@ class KernelDocDirective(Directive):
filename = env.config.kerneldoc_srctree + '/' + self.arguments[0]
export_file_patterns = []
+ identifiers = None
# Tell sphinx of the dependency
env.note_dependency(os.path.abspath(filename))
@@ -86,19 +88,22 @@ class KernelDocDirective(Directive):
export_file_patterns = str(self.options.get('internal')).split()
elif 'doc' in self.options:
cmd += ['-function', str(self.options.get('doc'))]
+ elif 'identifiers' in self.options:
+ identifiers = self.options.get('identifiers').split()
elif 'functions' in self.options:
- functions = self.options.get('functions').split()
- if functions:
- for f in functions:
- cmd += ['-function', f]
- else:
- cmd += ['-no-doc-sections']
+ identifiers = self.options.get('functions').split()
for pattern in export_file_patterns:
for f in glob.glob(env.config.kerneldoc_srctree + '/' + pattern):
env.note_dependency(os.path.abspath(f))
cmd += ['-export-file', f]
+ if identifiers:
+ for i in identifiers:
+ cmd += ['-function', i]
+ elif identifiers is not None:
+ cmd += ['-no-doc-sections']
+
cmd += [filename]
try:
--
2.20.1
Hi,
Here are the 3rd version of kselftest fixes some on 32bit arch
(e.g. arm)
In this version, I updated [1/5] to make va_max 1MB unconditionally
according to Alexey's comment.
When I built the ksefltest on arm, I hit some 32bit related warnings.
Here are the patches to fix those issues.
- [1/5] va_max was set 2^32 even on 32bit arch. This can make
va_max == 0 and always fail. Make it 1GB unconditionally.
- [2/5] Some VM tests requires 64bit user space, which should
not run on 32bit arch.
- [3/5] For counting the size of large file, we should use
size_t instead of unsinged long.
- [4/5] Gcc warns printf format for size_t and int64_t on
32bit arch. Use %llu and cast it.
- [5/5] Gcc warns __u64 and pointer type castings. It should
once translated to unsigned long.
Thank you,
---
Masami Hiramatsu (5):
selftests: proc: Make va_max 1MB
selftests: vm: Build/Run 64bit tests only on 64bit arch
selftests: net: Use size_t and ssize_t for counting file size
selftests: net: Fix printf format warnings on arm
selftests: sync: Fix cast warnings on arm
tools/testing/selftests/net/so_txtime.c | 4 ++--
tools/testing/selftests/net/tcp_mmap.c | 8 ++++----
tools/testing/selftests/net/udpgso.c | 3 ++-
tools/testing/selftests/net/udpgso_bench_tx.c | 3 ++-
.../selftests/proc/proc-self-map-files-002.c | 6 +++++-
tools/testing/selftests/sync/sync.c | 6 +++---
tools/testing/selftests/vm/Makefile | 5 +++++
tools/testing/selftests/vm/run_vmtests | 10 ++++++++++
8 files changed, 33 insertions(+), 12 deletions(-)
--
Masami Hiramatsu (Linaro) <mhiramat(a)kernel.org>
Hi Shua,
Here is the set with cleanup as suggested by Kees on v3.
Configured, built, and tested all modules loaded by
tools/testing/selftests/lib/*.sh
>From previous cover letters ...
While doing the testing for strscpy_pad() it was noticed that there is
duplication in how test modules are being fed to kselftest and also in
the test modules themselves.
This set makes an attempt at adding a framework to kselftest for writing
kernel test modules. It also adds a script for use in creating script
test runners for kselftest. My macro-foo is not great, all criticism
and suggestions very much appreciated. The design is based on test
modules lib/test_printf.c, lib/test_bitmap.c, lib/test_xarray.c.
Changes since last version:
- Remove dependency on Bash (thanks Kees)
- Use oneliner to implement kselftest test runners (thanks Kees)
- Squash patch that adds kselftest script creator script with patch
that uses it.
- Fix typos (thanks Randy)
- Add Kees' Acked-by tags to all patches
thanks,
Tobin.
Tobin C. Harding (6):
lib/test_printf: Add empty module_exit function
kselftest: Add test runner creation script
kselftest: Add test module framework header
lib: Use new kselftest header
lib/string: Add strscpy_pad() function
lib: Add test module for strscpy_pad
Documentation/dev-tools/kselftest.rst | 94 +++++++++++-
include/linux/string.h | 4 +
lib/Kconfig.debug | 3 +
lib/Makefile | 1 +
lib/string.c | 47 +++++-
lib/test_bitmap.c | 20 +--
lib/test_printf.c | 17 +--
lib/test_strscpy.c | 150 +++++++++++++++++++
tools/testing/selftests/kselftest_module.h | 48 ++++++
tools/testing/selftests/kselftest_module.sh | 84 +++++++++++
tools/testing/selftests/lib/Makefile | 2 +-
tools/testing/selftests/lib/bitmap.sh | 18 +--
tools/testing/selftests/lib/config | 1 +
tools/testing/selftests/lib/prime_numbers.sh | 17 +--
tools/testing/selftests/lib/printf.sh | 19 +--
tools/testing/selftests/lib/strscpy.sh | 3 +
16 files changed, 440 insertions(+), 88 deletions(-)
create mode 100644 lib/test_strscpy.c
create mode 100644 tools/testing/selftests/kselftest_module.h
create mode 100755 tools/testing/selftests/kselftest_module.sh
create mode 100755 tools/testing/selftests/lib/strscpy.sh
--
2.21.0
This patchset is being developed here:
<https://github.com/cyphar/linux/tree/openat2/master>
Patch changelog:
v14: [<https://lore.kernel.org/lkml/20191010054140.8483-1-cyphar@cyphar.com/>]
* The magic-link changes (and O_EMPTYPATH) have been dropped from this series
-- they will be developed and sent separately. The main reason is that we
need to restrict things other than open(2) (examples include truncate(2) as
well as mount(MS_BIND)). This will require a fair amount of extra work, and
there's no point stalling openat2(2) for that work to be completed.
* Minor rework of 'struct open_how':
* To avoid future headaches, make it a non-const argument.
* Expand ->flags and ->resolve to 64-bit fields to allow for more flag
extensions without needing to add separate fields too early. This
requires adding a bit of explicit padding (32 bits) to avoid userspace
putting garbage in the alignment padding -- this can be repurposed for
future extensions.
* upgrade_mask is dropped (and will be a separate field when we add it
again in the future) to avoid userspace foot-guns.
* Expand -EINVAL checks in build_open_flags(). Rather than silently
ignoring silly flag combinations (such as O_TMPFILE|O_PATH or
O_PATH|<most flags>), give an -EINVAL. All of the silent ignore semantics
were added to open(2) because we couldn't return -EINVAL -- but we can
now!
* open(2) and openat(2) clean up their flags before passing them to
build_open_flags(), so all mixed flags will continue to work. There is
one exception which is (O_PATH|O_TMPFILE) -- this is no longer
permitted (as far as I can tell this appears to be a bug, and there are
no userspace users that I've hit after running this code for a few
days). If it turns out that userspace does depend on (O_PATH|O_TMPFILE)
working, we can only disallow it for openat2(2).
* Don't zero out nd->root in complete_walk() for RCU-walk if we're doing a
scoped-lookup (this prevents a needless REF-walk retry).
* Attempt all tests on kernels that don't have openat2(2), rather than just
skipping everything.
v13: <https://lore.kernel.org/lkml/20190930183316.10190-1-cyphar@cyphar.com/>
v12: <https://lore.kernel.org/lkml/20190904201933.10736-1-cyphar@cyphar.com/>
v11: <https://lore.kernel.org/lkml/20190820033406.29796-1-cyphar@cyphar.com/>
<https://lore.kernel.org/lkml/20190728010207.9781-1-cyphar@cyphar.com/>
v10: <https://lore.kernel.org/lkml/20190719164225.27083-1-cyphar@cyphar.com/>
v09: <https://lore.kernel.org/lkml/20190706145737.5299-1-cyphar@cyphar.com/>
v08: <https://lore.kernel.org/lkml/20190520133305.11925-1-cyphar@cyphar.com/>
v07: <https://lore.kernel.org/lkml/20190507164317.13562-1-cyphar@cyphar.com/>
v06: <https://lore.kernel.org/lkml/20190506165439.9155-1-cyphar@cyphar.com/>
v05: <https://lore.kernel.org/lkml/20190320143717.2523-1-cyphar@cyphar.com/>
v04: <https://lore.kernel.org/lkml/20181112142654.341-1-cyphar@cyphar.com/>
v03: <https://lore.kernel.org/lkml/20181009070230.12884-1-cyphar@cyphar.com/>
v02: <https://lore.kernel.org/lkml/20181009065300.11053-1-cyphar@cyphar.com/>
v01: <https://lore.kernel.org/lkml/20180929103453.12025-1-cyphar@cyphar.com/>
For a very long time, extending openat(2) with new features has been
incredibly frustrating. This stems from the fact that openat(2) is
possibly the most famous counter-example to the mantra "don't silently
accept garbage from userspace" -- it doesn't check whether unknown flags
are present[1].
This means that (generally) the addition of new flags to openat(2) has
been fraught with backwards-compatibility issues (O_TMPFILE has to be
defined as __O_TMPFILE|O_DIRECTORY|[O_RDWR or O_WRONLY] to ensure old
kernels gave errors, since it's insecure to silently ignore the
flag[2]). All new security-related flags therefore have a tough road to
being added to openat(2).
Furthermore, the need for some sort of control over VFS's path resolution (to
avoid malicious paths resulting in inadvertent breakouts) has been a very
long-standing desire of many userspace applications. This patchset is a revival
of Al Viro's old AT_NO_JUMPS[3] patchset (which was a variant of David
Drysdale's O_BENEATH patchset[4] which was a spin-off of the Capsicum
project[5]) with a few additions and changes made based on the previous
discussion within [6] as well as others I felt were useful.
In line with the conclusions of the original discussion of AT_NO_JUMPS, the
flag has been split up into separate flags. However, instead of being an
openat(2) flag it is provided through a new syscall openat2(2) which provides
several other improvements to the openat(2) interface (see the patch
description for more details). The following new LOOKUP_* flags are added:
* LOOKUP_NO_XDEV blocks all mountpoint crossings (upwards, downwards,
or through absolute links). Absolute pathnames alone in openat(2) do not
trigger this. Magic-link traversal which implies a vfsmount jump is also
blocked (though magic-link jumps on the same vfsmount are permitted).
* LOOKUP_NO_MAGICLINKS blocks resolution through /proc/$pid/fd-style
links. This is done by blocking the usage of nd_jump_link() during
resolution in a filesystem. The term "magic-links" is used to match
with the only reference to these links in Documentation/, but I'm
happy to change the name.
It should be noted that this is different to the scope of
~LOOKUP_FOLLOW in that it applies to all path components. However,
you can do openat2(NO_FOLLOW|NO_MAGICLINKS) on a magic-link and it
will *not* fail (assuming that no parent component was a
magic-link), and you will have an fd for the magic-link.
In order to correctly detect magic-links, the introduction of a new
LOOKUP_MAGICLINK_JUMPED state flag was required.
* LOOKUP_BENEATH disallows escapes to outside the starting dirfd's
tree, using techniques such as ".." or absolute links. Absolute
paths in openat(2) are also disallowed. Conceptually this flag is to
ensure you "stay below" a certain point in the filesystem tree --
but this requires some additional to protect against various races
that would allow escape using "..".
Currently LOOKUP_BENEATH implies LOOKUP_NO_MAGICLINKS, because it
can trivially beam you around the filesystem (breaking the
protection). In future, there might be similar safety checks done as
in LOOKUP_IN_ROOT, but that requires more discussion.
In addition, two new flags are added that expand on the above ideas:
* LOOKUP_NO_SYMLINKS does what it says on the tin. No symlink
resolution is allowed at all, including magic-links. Just as with
LOOKUP_NO_MAGICLINKS this can still be used with NOFOLLOW to open an
fd for the symlink as long as no parent path had a symlink
component.
* LOOKUP_IN_ROOT is an extension of LOOKUP_BENEATH that, rather than
blocking attempts to move past the root, forces all such movements
to be scoped to the starting point. This provides chroot(2)-like
protection but without the cost of a chroot(2) for each filesystem
operation, as well as being safe against race attacks that chroot(2)
is not.
If a race is detected (as with LOOKUP_BENEATH) then an error is
generated, and similar to LOOKUP_BENEATH it is not permitted to cross
magic-links with LOOKUP_IN_ROOT.
The primary need for this is from container runtimes, which
currently need to do symlink scoping in userspace[7] when opening
paths in a potentially malicious container. There is a long list of
CVEs that could have bene mitigated by having RESOLVE_THIS_ROOT
(such as CVE-2017-1002101, CVE-2017-1002102, CVE-2018-15664, and
CVE-2019-5736, just to name a few).
In order to make all of the above more usable, I'm working on
libpathrs[8] which is a C-friendly library for safe path resolution. It
features a userspace-emulated backend if the kernel doesn't support
openat2(2). Hopefully we can get userspace to switch to using it, and
thus get openat2(2) support for free once it's ready.
[1]: https://lwn.net/Articles/588444/
[2]: https://lore.kernel.org/lkml/CA+55aFyyxJL1LyXZeBsf2ypriraj5ut1XkNDsunRBqgVj…
[3]: https://lore.kernel.org/lkml/20170429220414.GT29622@ZenIV.linux.org.uk
[4]: https://lore.kernel.org/lkml/1415094884-18349-1-git-send-email-drysdale@goo…
[5]: https://lore.kernel.org/lkml/1404124096-21445-1-git-send-email-drysdale@goo…
[6]: https://lwn.net/Articles/723057/
[7]: https://github.com/cyphar/filepath-securejoin
[8]: https://github.com/openSUSE/libpathrs
The current draft of the openat2(2) man-page is included below.
--8<---------------------------------------------------------------------------
OPENAT2(2) Linux Programmer's Manual OPENAT2(2)
NAME
openat2 - open and possibly create a file (extended)
SYNOPSIS
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
int openat2(int dirfd, const char *pathname, struct open_how *how, size_t size);
Note: There is no glibc wrapper for this system call; see NOTES.
DESCRIPTION
The openat2() system call opens the file specified by pathname. If the specified file
does not exist, it may optionally (if O_CREAT is specified in how.flags) be created by
openat2().
As with openat(2), if pathname is relative, then it is interpreted relative to the direc-
tory referred to by the file descriptor dirfd (or the current working directory of the
calling process, if dirfd is the special value AT_FDCWD.) If pathname is absolute, then
dirfd is ignored (unless how.resolve contains RESOLVE_IN_ROOT, in which case pathname is
resolved relative to dirfd.)
The openat2() system call is an extension of openat(2) and provides a superset of its
functionality. Rather than taking a single flag argument, an extensible structure (how)
is passed instead to allow for future extensions. size must be set to sizeof(struct
open_how), to facilitate future extensions (see the "Extensibility" section of the NOTES
for more detail on how extensions are handled.)
The open_how structure
The following structure indicates how pathname should be opened, and acts as a superset of
the flag and mode arguments to openat(2).
struct open_how {
__aligned_u64 flags; /* O_* flags. */
__u16 mode; /* Mode for O_{CREAT,TMPFILE}. */
__u16 __padding[3]; /* Must be zeroed. */
__aligned_u64 resolve; /* RESOLVE_* flags. */
};
Any future extensions to openat2() will be implemented as new fields appended to the above
structure (or through reuse of pre-existing padding space), with the zero value of the new
fields acting as though the extension were not present.
The meaning of each field is as follows:
flags
The file creation and status flags to use for this operation. All of the
O_* flags defined for openat(2) are valid openat2() flag values.
Unlike openat(2), it is an error to provide openat2() unknown or conflicting
flags in flags.
mode
File mode for the new file, with identical semantics to the mode argument to
openat(2). However, unlike openat(2), it is an error to provide openat2()
with a mode which contains bits other than 0777.
It is an error to provide openat2() a non-zero mode if flags does not con-
tain O_CREAT or O_TMPFILE.
resolve
Change how the components of pathname will be resolved (see path_resolu-
tion(7) for background information.) The primary use case for these flags
is to allow trusted programs to restrict how untrusted paths (or paths in-
side untrusted directories) are resolved. The full list of resolve flags is
given below.
RESOLVE_NO_XDEV
Disallow traversal of mount points during path resolution (including
all bind mounts).
Users of this flag are encouraged to make its use configurable (un-
less it is used for a specific security purpose), as bind mounts are
very widely used by end-users. Setting this flag indiscrimnately for
all uses of openat2() may result in spurious errors on previously-
functional systems.
RESOLVE_NO_SYMLINKS
Disallow resolution of symbolic links during path resolution. This
option implies RESOLVE_NO_MAGICLINKS.
If the trailing component is a symbolic link, and flags contains both
O_PATH and O_NOFOLLOW, then an O_PATH file descriptor referencing the
symbolic link will be returned.
Users of this flag are encouraged to make its use configurable (un-
less it is used for a specific security purpose), as symbolic links
are very widely used by end-users. Setting this flag indiscrimnately
for all uses of openat2() may result in spurious errors on previ-
ously-functional systems.
RESOLVE_NO_MAGICLINKS
Disallow all magic link resolution during path resolution.
If the trailing component is a magic link, and flags contains both
O_PATH and O_NOFOLLOW, then an O_PATH file descriptor referencing the
magic link will be returned.
Magic-links are symbolic link-like objects that are most notably
found in proc(5) (examples include /proc/[pid]/exe and
/proc/[pid]/fd/*.) Due to the potential danger of unknowingly open-
ing these magic links, it may be preferable for users to disable
their resolution entirely (see symboliclink(7) for more details.)
RESOLVE_BENEATH
Do not permit the path resolution to succeed if any component of the
resolution is not a descendant of the directory indicated by dirfd.
This results in absolute symbolic links (and absolute values of path-
name) to be rejected.
Currently, this flag also disables magic link resolution. However,
this may change in the future. The caller should explicitly specify
RESOLVE_NO_MAGICLINKS to ensure that magic links are not resolved.
RESOLVE_IN_ROOT
Treat dirfd as the root directory while resolving pathname (as though
the user called chroot(2) with dirfd as the argument.) Absolute sym-
bolic links and ".." path components will be scoped to dirfd. If
pathname is an absolute path, it is also treated relative to dirfd.
However, unlike chroot(2) (which changes the filesystem root perma-
nently for a process), RESOLVE_IN_ROOT allows a program to effi-
ciently restrict path resolution for only certain operations. It
also has several hardening features (such detecting escape attempts
during .. resolution) which chroot(2) does not.
Currently, this flag also disables magic link resolution. However,
this may change in the future. The caller should explicitly specify
RESOLVE_NO_MAGICLINKS to ensure that magic links are not resolved.
It is an error to provide openat2() unknown flags in resolve.
RETURN VALUE
On success, a new file descriptor is returned. On error, -1 is returned, and errno is set
appropriately.
ERRORS
The set of errors returned by openat2() includes all of the errors returned by openat(2),
as well as the following additional errors:
EINVAL An unknown flag or invalid value was specified in how.
EINVAL mode is non-zero, but flags does not contain O_CREAT or O_TMPFILE.
EINVAL size was smaller than any known version of struct open_how.
E2BIG An extension was specified in how, which the current kernel does not support (see
the "Extensibility" section of the NOTES for more detail on how extensions are han-
dled.)
EAGAIN resolve contains either RESOLVE_IN_ROOT or RESOLVE_BENEATH, and the kernel could
not ensure that a ".." component didn't escape (due to a race condition or poten-
tial attack.) Callers may choose to retry the openat2() call.
EXDEV resolve contains either RESOLVE_IN_ROOT or RESOLVE_BENEATH, and an escape from the
root during path resolution was detected.
EXDEV resolve contains RESOLVE_NO_XDEV, and a path component attempted to cross a mount
point.
ELOOP resolve contains RESOLVE_NO_SYMLINKS, and one of the path components was a symbolic
link (or magic link).
ELOOP resolve contains RESOLVE_NO_MAGICLINKS, and one of the path components was a magic
link.
VERSIONS
openat2() was added to Linux in kernel 5.FOO.
CONFORMING TO
This system call is Linux-specific.
The semantics of RESOLVE_BENEATH were modelled after FreeBSD's O_BENEATH.
NOTES
Glibc does not provide a wrapper for this system call; call it using systemcall(2).
Extensibility
In order to allow for struct open_how to be extended in future kernel revisions, openat2()
requires userspace to specify the size of struct open_how structure they are passing. By
providing this information, it is possible for openat2() to provide both forwards- and
backwards-compatibility — with size acting as an implicit version number (because new ex-
tension fields will always be appended, the size will always increase.) This extensibil-
ity design is very similar to other system calls such as perf_setattr(2),
perf_event_open(2), and clone(3).
If we let usize be the size of the structure according to userspace and ksize be the size
of the structure which the kernel supports, then there are only three cases to consider:
* If ksize equals usize, then there is no version mismatch and how can be used
verbatim.
* If ksize is larger than usize, then there are some extensions the kernel sup-
ports which the userspace program is unaware of. Because all extensions must
have their zero values be a no-op, the kernel treats all of the extension fields
not set by userspace to have zero values. This provides backwards-compatibil-
ity.
* If ksize is smaller than usize, then there are some extensions which the
userspace program is aware of but the kernel does not support. Because all ex-
tensions must have their zero values be a no-op, the kernel can safely ignore
the unsupported extension fields if they are all-zero. If any unsupported ex-
tension fields are non-zero, then -1 is returned and errno is set to E2BIG.
This provides forwards-compatibility.
Therefore, most userspace programs will not need to have any special handling of exten-
sions. However, if a userspace program wishes to determine what extensions the running
kernel supports, they may conduct a binary search on size (to find the largest value which
doesn't produce an error of E2BIG.)
SEE ALSO
openat(2), path_resolution(7), symlink(7)
Linux 2019-10-27 OPENAT2(2)
--8<---------------------------------------------------------------------------
Aleksa Sarai (6):
namei: O_BENEATH-style resolution restriction flags
namei: LOOKUP_IN_ROOT: chroot-like path resolution
namei: permit ".." resolution with LOOKUP_{IN_ROOT,BENEATH}
open: introduce openat2(2) syscall
selftests: add openat2(2) selftests
Documentation: path-lookup: mention LOOKUP_MAGICLINK_JUMPED
CREDITS | 4 +-
Documentation/filesystems/path-lookup.rst | 18 +-
arch/alpha/kernel/syscalls/syscall.tbl | 1 +
arch/arm/tools/syscall.tbl | 1 +
arch/arm64/include/asm/unistd.h | 2 +-
arch/arm64/include/asm/unistd32.h | 2 +
arch/ia64/kernel/syscalls/syscall.tbl | 1 +
arch/m68k/kernel/syscalls/syscall.tbl | 1 +
arch/microblaze/kernel/syscalls/syscall.tbl | 1 +
arch/mips/kernel/syscalls/syscall_n32.tbl | 1 +
arch/mips/kernel/syscalls/syscall_n64.tbl | 1 +
arch/mips/kernel/syscalls/syscall_o32.tbl | 1 +
arch/parisc/kernel/syscalls/syscall.tbl | 1 +
arch/powerpc/kernel/syscalls/syscall.tbl | 1 +
arch/s390/kernel/syscalls/syscall.tbl | 1 +
arch/sh/kernel/syscalls/syscall.tbl | 1 +
arch/sparc/kernel/syscalls/syscall.tbl | 1 +
arch/x86/entry/syscalls/syscall_32.tbl | 1 +
arch/x86/entry/syscalls/syscall_64.tbl | 1 +
arch/xtensa/kernel/syscalls/syscall.tbl | 1 +
fs/namei.c | 167 +++++-
fs/open.c | 154 ++++--
include/linux/fcntl.h | 12 +-
include/linux/namei.h | 12 +
include/linux/syscalls.h | 3 +
include/uapi/asm-generic/unistd.h | 5 +-
include/uapi/linux/fcntl.h | 41 ++
tools/testing/selftests/Makefile | 1 +
tools/testing/selftests/openat2/.gitignore | 1 +
tools/testing/selftests/openat2/Makefile | 8 +
tools/testing/selftests/openat2/helpers.c | 109 ++++
tools/testing/selftests/openat2/helpers.h | 107 ++++
.../testing/selftests/openat2/openat2_test.c | 297 ++++++++++
.../selftests/openat2/rename_attack_test.c | 160 ++++++
.../testing/selftests/openat2/resolve_test.c | 523 ++++++++++++++++++
35 files changed, 1571 insertions(+), 71 deletions(-)
create mode 100644 tools/testing/selftests/openat2/.gitignore
create mode 100644 tools/testing/selftests/openat2/Makefile
create mode 100644 tools/testing/selftests/openat2/helpers.c
create mode 100644 tools/testing/selftests/openat2/helpers.h
create mode 100644 tools/testing/selftests/openat2/openat2_test.c
create mode 100644 tools/testing/selftests/openat2/rename_attack_test.c
create mode 100644 tools/testing/selftests/openat2/resolve_test.c
--
2.23.0
From: John Hubbard <jhubbard(a)nvidia.com>
[ Upstream commit 6f24c8d30d08f270b54f4c2cb9b08dfccbe59c57 ]
Even though gup_benchmark.c has code to handle the -w command-line option,
the "w" is not part of the getopt string. It looks as if it has been
missing the whole time.
On my machine, this leads naturally to the following predictable result:
$ sudo ./gup_benchmark -w
./gup_benchmark: invalid option -- 'w'
...which is fixed with this commit.
Link: http://lkml.kernel.org/r/20191014184639.1512873-2-jhubbard@nvidia.com
Signed-off-by: John Hubbard <jhubbard(a)nvidia.com>
Acked-by: Kirill A. Shutemov <kirill.shutemov(a)linux.intel.com>
Cc: Keith Busch <keith.busch(a)intel.com>
Cc: Shuah Khan <shuah(a)kernel.org>
Cc: Christoph Hellwig <hch(a)infradead.org>
Cc: "Aneesh Kumar K . V" <aneesh.kumar(a)linux.ibm.com>
Cc: Ira Weiny <ira.weiny(a)intel.com>
Cc: Christoph Hellwig <hch(a)lst.de>
Cc: kbuild test robot <lkp(a)intel.com>
Signed-off-by: Andrew Morton <akpm(a)linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds(a)linux-foundation.org>
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
---
tools/testing/selftests/vm/gup_benchmark.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tools/testing/selftests/vm/gup_benchmark.c b/tools/testing/selftests/vm/gup_benchmark.c
index c0534e298b512..cb3fc09645c48 100644
--- a/tools/testing/selftests/vm/gup_benchmark.c
+++ b/tools/testing/selftests/vm/gup_benchmark.c
@@ -37,7 +37,7 @@ int main(int argc, char **argv)
char *file = "/dev/zero";
char *p;
- while ((opt = getopt(argc, argv, "m:r:n:f:tTLUSH")) != -1) {
+ while ((opt = getopt(argc, argv, "m:r:n:f:tTLUwSH")) != -1) {
switch (opt) {
case 'm':
size = atoi(optarg) * MB;
--
2.20.1
From: Jiri Benc <jbenc(a)redhat.com>
[ Upstream commit fd418b01fe26c2430b1091675cceb3ab2b52e1e0 ]
Many distributions enable rp_filter. However, the flow dissector test
generates packets that have 1.1.1.1 set as (inner) source address without
this address being reachable. This causes the selftest to fail.
The selftests should not assume a particular initial configuration. Switch
off rp_filter.
Fixes: 50b3ed57dee9 ("selftests/bpf: test bpf flow dissection")
Signed-off-by: Jiri Benc <jbenc(a)redhat.com>
Signed-off-by: Daniel Borkmann <daniel(a)iogearbox.net>
Acked-by: Petar Penkov <ppenkov(a)google.com>
Link: https://lore.kernel.org/bpf/513a298f53e99561d2f70b2e60e2858ea6cda754.157053…
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
---
tools/testing/selftests/bpf/test_flow_dissector.sh | 3 +++
1 file changed, 3 insertions(+)
diff --git a/tools/testing/selftests/bpf/test_flow_dissector.sh b/tools/testing/selftests/bpf/test_flow_dissector.sh
index d23d4da66b834..e2d06191bd35c 100755
--- a/tools/testing/selftests/bpf/test_flow_dissector.sh
+++ b/tools/testing/selftests/bpf/test_flow_dissector.sh
@@ -63,6 +63,9 @@ fi
# Setup
tc qdisc add dev lo ingress
+echo 0 > /proc/sys/net/ipv4/conf/default/rp_filter
+echo 0 > /proc/sys/net/ipv4/conf/all/rp_filter
+echo 0 > /proc/sys/net/ipv4/conf/lo/rp_filter
echo "Testing IPv4..."
# Drops all IP/UDP packets coming from port 9
--
2.20.1
Commit 852c8cbf34d3 ("selftests/kselftest/runner.sh: Add 45 second
timeout per test") introduced a timeout per test. Livepatch tests could
run longer than 45 seconds, especially on slower machines. They do not
hang and they detect if something goes awry with internal accounting.
Better than looking for an arbitrary value, just disable the timeout for
livepatch selftests.
Signed-off-by: Miroslav Benes <mbenes(a)suse.cz>
---
tools/testing/selftests/livepatch/settings | 1 +
1 file changed, 1 insertion(+)
create mode 100644 tools/testing/selftests/livepatch/settings
diff --git a/tools/testing/selftests/livepatch/settings b/tools/testing/selftests/livepatch/settings
new file mode 100644
index 000000000000..e7b9417537fb
--- /dev/null
+++ b/tools/testing/selftests/livepatch/settings
@@ -0,0 +1 @@
+timeout=0
--
2.23.0
Hi
this patchset aims to add the initial arch-specific arm64 support to
kselftest starting with signals-related test-cases.
This series is based on v5.4-rc2.
A common internal test-case layout is proposed for signal tests and it is
wired-up to the toplevel kselftest Makefile, so that it should be possible
at the end to run it on an arm64 target in the usual way with KSFT.
~/linux# make TARGETS=arm64 kselftest
New KSFT arm64 testcases live inside tools/testing/selftests/arm64 grouped
by family inside subdirectories: arm64/signal is the first family proposed
with this series.
This series converts also to this subdirectory scheme the pre-existing
KSFT arm64 tags tests (already merged in v5.3), moving them into their own
arm64/tags subdirectory.
Thanks
Cristian
Notes:
-----
- further details in the included READMEs
- more tests still to be written (current strategy is going through the
related Kernel signal-handling code and write a test for each possible
and sensible code-path)
A few ideas for more TODO testcases:
- mangle_pstate_invalid_ssbs_regs: mess with SSBS bits on every
possible configured behavior
- fake_sigreturn_unmapped_sp: SP into unmapped addrs
- fake_sigreturn_kernelspace_sp: SP into kernel addrs
- fake_sigreturn_sve_bad_extra_context: SVE extra context badly formed
- fake_sigreturn_misaligned_sp_4: misaligned SP by 4
(i.e., __alignof__(struct _aarch64_ctx))
- fake_sigreturn_misaligned_sp_8: misaligned SP by 8
(i.e., sizeof(struct _aarch64_ctx))
- fake_sigreturn_bad_size_non_aligned: a size that doesn't overflow
__reserved[], but is not a multiple of 16
- fake_sigreturn_bad_size_tiny: a size that is less than 16
- fake_sigreturn_bad_size_overflow_tiny: a size that does overflow
__reserved[], but by less than 16 bytes?
- mangle_sve_invalid_extra_context: SVE extra_context invalid
- SVE signal testcases and special handling will be part of an additional patch
still to be released
- KSFT arm64 tags test patch
https://lore.kernel.org/linux-arm-kernel/c1e6aad230658bc175b42d92daeff2e300…
is relocated into its own directory under tools/testing/selftests/arm64/tags
Changes:
--------
v8-->v9:
- fixed a couple of misplaced .gitignore
v7-->v8:
- removed SSBS test case
- split remnants of SSBS patch (v7 05/11), containing some helpers,
into two distinct patches
v6-->v7:
- rebased on v5.4-rc2
- renamed SUBTARGETS arm64/ toplevel Makefile ENV to ARM64_SUBTARGETS
- fixed fake_sigreturn alignment routines (off by one)
- fixed SSBS test: avoid using MRS/MSR as whole and SKIP when SSBS not
supported
- reporting KSFT_SKIP when needed (usually if test_init(0 fails)
- using ID_AA64PFR1_EL1.SSBS to check SSBS support instead of HWCAP_SSBS
v5-->v6:
- added arm64 toplevel Makefile SUBTARGETS env var to be able to selectively
build only some arm64/ tests subdirectories
- removed unneed toplevel Makefile exports and fixed Copyright
- better checks for supported features and features names helpers
- converted some run-time critical assert() to abort() to avoid
issues when -NDEBUG is set
- default_handler() signal handler refactored and split
- using SIGTRAP for get_current_context()
- use volatile where proper
- refactor and relocate test_init() invocation
- review usage of MRS SSBS instructions depending on HW_SSBS
- cleanup fake_sigreturn trampoline
- cleanup get_starting_header helper
- avoiding timeout test failures wherever possible (fail immediately
if possible)
v4-->v5:
- rebased on arm64/for-next-core merging 01/11 with KSFT tags tests:
commit 9ce1263033cd ("selftests, arm64: add a selftest for passing tagged pointers to kernel")
- moved .gitignore up on elevel
- moved kernel header search mechanism into KSFT arm64 toplevel Makefile
so that it can be used easily also by each arm64 KSFT subsystem inside
subdirs of arm64
v3-->v4:
- rebased on v5.3-rc6
- added test descriptions
- fixed commit messages (imperative mood)
- added missing includes and removed unneeded ones
- added/used new get_starting_head() helper
- fixed/simplified signal.S::fakke_sigreturn()
- added set_regval() macro and .init initialization func
- better synchonization in get_current_context()
- macroization of mangle_pstate_invalid_mode_el
- split mangle_pstate_invalid_mode_el h/t
- removed standalone mode
- simplified CPU features checks
- fixed/refactored get_header() and validation routines
- simplfied docs
v2-->v3:
- rebased on v5.3-rc2
- better test result characterization looking for
SEGV_ACCERR in si_code on SIGSEGV
- using KSFT Framework macros for retvalues
- removed SAFE_WRITE()/dump_uc: buggy, un-needed and unused
- reviewed generation process of test_arm64_signals.sh runner script
- re-added a fixed fake_sigreturn_misaligned_sp testcase and a properly
extended fake_sigreturn() helper
- added tests' TODO notes
v1-->v2:
- rebased on 5.2-rc7
- various makefile's cleanups
- mixed READMEs fixes
- fixed test_arm64_signals.sh runner script
- cleaned up assembly code in signal.S
- improved get_current_context() logic
- fixed SAFE_WRITE()
- common support code split into more chunks, each one introduced when
needed by some new testcases
- fixed some headers validation routines in testcases.c
- removed some still broken/immature tests:
+ fake_sigreturn_misaligned
+ fake_sigreturn_overflow_reserved
+ mangle_pc_invalid
+ mangle_sp_misaligned
- fixed some other testcases:
+ mangle_pstate_ssbs_regs: better checks of SSBS bit when feature unsupported
+ mangle_pstate_invalid_compat_toggle: name fix
+ mangle_pstate_invalid_mode_el[1-3]: precautionary zeroing PSTATE.MODE
+ fake_sigreturn_bad_magic, fake_sigreturn_bad_size,
fake_sigreturn_bad_size_for_magic0:
- accounting for available space...dropping extra when needed
- keeping alignent
- new testcases on FPSMID context:
+ fake_sigreturn_missing_fpsimd
+ fake_sigreturn_duplicated_fpsimd
Cristian Marussi (12):
kselftest: arm64: extend toplevel skeleton Makefile
kselftest: arm64: mangle_pstate_invalid_compat_toggle and common utils
kselftest: arm64: mangle_pstate_invalid_daif_bits
kselftest: arm64: mangle_pstate_invalid_mode_el[123][ht]
kselftest: arm64: extend test_init functionalities
kselftest: arm64: add helper get_current_context
kselftest: arm64: fake_sigreturn_bad_magic
kselftest: arm64: fake_sigreturn_bad_size_for_magic0
kselftest: arm64: fake_sigreturn_missing_fpsimd
kselftest: arm64: fake_sigreturn_duplicated_fpsimd
kselftest: arm64: fake_sigreturn_bad_size
kselftest: arm64: fake_sigreturn_misaligned_sp
tools/testing/selftests/Makefile | 1 +
tools/testing/selftests/arm64/Makefile | 64 +++-
tools/testing/selftests/arm64/README | 25 ++
.../testing/selftests/arm64/signal/.gitignore | 3 +
tools/testing/selftests/arm64/signal/Makefile | 32 ++
tools/testing/selftests/arm64/signal/README | 59 +++
.../testing/selftests/arm64/signal/signals.S | 64 ++++
.../selftests/arm64/signal/test_signals.c | 29 ++
.../selftests/arm64/signal/test_signals.h | 116 ++++++
.../arm64/signal/test_signals_utils.c | 340 ++++++++++++++++++
.../arm64/signal/test_signals_utils.h | 120 +++++++
.../testcases/fake_sigreturn_bad_magic.c | 52 +++
.../testcases/fake_sigreturn_bad_size.c | 77 ++++
.../fake_sigreturn_bad_size_for_magic0.c | 46 +++
.../fake_sigreturn_duplicated_fpsimd.c | 50 +++
.../testcases/fake_sigreturn_misaligned_sp.c | 37 ++
.../testcases/fake_sigreturn_missing_fpsimd.c | 50 +++
.../mangle_pstate_invalid_compat_toggle.c | 31 ++
.../mangle_pstate_invalid_daif_bits.c | 35 ++
.../mangle_pstate_invalid_mode_el1h.c | 15 +
.../mangle_pstate_invalid_mode_el1t.c | 15 +
.../mangle_pstate_invalid_mode_el2h.c | 15 +
.../mangle_pstate_invalid_mode_el2t.c | 15 +
.../mangle_pstate_invalid_mode_el3h.c | 15 +
.../mangle_pstate_invalid_mode_el3t.c | 15 +
.../mangle_pstate_invalid_mode_template.h | 28 ++
.../arm64/signal/testcases/testcases.c | 196 ++++++++++
.../arm64/signal/testcases/testcases.h | 104 ++++++
.../selftests/arm64/{ => tags}/.gitignore | 0
tools/testing/selftests/arm64/tags/Makefile | 7 +
.../arm64/{ => tags}/run_tags_test.sh | 0
.../selftests/arm64/{ => tags}/tags_test.c | 0
32 files changed, 1651 insertions(+), 5 deletions(-)
create mode 100644 tools/testing/selftests/arm64/README
create mode 100644 tools/testing/selftests/arm64/signal/.gitignore
create mode 100644 tools/testing/selftests/arm64/signal/Makefile
create mode 100644 tools/testing/selftests/arm64/signal/README
create mode 100644 tools/testing/selftests/arm64/signal/signals.S
create mode 100644 tools/testing/selftests/arm64/signal/test_signals.c
create mode 100644 tools/testing/selftests/arm64/signal/test_signals.h
create mode 100644 tools/testing/selftests/arm64/signal/test_signals_utils.c
create mode 100644 tools/testing/selftests/arm64/signal/test_signals_utils.h
create mode 100644 tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_magic.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size_for_magic0.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_duplicated_fpsimd.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_misaligned_sp.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_missing_fpsimd.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/mangle_pstate_invalid_compat_toggle.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/mangle_pstate_invalid_daif_bits.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/mangle_pstate_invalid_mode_el1h.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/mangle_pstate_invalid_mode_el1t.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/mangle_pstate_invalid_mode_el2h.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/mangle_pstate_invalid_mode_el2t.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/mangle_pstate_invalid_mode_el3h.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/mangle_pstate_invalid_mode_el3t.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/mangle_pstate_invalid_mode_template.h
create mode 100644 tools/testing/selftests/arm64/signal/testcases/testcases.c
create mode 100644 tools/testing/selftests/arm64/signal/testcases/testcases.h
rename tools/testing/selftests/arm64/{ => tags}/.gitignore (100%)
create mode 100644 tools/testing/selftests/arm64/tags/Makefile
rename tools/testing/selftests/arm64/{ => tags}/run_tags_test.sh (100%)
rename tools/testing/selftests/arm64/{ => tags}/tags_test.c (100%)
--
2.17.1
This patchset is being developed here:
<https://github.com/cyphar/linux/tree/openat2/master>
Patch changelog:
v14:
* The magic-link changes (and O_EMPTYPATH) have been dropped from this series
-- they will be developed and sent separately. The main reason is that we
need to restrict things other than open(2) (examples include truncate(2) as
well as mount(MS_BIND)). This will require a fair amount of extra work, and
there's no point stalling openat2(2) for that work to be completed.
* Minor rework of 'struct open_how':
* To avoid future headaches, make it a non-const argument.
* Expand ->flags and ->resolve to 64-bit fields to allow for more flag
extensions without needing to add separate fields too early. This
requires adding a bit of explicit padding (32 bits) to avoid userspace
putting garbage in the alignment padding -- this can be repurposed for
future extensions.
* upgrade_mask is dropped (and will be a separate field when we add it
again in the future) to avoid userspace foot-guns.
* Expand -EINVAL checks in build_open_flags(). Rather than silently
ignoring silly flag combinations (such as O_TMPFILE|O_PATH or
O_PATH|<most flags>), give an -EINVAL. All of the silent ignore semantics
were added to open(2) because we couldn't return -EINVAL -- but we can
now!
* open(2) and openat(2) clean up their flags before passing them to
build_open_flags(), so all mixed flags will continue to work. There is
one exception which is (O_PATH|O_TMPFILE) -- this is no longer
permitted (as far as I can tell this appears to be a bug, and there are
no userspace users that I've hit after running this code for a few
days). If it turns out that userspace does depend on (O_PATH|O_TMPFILE)
working, we can only disallow it for openat2(2).
* Don't zero out nd->root in complete_walk() for RCU-walk if we're doing a
scoped-lookup (this prevents a needless REF-walk retry).
* Attempt all tests on kernels that don't have openat2(2), rather than just
skipping everything.
v13: <https://lore.kernel.org/lkml/20190930183316.10190-1-cyphar@cyphar.com/>
v12: <https://lore.kernel.org/lkml/20190904201933.10736-1-cyphar@cyphar.com/>
v11: <https://lore.kernel.org/lkml/20190820033406.29796-1-cyphar@cyphar.com/>
<https://lore.kernel.org/lkml/20190728010207.9781-1-cyphar@cyphar.com/>
v10: <https://lore.kernel.org/lkml/20190719164225.27083-1-cyphar@cyphar.com/>
v09: <https://lore.kernel.org/lkml/20190706145737.5299-1-cyphar@cyphar.com/>
v08: <https://lore.kernel.org/lkml/20190520133305.11925-1-cyphar@cyphar.com/>
v07: <https://lore.kernel.org/lkml/20190507164317.13562-1-cyphar@cyphar.com/>
v06: <https://lore.kernel.org/lkml/20190506165439.9155-1-cyphar@cyphar.com/>
v05: <https://lore.kernel.org/lkml/20190320143717.2523-1-cyphar@cyphar.com/>
v04: <https://lore.kernel.org/lkml/20181112142654.341-1-cyphar@cyphar.com/>
v03: <https://lore.kernel.org/lkml/20181009070230.12884-1-cyphar@cyphar.com/>
v02: <https://lore.kernel.org/lkml/20181009065300.11053-1-cyphar@cyphar.com/>
v01: <https://lore.kernel.org/lkml/20180929103453.12025-1-cyphar@cyphar.com/>
For a very long time, extending openat(2) with new features has been
incredibly frustrating. This stems from the fact that openat(2) is
possibly the most famous counter-example to the mantra "don't silently
accept garbage from userspace" -- it doesn't check whether unknown flags
are present[1].
This means that (generally) the addition of new flags to openat(2) has
been fraught with backwards-compatibility issues (O_TMPFILE has to be
defined as __O_TMPFILE|O_DIRECTORY|[O_RDWR or O_WRONLY] to ensure old
kernels gave errors, since it's insecure to silently ignore the
flag[2]). All new security-related flags therefore have a tough road to
being added to openat(2).
Furthermore, the need for some sort of control over VFS's path resolution (to
avoid malicious paths resulting in inadvertent breakouts) has been a very
long-standing desire of many userspace applications. This patchset is a revival
of Al Viro's old AT_NO_JUMPS[3] patchset (which was a variant of David
Drysdale's O_BENEATH patchset[4] which was a spin-off of the Capsicum
project[5]) with a few additions and changes made based on the previous
discussion within [6] as well as others I felt were useful.
In line with the conclusions of the original discussion of AT_NO_JUMPS, the
flag has been split up into separate flags. However, instead of being an
openat(2) flag it is provided through a new syscall openat2(2) which provides
several other improvements to the openat(2) interface (see the patch
description for more details). The following new LOOKUP_* flags are added:
* LOOKUP_NO_XDEV blocks all mountpoint crossings (upwards, downwards,
or through absolute links). Absolute pathnames alone in openat(2) do not
trigger this. Magic-link traversal which implies a vfsmount jump is also
blocked (though magic-link jumps on the same vfsmount are permitted).
* LOOKUP_NO_MAGICLINKS blocks resolution through /proc/$pid/fd-style
links. This is done by blocking the usage of nd_jump_link() during
resolution in a filesystem. The term "magic-links" is used to match
with the only reference to these links in Documentation/, but I'm
happy to change the name.
It should be noted that this is different to the scope of
~LOOKUP_FOLLOW in that it applies to all path components. However,
you can do openat2(NO_FOLLOW|NO_MAGICLINKS) on a magic-link and it
will *not* fail (assuming that no parent component was a
magic-link), and you will have an fd for the magic-link.
In order to correctly detect magic-links, the introduction of a new
LOOKUP_MAGICLINK_JUMPED state flag was required.
* LOOKUP_BENEATH disallows escapes to outside the starting dirfd's
tree, using techniques such as ".." or absolute links. Absolute
paths in openat(2) are also disallowed. Conceptually this flag is to
ensure you "stay below" a certain point in the filesystem tree --
but this requires some additional to protect against various races
that would allow escape using "..".
Currently LOOKUP_BENEATH implies LOOKUP_NO_MAGICLINKS, because it
can trivially beam you around the filesystem (breaking the
protection). In future, there might be similar safety checks done as
in LOOKUP_IN_ROOT, but that requires more discussion.
In addition, two new flags are added that expand on the above ideas:
* LOOKUP_NO_SYMLINKS does what it says on the tin. No symlink
resolution is allowed at all, including magic-links. Just as with
LOOKUP_NO_MAGICLINKS this can still be used with NOFOLLOW to open an
fd for the symlink as long as no parent path had a symlink
component.
* LOOKUP_IN_ROOT is an extension of LOOKUP_BENEATH that, rather than
blocking attempts to move past the root, forces all such movements
to be scoped to the starting point. This provides chroot(2)-like
protection but without the cost of a chroot(2) for each filesystem
operation, as well as being safe against race attacks that chroot(2)
is not.
If a race is detected (as with LOOKUP_BENEATH) then an error is
generated, and similar to LOOKUP_BENEATH it is not permitted to cross
magic-links with LOOKUP_IN_ROOT.
The primary need for this is from container runtimes, which
currently need to do symlink scoping in userspace[7] when opening
paths in a potentially malicious container. There is a long list of
CVEs that could have bene mitigated by having RESOLVE_THIS_ROOT
(such as CVE-2017-1002101, CVE-2017-1002102, CVE-2018-15664, and
CVE-2019-5736, just to name a few).
In order to make all of the above more usable, I'm working on
libpathrs[8] which is a C-friendly library for safe path resolution. It
features a userspace-emulated backend if the kernel doesn't support
openat2(2). Hopefully we can get userspace to switch to using it, and
thus get openat2(2) support for free once it's ready.
[1]: https://lwn.net/Articles/588444/
[2]: https://lore.kernel.org/lkml/CA+55aFyyxJL1LyXZeBsf2ypriraj5ut1XkNDsunRBqgVj…
[3]: https://lore.kernel.org/lkml/20170429220414.GT29622@ZenIV.linux.org.uk
[4]: https://lore.kernel.org/lkml/1415094884-18349-1-git-send-email-drysdale@goo…
[5]: https://lore.kernel.org/lkml/1404124096-21445-1-git-send-email-drysdale@goo…
[6]: https://lwn.net/Articles/723057/
[7]: https://github.com/cyphar/filepath-securejoin
[8]: https://github.com/openSUSE/libpathrs
The current draft of the openat2(2) man-page is included below.
--8<---------------------------------------------------------------------------
OPENAT2(2) Linux Programmer's Manual OPENAT2(2)
NAME
openat2 - open and possibly create a file (extended)
SYNOPSIS
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
int openat2(int dirfd, const char *pathname, struct open_how *how, size_t size);
Note: There is no glibc wrapper for this system call; see NOTES.
DESCRIPTION
The openat2() system call opens the file specified by pathname. If the specified file
does not exist, it may optionally (if O_CREAT is specified in how.flags) be created by
openat2().
As with openat(2), if pathname is relative, then it is interpreted relative to the
directory referred to by the file descriptor dirfd (or the current working directory of
the calling process, if dirfd is the special value AT_FDCWD.) If pathname is absolute,
then dirfd is ignored (unless how.resolve contains RESOLVE_IN_ROOT, in which case pathname
is resolved relative to dirfd.)
The openat2() system call is an extension of openat(2) and provides a superset of its
functionality. Rather than taking a single flag argument, an extensible structure (how)
is passed instead to allow for future extensions. size must be set to sizeof(struct
open_how), to facilitate future extensions (see the "Extensibility" section of the NOTES
for more detail on how extensions are handled.)
The open_how structure
The following structure indicates how pathname should be opened, and acts as a superset of
the flag and mode arguments to openat(2).
struct open_how {
__aligned_u64 flags; /* O_* flags. */
__u16 mode; /* Mode for O_{CREAT,TMPFILE}. */
__u16 __padding[3]; /* Must be zeroed. */
__aligned_u64 resolve; /* RESOLVE_* flags. */
};
Any future extensions to openat2() will be implemented as new fields appended to the above
structure (or through reuse of pre-existing padding space), with the zero value of the new
fields acting as though the extension were not present.
The meaning of each field is as follows:
flags
The file creation and status flags to use for this operation. All of the
O_* flags defined for openat(2) are valid openat2() flag values.
Unlike openat(2), it is an error to provide openat2() unknown or conflicting
flags in flags.
mode
File mode for the new file, with identical semantics to the mode argument to
openat(2). However, unlike openat(2), it is an error to provide openat2()
with a mode which contains bits other than 0777.
It is an error to provide openat2() a non-zero mode if flags does not
contain O_CREAT or O_TMPFILE.
resolve
Change how the components of pathname will be resolved (see
path_resolution(7) for background information.) The primary use case for
these flags is to allow trusted programs to restrict how untrusted paths (or
paths inside untrusted directories) are resolved. The full list of resolve
flags is given below.
RESOLVE_NO_XDEV
Disallow traversal of mount points during path resolution (including
all bind mounts).
Users of this flag are encouraged to make its use configurable
(unless it is used for a specific security purpose), as bind mounts
are very widely used by end-users. Setting this flag indiscrimnately
for all uses of openat2() may result in spurious errors on
previously-functional systems.
RESOLVE_NO_SYMLINKS
Disallow resolution of symbolic links during path resolution. This
option implies RESOLVE_NO_MAGICLINKS.
If the trailing component is a symbolic link, and flags contains both
O_PATH and O_NOFOLLOW, then an O_PATH file descriptor referencing the
symbolic link will be returned.
Users of this flag are encouraged to make its use configurable
(unless it is used for a specific security purpose), as symbolic
links are very widely used by end-users. Setting this flag
indiscrimnately for all uses of openat2() may result in spurious
errors on previously-functional systems.
RESOLVE_NO_MAGICLINKS
Disallow all magic link resolution during path resolution.
If the trailing component is a magic link, and flags contains both
O_PATH and O_NOFOLLOW, then an O_PATH file descriptor referencing the
magic link will be returned.
Magic-links are symbolic link-like objects that are most notably
found in proc(5) (examples include /proc/[pid]/exe and
/proc/[pid]/fd/*.) Due to the potential danger of unknowingly
opening these magic links, it may be preferable for users to disable
their resolution entirely (see symboliclink(7) for more details.)
RESOLVE_BENEATH
Do not permit the path resolution to succeed if any component of the
resolution is not a descendant of the directory indicated by dirfd.
This results in absolute symbolic links (and absolute values of
pathname) to be rejected.
Currently, this flag also disables magic link resolution. However,
this may change in the future. The caller should explicitly specify
RESOLVE_NO_MAGICLINKS to ensure that magic links are not resolved.
RESOLVE_IN_ROOT
Treat dirfd as the root directory while resolving pathname (as though
the user called chroot(2) with dirfd as the argument.) Absolute
symbolic links and ".." path components will be scoped to dirfd. If
pathname is an absolute path, it is also treated relative to dirfd.
However, unlike chroot(2) (which changes the filesystem root
permanently for a process), RESOLVE_IN_ROOT allows a program to
efficiently restrict path resolution for only certain operations. It
also has several hardening features (such detecting escape attempts
during .. resolution) which chroot(2) does not.
Currently, this flag also disables magic link resolution. However,
this may change in the future. The caller should explicitly specify
RESOLVE_NO_MAGICLINKS to ensure that magic links are not resolved.
It is an error to provide openat2() unknown flags in resolve.
RETURN VALUE
On success, a new file descriptor is returned. On error, -1 is returned, and errno is set
appropriately.
ERRORS
The set of errors returned by openat2() includes all of the errors returned by openat(2),
as well as the following additional errors:
EINVAL An unknown flag or invalid value was specified in how.
EINVAL mode is non-zero, but flags does not contain O_CREAT or O_TMPFILE.
EINVAL size was smaller than any known version of struct open_how.
E2BIG An extension was specified in how, which the current kernel does not support (see
the "Extensibility" section of the NOTES for more detail on how extensions are
handled.)
EAGAIN resolve contains either RESOLVE_IN_ROOT or RESOLVE_BENEATH, and the kernel could
not ensure that a ".." component didn't escape (due to a race condition or
potential attack.) Callers may choose to retry the openat2() call.
EXDEV resolve contains either RESOLVE_IN_ROOT or RESOLVE_BENEATH, and an escape from the
root during path resolution was detected.
EXDEV resolve contains RESOLVE_NO_XDEV, and a path component attempted to cross a mount
point.
ELOOP resolve contains RESOLVE_NO_SYMLINKS, and one of the path components was a symbolic
link (or magic link).
ELOOP resolve contains RESOLVE_NO_MAGICLINKS, and one of the path components was a magic
link.
VERSIONS
openat2() was added to Linux in kernel 5.FOO.
CONFORMING TO
This system call is Linux-specific.
The semantics of RESOLVE_BENEATH were modelled after FreeBSD's O_BENEATH.
NOTES
Glibc does not provide a wrapper for this system call; call it using systemcall(2).
Extensibility
In order to allow for struct open_how to be extended in future kernel revisions, openat2()
requires userspace to specify the size of struct open_how structure they are passing. By
providing this information, it is possible for openat2() to provide both forwards- and
backwards-compatibility — with size acting as an implicit version number (because new
extension fields will always be appended, the size will always increase.) This
extensibility design is very similar to other system calls such as perf_setattr(2),
perf_event_open(2), and clone(3).
If we let usize be the size of the structure according to userspace and ksize be the size
of the structure which the kernel supports, then there are only three cases to consider:
* If ksize equals usize, then there is no version mismatch and how can be used
verbatim.
* If ksize is larger than usize, then there are some extensions the kernel
supports which the userspace program is unaware of. Because all extensions must
have their zero values be a no-op, the kernel treats all of the extension fields
not set by userspace to have zero values. This provides backwards-
compatibility.
* If ksize is smaller than usize, then there are some extensions which the
userspace program is aware of but the kernel does not support. Because all
extensions must have their zero values be a no-op, the kernel can safely ignore
the unsupported extension fields if they are all-zero. If any unsupported
extension fields are non-zero, then -1 is returned and errno is set to E2BIG.
This provides forwards-compatibility.
Therefore, most userspace programs will not need to have any special handling of
extensions. However, if a userspace program wishes to determine what extensions the
running kernel supports, they may conduct a binary search on size (to find the largest
value which doesn't produce an error of E2BIG.)
SEE ALSO
openat(2), path_resolution(7), symboliclink(7)
Linux 2019-10-10 OPENAT2(2)
--8<---------------------------------------------------------------------------
Aleksa Sarai (6):
namei: O_BENEATH-style resolution restriction flags
namei: LOOKUP_IN_ROOT: chroot-like path resolution
namei: permit ".." resolution with LOOKUP_{IN_ROOT,BENEATH}
open: introduce openat2(2) syscall
selftests: add openat2(2) selftests
Documentation: path-lookup: mention LOOKUP_MAGICLINK_JUMPED
CREDITS | 4 +-
Documentation/filesystems/path-lookup.rst | 18 +-
arch/alpha/kernel/syscalls/syscall.tbl | 1 +
arch/arm/tools/syscall.tbl | 1 +
arch/arm64/include/asm/unistd.h | 2 +-
arch/arm64/include/asm/unistd32.h | 2 +
arch/ia64/kernel/syscalls/syscall.tbl | 1 +
arch/m68k/kernel/syscalls/syscall.tbl | 1 +
arch/microblaze/kernel/syscalls/syscall.tbl | 1 +
arch/mips/kernel/syscalls/syscall_n32.tbl | 1 +
arch/mips/kernel/syscalls/syscall_n64.tbl | 1 +
arch/mips/kernel/syscalls/syscall_o32.tbl | 1 +
arch/parisc/kernel/syscalls/syscall.tbl | 1 +
arch/powerpc/kernel/syscalls/syscall.tbl | 1 +
arch/s390/kernel/syscalls/syscall.tbl | 1 +
arch/sh/kernel/syscalls/syscall.tbl | 1 +
arch/sparc/kernel/syscalls/syscall.tbl | 1 +
arch/x86/entry/syscalls/syscall_32.tbl | 1 +
arch/x86/entry/syscalls/syscall_64.tbl | 1 +
arch/xtensa/kernel/syscalls/syscall.tbl | 1 +
fs/namei.c | 167 +++++-
fs/open.c | 154 ++++--
include/linux/fcntl.h | 12 +-
include/linux/namei.h | 12 +
include/linux/syscalls.h | 3 +
include/uapi/asm-generic/unistd.h | 5 +-
include/uapi/linux/fcntl.h | 41 ++
tools/testing/selftests/Makefile | 1 +
tools/testing/selftests/openat2/.gitignore | 1 +
tools/testing/selftests/openat2/Makefile | 8 +
tools/testing/selftests/openat2/helpers.c | 109 ++++
tools/testing/selftests/openat2/helpers.h | 107 ++++
.../testing/selftests/openat2/openat2_test.c | 297 ++++++++++
.../selftests/openat2/rename_attack_test.c | 160 ++++++
.../testing/selftests/openat2/resolve_test.c | 523 ++++++++++++++++++
35 files changed, 1571 insertions(+), 71 deletions(-)
create mode 100644 tools/testing/selftests/openat2/.gitignore
create mode 100644 tools/testing/selftests/openat2/Makefile
create mode 100644 tools/testing/selftests/openat2/helpers.c
create mode 100644 tools/testing/selftests/openat2/helpers.h
create mode 100644 tools/testing/selftests/openat2/openat2_test.c
create mode 100644 tools/testing/selftests/openat2/rename_attack_test.c
create mode 100644 tools/testing/selftests/openat2/resolve_test.c
--
2.23.0
The current kunit execution model is to provide base kunit functionality
and tests built-in to the kernel. The aim of this series is to allow
building kunit itself and tests as modules. This in turn allows a
simple form of selective execution; load the module you wish to test.
In doing so, kunit itself (if also built as a module) will be loaded as
an implicit dependency.
Because this requires a core API modification - if a module delivers
multiple suites, they must be declared with the kunit_test_suites()
macro - we're proposing this patch as a candidate to be applied to the
test tree before too many kunit consumers appear. We attempt to deal
with existing consumers in patch 1.
Changes since v1:
- sent correct patch set; apologies, previous patch set was built
prior to kunit move to lib/ and should be ignored.
Patch 1 consists changes needed to support loading tests as modules.
Patch 2 allows kunit itself to be loaded as a module.
Patch 3 documents module support.
Alan Maguire (3):
kunit: allow kunit tests to be loaded as a module
kunit: allow kunit to be loaded as a module
kunit: update documentation to describe module-based build
Documentation/dev-tools/kunit/faq.rst | 3 ++-
Documentation/dev-tools/kunit/index.rst | 3 +++
Documentation/dev-tools/kunit/usage.rst | 16 ++++++++++++++++
include/kunit/test.h | 30 +++++++++++++++++++++++-------
kernel/sysctl-test.c | 6 +++++-
lib/Kconfig.debug | 4 ++--
lib/kunit/Kconfig | 6 +++---
lib/kunit/Makefile | 4 +++-
lib/kunit/assert.c | 8 ++++++++
lib/kunit/example-test.c | 6 +++++-
lib/kunit/string-stream-test.c | 9 +++++++--
lib/kunit/string-stream.c | 7 +++++++
lib/kunit/test-test.c | 8 ++++++--
lib/kunit/test.c | 12 ++++++++++++
lib/kunit/try-catch.c | 8 ++++++--
15 files changed, 108 insertions(+), 22 deletions(-)
--
1.8.3.1
Hi,
Here are the 2nd version of kselftest fixes some on 32bit arch
(e.g. arm). In this version, I updated [1/5] to make va_max 1GB
instead of 3GB, according to Alexey's comment.
When I built the ksefltest on arm, I hit some 32bit related warnings.
Here are the patches to fix those issues.
- [1/5] va_max was set 2^32 even on 32bit arch. This can make
va_max == 0 and always fail. Make it 3GB on 32bit.
- [2/5] Some VM tests requires 64bit user space, which should
not run on 32bit arch.
- [3/5] For counting the size of large file, we should use
size_t instead of unsinged long.
- [4/5] Gcc warns printf format for size_t and int64_t on
32bit arch. Use %llu and cast it.
- [5/5] Gcc warns __u64 and pointer type castings. It should
once translated to unsigned long.
Thank you,
---
Masami Hiramatsu (5):
selftests: proc: Make va_max 1GB on 32bit arch
selftests: vm: Build/Run 64bit tests only on 64bit arch
selftests: net: Use size_t and ssize_t for counting file size
selftests: net: Fix printf format warnings on arm
selftests: sync: Fix cast warnings on arm
tools/testing/selftests/net/so_txtime.c | 4 ++--
tools/testing/selftests/net/tcp_mmap.c | 8 ++++----
tools/testing/selftests/net/udpgso.c | 3 ++-
tools/testing/selftests/net/udpgso_bench_tx.c | 3 ++-
.../selftests/proc/proc-self-map-files-002.c | 11 ++++++++++-
tools/testing/selftests/sync/sync.c | 6 +++---
tools/testing/selftests/vm/Makefile | 5 +++++
tools/testing/selftests/vm/run_vmtests | 10 ++++++++++
8 files changed, 38 insertions(+), 12 deletions(-)
--
Masami Hiramatsu (Linaro) <mhiramat(a)kernel.org>
Hi,
Here's another gup_benchmark.c fix, which I ran into while adding
support for the upcoming FOLL_PIN work. Anyway, the problem is
clearly described in the patch commit description, and the fix seems
like the best way to me, but the fix is not *completely* black and
white.
This fix forces MAP_ANONYMOUS for the MAP_HUGETLB case. However,
another way to do it might be to mmap() against a valid hugetlb
page file, instead of /dev/zero. But that seems like a lot of
trouble and if I'm reading the intent correctly, MAP_ANONYMOUS
is what's desired anyway.
John Hubbard (1):
mm/gup_benchmark: fix MAP_HUGETLB case
tools/testing/selftests/vm/gup_benchmark.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
2.23.0
BugLink: https://bugs.launchpad.net/bugs/1849281
The ifndef for SECCOMP_USER_NOTIF_FLAG_CONTINUE was placed under the
ifndef for the SECCOMP_FILTER_FLAG_NEW_LISTENER feature. This will not
work on systems that do support SECCOMP_FILTER_FLAG_NEW_LISTENER but do not
support SECCOMP_USER_NOTIF_FLAG_CONTINUE. So move the latter ifndef out of
the former ifndef's scope.
2019-10-20 11:14:01 make run_tests -C seccomp
make: Entering directory '/usr/src/perf_selftests-x86_64-rhel-7.6-0eebfed2954f152259cae0ad57b91d3ea92968e8/tools/testing/selftests/seccomp'
gcc -Wl,-no-as-needed -Wall seccomp_bpf.c -lpthread -o seccomp_bpf
seccomp_bpf.c: In function ‘user_notification_continue’:
seccomp_bpf.c:3562:15: error: ‘SECCOMP_USER_NOTIF_FLAG_CONTINUE’ undeclared (first use in this function)
resp.flags = SECCOMP_USER_NOTIF_FLAG_CONTINUE;
^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
seccomp_bpf.c:3562:15: note: each undeclared identifier is reported only once for each function it appears in
Makefile:12: recipe for target 'seccomp_bpf' failed
make: *** [seccomp_bpf] Error 1
make: Leaving directory '/usr/src/perf_selftests-x86_64-rhel-7.6-0eebfed2954f152259cae0ad57b91d3ea92968e8/tools/testing/selftests/seccomp'
Reported-by: kernel test robot <rong.a.chen(a)intel.com>
Fixes: 0eebfed2954f ("seccomp: test SECCOMP_USER_NOTIF_FLAG_CONTINUE")
Cc: linux-kselftest(a)vger.kernel.org
Signed-off-by: Christian Brauner <christian.brauner(a)ubuntu.com>
Reviewed-by: Tycho Andersen <tycho(a)tycho.ws>
Link: https://lore.kernel.org/r/20191021091055.4644-1-christian.brauner@ubuntu.com
Signed-off-by: Kees Cook <keescook(a)chromium.org>
(cherry picked from commit 2aa8d8d04ca29c3269154e1d48855e498be8882f
https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git)
Signed-off-by: Christian Brauner <christian.brauner(a)ubuntu.com>
---
tools/testing/selftests/seccomp/seccomp_bpf.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/tools/testing/selftests/seccomp/seccomp_bpf.c b/tools/testing/selftests/seccomp/seccomp_bpf.c
index 6021baecb386..bf834ee02b69 100644
--- a/tools/testing/selftests/seccomp/seccomp_bpf.c
+++ b/tools/testing/selftests/seccomp/seccomp_bpf.c
@@ -167,10 +167,6 @@ struct seccomp_metadata {
#define SECCOMP_RET_USER_NOTIF 0x7fc00000U
-#ifndef SECCOMP_USER_NOTIF_FLAG_CONTINUE
-#define SECCOMP_USER_NOTIF_FLAG_CONTINUE 0x00000001
-#endif
-
#define SECCOMP_IOC_MAGIC '!'
#define SECCOMP_IO(nr) _IO(SECCOMP_IOC_MAGIC, nr)
#define SECCOMP_IOR(nr, type) _IOR(SECCOMP_IOC_MAGIC, nr, type)
@@ -209,6 +205,10 @@ struct seccomp_notif_sizes {
#define PTRACE_EVENTMSG_SYSCALL_EXIT 2
#endif
+#ifndef SECCOMP_USER_NOTIF_FLAG_CONTINUE
+#define SECCOMP_USER_NOTIF_FLAG_CONTINUE 0x00000001
+#endif
+
#ifndef seccomp
int seccomp(unsigned int op, unsigned int flags, void *args)
{
--
2.23.0
Reset all signal handlers of the child not set to SIG_IGN to SIG_DFL.
Mutually exclusive with CLONE_SIGHAND to not disturb other thread's
signal handler.
In the spirit of closer cooperation between glibc developers and kernel
developers (cf. [2]) this patchset came out of a discussion on the glibc
mailing list for improving posix_spawn() (cf. [1], [3], [4]). Kernel
support for this feature has been explicitly requested by glibc and I
see no reason not to help them with this.
The child helper process on Linux posix_spawn must ensure that no signal
handlers are enabled, so the signal disposition must be either SIG_DFL
or SIG_IGN. However, it requires a sigprocmask to obtain the current
signal mask and at least _NSIG sigaction calls to reset the signal
handlers for each posix_spawn call or complex state tracking that might
lead to data corruption in glibc. Adding this flags lets glibc avoid
these problems.
[1]: https://www.sourceware.org/ml/libc-alpha/2019-10/msg00149.html
[3]: https://www.sourceware.org/ml/libc-alpha/2019-10/msg00158.html
[4]: https://www.sourceware.org/ml/libc-alpha/2019-10/msg00160.html
[2]: https://lwn.net/Articles/799331/
'[...] by asking for better cooperation with the C-library projects
in general. They should be copied on patches containing ABI
changes, for example. I noted that there are often times where
C-library developers wish the kernel community had done things
differently; how could those be avoided in the future? Members of
the audience suggested that more glibc developers should perhaps
join the linux-api list. The other suggestion was to "copy Florian
on everything".'
Cc: Oleg Nesterov <oleg(a)redhat.com>
Cc: Florian Weimer <fweimer(a)redhat.com>
Cc: libc-alpha(a)sourceware.org
Cc: linux-api(a)vger.kernel.org
Signed-off-by: Christian Brauner <christian.brauner(a)ubuntu.com>
---
/* v1 */
Link: https://lore.kernel.org/r/20191010133518.5420-1-christian.brauner@ubuntu.com
/* v2 */
Link: https://lore.kernel.org/r/20191011102537.27502-1-christian.brauner@ubuntu.c…
- Florian Weimer <fweimer(a)redhat.com>:
- update comment in clone3_args_valid()
/* v3 */
- "Michael Kerrisk (man-pages)" <mtk.manpages(a)gmail.com>:
- s/CLONE3_CLEAR_SIGHAND/CLONE_CLEAR_SIGHAND/g
---
include/uapi/linux/sched.h | 3 +++
kernel/fork.c | 16 +++++++++++-----
2 files changed, 14 insertions(+), 5 deletions(-)
diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h
index 99335e1f4a27..1d500ed03c63 100644
--- a/include/uapi/linux/sched.h
+++ b/include/uapi/linux/sched.h
@@ -33,6 +33,9 @@
#define CLONE_NEWNET 0x40000000 /* New network namespace */
#define CLONE_IO 0x80000000 /* Clone io context */
+/* Flags for the clone3() syscall. */
+#define CLONE_CLEAR_SIGHAND 0x100000000ULL /* Clear any signal handler and reset to SIG_DFL. */
+
#ifndef __ASSEMBLY__
/**
* struct clone_args - arguments for the clone3 syscall
diff --git a/kernel/fork.c b/kernel/fork.c
index 1f6c45f6a734..aa5b5137f071 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1517,6 +1517,11 @@ static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk)
spin_lock_irq(¤t->sighand->siglock);
memcpy(sig->action, current->sighand->action, sizeof(sig->action));
spin_unlock_irq(¤t->sighand->siglock);
+
+ /* Reset all signal handler not set to SIG_IGN to SIG_DFL. */
+ if (clone_flags & CLONE_CLEAR_SIGHAND)
+ flush_signal_handlers(tsk, 0);
+
return 0;
}
@@ -2563,11 +2568,8 @@ noinline static int copy_clone_args_from_user(struct kernel_clone_args *kargs,
static bool clone3_args_valid(const struct kernel_clone_args *kargs)
{
- /*
- * All lower bits of the flag word are taken.
- * Verify that no other unknown flags are passed along.
- */
- if (kargs->flags & ~CLONE_LEGACY_FLAGS)
+ /* Verify that no unknown flags are passed along. */
+ if (kargs->flags & ~(CLONE_LEGACY_FLAGS | CLONE_CLEAR_SIGHAND))
return false;
/*
@@ -2577,6 +2579,10 @@ static bool clone3_args_valid(const struct kernel_clone_args *kargs)
if (kargs->flags & (CLONE_DETACHED | CSIGNAL))
return false;
+ if ((kargs->flags & (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND)) ==
+ (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND))
+ return false;
+
if ((kargs->flags & (CLONE_THREAD | CLONE_PARENT)) &&
kargs->exit_signal)
return false;
--
2.23.0
These counters will track hugetlb reservations rather than hugetlb
memory faulted in. This patch only adds the counter, following patches
add the charging and uncharging of the counter.
Problem:
Currently tasks attempting to allocate more hugetlb memory than is available get
a failure at mmap/shmget time. This is thanks to Hugetlbfs Reservations [1].
However, if a task attempts to allocate hugetlb memory only more than its
hugetlb_cgroup limit allows, the kernel will allow the mmap/shmget call,
but will SIGBUS the task when it attempts to fault the memory in.
We have developers interested in using hugetlb_cgroups, and they have expressed
dissatisfaction regarding this behavior. We'd like to improve this
behavior such that tasks violating the hugetlb_cgroup limits get an error on
mmap/shmget time, rather than getting SIGBUS'd when they try to fault
the excess memory in.
The underlying problem is that today's hugetlb_cgroup accounting happens
at hugetlb memory *fault* time, rather than at *reservation* time.
Thus, enforcing the hugetlb_cgroup limit only happens at fault time, and
the offending task gets SIGBUS'd.
Proposed Solution:
A new page counter named hugetlb.xMB.reservation_[limit|usage]_in_bytes. This
counter has slightly different semantics than
hugetlb.xMB.[limit|usage]_in_bytes:
- While usage_in_bytes tracks all *faulted* hugetlb memory,
reservation_usage_in_bytes tracks all *reserved* hugetlb memory and
hugetlb memory faulted in without a prior reservation.
- If a task attempts to reserve more memory than limit_in_bytes allows,
the kernel will allow it to do so. But if a task attempts to reserve
more memory than reservation_limit_in_bytes, the kernel will fail this
reservation.
This proposal is implemented in this patch series, with tests to verify
functionality and show the usage. We also added cgroup-v2 support to
hugetlb_cgroup so that the new use cases can be extended to v2.
Alternatives considered:
1. A new cgroup, instead of only a new page_counter attached to
the existing hugetlb_cgroup. Adding a new cgroup seemed like a lot of code
duplication with hugetlb_cgroup. Keeping hugetlb related page counters under
hugetlb_cgroup seemed cleaner as well.
2. Instead of adding a new counter, we considered adding a sysctl that modifies
the behavior of hugetlb.xMB.[limit|usage]_in_bytes, to do accounting at
reservation time rather than fault time. Adding a new page_counter seems
better as userspace could, if it wants, choose to enforce different cgroups
differently: one via limit_in_bytes, and another via
reservation_limit_in_bytes. This could be very useful if you're
transitioning how hugetlb memory is partitioned on your system one
cgroup at a time, for example. Also, someone may find usage for both
limit_in_bytes and reservation_limit_in_bytes concurrently, and this
approach gives them the option to do so.
Testing:
- Added tests passing.
- libhugetlbfs tests mostly passing, but some tests have trouble with and
without this patch series. Seems environment issue rather than code:
- Overall results:
********** TEST SUMMARY
* 2M
* 32-bit 64-bit
* Total testcases: 84 0
* Skipped: 0 0
* PASS: 66 0
* FAIL: 14 0
* Killed by signal: 0 0
* Bad configuration: 4 0
* Expected FAIL: 0 0
* Unexpected PASS: 0 0
* Test not present: 0 0
* Strange test result: 0 0
**********
- Failing tests:
- elflink_rw_and_share_test("linkhuge_rw") segfaults with and without this
patch series.
- LD_PRELOAD=libhugetlbfs.so HUGETLB_MORECORE=yes malloc (2M: 32):
FAIL Address is not hugepage
- LD_PRELOAD=libhugetlbfs.so HUGETLB_RESTRICT_EXE=unknown:malloc
HUGETLB_MORECORE=yes malloc (2M: 32):
FAIL Address is not hugepage
- LD_PRELOAD=libhugetlbfs.so HUGETLB_MORECORE=yes malloc_manysmall (2M: 32):
FAIL Address is not hugepage
- GLIBC_TUNABLES=glibc.malloc.tcache_count=0 LD_PRELOAD=libhugetlbfs.so
HUGETLB_MORECORE=yes heapshrink (2M: 32):
FAIL Heap not on hugepages
- GLIBC_TUNABLES=glibc.malloc.tcache_count=0 LD_PRELOAD=libhugetlbfs.so
libheapshrink.so HUGETLB_MORECORE=yes heapshrink (2M: 32):
FAIL Heap not on hugepages
- HUGETLB_ELFMAP=RW linkhuge_rw (2M: 32): FAIL small_data is not hugepage
- HUGETLB_ELFMAP=RW HUGETLB_MINIMAL_COPY=no linkhuge_rw (2M: 32):
FAIL small_data is not hugepage
- alloc-instantiate-race shared (2M: 32):
Bad configuration: sched_setaffinity(cpu1): Invalid argument -
FAIL Child 1 killed by signal Killed
- shmoverride_linked (2M: 32):
FAIL shmget failed size 2097152 from line 176: Invalid argument
- HUGETLB_SHM=yes shmoverride_linked (2M: 32):
FAIL shmget failed size 2097152 from line 176: Invalid argument
- shmoverride_linked_static (2M: 32):
FAIL shmget failed size 2097152 from line 176: Invalid argument
- HUGETLB_SHM=yes shmoverride_linked_static (2M: 32):
FAIL shmget failed size 2097152 from line 176: Invalid argument
- LD_PRELOAD=libhugetlbfs.so shmoverride_unlinked (2M: 32):
FAIL shmget failed size 2097152 from line 176: Invalid argument
- LD_PRELOAD=libhugetlbfs.so HUGETLB_SHM=yes shmoverride_unlinked (2M: 32):
FAIL shmget failed size 2097152 from line 176: Invalid argument
[1]: https://www.kernel.org/doc/html/latest/vm/hugetlbfs_reserv.html
Signed-off-by: Mina Almasry <almasrymina(a)google.com>
Acked-by: Hillf Danton <hdanton(a)sina.com>
---
include/linux/hugetlb.h | 23 ++++++++-
mm/hugetlb_cgroup.c | 111 ++++++++++++++++++++++++++++++----------
2 files changed, 107 insertions(+), 27 deletions(-)
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 53fc34f930d08..9c49a0ba894d3 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -320,6 +320,27 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
#ifdef CONFIG_HUGETLB_PAGE
+enum {
+ /* Tracks hugetlb memory faulted in. */
+ HUGETLB_RES_USAGE,
+ /* Tracks hugetlb memory reserved. */
+ HUGETLB_RES_RESERVATION_USAGE,
+ /* Limit for hugetlb memory faulted in. */
+ HUGETLB_RES_LIMIT,
+ /* Limit for hugetlb memory reserved. */
+ HUGETLB_RES_RESERVATION_LIMIT,
+ /* Max usage for hugetlb memory faulted in. */
+ HUGETLB_RES_MAX_USAGE,
+ /* Max usage for hugetlb memory reserved. */
+ HUGETLB_RES_RESERVATION_MAX_USAGE,
+ /* Faulted memory accounting fail count. */
+ HUGETLB_RES_FAILCNT,
+ /* Reserved memory accounting fail count. */
+ HUGETLB_RES_RESERVATION_FAILCNT,
+ HUGETLB_RES_NULL,
+ HUGETLB_RES_MAX,
+};
+
#define HSTATE_NAME_LEN 32
/* Defines one hugetlb page size */
struct hstate {
@@ -340,7 +361,7 @@ struct hstate {
unsigned int surplus_huge_pages_node[MAX_NUMNODES];
#ifdef CONFIG_CGROUP_HUGETLB
/* cgroup control files */
- struct cftype cgroup_files[5];
+ struct cftype cgroup_files[HUGETLB_RES_MAX];
#endif
char name[HSTATE_NAME_LEN];
};
diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c
index f1930fa0b445d..1ed4448ca41d3 100644
--- a/mm/hugetlb_cgroup.c
+++ b/mm/hugetlb_cgroup.c
@@ -25,6 +25,10 @@ struct hugetlb_cgroup {
* the counter to account for hugepages from hugetlb.
*/
struct page_counter hugepage[HUGE_MAX_HSTATE];
+ /*
+ * the counter to account for hugepage reservations from hugetlb.
+ */
+ struct page_counter reserved_hugepage[HUGE_MAX_HSTATE];
};
#define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
@@ -33,6 +37,14 @@ struct hugetlb_cgroup {
static struct hugetlb_cgroup *root_h_cgroup __read_mostly;
+static inline struct page_counter *
+hugetlb_cgroup_get_counter(struct hugetlb_cgroup *h_cg, int idx, bool reserved)
+{
+ if (reserved)
+ return &h_cg->reserved_hugepage[idx];
+ return &h_cg->hugepage[idx];
+}
+
static inline
struct hugetlb_cgroup *hugetlb_cgroup_from_css(struct cgroup_subsys_state *s)
{
@@ -254,30 +266,33 @@ void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages,
return;
}
-enum {
- RES_USAGE,
- RES_LIMIT,
- RES_MAX_USAGE,
- RES_FAILCNT,
-};
-
static u64 hugetlb_cgroup_read_u64(struct cgroup_subsys_state *css,
struct cftype *cft)
{
struct page_counter *counter;
+ struct page_counter *reserved_counter;
struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css);
counter = &h_cg->hugepage[MEMFILE_IDX(cft->private)];
+ reserved_counter = &h_cg->reserved_hugepage[MEMFILE_IDX(cft->private)];
switch (MEMFILE_ATTR(cft->private)) {
- case RES_USAGE:
+ case HUGETLB_RES_USAGE:
return (u64)page_counter_read(counter) * PAGE_SIZE;
- case RES_LIMIT:
+ case HUGETLB_RES_RESERVATION_USAGE:
+ return (u64)page_counter_read(reserved_counter) * PAGE_SIZE;
+ case HUGETLB_RES_LIMIT:
return (u64)counter->max * PAGE_SIZE;
- case RES_MAX_USAGE:
+ case HUGETLB_RES_RESERVATION_LIMIT:
+ return (u64)reserved_counter->max * PAGE_SIZE;
+ case HUGETLB_RES_MAX_USAGE:
return (u64)counter->watermark * PAGE_SIZE;
- case RES_FAILCNT:
+ case HUGETLB_RES_RESERVATION_MAX_USAGE:
+ return (u64)reserved_counter->watermark * PAGE_SIZE;
+ case HUGETLB_RES_FAILCNT:
return counter->failcnt;
+ case HUGETLB_RES_RESERVATION_FAILCNT:
+ return reserved_counter->failcnt;
default:
BUG();
}
@@ -291,6 +306,7 @@ static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of,
int ret, idx;
unsigned long nr_pages;
struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of));
+ bool reserved = false;
if (hugetlb_cgroup_is_root(h_cg)) /* Can't set limit on root */
return -EINVAL;
@@ -304,9 +320,14 @@ static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of,
nr_pages = round_down(nr_pages, 1 << huge_page_order(&hstates[idx]));
switch (MEMFILE_ATTR(of_cft(of)->private)) {
- case RES_LIMIT:
+ case HUGETLB_RES_RESERVATION_LIMIT:
+ reserved = true;
+ /* Fall through. */
+ case HUGETLB_RES_LIMIT:
mutex_lock(&hugetlb_limit_mutex);
- ret = page_counter_set_max(&h_cg->hugepage[idx], nr_pages);
+ ret = page_counter_set_max(hugetlb_cgroup_get_counter(h_cg, idx,
+ reserved),
+ nr_pages);
mutex_unlock(&hugetlb_limit_mutex);
break;
default:
@@ -320,18 +341,26 @@ static ssize_t hugetlb_cgroup_reset(struct kernfs_open_file *of,
char *buf, size_t nbytes, loff_t off)
{
int ret = 0;
- struct page_counter *counter;
+ struct page_counter *counter, *reserved_counter;
struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of));
counter = &h_cg->hugepage[MEMFILE_IDX(of_cft(of)->private)];
+ reserved_counter =
+ &h_cg->reserved_hugepage[MEMFILE_IDX(of_cft(of)->private)];
switch (MEMFILE_ATTR(of_cft(of)->private)) {
- case RES_MAX_USAGE:
+ case HUGETLB_RES_MAX_USAGE:
page_counter_reset_watermark(counter);
break;
- case RES_FAILCNT:
+ case HUGETLB_RES_RESERVATION_MAX_USAGE:
+ page_counter_reset_watermark(reserved_counter);
+ break;
+ case HUGETLB_RES_FAILCNT:
counter->failcnt = 0;
break;
+ case HUGETLB_RES_RESERVATION_FAILCNT:
+ reserved_counter->failcnt = 0;
+ break;
default:
ret = -EINVAL;
break;
@@ -357,37 +386,67 @@ static void __init __hugetlb_cgroup_file_init(int idx)
struct hstate *h = &hstates[idx];
/* format the size */
- mem_fmt(buf, 32, huge_page_size(h));
+ mem_fmt(buf, sizeof(buf), huge_page_size(h));
/* Add the limit file */
- cft = &h->cgroup_files[0];
+ cft = &h->cgroup_files[HUGETLB_RES_LIMIT];
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.limit_in_bytes", buf);
- cft->private = MEMFILE_PRIVATE(idx, RES_LIMIT);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_LIMIT);
+ cft->read_u64 = hugetlb_cgroup_read_u64;
+ cft->write = hugetlb_cgroup_write;
+
+ /* Add the reservation limit file */
+ cft = &h->cgroup_files[HUGETLB_RES_RESERVATION_LIMIT];
+ snprintf(cft->name, MAX_CFTYPE_NAME, "%s.reservation_limit_in_bytes",
+ buf);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_RESERVATION_LIMIT);
cft->read_u64 = hugetlb_cgroup_read_u64;
cft->write = hugetlb_cgroup_write;
/* Add the usage file */
- cft = &h->cgroup_files[1];
+ cft = &h->cgroup_files[HUGETLB_RES_USAGE];
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.usage_in_bytes", buf);
- cft->private = MEMFILE_PRIVATE(idx, RES_USAGE);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_USAGE);
+ cft->read_u64 = hugetlb_cgroup_read_u64;
+
+ /* Add the reservation usage file */
+ cft = &h->cgroup_files[HUGETLB_RES_RESERVATION_USAGE];
+ snprintf(cft->name, MAX_CFTYPE_NAME, "%s.reservation_usage_in_bytes",
+ buf);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_RESERVATION_USAGE);
cft->read_u64 = hugetlb_cgroup_read_u64;
/* Add the MAX usage file */
- cft = &h->cgroup_files[2];
+ cft = &h->cgroup_files[HUGETLB_RES_MAX_USAGE];
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.max_usage_in_bytes", buf);
- cft->private = MEMFILE_PRIVATE(idx, RES_MAX_USAGE);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_MAX_USAGE);
+ cft->write = hugetlb_cgroup_reset;
+ cft->read_u64 = hugetlb_cgroup_read_u64;
+
+ /* Add the MAX reservation usage file */
+ cft = &h->cgroup_files[HUGETLB_RES_RESERVATION_MAX_USAGE];
+ snprintf(cft->name, MAX_CFTYPE_NAME,
+ "%s.reservation_max_usage_in_bytes", buf);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_RESERVATION_MAX_USAGE);
cft->write = hugetlb_cgroup_reset;
cft->read_u64 = hugetlb_cgroup_read_u64;
/* Add the failcntfile */
- cft = &h->cgroup_files[3];
+ cft = &h->cgroup_files[HUGETLB_RES_FAILCNT];
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.failcnt", buf);
- cft->private = MEMFILE_PRIVATE(idx, RES_FAILCNT);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_FAILCNT);
+ cft->write = hugetlb_cgroup_reset;
+ cft->read_u64 = hugetlb_cgroup_read_u64;
+
+ /* Add the reservation failcntfile */
+ cft = &h->cgroup_files[HUGETLB_RES_RESERVATION_FAILCNT];
+ snprintf(cft->name, MAX_CFTYPE_NAME, "%s.reservation_failcnt", buf);
+ cft->private = MEMFILE_PRIVATE(idx, HUGETLB_RES_RESERVATION_FAILCNT);
cft->write = hugetlb_cgroup_reset;
cft->read_u64 = hugetlb_cgroup_read_u64;
/* NULL terminate the last cft */
- cft = &h->cgroup_files[4];
+ cft = &h->cgroup_files[HUGETLB_RES_NULL];
memset(cft, 0, sizeof(*cft));
WARN_ON(cgroup_add_legacy_cftypes(&hugetlb_cgrp_subsys,
--
2.23.0.700.g56cf767bdb-goog
The ifndef for SECCOMP_USER_NOTIF_FLAG_CONTINUE was placed under the
ifndef for the SECCOMP_FILTER_FLAG_NEW_LISTENER feature. This will not
work on systems that do support SECCOMP_FILTER_FLAG_NEW_LISTENER but do not
support SECCOMP_USER_NOTIF_FLAG_CONTINUE. So move the latter ifndef out of
the former ifndef's scope.
2019-10-20 11:14:01 make run_tests -C seccomp
make: Entering directory '/usr/src/perf_selftests-x86_64-rhel-7.6-0eebfed2954f152259cae0ad57b91d3ea92968e8/tools/testing/selftests/seccomp'
gcc -Wl,-no-as-needed -Wall seccomp_bpf.c -lpthread -o seccomp_bpf
seccomp_bpf.c: In function ‘user_notification_continue’:
seccomp_bpf.c:3562:15: error: ‘SECCOMP_USER_NOTIF_FLAG_CONTINUE’ undeclared (first use in this function)
resp.flags = SECCOMP_USER_NOTIF_FLAG_CONTINUE;
^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
seccomp_bpf.c:3562:15: note: each undeclared identifier is reported only once for each function it appears in
Makefile:12: recipe for target 'seccomp_bpf' failed
make: *** [seccomp_bpf] Error 1
make: Leaving directory '/usr/src/perf_selftests-x86_64-rhel-7.6-0eebfed2954f152259cae0ad57b91d3ea92968e8/tools/testing/selftests/seccomp'
Reported-by: kernel test robot <rong.a.chen(a)intel.com>
Fixes: 0eebfed2954f ("seccomp: test SECCOMP_USER_NOTIF_FLAG_CONTINUE")
Cc: linux-kselftest(a)vger.kernel.org
Signed-off-by: Christian Brauner <christian.brauner(a)ubuntu.com>
---
tools/testing/selftests/seccomp/seccomp_bpf.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/tools/testing/selftests/seccomp/seccomp_bpf.c b/tools/testing/selftests/seccomp/seccomp_bpf.c
index 2519377ebda3..9669b81086cf 100644
--- a/tools/testing/selftests/seccomp/seccomp_bpf.c
+++ b/tools/testing/selftests/seccomp/seccomp_bpf.c
@@ -168,10 +168,6 @@ struct seccomp_metadata {
#define SECCOMP_RET_USER_NOTIF 0x7fc00000U
-#ifndef SECCOMP_USER_NOTIF_FLAG_CONTINUE
-#define SECCOMP_USER_NOTIF_FLAG_CONTINUE 0x00000001
-#endif
-
#define SECCOMP_IOC_MAGIC '!'
#define SECCOMP_IO(nr) _IO(SECCOMP_IOC_MAGIC, nr)
#define SECCOMP_IOR(nr, type) _IOR(SECCOMP_IOC_MAGIC, nr, type)
@@ -205,6 +201,10 @@ struct seccomp_notif_sizes {
};
#endif
+#ifndef SECCOMP_USER_NOTIF_FLAG_CONTINUE
+#define SECCOMP_USER_NOTIF_FLAG_CONTINUE 0x00000001
+#endif
+
#ifndef seccomp
int seccomp(unsigned int op, unsigned int flags, void *args)
{
--
2.23.0
Hi,
Here are some patches to fix some warnings/issues on 32bit arch
(e.g. arm).
When I built the ksefltest on arm, I hit some 32bit related warnings.
Here are the patches to fix those issues.
- [1/5] va_max was set 2^32 even on 32bit arch. This can make
va_max == 0 and always fail. Make it 3GB on 32bit.
- [2/5] Some VM tests requires 64bit user space, which should
not run on 32bit arch.
- [3/5] For counting the size of large file, we should use
size_t instead of unsinged long.
- [4/5] Gcc warns printf format for size_t and int64_t on
32bit arch. Use %llu and cast it.
- [5/5] Gcc warns __u64 and pointer type castings. It should
once translated to unsigned long.
Thank you,
---
Masami Hiramatsu (5):
selftests: proc: Make va_max 3GB on 32bit arch
selftests: vm: Build/Run 64bit tests only on 64bit arch
selftests: net: Use size_t and ssize_t for counting file size
selftests: net: Fix printf format warnings on arm
selftests: sync: Fix cast warnings on arm
tools/testing/selftests/net/so_txtime.c | 4 ++--
tools/testing/selftests/net/tcp_mmap.c | 8 ++++----
tools/testing/selftests/net/udpgso.c | 3 ++-
tools/testing/selftests/net/udpgso_bench_tx.c | 3 ++-
.../selftests/proc/proc-self-map-files-002.c | 11 ++++++++++-
tools/testing/selftests/sync/sync.c | 6 +++---
tools/testing/selftests/vm/Makefile | 5 +++++
tools/testing/selftests/vm/run_vmtests | 10 ++++++++++
8 files changed, 38 insertions(+), 12 deletions(-)
--
Masami Hiramatsu (Linaro) <mhiramat(a)kernel.org>
Hi All.
The patch 5c069b6dedef "selftests: Move test output to diagnostic lines"
from Apr 24, 2019,
leads to `make run_tests -C bpf` hanging forever.
Bpf includes many subtest, when cmd `make run_tests -C bpf` runs to
test_lwt_seg6local.sh, task will hang and runner.sh never run next task.
I checked ps aux, prefix.pl will never exit.
```
91058 [ 811.451584] # [25] VAR __license type_id=24 linkage=1
91059 [ 811.451586]-
91060 [ 811.455365] # [26] DATASEC license size=0 vlen=1 size == 0
91061 [ 811.455367]-
91062 [ 811.457424] #-
91063 [ 811.457425]-
91064 [ 811.460912] # selftests: test_lwt_seg6local [PASS]
91065 [ 811.460914]-
91066 [ 3620.461986] Thu Oct 17 14:54:05 CST 2019 detected soft_timeout
```
Ignore test_lwt_seg6local and run `make run_tests -C bpf` again, task
will hang on test_tc_tunnel.sh.
Kushwaha also meet this issue, `make run_tests -C bpf` hang on
test_lwt_ip_encap.sh (This test failed on my localhost).
--
Best regards.
Liu Yiding
Hi,friend,
This is Daniel Murray and i am purchasing manager from Sinara Group Co.,LTD in Russia.
We are glad to know about your company from the web and we are interested in your products.
Could you kindly send us your Latest catalog and price list for our trial order.
Thanks and Best Regards,
Daniel Murray
Purchasing Manager
Sinara Group Co.,LTD
Hi All,
I am trying to build kselftest on Linux-5.4-rc3+ on ubuntu 18.04. I
installed LLVM-9.0.0 and Clang-9.0.0 from below links after following
steps from [1] because of discussion [2]
https://releases.llvm.org/9.0.0/llvm-9.0.0.src.tar.xzhttps://releases.llvm.org/9.0.0/clang-tools-extra-9.0.0.src.tar.xzhttps://releases.llvm.org/9.0.0/cfe-9.0.0.src.tar.xz
After that I started this error.
make[2]: Leaving directory '/usr/src/tovards/linux/tools/lib/bpf'
(clang -I. -I./include/uapi -I../../../include/uapi
-I/usr/src/tovards/linux/tools/testing/selftests/bpf/../usr/include
-D__TARGET_ARCH_arm64 -idirafter /usr/local/include -idirafter
/usr/local/lib/clang/9.0.0/include -idirafter
/usr/include/aarch64-linux-gnu -idirafter /usr/include
-Wno-compare-distinct-pointer-types -O2 -target bpf -emit-llvm \
-c progs/test_core_reloc_ints.c -o - || echo "clang failed") | \
llc -march=arm64 -mcpu=generic -filetype=obj -o
/usr/src/tovards/linux/tools/testing/selftests/bpf/test_core_reloc_ints.o
progs/test_core_reloc_ints.c:32:6: error: using
builtin_preserve_access_index() without -g
if (BPF_CORE_READ(&out->u8_field, &in->u8_field) ||
^
./bpf_helpers.h:533:10: note: expanded from macro 'BPF_CORE_READ'
__builtin_preserve_access_index(src))
^
progs/test_core_reloc_ints.c:33:6: error: using
builtin_preserve_access_index() without -g
BPF_CORE_READ(&out->s8_field, &in->s8_field) ||
^
./bpf_helpers.h:533:10: note: expanded from macro 'BPF_CORE_READ'
__builtin_preserve_access_index(src))
^
progs/test_core_reloc_ints.c:34:6: error: using
builtin_preserve_access_index() without -g
BPF_CORE_READ(&out->u16_field, &in->u16_field) ||
^
./bpf_helpers.h:533:10: note: expanded from macro 'BPF_CORE_READ'
__builtin_preserve_access_index(src))
^
progs/test_core_reloc_ints.c:35:6: error: using
builtin_preserve_access_index() without -g
BPF_CORE_READ(&out->s16_field, &in->s16_field) ||
^
./bpf_helpers.h:533:10: note: expanded from macro 'BPF_CORE_READ'
__builtin_preserve_access_index(src))
^
progs/test_core_reloc_ints.c:36:6: error: using
builtin_preserve_access_index() without -g
BPF_CORE_READ(&out->u32_field, &in->u32_field) ||
^
./bpf_helpers.h:533:10: note: expanded from macro 'BPF_CORE_READ'
__builtin_preserve_access_index(src))
^
progs/test_core_reloc_ints.c:37:6: error: using
builtin_preserve_access_index() without -g
BPF_CORE_READ(&out->s32_field, &in->s32_field) ||
^
./bpf_helpers.h:533:10: note: expanded from macro 'BPF_CORE_READ'
__builtin_preserve_access_index(src))
^
progs/test_core_reloc_ints.c:38:6: error: using
builtin_preserve_access_index() without -g
BPF_CORE_READ(&out->u64_field, &in->u64_field) ||
^
./bpf_helpers.h:533:10: note: expanded from macro 'BPF_CORE_READ'
__builtin_preserve_access_index(src))
^
progs/test_core_reloc_ints.c:39:6: error: using
builtin_preserve_access_index() without -g
BPF_CORE_READ(&out->s64_field, &in->s64_field))
^
./bpf_helpers.h:533:10: note: expanded from macro 'BPF_CORE_READ'
__builtin_preserve_access_index(src))
^
8 errors generated.
llc: error: llc: <stdin>:1:1: error: expected top-level entity
clang failed
In order to solve this error, I modifed bpf/Makefile as
CLANG_CFLAGS = $(CLANG_SYS_INCLUDES) \
- -Wno-compare-distinct-pointer-types
+ -Wno-compare-distinct-pointer-types -g
Now I am getting this error
(clang -I. -I./include/uapi -I../../../include/uapi
-I/usr/src/tovards/linux/tools/testing/selftests/bpf/../usr/include
-D__TARGET_ARCH_arm64 -idirafter /usr/local/include -idirafter
/usr/local/lib/clang/9.0.0/include -idirafter
/usr/include/aarch64-linux-gnu -idirafter /usr/include
-Wno-compare-distinct-pointer-types -g -O2 -target bpf -emit-llvm \
-c progs/test_core_reloc_ints.c -o - || echo "clang failed") | \
llc -march=arm64 -mcpu=generic -filetype=obj -o
/usr/src/tovards/linux/tools/testing/selftests/bpf/test_core_reloc_ints.o
LLVM ERROR: Cannot select: intrinsic %llvm.preserve.struct.access.index
Makefile:267: recipe for target
'/usr/src/tovards/linux/tools/testing/selftests/bpf/test_core_reloc_ints.o'
failed
make[1]: ***
[/usr/src/tovards/linux/tools/testing/selftests/bpf/test_core_reloc_ints.o]
Error 1
Please suggest!!
--prabhakar(pk)
[1]
https://stackoverflow.com/questions/47255526/how-to-build-the-latest-clang-…
[2] https://www.mail-archive.com/netdev@vger.kernel.org/msg315096.html
Linux top-commit
----------------
commit bc88f85c6c09306bd21917e1ae28205e9cd775a7 (HEAD -> master,
origin/master, origin/HEAD)
Author: Ben Dooks <ben.dooks(a)codethink.co.uk>
Date: Wed Oct 16 12:24:58 2019 +0100
kthread: make __kthread_queue_delayed_work static
The __kthread_queue_delayed_work is not exported so
make it static, to avoid the following sparse warning:
kernel/kthread.c:869:6: warning: symbol
'__kthread_queue_delayed_work' was not declared. Should it be static?
Signed-off-by: Ben Dooks <ben.dooks(a)codethink.co.uk>
Signed-off-by: Linus Torvalds <torvalds(a)linux-foundation.org>
Hey Knut and Shuah,
Following up on our offline discussion on Wednesday night:
We decided that it would make sense for Knut to try to implement Hybrid
Testing (testing that crosses the kernel userspace boundary) that he
introduced here[1] on top of the existing KUnit infrastructure.
We discussed several possible things in the kernel that Knut could test
with the new Hybrid Testing feature as an initial example. Those were
(in reverse order of expected difficulty):
1. RDS (Reliable Datagram Sockets) - We decided that, although this was
one of the more complicated subsystems to work with, it was probably
the best candidate for Knut to start with because it was in desperate
need of better testing, much of the testing would require crossing
the kernel userspace boundary to be effective, and Knut has access to
RDS (since he works at Oracle).
2. KMOD - Probably much simpler than RDS, and the maintainer, Luis
Chamberlain (CC'ed) would like to see better testing here, but
probably still not as good as RDS because it is in less dire need of
testing, collaboration on this would be more difficult, and Luis is
currently on an extended vacation. Luis and I had already been
discussing testing KMOD here[2].
3. IP over USB - Least desirable option, but still possible. More
complicated than KMOD, and not as easy to collaborate on as RDS.
I don't really think we discussed how this would work. I remember that I
mentioned that it would be easier if I sent out a patch that
centralizes where KUnit tests are dispatched from in the kernel; I will
try to get an RFC for that out, probably sometime next week. That should
provide a pretty straightforward place for Knut to move his work on top
of.
The next question is what the userspace component of this should look
like. To me it seems like we should probably have the kselftest test
runner manage when the test gets run, and collecting and reporting the
result of the test, but I think Knut has thought more about this than I,
and Shuah is the kselftest maintainer, so I am guessing this will
probably mostly be a discussion between the two of you.
So I think we have a couple of TODOs between us:
Brendan:
- Need to send out patch that provides a single place where all tests
are dispatched from.
Knut:
- Start splitting out the hybrid test stuff from the rest of the RFC
you sent previously.
Knut and Shuah:
- Start figuring out what the userspace component of this will look
like.
Cheers!
[1] https://lore.kernel.org/linux-kselftest/524b4e062500c6a240d4d7c0e1d0a299680…
[2] https://groups.google.com/forum/#!topic/kunit-dev/CdIytJtii00
Sorry for the delayed reply. I was on vacation.
On Fri, Oct 11, 2019 at 5:16 AM Theodore Ts'o <theodore.tso(a)gmail.com> wrote:
>
>
>
> On Friday, October 11, 2019 at 7:19:49 AM UTC-4, Brendan Higgins wrote:
>>
>>
>> Should we maybe drop `--build_dir` in favor of `O`?
>
>
> How about if "make kunit" results in "./tools/testing/kunit/kunit.py run --build_dir=/.kunit --allconfig"
>
> ... where --allconfig automatically creates kunitconfig but in includes all of the CONFIG options which depends on CONFIG_KUNIT, so that all unit tests are run? That way, we make it super easy for people to run the unit tests. Since most users are used using make targets, this I bet will significantly increase the number of developers using kunit, because it will be super-duper convenient for them.
>
> Also, it would be nice if kunit.py first looks for kunitconfig in build_dir, and then in the top-level of the kernel sources, and we put .kunit in .gitignore. That will make "git status" look nice and clean.
>
> What do folks think?
Having something like --allconfig is the ultimate goal. I had been
talking to Luis and Shuah about this for some time.
I think the best way to make this work would be for kunit_tool to be
able to detect all the tests with CONFIG_KUNIT as you suggest (or
something like it). Luis actually already suggested it; however, we
identified that this would likely not be as easy as it sounds as it is
possible to have mutually exclusive CONFIGs. Luis pointed out that
some researchers are currently working on a sat solver for Kconfig
that we could use to potentially address this problem. Nevertheless, a
complete solution in this regard is actually somewhat difficult.
Shuah's solution was just to use CONFIG fragments in the meantime
similar to what kselftest already does. I was leaning in that
direction since kselftest already does that and we know that it works.
Shuah, Luis, does this still match what you have been thinking?
+open list:KERNEL SELFTEST FRAMEWORK In case anyone in kselftest has
any thoughts.
On Thu, Oct 10, 2019 at 7:05 PM Theodore Ts'o <theodore.tso(a)gmail.com> wrote:
>
> I've been experimenting with the ext4 kunit test case, and something that would be really helpful is if the default is to store the object files for the ARCM=um kernel and its .config file in the top-level directory .kunit. That is, that the default for --build_dir should be .kunit.
>
> Why does this important? Because the kernel developer will want to be running unit tests as well as building kernels that can be run under whatever architecture they are normally developing for (for example, an x86 kernel that can be run using kvm; or a arm64 kernel that gets run on an Android device by using the "fastboot" command). So that means we don't want to be overwriting the object files and .config files for building the kernel for x86 when building the kunit kernel using the um arch. For example, for ext4, my ideal workflow might go something like this:
That's a good point.
> <hack hack hack>
> % ./tools/testing/kunit/kunit.py run
> <watch to see that unit tests succeed, and since most of the object files have already been built for the kunit kernel in be stored in the .kunit directory, this will be fast, since only the modified files will need to be recompiled>
> % kbuild
> <this is a script that builds an x86 kernel in /build/ext4-64 that is designed to be run under either kvm or in a GCE VM; since the kunit object files are stored in /build/ext4-kunit, the pre-existing files when building for x86_64 haven't been disturbed, so this build is fast as well>
> % kvm-xfstests smoke
> <this will run xfstests using the kernel plucked from /build/ext-64, using kvm>
>
> The point is when I'm developing an ext4 feature, or reviewing and merging ext4 commits, I need to be able to maintain separate build trees and separate config files for ARCH=um as well as ARCH=x86_64, and if the ARCH=um are stored in the kernel sources, then building with O=... doesn't work:
>
> <tytso@lambda> {/usr/projects/linux/kunit} (kunit)
> 1084% make O=/build/test-dir
> make[1]: Entering directory '/build/test-dir'
> ***
> *** The source tree is not clean, please run 'make mrproper'
> *** in /usr/projects/linux/kunit
> ***
Should we maybe drop `--build_dir` in favor of `O`?
> One of the other reasons why it would be good to use --build_dir by default is that way, building with a separate O= build directory is regularly tested. Right now, "kunit.py --build_dir=" seems to be broken.
Good point.
> % ./tools/testing/kunit/kunit.py run --build_dir=/build/ext4-kunit
> Generating .config ...
> [22:04:12] Building KUnit Kernel ...
> /usr/projects/linux/kunit/arch/x86/um/user-offsets.c:20:10: fatal error: asm/syscalls_64.h: No such file or directory
> 20 | #include <asm/syscalls_64.h>
> | ^~~~~~~~~~~~~~~~~~~
> compilation terminated.
>
> (This appears to be an ARCH=um bug, not a kunit bug, though.)
Yeah, I encountered this before. Some file is not getting properly
cleaned up by `make mrproper`. It works if you do `git clean -fdx` (I
know that's not a real solution for most people). Nevertheless, it
sounds like we need to sit down and actually solve this problem since
it is affecting users now.
I think you make a compelling argument. Anyone else have any thoughts on this?
From: Kees Cook <keescook(a)chromium.org>
[ Upstream commit 852c8cbf34d3b3130a05c38064dd98614f97d3a8 ]
Commit a745f7af3cbd ("selftests/harness: Add 30 second timeout per
test") solves the problem of kselftest_harness.h-using binary tests
possibly hanging forever. However, scripts and other binaries can still
hang forever. This adds a global timeout to each test script run.
To make this configurable (e.g. as needed in the "rtc" test case),
include a new per-test-directory "settings" file (similar to "config")
that can contain kselftest-specific settings. The first recognized field
is "timeout".
Additionally, this splits the reporting for timeouts into a specific
"TIMEOUT" not-ok (and adds exit code reporting in the remaining case).
Signed-off-by: Kees Cook <keescook(a)chromium.org>
Signed-off-by: Shuah Khan <skhan(a)linuxfoundation.org>
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
---
tools/testing/selftests/kselftest/runner.sh | 36 +++++++++++++++++++--
tools/testing/selftests/rtc/settings | 1 +
2 files changed, 34 insertions(+), 3 deletions(-)
create mode 100644 tools/testing/selftests/rtc/settings
diff --git a/tools/testing/selftests/kselftest/runner.sh b/tools/testing/selftests/kselftest/runner.sh
index 00c9020bdda8b..84de7bc74f2cf 100644
--- a/tools/testing/selftests/kselftest/runner.sh
+++ b/tools/testing/selftests/kselftest/runner.sh
@@ -3,9 +3,14 @@
#
# Runs a set of tests in a given subdirectory.
export skip_rc=4
+export timeout_rc=124
export logfile=/dev/stdout
export per_test_logging=
+# Defaults for "settings" file fields:
+# "timeout" how many seconds to let each test run before failing.
+export kselftest_default_timeout=45
+
# There isn't a shell-agnostic way to find the path of a sourced file,
# so we must rely on BASE_DIR being set to find other tools.
if [ -z "$BASE_DIR" ]; then
@@ -24,6 +29,16 @@ tap_prefix()
fi
}
+tap_timeout()
+{
+ # Make sure tests will time out if utility is available.
+ if [ -x /usr/bin/timeout ] ; then
+ /usr/bin/timeout "$kselftest_timeout" "$1"
+ else
+ "$1"
+ fi
+}
+
run_one()
{
DIR="$1"
@@ -32,6 +47,18 @@ run_one()
BASENAME_TEST=$(basename $TEST)
+ # Reset any "settings"-file variables.
+ export kselftest_timeout="$kselftest_default_timeout"
+ # Load per-test-directory kselftest "settings" file.
+ settings="$BASE_DIR/$DIR/settings"
+ if [ -r "$settings" ] ; then
+ while read line ; do
+ field=$(echo "$line" | cut -d= -f1)
+ value=$(echo "$line" | cut -d= -f2-)
+ eval "kselftest_$field"="$value"
+ done < "$settings"
+ fi
+
TEST_HDR_MSG="selftests: $DIR: $BASENAME_TEST"
echo "# $TEST_HDR_MSG"
if [ ! -x "$TEST" ]; then
@@ -44,14 +71,17 @@ run_one()
echo "not ok $test_num $TEST_HDR_MSG"
else
cd `dirname $TEST` > /dev/null
- (((((./$BASENAME_TEST 2>&1; echo $? >&3) |
+ ((((( tap_timeout ./$BASENAME_TEST 2>&1; echo $? >&3) |
tap_prefix >&4) 3>&1) |
(read xs; exit $xs)) 4>>"$logfile" &&
echo "ok $test_num $TEST_HDR_MSG") ||
- (if [ $? -eq $skip_rc ]; then \
+ (rc=$?; \
+ if [ $rc -eq $skip_rc ]; then \
echo "not ok $test_num $TEST_HDR_MSG # SKIP"
+ elif [ $rc -eq $timeout_rc ]; then \
+ echo "not ok $test_num $TEST_HDR_MSG # TIMEOUT"
else
- echo "not ok $test_num $TEST_HDR_MSG"
+ echo "not ok $test_num $TEST_HDR_MSG # exit=$rc"
fi)
cd - >/dev/null
fi
diff --git a/tools/testing/selftests/rtc/settings b/tools/testing/selftests/rtc/settings
new file mode 100644
index 0000000000000..ba4d85f74cd6b
--- /dev/null
+++ b/tools/testing/selftests/rtc/settings
@@ -0,0 +1 @@
+timeout=90
--
2.20.1
From: Cristian Marussi <cristian.marussi(a)arm.com>
[ Upstream commit 131b30c94fbc0adb15f911609884dd39dada8f00 ]
A TARGET which failed to be built/installed should not be included in the
runlist generated inside the run_kselftest.sh script.
Signed-off-by: Cristian Marussi <cristian.marussi(a)arm.com>
Signed-off-by: Shuah Khan <skhan(a)linuxfoundation.org>
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
---
tools/testing/selftests/Makefile | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
index 25b43a8c2b159..1779923d7a7b8 100644
--- a/tools/testing/selftests/Makefile
+++ b/tools/testing/selftests/Makefile
@@ -198,8 +198,12 @@ ifdef INSTALL_PATH
echo " cat /dev/null > \$$logfile" >> $(ALL_SCRIPT)
echo "fi" >> $(ALL_SCRIPT)
+ @# While building run_kselftest.sh skip also non-existent TARGET dirs:
+ @# they could be the result of a build failure and should NOT be
+ @# included in the generated runlist.
for TARGET in $(TARGETS); do \
BUILD_TARGET=$$BUILD/$$TARGET; \
+ [ ! -d $$INSTALL_PATH/$$TARGET ] && echo "Skipping non-existent dir: $$TARGET" && continue; \
echo "[ -w /dev/kmsg ] && echo \"kselftest: Running tests in $$TARGET\" >> /dev/kmsg" >> $(ALL_SCRIPT); \
echo "cd $$TARGET" >> $(ALL_SCRIPT); \
echo -n "run_many" >> $(ALL_SCRIPT); \
--
2.20.1
Currently, when a task is dead we still print the pid it used to use in
the fdinfo files of its pidfds. This doesn't make much sense since the
pid may have already been reused. So verify that the task is still
alive. If the task is not alive anymore, we will print -1. This allows
us to differentiate between a task not being present in a given pid
namespace - in which case we already print 0 - and a task having been
reaped.
Note that this uses PIDTYPE_PID for the check. Technically, we could've
checked PIDTYPE_TGID since pidfds currently only refer to thread-group
leaders but if they won't anymore in the future then this check becomes
problematic without it being immediately obvious to non-experts imho. If
a thread is created via clone(CLONE_THREAD) than struct pid has a single
non-empty list pid->tasks[PIDTYPE_PID] and this pid can't be used as a
PIDTYPE_TGID meaning pid->tasks[PIDTYPE_TGID] will return NULL even
though the thread-group leader might still be very much alive. We could
be more complicated and do something like:
bool alive = false;
rcu_read_lock();
struct task_struct *tsk = pid_task(pid, PIDTYPE_PID);
if (tsk && task_tgid(tsk))
alive = true;
rcu_read_unlock();
but it's really not worth it. We already have created a pidfd and we
thus know it refers to a thread-group leader. Checking PIDTYPE_PID is
fine and is easier to maintain should we ever allow pidfds to refer to
threads.
Cc: Jann Horn <jannh(a)google.com>
Cc: Christian Kellner <christian(a)kellner.me>
Cc: Oleg Nesterov <oleg(a)redhat.com>
Cc: linux-api(a)vger.kernel.org
Signed-off-by: Christian Brauner <christian.brauner(a)ubuntu.com>
---
kernel/fork.c | 29 +++++++++++++++++++++++------
1 file changed, 23 insertions(+), 6 deletions(-)
diff --git a/kernel/fork.c b/kernel/fork.c
index 782986962d47..a67944a5e542 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1695,6 +1695,18 @@ static int pidfd_release(struct inode *inode, struct file *file)
}
#ifdef CONFIG_PROC_FS
+static inline bool task_alive(struct pid *pid)
+{
+ bool alive = true;
+
+ rcu_read_lock();
+ if (!pid_task(pid, PIDTYPE_PID))
+ alive = false;
+ rcu_read_unlock();
+
+ return alive;
+}
+
/**
* pidfd_show_fdinfo - print information about a pidfd
* @m: proc fdinfo file
@@ -1732,15 +1744,20 @@ static int pidfd_release(struct inode *inode, struct file *file)
*/
static void pidfd_show_fdinfo(struct seq_file *m, struct file *f)
{
- struct pid_namespace *ns = proc_pid_ns(file_inode(m->file));
struct pid *pid = f->private_data;
- pid_t nr = pid_nr_ns(pid, ns);
+ struct pid_namespace *ns;
+ pid_t nr = -1;
+
+ if (likely(task_alive(pid))) {
+ ns = proc_pid_ns(file_inode(m->file));
+ nr = pid_nr_ns(pid, ns);
+ }
- seq_put_decimal_ull(m, "Pid:\t", nr);
+ seq_put_decimal_ll(m, "Pid:\t", nr);
#ifdef CONFIG_PID_NS
- seq_put_decimal_ull(m, "\nNSpid:\t", nr);
- if (nr) {
+ seq_put_decimal_ll(m, "\nNSpid:\t", nr);
+ if (nr > 0) {
int i;
/* If nr is non-zero it means that 'pid' is valid and that
@@ -1749,7 +1766,7 @@ static void pidfd_show_fdinfo(struct seq_file *m, struct file *f)
* Start at one below the already printed level.
*/
for (i = ns->level + 1; i <= pid->level; i++)
- seq_put_decimal_ull(m, "\t", pid->numbers[i].nr);
+ seq_put_decimal_ll(m, "\t", pid->numbers[i].nr);
}
#endif
seq_putc(m, '\n');
--
2.23.0
Add kselftest-all target to build tests from the top level
Makefile. This is to simplify kselftest use-cases for CI and
distributions where build and test systems are different.
Current kselftest target builds and runs tests on a development
system which is a developer use-case.
Add kselftest-install target to install tests from the top level
Makefile. This is to simplify kselftest use-cases for CI and
distributions where build and test systems are different.
This change addresses requests from developers and testers to add
support for installing kselftest from the main Makefile.
In addition, make the install directory the same when install is
run using "make kselftest-install" or by running kselftest_install.sh.
Also fix the INSTALL_PATH variable conflict between main Makefile and
selftests Makefile.
Signed-off-by: Shuah Khan <skhan(a)linuxfoundation.org>
---
Changes since v1:
- Collpased two patches that added separate targets to
build and install into one patch using pattern rule to
invoke all, install, and clean targets from main Makefile.
Makefile | 5 ++---
tools/testing/selftests/Makefile | 8 ++++++--
tools/testing/selftests/kselftest_install.sh | 4 ++--
3 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/Makefile b/Makefile
index d456746da347..ec296c60c1af 100644
--- a/Makefile
+++ b/Makefile
@@ -1237,9 +1237,8 @@ PHONY += kselftest
kselftest:
$(Q)$(MAKE) -C $(srctree)/tools/testing/selftests run_tests
-PHONY += kselftest-clean
-kselftest-clean:
- $(Q)$(MAKE) -C $(srctree)/tools/testing/selftests clean
+kselftest-%: FORCE
+ $(Q)$(MAKE) -C $(srctree)/tools/testing/selftests $*
PHONY += kselftest-merge
kselftest-merge:
diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
index c3feccb99ff5..bad18145ed1a 100644
--- a/tools/testing/selftests/Makefile
+++ b/tools/testing/selftests/Makefile
@@ -171,9 +171,12 @@ run_pstore_crash:
# 1. output_dir=kernel_src
# 2. a separate output directory is specified using O= KBUILD_OUTPUT
# 3. a separate output directory is specified using KBUILD_OUTPUT
+# Avoid conflict with INSTALL_PATH set by the main Makefile
#
-INSTALL_PATH ?= $(BUILD)/install
-INSTALL_PATH := $(abspath $(INSTALL_PATH))
+KSFT_INSTALL_PATH ?= $(BUILD)/kselftest_install
+KSFT_INSTALL_PATH := $(abspath $(KSFT_INSTALL_PATH))
+# Avoid changing the rest of the logic here and lib.mk.
+INSTALL_PATH := $(KSFT_INSTALL_PATH)
ALL_SCRIPT := $(INSTALL_PATH)/run_kselftest.sh
install: all
@@ -203,6 +206,7 @@ ifdef INSTALL_PATH
echo "[ -w /dev/kmsg ] && echo \"kselftest: Running tests in $$TARGET\" >> /dev/kmsg" >> $(ALL_SCRIPT); \
echo "cd $$TARGET" >> $(ALL_SCRIPT); \
echo -n "run_many" >> $(ALL_SCRIPT); \
+ echo -n "Emit Tests for $$TARGET\n"; \
$(MAKE) -s --no-print-directory OUTPUT=$$BUILD_TARGET -C $$TARGET emit_tests >> $(ALL_SCRIPT); \
echo "" >> $(ALL_SCRIPT); \
echo "cd \$$ROOT" >> $(ALL_SCRIPT); \
diff --git a/tools/testing/selftests/kselftest_install.sh b/tools/testing/selftests/kselftest_install.sh
index ec304463883c..e2e1911d62d5 100755
--- a/tools/testing/selftests/kselftest_install.sh
+++ b/tools/testing/selftests/kselftest_install.sh
@@ -24,12 +24,12 @@ main()
echo "$0: Installing in specified location - $install_loc ..."
fi
- install_dir=$install_loc/kselftest
+ install_dir=$install_loc/kselftest_install
# Create install directory
mkdir -p $install_dir
# Build tests
- INSTALL_PATH=$install_dir make install
+ KSFT_INSTALL_PATH=$install_dir make install
}
main "$@"
--
2.20.1
Livepatch uses ftrace for redirection to new patched functions. It means
that if ftrace is disabled, all live patched functions are disabled as
well. Toggling global 'ftrace_enabled' sysctl thus affect it directly.
It is not a problem per se, because only administrator can set sysctl
values, but it still may be surprising.
Introduce PERMANENT ftrace_ops flag to amend this. If the
FTRACE_OPS_FL_PERMANENT is set on any ftrace ops, the tracing cannot be
disabled by disabling ftrace_enabled. Equally, a callback with the flag
set cannot be registered if ftrace_enabled is disabled.
v2->v3:
- ftrace_enabled explicitly set to true
- selftest from Joe Lawrence (I just split it to two patches)
- typo fix
v1->v2:
- different logic, proposed by Joe Lawrence
Joe Lawrence (2):
selftests/livepatch: Make dynamic debug setup and restore generic
selftests/livepatch: Test interaction with ftrace_enabled
Miroslav Benes (1):
ftrace: Introduce PERMANENT ftrace_ops flag
Documentation/trace/ftrace-uses.rst | 8 +++
Documentation/trace/ftrace.rst | 4 +-
include/linux/ftrace.h | 3 +
kernel/livepatch/patch.c | 3 +-
kernel/trace/ftrace.c | 23 ++++++-
tools/testing/selftests/livepatch/Makefile | 3 +-
.../testing/selftests/livepatch/functions.sh | 34 +++++++---
.../selftests/livepatch/test-callbacks.sh | 2 +-
.../selftests/livepatch/test-ftrace.sh | 65 +++++++++++++++++++
.../selftests/livepatch/test-livepatch.sh | 2 +-
.../selftests/livepatch/test-shadow-vars.sh | 2 +-
11 files changed, 132 insertions(+), 17 deletions(-)
create mode 100755 tools/testing/selftests/livepatch/test-ftrace.sh
--
2.23.0
On 2019-10-07 17:21:50, brendanhiggins(a)google.com wrote:
> On Fri, Oct 4, 2019 at 11:55 AM Konstantin Ryabitsev via RT
> <kernel-helpdesk(a)rt.linuxfoundation.org> wrote:
> >
> > On 2019-10-02 17:53:58, brendanhiggins(a)google.com wrote:
> > > Hi,
> > >
> > > I am thinking about requesting a Bugzilla component for my kernel
> > > project KUnit. I am not sure if this is the right place for it.
> > > Some
> > > background on KUnit: We are working on adding unit testing for the
> > > Linux
> > > kernel[1][2]. We have our initial patchset that introduces the
> > > subsystem
> > > in the process of being merged (Linus sent our PR back to us for a
> > > minor
> > > fix[3], so it should be in either 5.4-rc2 or 5.5, but is
> > > nevertheless
> > > in
> > > linux-next). However, we also have a staging repo that people are
> > > using
> > > and some supporting code that lives outside of the kernel.
> > >
> > > So I am trying to figure out:
> > >
> > > 1. Is it appropriate to request a Bugzilla component before our
> > > subsystem has been merged into torvalds/master? I would just
> > > wait,
> > > but I have some users looking to file issues, so I would prefer
> > > to
> > > provide them something sooner rather than later.
> > >
> > > 2. Is it appropriate to use the kernel's Bugzilla to track issues
> > > outside of the Linux kernel? As I mention above, we have code
> > > that
> > > lives outside of the kernel; is it appropriate to use
> > > kernel.org's
> > > Bugzilla for this?
> > >
> > > 3. Does Bugzilla match my planned usage model? It doesn't look like
> > > Bugzilla get's much usage aside from reporting bugs. I want to
> > > use
> > > it for tracking feature progress and things like that. Is that
> > > okay?
> >
> > Yes, we can certainly host this on bugzilla.kernel.org. Would you be
> > okay with Tools/KUnit as product/category?
>
> Cool, well as long as none of my above points are an issue for you.
> Then yes, can you create me a component? I am fine with Tools/KUnit as
> the product/category.
Apologies for the delay, mostly due to Thanksgiving in Canada. You should be able to start using Tools/KUnit now. It uses a default virtual assignee tools_kunit(a)kernel-bugs.kernel.org, so to start receiving bugmail for it, please follow instructions on this page:
https://korg.wiki.kernel.org/userdoc/bugzilla#to_start_getting_bug_reports_…
Best regards,
--
Konstantin Ryabitsev
Director, LF Projects IT
The Linux Foundation
"
On Wed, Oct 16, 2019 at 3:59 PM Prabhakar Kushwaha
<prabhakar.pkin(a)gmail.com> wrote:
>
> On Tue, Sep 24, 2019 at 12:56:53PM -0600, Shuah Khan wrote:
> > On 9/24/19 12:49 PM, Daniel Borkmann wrote:
> > > On Tue, Sep 24, 2019 at 09:48:35AM -0600, Shuah Khan wrote:
> > > > On 9/24/19 9:43 AM, Yonghong Song wrote:
> > > > > On 9/24/19 8:26 AM, Shuah Khan wrote:
> > > > > > Hi Alexei and Daniel,
> > > > > >
> > > > > > bpf test doesn't build on Linux 5.4 mainline. Do you know what's
> > > > > > happening here.
> > > > > >
> > > > > > make -C tools/testing/selftests/bpf/
> > > > > >
> > > > > > -c progs/test_core_reloc_ptr_as_arr.c -o - || echo "clang failed") | \
> > > > > > llc -march=bpf -mcpu=generic -filetype=obj -o
> > > > > > /mnt/data/lkml/linux_5.4/tools/testing/selftests/bpf/test_core_reloc_ptr_as_arr.o
> > > > > >
> > > > > > progs/test_core_reloc_ptr_as_arr.c:25:6: error: use of unknown builtin
> > > > > > '__builtin_preserve_access_index'
> > > > > > [-Wimplicit-function-declaration]
> > > > > > if (BPF_CORE_READ(&out->a, &in[2].a))
> > > > > > ^
> > > > > > ./bpf_helpers.h:533:10: note: expanded from macro 'BPF_CORE_READ'
> > > > > > __builtin_preserve_access_index(src))
> > > > > > ^
> > > > > > progs/test_core_reloc_ptr_as_arr.c:25:6: warning: incompatible
> > > > > > integer to
> > > > > > pointer conversion passing 'int' to parameter of type 'const
> > > > > > void *'
> > > > > > [-Wint-conversion]
> > > > > > if (BPF_CORE_READ(&out->a, &in[2].a))
> > > > > > ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> > > > > > ./bpf_helpers.h:533:10: note: expanded from macro 'BPF_CORE_READ'
> > > > > > __builtin_preserve_access_index(src))
> > > > > > ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> > > > > > 1 warning and 1 error generated.
> > > > > > llc: error: llc: <stdin>:1:1: error: expected top-level entity
> > > > > > clang failed
> > > > > >
> > > > > > Also
> > > > > >
> > > > > > make TARGETS=bpf kselftest fails as well. Dependency between
> > > > > > tools/lib/bpf and the test. How can we avoid this type of
> > > > > > dependency or resolve it in a way it doesn't result in build
> > > > > > failures?
> > > > >
> > > > > Thanks, Shuah.
> > > > >
> > > > > The clang __builtin_preserve_access_index() intrinsic is
> > > > > introduced in LLVM9 (which just released last week) and
> > > > > the builtin and other CO-RE features are only supported
> > > > > in LLVM10 (current development branch) with more bug fixes
> > > > > and added features.
> > > > >
> > > > > I think we should do a feature test for llvm version and only
> > > > > enable these tests when llvm version >= 10.
> > > >
> > > > Yes. If new tests depend on a particular llvm revision, the failing
> > > > the build is a regression. I would like to see older tests that don't
> > > > have dependency build and run.
> > >
> > > So far we haven't made it a requirement as majority of BPF contributors
> > > that would run/add tests in here are also on bleeding edge LLVM anyway
> > > and other CIs like 0-day bot have simply upgraded their LLVM version
> > > from git whenever there was a failure similar to the one here so its
> > > ensured that really /all/ test cases are running and nothing would be
> > > skipped. There is worry to some degree that CIs just keep sticking to
> > > an old compiler since tests "just" pass and regressions wouldn't be
> > > caught on new releases for those that are skipped. >
> >
> > Sure. Bleeding edge is developer mode. We still have to be concerned
> > about users that might not upgrade quickly.
> >
> > > That said, for the C based tests, it should actually be straight forward
> > > to categorize them based on built-in macros like ...
> > >
> > > $ echo | clang -dM -E -
> > > [...]
> > > #define __clang_major__ 10
> > > #define __clang_minor__ 0
> > > [...]
> >
> > What would nice running the tests you can run and then say some tests
> > aren't going to run. Is this something you can support?
>
> Once there is such infra in place, should be possible.
>
> > > ... given there is now also bpf-gcc, the test matrix gets bigger anyway,
> > > so it might be worth rethinking to run the suite multiple times with
> > > different major llvm{,gcc} versions at some point to make sure their
> > > generated BPF bytecode keeps passing the verifier, and yell loudly if
> > > newer features had to be skipped due to lack of recent compiler version.
> > > This would be a super set of /just/ skipping tests and improve coverage
> > > at the same time.
> >
> > Probably. Reality is most users will just quit and add bpf to "hard to
> > run category" of tests.
>
> I don't really worry too much about such users at this point, more important
> is that we have a way to test bpf-gcc and llvm behavior side by side to
> make sure behavior is consistent and to have some sort of automated CI
> integration that runs BPF kselftests before we even stare at a patch for
> review. These are right now the two highest prio items from BPF testing
> side where we need to get to.
>
I am also facing same issue with Linux-5.4 for ARM64 platforms.
I am using ubuntu-18.04 which has LLVM version 6.0.0. Looks like
there is no LLVM of version 9.0 or 10.0 available for ARM64.
https://apt.llvm.org/
I also tried "sudo ./llvm.sh 9". But it does not install llvm version.
root@ubuntu$ sudo ./llvm.sh 9
+ LLVM_VERSION=9
+ '[' 1 -eq 1 ']'
+ LLVM_VERSION=9
++ lsb_release -is
+ DISTRO=Ubuntu
++ lsb_release -sr
+ VERSION=18.04
+ DIST_VERSION=Ubuntu_18.04
+ [[ 0 -ne 0 ]]
+ declare -A LLVM_VERSION_PATTERNS
+ LLVM_VERSION_PATTERNS[8]=-8
+ LLVM_VERSION_PATTERNS[9]=-9
+ LLVM_VERSION_PATTERNS[10]=
+ '[' '!' _ ']'
+ LLVM_VERSION_STRING=-9
+ case "$DIST_VERSION" in
+ REPO_NAME='deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main'
+ wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key
+ apt-key add -
--2019-10-16 03:33:34-- https://apt.llvm.org/llvm-snapshot.gpg.key
Resolving apt.llvm.org (apt.llvm.org)... 151.101.190.49, 2a04:4e42:a::561
Connecting to apt.llvm.org (apt.llvm.org)|151.101.190.49|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 3145 (3.1K) [application/octet-stream]
Saving to: ?STDOUT?
-
100%[====================================================================================================>]
3.07K --.-KB/s in 0s
2019-10-16 03:33:35 (74.8 MB/s) - written to stdout [3145/3145]
OK
+ add-apt-repository 'deb http://apt.llvm.org/bionic/
llvm-toolchain-bionic-9 main'
Hit:1 http://apt.llvm.org/bionic llvm-toolchain-bionic InRelease
Hit:2 http://apt.llvm.org/bionic llvm-toolchain-bionic-9 InRelease
Hit:3 http://apt.llvm.org/bionic llvm-toolchain-bionic-8 InRelease
Hit:4 http://us.ports.ubuntu.com/ubuntu-ports bionic InRelease
Hit:5 http://us.ports.ubuntu.com/ubuntu-ports bionic-updates InRelease
Hit:6 http://ports.ubuntu.com/ubuntu-ports bionic-security InRelease
Hit:7 http://us.ports.ubuntu.com/ubuntu-ports bionic-backports InRelease
Reading package lists... Done
N: Skipping acquire of configured file 'main/binary-arm64/Packages' as
repository 'http://apt.llvm.org/bionic llvm-toolchain-bionic
InRelease' doesn't support architecture 'arm64'
N: Skipping acquire of configured file 'main/binary-arm64/Packages' as
repository 'http://apt.llvm.org/bionic llvm-toolchain-bionic-9
InRelease' doesn't support architecture 'arm64'
N: Skipping acquire of configured file 'main/binary-arm64/Packages' as
repository 'http://apt.llvm.org/bionic llvm-toolchain-bionic-8
InRelease' doesn't support architecture 'arm64'
+ apt-get update
Hit:1 http://apt.llvm.org/bionic llvm-toolchain-bionic InRelease
Hit:2 http://apt.llvm.org/bionic llvm-toolchain-bionic-9 InRelease
Hit:3 http://apt.llvm.org/bionic llvm-toolchain-bionic-8 InRelease
Hit:4 http://us.ports.ubuntu.com/ubuntu-ports bionic InRelease
Hit:5 http://us.ports.ubuntu.com/ubuntu-ports bionic-updates InRelease
Hit:6 http://ports.ubuntu.com/ubuntu-ports bionic-security InRelease
Hit:7 http://us.ports.ubuntu.com/ubuntu-ports bionic-backports InRelease
Reading package lists... Done
N: Skipping acquire of configured file 'main/binary-arm64/Packages' as
repository 'http://apt.llvm.org/bionic llvm-toolchain-bionic
InRelease' doesn't support architecture 'arm64'
N: Skipping acquire of configured file 'main/binary-arm64/Packages' as
repository 'http://apt.llvm.org/bionic llvm-toolchain-bionic-9
InRelease' doesn't support architecture 'arm64'
N: Skipping acquire of configured file 'main/binary-arm64/Packages' as
repository 'http://apt.llvm.org/bionic llvm-toolchain-bionic-8
InRelease' doesn't support architecture 'arm64'
+ apt-get install -y clang-9 lldb-9 lld-9 clangd-9
Reading package lists... Done
Building dependency tree
Reading state information... Done
E: Unable to locate package clang-9
E: Unable to locate package lldb-9
E: Unable to locate package lld-9
E: Unable to locate package clangd-9
--prabhakar(pk)
From: Christian Kellner <christian(a)kellner.me>
The pidfd_{open,poll}_test.c files both include `linux/wait.h` and
later `sys/wait.h`. The former has `#define P_ALL 0`, but in the
latter P_ALL is part of idtype_t enum, where it gets substituted
due to the aforementioned define to be `0`, which then results in
`typedef enum {0, ...`, which then results into a compiler error:
"error: expected identifier before numeric constant".
Since we need `sys/wait.h` for waitpid, drop `linux/wait.h`.
Signed-off-by: Christian Kellner <christian(a)kellner.me>
---
tools/testing/selftests/pidfd/pidfd_open_test.c | 1 -
tools/testing/selftests/pidfd/pidfd_poll_test.c | 1 -
2 files changed, 2 deletions(-)
diff --git a/tools/testing/selftests/pidfd/pidfd_open_test.c b/tools/testing/selftests/pidfd/pidfd_open_test.c
index b9fe75fc3e51..8a59438ccc78 100644
--- a/tools/testing/selftests/pidfd/pidfd_open_test.c
+++ b/tools/testing/selftests/pidfd/pidfd_open_test.c
@@ -6,7 +6,6 @@
#include <inttypes.h>
#include <limits.h>
#include <linux/types.h>
-#include <linux/wait.h>
#include <sched.h>
#include <signal.h>
#include <stdbool.h>
diff --git a/tools/testing/selftests/pidfd/pidfd_poll_test.c b/tools/testing/selftests/pidfd/pidfd_poll_test.c
index 4b115444dfe9..610811275357 100644
--- a/tools/testing/selftests/pidfd/pidfd_poll_test.c
+++ b/tools/testing/selftests/pidfd/pidfd_poll_test.c
@@ -3,7 +3,6 @@
#define _GNU_SOURCE
#include <errno.h>
#include <linux/types.h>
-#include <linux/wait.h>
#include <poll.h>
#include <signal.h>
#include <stdbool.h>
--
2.21.0
I created a new kernel.org repository for compiling and preserving
Autogenerated regression tests. This is an outcome based on decision
made at the Maintainer Summit to preserve and use the autogenerated
reproducers for regression testing.
Linux Kernel Autogenerated Regression Tests
===========================================
linux-arts repository is dedicated to preserving the autogenerated
regression tests for regression testing.
Automates testing entities such as Syzbot generate reproducers to test
the kernel. It is a manual process to download reproducers and once the
bug is fixed, kernel developers loose track of these reproducers.
As per the decision made at a session at the Maintainers Summit 2019,
this git repository is created to preserve the reproducers.
Please find the details on the discussion that lead to this decision
at https://lwn.net/Articles/799162/
Please find the new git repo at:
https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-arts.git/
The initial commit pulls in Syzkaller reproducers from Dmitry Vyukov.
As per Dmitry's request, the run and build scripts have been updated
to with just the syzbot link.
Thanks Dmitry for compiling these reproducers and working on the
framework of running and building them.
As for reproducers, each .c is specific to syzbot report and named
with the syzbot bug report ID.
Please play with it and let me know if you have questions. README
has the details on where to send patches and regressions tests.
thanks,
-- Shuah
Hi All,
I am trying to understand test_gso test for IPv4 and IPv6 with following
piece of code
# listen on IPv*_DST, capture TCP into $TMPFILE
if [ "${PROTO}" == "IPv4" ] ; then
IP_DST=${IPv4_DST}
ip netns exec ${NS3} bash -c \
"nc -4 -l -s ${IPv4_DST} -p 9000 > ${TMPFILE} &"
elif [ "${PROTO}" == "IPv6" ] ; then
IP_DST=${IPv6_DST}
ip netns exec ${NS3} bash -c \
"nc -6 -l -s ${IPv6_DST} -p 9000 > ${TMPFILE} &"
RET=$?
else
echo " test_gso: unknown PROTO: ${PROTO}"
fi
I have couple of queries around it
a) why -l is being used for reading listen on IPv*_DST with -s option.
I was looking at https://www.computerhope.com/unix/nc.htm, following has
been mentioned:
-l : Used to specify that nc should listen for an incoming connection
rather than initiate a connection to a
remote host. It is an error to use this option in conjunction with the
-p, -s, or -z options.
Additionally, any timeouts specified with the -w option are ignored.
b) Even if there is requirement of -l option to use. can we provide
timeout option also. how? as -w dont work with -l.
I am facing an issue with Linux-5.3 Kselftest where even if
test_lwt_ip_encap.sh "exit", bpf/runner.sh is not
running next test case. It just wait until CRTL + c is pressed.
If I comment above code things work fine.
Please sugggest
--prabhakar (pk)
Hi,
I am thinking about requesting a Bugzilla component for my kernel
project KUnit. I am not sure if this is the right place for it. Some
background on KUnit: We are working on adding unit testing for the Linux
kernel[1][2]. We have our initial patchset that introduces the subsystem
in the process of being merged (Linus sent our PR back to us for a minor
fix[3], so it should be in either 5.4-rc2 or 5.5, but is nevertheless in
linux-next). However, we also have a staging repo that people are using
and some supporting code that lives outside of the kernel.
So I am trying to figure out:
1. Is it appropriate to request a Bugzilla component before our
subsystem has been merged into torvalds/master? I would just wait,
but I have some users looking to file issues, so I would prefer to
provide them something sooner rather than later.
2. Is it appropriate to use the kernel's Bugzilla to track issues
outside of the Linux kernel? As I mention above, we have code that
lives outside of the kernel; is it appropriate to use kernel.org's
Bugzilla for this?
3. Does Bugzilla match my planned usage model? It doesn't look like
Bugzilla get's much usage aside from reporting bugs. I want to use
it for tracking feature progress and things like that. Is that okay?
If kernel.org's Bugzilla is not a fit for what I want to do, that's
fine. I just want to make sure before I go off and potentially fracture
a central bug repository by creating my own somewhere else.
Thanks!
[1] https://lwn.net/Articles/780985/
[2] https://google.github.io/kunit-docs/third_party/kernel/docs/index.html
[3] https://lore.kernel.org/lkml/be8059f4-8e8f-cd18-0978-a9c861f6396b@linuxfoun…
Hi,
Changes since v1:
1) Fixed a krobot-reported mistake, which also uncovered a pre-existing
bug. Thanks to Kirill for recommending the fix.
2) Added another small fix: changed the data type to unsigned int, as
pointed out by Ira.
2) Added a "Fixes:" line, thanks to Kirill and Aneesh for pinpointing the
commit.
3) Collected Acked-by and Suggested-by's.
Original cover letter, edited slightly
======================================
These trivial fixes apply to today's linux.git (5.4-rc3).
I found these while polishing up the Next And Final get_user_pages()+dma
tracking patchset (which is in final testing and passing nicely...so far).
Anyway, as these two patches apply cleanly both before and after the larger
gup/dma upcoming patchset, I thought it best to send this out separately,
in order to avoid muddying the waters more than usual.
Cc: Christoph Hellwig <hch(a)lst.de>
Cc: Aneesh Kumar K.V <aneesh.kumar(a)linux.ibm.com>
Cc: Kirill A. Shutemov <kirill.shutemov(a)linux.intel.com>
Cc: Keith Busch <keith.busch(a)intel.com>
Cc: Ira Weiny <ira.weiny(a)intel.com>
Cc: Shuah Khan <shuah(a)kernel.org>
Cc: linux-kselftest(a)vger.kernel.org
John Hubbard (2):
mm/gup_benchmark: add a missing "w" to getopt string
mm/gup: fix a misnamed "write" argument, and a related bug
mm/gup.c | 14 ++++++++------
tools/testing/selftests/vm/gup_benchmark.c | 2 +-
2 files changed, 9 insertions(+), 7 deletions(-)
--
2.23.0
Patch series implements hugetlb_cgroup reservation usage and limits, which
track hugetlb reservations rather than hugetlb memory faulted in. Details of
the approach is 1/7.
Changes in v5:
- Moved the bulk of the description to the first patch in the series.
- Clang formatted the entire series.
- Split off 'hugetlb: remove duplicated code' and 'hugetlb: region_chg provides
only cache entry' into their own patch series.
- Added comments to HUGETLB_RES enum.
- Fixed bug in 'hugetlb: disable region_add file_region coalescing' calculating
the wrong number of regions_needed in some cases.
- Changed sleeps in test to proper conditions.
- Misc fixes in test based on shuah@ review.
Changes in v4:
- Split up 'hugetlb_cgroup: add accounting for shared mappings' into 4 patches
for better isolation and context on the indvidual changes:
- hugetlb_cgroup: add accounting for shared mappings
- hugetlb: disable region_add file_region coalescing
- hugetlb: remove duplicated code
- hugetlb: region_chg provides only cache entry
- Fixed resv->adds_in_progress accounting.
- Retained behavior that region_add never fails, in earlier patchsets region_add
could return failure.
- Fixed libhugetlbfs failure.
- Minor fix to the added tests that was preventing them from running on some
environments.
Changes in v3:
- Addressed comments of Hillf Danton:
- Added docs.
- cgroup_files now uses enum.
- Various readability improvements.
- Addressed comments of Mike Kravetz.
- region_* functions no longer coalesce file_region entries in the resv_map.
- region_add() and region_chg() refactored to make them much easier to
understand and remove duplicated code so this patch doesn't add too much
complexity.
- Refactored common functionality into helpers.
Changes in v2:
- Split the patch into a 5 patch series.
- Fixed patch subject.
Mina Almasry (7):
hugetlb_cgroup: Add hugetlb_cgroup reservation counter
hugetlb_cgroup: add interface for charge/uncharge hugetlb reservations
hugetlb_cgroup: add reservation accounting for private mappings
hugetlb: disable region_add file_region coalescing
hugetlb_cgroup: add accounting for shared mappings
hugetlb_cgroup: Add hugetlb_cgroup reservation tests
hugetlb_cgroup: Add hugetlb_cgroup reservation docs
.../admin-guide/cgroup-v1/hugetlb.rst | 85 +++-
include/linux/hugetlb.h | 31 +-
include/linux/hugetlb_cgroup.h | 33 +-
mm/hugetlb.c | 423 +++++++++++-----
mm/hugetlb_cgroup.c | 190 ++++++--
tools/testing/selftests/vm/.gitignore | 1 +
tools/testing/selftests/vm/Makefile | 1 +
.../selftests/vm/charge_reserved_hugetlb.sh | 461 ++++++++++++++++++
.../selftests/vm/write_hugetlb_memory.sh | 22 +
.../testing/selftests/vm/write_to_hugetlbfs.c | 250 ++++++++++
10 files changed, 1306 insertions(+), 191 deletions(-)
create mode 100755 tools/testing/selftests/vm/charge_reserved_hugetlb.sh
create mode 100644 tools/testing/selftests/vm/write_hugetlb_memory.sh
create mode 100644 tools/testing/selftests/vm/write_to_hugetlbfs.c
--
2.23.0.351.gc4317032e6-goog
Hi,
These trivial fixes apply to today's linux.git (5.4-rc3, or maybe -rc4, by
the time I send this).
I found these while polishing up the Next And Final get_user_pages()+dma
tracking patchset (which is in final testing and passing nicely...so far).
Anyway, as these two patches apply cleanly both before and after the larger
gup/dma upcoming patchset, I thought it best to send this out separately,
in order to avoid muddying the waters more than usual.
Cc: Christoph Hellwig <hch(a)lst.de>
Cc: Aneesh Kumar K.V <aneesh.kumar(a)linux.ibm.com>
Cc: Kirill A. Shutemov <kirill.shutemov(a)linux.intel.com>
Cc: Keith Busch <keith.busch(a)intel.com>
Cc: Shuah Khan <shuah(a)kernel.org>
Cc: linux-kselftest(a)vger.kernel.org
John Hubbard (2):
mm/gup_benchmark: add a missing "w" to getopt string
mm/gup: fix a misnamed "write" argument: should be "flags"
mm/gup.c | 12 +++++++-----
tools/testing/selftests/vm/gup_benchmark.c | 2 +-
2 files changed, 8 insertions(+), 6 deletions(-)
--
2.23.0
Reset all signal handlers of the child not set to SIG_IGN to SIG_DFL.
Mutually exclusive with CLONE_SIGHAND to not disturb other thread's
signal handler.
In the spirit of closer cooperation between glibc developers and kernel
developers (cf. [2]) this patchset came out of a discussion on the glibc
mailing list for improving posix_spawn() (cf. [1], [3], [4]). Kernel
support for this feature has been explicitly requested by glibc and I
see no reason not to help them with this.
The child helper process on Linux posix_spawn must ensure that no signal
handlers are enabled, so the signal disposition must be either SIG_DFL
or SIG_IGN. However, it requires a sigprocmask to obtain the current
signal mask and at least _NSIG sigaction calls to reset the signal
handlers for each posix_spawn call or complex state tracking that might
lead to data corruption in glibc. Adding this flags lets glibc avoid
these problems.
[1]: https://www.sourceware.org/ml/libc-alpha/2019-10/msg00149.html
[3]: https://www.sourceware.org/ml/libc-alpha/2019-10/msg00158.html
[4]: https://www.sourceware.org/ml/libc-alpha/2019-10/msg00160.html
[2]: https://lwn.net/Articles/799331/
'[...] by asking for better cooperation with the C-library projects
in general. They should be copied on patches containing ABI
changes, for example. I noted that there are often times where
C-library developers wish the kernel community had done things
differently; how could those be avoided in the future? Members of
the audience suggested that more glibc developers should perhaps
join the linux-api list. The other suggestion was to "copy Florian
on everything".'
Cc: Oleg Nesterov <oleg(a)redhat.com>
Cc: Florian Weimer <fweimer(a)redhat.com>
Cc: libc-alpha(a)sourceware.org
Signed-off-by: Christian Brauner <christian.brauner(a)ubuntu.com>
---
include/uapi/linux/sched.h | 3 +++
kernel/fork.c | 11 ++++++++++-
2 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h
index 99335e1f4a27..c583720f689f 100644
--- a/include/uapi/linux/sched.h
+++ b/include/uapi/linux/sched.h
@@ -33,6 +33,9 @@
#define CLONE_NEWNET 0x40000000 /* New network namespace */
#define CLONE_IO 0x80000000 /* Clone io context */
+/* Flags for the clone3() syscall */
+#define CLONE3_CLEAR_SIGHAND 0x100000000ULL /* Clear any signal handler and reset to SIG_DFL. */
+
#ifndef __ASSEMBLY__
/**
* struct clone_args - arguments for the clone3 syscall
diff --git a/kernel/fork.c b/kernel/fork.c
index 1f6c45f6a734..661f8d1f3881 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1517,6 +1517,11 @@ static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk)
spin_lock_irq(¤t->sighand->siglock);
memcpy(sig->action, current->sighand->action, sizeof(sig->action));
spin_unlock_irq(¤t->sighand->siglock);
+
+ /* Reset all signal handler not set to SIG_IGN to SIG_DFL. */
+ if (clone_flags & CLONE3_CLEAR_SIGHAND)
+ flush_signal_handlers(tsk, 0);
+
return 0;
}
@@ -2567,7 +2572,7 @@ static bool clone3_args_valid(const struct kernel_clone_args *kargs)
* All lower bits of the flag word are taken.
* Verify that no other unknown flags are passed along.
*/
- if (kargs->flags & ~CLONE_LEGACY_FLAGS)
+ if (kargs->flags & ~(CLONE_LEGACY_FLAGS | CLONE3_CLEAR_SIGHAND))
return false;
/*
@@ -2577,6 +2582,10 @@ static bool clone3_args_valid(const struct kernel_clone_args *kargs)
if (kargs->flags & (CLONE_DETACHED | CSIGNAL))
return false;
+ if ((kargs->flags & (CLONE_SIGHAND | CLONE3_CLEAR_SIGHAND)) ==
+ (CLONE_SIGHAND | CLONE3_CLEAR_SIGHAND))
+ return false;
+
if ((kargs->flags & (CLONE_THREAD | CLONE_PARENT)) &&
kargs->exit_signal)
return false;
--
2.23.0