The first version of this patch is a part of mmc non-blocking v9 patchset: [PATCH v9 11/12] mmc: core: add random fault injection
change log: v2 - Resolve build issue in mmc core.c due to multiple init_module by removing the fault inject module. - Export fault injection functions to make them available for modules - Update fault injection documentation on MMC IO v3 - add function descriptions in core.c - use export GPL for fault injection functions
Per Forlin (3): fault-inject: make fault injection available for modules mmc: core: add random fault injection fault injection: add documentation on MMC IO fault injection
Documentation/fault-injection/fault-injection.txt | 5 ++ drivers/mmc/core/core.c | 69 +++++++++++++++++++++ drivers/mmc/core/debugfs.c | 5 ++ include/linux/mmc/host.h | 3 + lib/Kconfig.debug | 11 +++ lib/fault-inject.c | 2 + 6 files changed, 95 insertions(+), 0 deletions(-)
export symbols should_fail() and init_fault_attr_dentries() in order to make modules use the fault injection functionality
Signed-off-by: Per Forlin per.forlin@linaro.org --- lib/fault-inject.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-)
diff --git a/lib/fault-inject.c b/lib/fault-inject.c index 7e65af7..27783da 100644 --- a/lib/fault-inject.c +++ b/lib/fault-inject.c @@ -131,6 +131,7 @@ bool should_fail(struct fault_attr *attr, ssize_t size)
return true; } +EXPORT_SYMBOL_GPL(should_fail);
#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
@@ -311,5 +312,6 @@ fail: cleanup_fault_attr_dentries(attr); return -ENOMEM; } +EXPORT_SYMBOL_GPL(init_fault_attr_dentries);
#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
This adds support to inject data errors after a completed host transfer. The mmc core will return error even though the host transfer is successful. This simple fault injection proved to be very useful to test the non-blocking error handling in the mmc_blk_issue_rw_rq(). Random faults can also test how the host driver handles pre_req() and post_req() in case of errors.
Signed-off-by: Per Forlin per.forlin@linaro.org --- drivers/mmc/core/core.c | 69 ++++++++++++++++++++++++++++++++++++++++++++ drivers/mmc/core/debugfs.c | 5 +++ include/linux/mmc/host.h | 3 ++ lib/Kconfig.debug | 11 +++++++ 4 files changed, 88 insertions(+), 0 deletions(-)
diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c index ab36c7b..dac2673 100644 --- a/drivers/mmc/core/core.c +++ b/drivers/mmc/core/core.c @@ -24,6 +24,11 @@ #include <linux/regulator/consumer.h> #include <linux/pm_runtime.h>
+#ifdef CONFIG_FAIL_MMC_REQUEST +#include <linux/fault-inject.h> +#include <linux/random.h> +#endif + #include <linux/mmc/card.h> #include <linux/mmc/host.h> #include <linux/mmc/mmc.h> @@ -82,6 +87,66 @@ static void mmc_flush_scheduled_work(void) flush_workqueue(workqueue); }
+#ifdef CONFIG_FAIL_MMC_REQUEST + +static DECLARE_FAULT_ATTR(fail_mmc_request); + +/* + * Internal function. Pass the boot param fail_mmc_request to + * the setup fault injection attributes routine. + */ +static int __init setup_fail_mmc_request(char *str) +{ + return setup_fault_attr(&fail_mmc_request, str); +} +__setup("fail_mmc_request=", setup_fail_mmc_request); + +/* + * Internal function. Inject random data errors. + * If mmc_data is NULL no errors are injected. + */ +static void mmc_should_fail_request(struct mmc_host *host, + struct mmc_request *mrq) +{ + struct mmc_command *cmd = mrq->cmd; + struct mmc_data *data = mrq->data; + static const int data_errors[] = { + -ETIMEDOUT, + -EILSEQ, + -EIO, + }; + + if (!data) + return; + + if (cmd->error || data->error || !host->make_it_fail || + !should_fail(&fail_mmc_request, data->blksz * data->blocks)) + return; + + data->error = data_errors[random32() % ARRAY_SIZE(data_errors)]; + data->bytes_xfered = (random32() % (data->bytes_xfered >> 9)) << 9; +} + +static int __init fail_mmc_request_debugfs(void) +{ + return init_fault_attr_dentries(&fail_mmc_request, + "fail_mmc_request"); +} + +#else /* CONFIG_FAIL_MMC_REQUEST */ + +static void mmc_should_fail_request(struct mmc_host *host, + struct mmc_request *mrq) +{ +} + +static int __init fail_mmc_request_debugfs(void) +{ + return 0; +} +#endif /* CONFIG_FAIL_MMC_REQUEST */ + + /** * mmc_request_done - finish processing an MMC request * @host: MMC host which completed request @@ -108,6 +173,8 @@ void mmc_request_done(struct mmc_host *host, struct mmc_request *mrq) cmd->error = 0; host->ops->request(host, mrq); } else { + mmc_should_fail_request(host, mrq); + led_trigger_event(host->led, LED_OFF);
pr_debug("%s: req done (CMD%u): %d: %08x %08x %08x %08x\n", @@ -2064,6 +2131,8 @@ static int __init mmc_init(void) if (ret) goto unregister_host_class;
+ fail_mmc_request_debugfs(); + return 0;
unregister_host_class: diff --git a/drivers/mmc/core/debugfs.c b/drivers/mmc/core/debugfs.c index 998797e..588e76f 100644 --- a/drivers/mmc/core/debugfs.c +++ b/drivers/mmc/core/debugfs.c @@ -188,6 +188,11 @@ void mmc_add_host_debugfs(struct mmc_host *host) root, &host->clk_delay)) goto err_node; #endif +#ifdef CONFIG_FAIL_MMC_REQUEST + if (!debugfs_create_u8("make-it-fail", S_IRUSR | S_IWUSR, + root, &host->make_it_fail)) + goto err_node; +#endif return;
err_node: diff --git a/include/linux/mmc/host.h b/include/linux/mmc/host.h index 771455f..250b46d 100644 --- a/include/linux/mmc/host.h +++ b/include/linux/mmc/host.h @@ -303,6 +303,9 @@ struct mmc_host {
struct mmc_async_req *areq; /* active async req */
+#ifdef CONFIG_FAIL_MMC_REQUEST + u8 make_it_fail; +#endif unsigned long private[0] ____cacheline_aligned; };
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index c768bcd..c2d1423 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1057,6 +1057,17 @@ config FAIL_IO_TIMEOUT Only works with drivers that use the generic timeout handling, for others it wont do anything.
+config FAIL_MMC_REQUEST + bool "Fault-injection capability for MMC IO" + select DEBUG_FS + depends on FAULT_INJECTION && MMC + help + Provide fault-injection capability for MMC IO. + This will make the mmc core return data errors. This is + useful to test the error handling in the mmc block device + and to test how the mmc host driver handles retries from + the block device. + config FAULT_INJECTION_DEBUG_FS bool "Debugfs entries for fault-injection capabilities" depends on FAULT_INJECTION && SYSFS && DEBUG_FS
2011/7/21 Per Forlin per.forlin@linaro.org:
This adds support to inject data errors after a completed host transfer. The mmc core will return error even though the host transfer is successful. This simple fault injection proved to be very useful to test the non-blocking error handling in the mmc_blk_issue_rw_rq(). Random faults can also test how the host driver handles pre_req() and post_req() in case of errors.
Looks good to me.
@@ -82,6 +87,66 @@ static void mmc_flush_scheduled_work(void) flush_workqueue(workqueue); }
+#ifdef CONFIG_FAIL_MMC_REQUEST
+static DECLARE_FAULT_ATTR(fail_mmc_request);
I think the fail_attr should be defined for each mmc_host like make_it_fail in struct mmc_host and debugfs entries should also be created in a subdirectory of mmc host debugfs.
And I know that init_fault_attr_dentries() can only create a subdirectory in debugfs root directory. But I have a patch which support for creating it in arbitrary directory. Could you take a look at this? (Note that this patch is based on mmotm and not yet tested)
On 25 July 2011 17:58, Akinobu Mita akinobu.mita@gmail.com wrote:
2011/7/21 Per Forlin per.forlin@linaro.org:
This adds support to inject data errors after a completed host transfer. The mmc core will return error even though the host transfer is successful. This simple fault injection proved to be very useful to test the non-blocking error handling in the mmc_blk_issue_rw_rq(). Random faults can also test how the host driver handles pre_req() and post_req() in case of errors.
Looks good to me.
Thanks,
@@ -82,6 +87,66 @@ static void mmc_flush_scheduled_work(void) flush_workqueue(workqueue); }
+#ifdef CONFIG_FAIL_MMC_REQUEST
+static DECLARE_FAULT_ATTR(fail_mmc_request);
I think the fail_attr should be defined for each mmc_host like make_it_fail in struct mmc_host and debugfs entries should also be created in a subdirectory of mmc host debugfs.
I looked at blk-core.c and used the same code here. Current code creates the entry under the debugfs root. This means one fail_attr for all hosts. I agree, it's more clean to move the fail_attr to the host-debugfs-entry which require the fail_attr to be stored same way as make_it_fail.
And I know that init_fault_attr_dentries() can only create a subdirectory in debugfs root directory. But I have a patch which support for creating it in arbitrary directory. Could you take a look at this? (Note that this patch is based on mmotm and not yet tested)
Thanks, I will check it out.
Regards, Per
Hi Akinobu,
On 25 July 2011 21:19, Per Forlin per.forlin@linaro.org wrote:
On 25 July 2011 17:58, Akinobu Mita akinobu.mita@gmail.com wrote:
2011/7/21 Per Forlin per.forlin@linaro.org:
This adds support to inject data errors after a completed host transfer. The mmc core will return error even though the host transfer is successful. This simple fault injection proved to be very useful to test the non-blocking error handling in the mmc_blk_issue_rw_rq(). Random faults can also test how the host driver handles pre_req() and post_req() in case of errors.
Looks good to me.
Thanks,
@@ -82,6 +87,66 @@ static void mmc_flush_scheduled_work(void) flush_workqueue(workqueue); }
+#ifdef CONFIG_FAIL_MMC_REQUEST
+static DECLARE_FAULT_ATTR(fail_mmc_request);
I think the fail_attr should be defined for each mmc_host like make_it_fail in struct mmc_host and debugfs entries should also be created in a subdirectory of mmc host debugfs.
I looked at blk-core.c and used the same code here. Current code creates the entry under the debugfs root. This means one fail_attr for all hosts. I agree, it's more clean to move the fail_attr to the host-debugfs-entry which require the fail_attr to be stored same way as make_it_fail.
And I know that init_fault_attr_dentries() can only create a subdirectory in debugfs root directory. But I have a patch which support for creating it in arbitrary directory. Could you take a look at this? (Note that this patch is based on mmotm and not yet tested)
I looked at your patch and it raised two questions. I can't use FAULT_ATTR_INITIALIZER since mmc_host is allocated on the heap. It looks like setup_fault_attr(attr, str) will fail if str is NULL. How can I initialise the fault_attrs if not stack allocated? About the boot param initialisation of fault attr. There can only be one fault_mmc_request boot param for the entire kernel but there is one fault_attr per host, and there may be many hosts. It would be convenient if setup_fault_attrs would take (attr, boot_param_name), look up boot_param_name and use that otherwise set default values.
Regards, Per
2011/7/26 Per Forlin per.forlin@linaro.org:
And I know that init_fault_attr_dentries() can only create a subdirectory in debugfs root directory. But I have a patch which support for creating it in arbitrary directory. Could you take a look at this? (Note that this patch is based on mmotm and not yet tested)
I looked at your patch and it raised two questions. I can't use FAULT_ATTR_INITIALIZER since mmc_host is allocated on the heap. It looks like setup_fault_attr(attr, str) will fail if str is NULL. How can I initialise the fault_attrs if not stack allocated? About the boot param initialisation of fault attr. There can only be one fault_mmc_request boot param for the entire kernel but there is one fault_attr per host, and there may be many hosts. It would be convenient if setup_fault_attrs would take (attr, boot_param_name), look up boot_param_name and use that otherwise set default values.
I think you can define one default fail_attr for boot time configuration and copy it to per-host fail_attr in mmc_add_host_debugfs().
/* pseudo-code */
static DECLARE_FAULT_ATTR(default_mmc_fail_attr);
static int __init setup_fail_mmc_request(char *str) { return setup_fault_attr(&default_mmc_fail_attr, str); } __setup("fail_mmc_request=", setup_fail_mmc_request);
...
void mmc_add_host_debugfs(struct mmc_host *host) { ...
#ifdef CONFIG_FAIL_MMC_REQUEST host->fail_attr = default_mmc_fail_attr; if (!debugfs_create_fault_attr("fail_mmc_request", root, &host->fail_attr)) goto err_node; #endif ... }
On 26 July 2011 03:41, Akinobu Mita akinobu.mita@gmail.com wrote:
2011/7/26 Per Forlin per.forlin@linaro.org:
And I know that init_fault_attr_dentries() can only create a subdirectory in debugfs root directory. But I have a patch which support for creating it in arbitrary directory. Could you take a look at this? (Note that this patch is based on mmotm and not yet tested)
I looked at your patch and it raised two questions. I can't use FAULT_ATTR_INITIALIZER since mmc_host is allocated on the heap. It looks like setup_fault_attr(attr, str) will fail if str is NULL. How can I initialise the fault_attrs if not stack allocated? About the boot param initialisation of fault attr. There can only be one fault_mmc_request boot param for the entire kernel but there is one fault_attr per host, and there may be many hosts. It would be convenient if setup_fault_attrs would take (attr, boot_param_name), look up boot_param_name and use that otherwise set default values.
I think you can define one default fail_attr for boot time configuration and copy it to per-host fail_attr in mmc_add_host_debugfs().
You're right. This works fine. I'll prepare a new version of my patch.
On 25 July 2011 17:58, Akinobu Mita akinobu.mita@gmail.com wrote:
2011/7/21 Per Forlin per.forlin@linaro.org:
This adds support to inject data errors after a completed host transfer. The mmc core will return error even though the host transfer is successful. This simple fault injection proved to be very useful to test the non-blocking error handling in the mmc_blk_issue_rw_rq(). Random faults can also test how the host driver handles pre_req() and post_req() in case of errors.
Looks good to me.
@@ -82,6 +87,66 @@ static void mmc_flush_scheduled_work(void) flush_workqueue(workqueue); }
+#ifdef CONFIG_FAIL_MMC_REQUEST
+static DECLARE_FAULT_ATTR(fail_mmc_request);
I think the fail_attr should be defined for each mmc_host like make_it_fail in struct mmc_host and debugfs entries should also be created in a subdirectory of mmc host debugfs.
And I know that init_fault_attr_dentries() can only create a subdirectory in debugfs root directory. But I have a patch which support for creating it in arbitrary directory. Could you take a look at this? (Note that this patch is based on mmotm and not yet tested)
I tested your patch on a Snowball board. I had two active mmc cards and did various fault injection configurations on the two cards together with dd. I did only test MMC IO fault injection and not any of the other fault injection clients. Tested-by: Per Forlin per.forlin@linaro.org
Regards, Per
Add description on how to enable random fault injection for MMC IO
Signed-off-by: Per Forlin per.forlin@linaro.org --- Documentation/fault-injection/fault-injection.txt | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-)
diff --git a/Documentation/fault-injection/fault-injection.txt b/Documentation/fault-injection/fault-injection.txt index 7be15e4..27eede4 100644 --- a/Documentation/fault-injection/fault-injection.txt +++ b/Documentation/fault-injection/fault-injection.txt @@ -21,6 +21,11 @@ o fail_make_request /sys/block/<device>/make-it-fail or /sys/block/<device>/<partition>/make-it-fail. (generic_make_request())
+o fail_mmc_request + + injects MMC data errors on devices permitted by setting + /sys/kernel/debug/mmc0/make-it-fail + Configure fault-injection capabilities behavior -----------------------------------------------
-
Akinobu Mita -
Per Forlin