From: Eric Biggers ebiggers@google.com

commit 9f480faec58cd6197a007ea1dcac6b7c3daf1139 upstream.

When chacha20_block() outputs the keystream block, it uses 'u32' stores directly. However, the callers (crypto/chacha20_generic.c and drivers/char/random.c) declare the keystream buffer as a 'u8' array, which is not guaranteed to have the needed alignment.

Fix it by having both callers declare the keystream as a 'u32' array. For now this is preferable to switching over to the unaligned access macros because chacha20_block() is only being used in cases where we can easily control the alignment (stack buffers).

Signed-off-by: Eric Biggers ebiggers@google.com Signed-off-by: Herbert Xu herbert@gondor.apana.org.au Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- crypto/chacha20_generic.c | 6 +++--- drivers/char/random.c | 24 ++++++++++++------------ include/crypto/chacha20.h | 3 ++- lib/chacha20.c | 2 +- 4 files changed, 18 insertions(+), 17 deletions(-)

--- a/crypto/chacha20_generic.c +++ b/crypto/chacha20_generic.c @@ -22,20 +22,20 @@ static inline u32 le32_to_cpuvp(const vo static void chacha20_docrypt(u32 *state, u8 *dst, const u8 *src, unsigned int bytes) { - u8 stream[CHACHA20_BLOCK_SIZE]; + u32 stream[CHACHA20_BLOCK_WORDS];

if (dst != src) memcpy(dst, src, bytes);

while (bytes >= CHACHA20_BLOCK_SIZE) { chacha20_block(state, stream); - crypto_xor(dst, stream, CHACHA20_BLOCK_SIZE); + crypto_xor(dst, (const u8 *)stream, CHACHA20_BLOCK_SIZE); bytes -= CHACHA20_BLOCK_SIZE; dst += CHACHA20_BLOCK_SIZE; } if (bytes) { chacha20_block(state, stream); - crypto_xor(dst, stream, bytes); + crypto_xor(dst, (const u8 *)stream, bytes); } }

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -433,9 +433,9 @@ static int crng_init_cnt = 0; static unsigned long crng_global_init_time = 0; #define CRNG_INIT_CNT_THRESH (2*CHACHA20_KEY_SIZE) static void _extract_crng(struct crng_state *crng, - __u8 out[CHACHA20_BLOCK_SIZE]); + __u32 out[CHACHA20_BLOCK_WORDS]); static void _crng_backtrack_protect(struct crng_state *crng, - __u8 tmp[CHACHA20_BLOCK_SIZE], int used); + __u32 tmp[CHACHA20_BLOCK_WORDS], int used); static void process_random_ready_list(void); static void _get_random_bytes(void *buf, int nbytes);

@@ -929,7 +929,7 @@ static void crng_reseed(struct crng_stat unsigned long flags; int i, num; union { - __u8 block[CHACHA20_BLOCK_SIZE]; + __u32 block[CHACHA20_BLOCK_WORDS]; __u32 key[8]; } buf;

@@ -976,7 +976,7 @@ static void crng_reseed(struct crng_stat }

static void _extract_crng(struct crng_state *crng, - __u8 out[CHACHA20_BLOCK_SIZE]) + __u32 out[CHACHA20_BLOCK_WORDS]) { unsigned long v, flags, init_time;

@@ -996,7 +996,7 @@ static void _extract_crng(struct crng_st spin_unlock_irqrestore(&crng->lock, flags); }

-static void extract_crng(__u8 out[CHACHA20_BLOCK_SIZE]) +static void extract_crng(__u32 out[CHACHA20_BLOCK_WORDS]) { _extract_crng(select_crng(), out); } @@ -1006,7 +1006,7 @@ static void extract_crng(__u8 out[CHACHA * enough) to mutate the CRNG key to provide backtracking protection. */ static void _crng_backtrack_protect(struct crng_state *crng, - __u8 tmp[CHACHA20_BLOCK_SIZE], int used) + __u32 tmp[CHACHA20_BLOCK_WORDS], int used) { unsigned long flags; __u32 *s, *d; @@ -1018,14 +1018,14 @@ static void _crng_backtrack_protect(stru used = 0; } spin_lock_irqsave(&crng->lock, flags); - s = (__u32 *) &tmp[used]; + s = &tmp[used / sizeof(__u32)]; d = &crng->state[4]; for (i=0; i < 8; i++) *d++ ^= *s++; spin_unlock_irqrestore(&crng->lock, flags); }

-static void crng_backtrack_protect(__u8 tmp[CHACHA20_BLOCK_SIZE], int used) +static void crng_backtrack_protect(__u32 tmp[CHACHA20_BLOCK_WORDS], int used) { _crng_backtrack_protect(select_crng(), tmp, used); } @@ -1033,7 +1033,7 @@ static void crng_backtrack_protect(__u8 static ssize_t extract_crng_user(void __user *buf, size_t nbytes) { ssize_t ret = 0, i = CHACHA20_BLOCK_SIZE; - __u8 tmp[CHACHA20_BLOCK_SIZE]; + __u32 tmp[CHACHA20_BLOCK_WORDS]; int large_request = (nbytes > 256);

while (nbytes) { @@ -1619,7 +1619,7 @@ static void _warn_unseeded_randomness(co */ static void _get_random_bytes(void *buf, int nbytes) { - __u8 tmp[CHACHA20_BLOCK_SIZE]; + __u32 tmp[CHACHA20_BLOCK_WORDS];

trace_get_random_bytes(nbytes, _RET_IP_);

@@ -2220,7 +2220,7 @@ u64 get_random_u64(void) batch = raw_cpu_ptr(&batched_entropy_u64); spin_lock_irqsave(&batch->batch_lock, flags); if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) { - extract_crng((u8 *)batch->entropy_u64); + extract_crng((__u32 *)batch->entropy_u64); batch->position = 0; } ret = batch->entropy_u64[batch->position++]; @@ -2244,7 +2244,7 @@ u32 get_random_u32(void) batch = raw_cpu_ptr(&batched_entropy_u32); spin_lock_irqsave(&batch->batch_lock, flags); if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) { - extract_crng((u8 *)batch->entropy_u32); + extract_crng(batch->entropy_u32); batch->position = 0; } ret = batch->entropy_u32[batch->position++]; --- a/include/crypto/chacha20.h +++ b/include/crypto/chacha20.h @@ -13,12 +13,13 @@ #define CHACHA20_IV_SIZE 16 #define CHACHA20_KEY_SIZE 32 #define CHACHA20_BLOCK_SIZE 64 +#define CHACHA20_BLOCK_WORDS (CHACHA20_BLOCK_SIZE / sizeof(u32))

struct chacha20_ctx { u32 key[8]; };

-void chacha20_block(u32 *state, void *stream); +void chacha20_block(u32 *state, u32 *stream); void crypto_chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv); int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keysize); --- a/lib/chacha20.c +++ b/lib/chacha20.c @@ -21,7 +21,7 @@ static inline u32 rotl32(u32 v, u8 n) return (v << n) | (v >> (sizeof(v) * 8 - n)); }

-extern void chacha20_block(u32 *state, void *stream) +void chacha20_block(u32 *state, u32 *stream) { u32 x[16], *out = stream; int i;

From: Andi Kleen ak@linux.intel.com

commit e8e8a2e47db6bb85bb0cb21e77b5c6aaedf864b4 upstream.

add_interrupt_randomess always wakes up code blocking on /dev/random. This wake up is done unconditionally. Unfortunately this means all interrupts take the wait queue spinlock, which can be rather expensive on large systems processing lots of interrupts.

We saw 1% cpu time spinning on this on a large macro workload running on a large system.

I believe it's a recent regression (?)

Always check if there is a waiter on the wait queue before waking up. This check can be done without taking a spinlock.

1.06% 10460 [kernel.vmlinux] [k] native_queued_spin_lock_slowpath | ---native_queued_spin_lock_slowpath | --0.57%--_raw_spin_lock_irqsave | --0.56%--__wake_up_common_lock credit_entropy_bits add_interrupt_randomness handle_irq_event_percpu handle_irq_event handle_edge_irq handle_irq do_IRQ common_interrupt

Signed-off-by: Andi Kleen ak@linux.intel.com Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -721,7 +721,8 @@ retry: }

/* should we wake readers? */ - if (entropy_bits >= random_read_wakeup_bits) { + if (entropy_bits >= random_read_wakeup_bits && + wq_has_sleeper(&random_read_wait)) { wake_up_interruptible(&random_read_wait); kill_fasync(&fasync, SIGIO, POLL_IN); }

From: "Tobin C. Harding" me@tobin.cc

commit 8ddd6efa56c3fe23df9fe4cf5e2b49cc55416ef4 upstream.

There are a couple of whitespace issues around the function get_random_bytes_arch(). In preparation for patching this function let's clean them up.

Acked-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Tobin C. Harding me@tobin.cc Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1742,7 +1742,7 @@ void get_random_bytes_arch(void *buf, in

if (!arch_get_random_long(&v)) break; - + memcpy(p, &v, chunk); p += chunk; nbytes -= chunk; @@ -1753,7 +1753,6 @@ void get_random_bytes_arch(void *buf, in } EXPORT_SYMBOL(get_random_bytes_arch);

- /* * init_std_data - initialize pool with system data *

From: Theodore Ts'o tytso@mit.edu

commit 39a8883a2b989d1d21bd8dd99f5557f0c5e89694 upstream.

This gives the user building their own kernel (or a Linux distribution) the option of deciding whether or not to trust the CPU's hardware random number generator (e.g., RDRAND for x86 CPU's) as being correctly implemented and not having a back door introduced (perhaps courtesy of a Nation State's law enforcement or intelligence agencies).

This will prevent getrandom(2) from blocking, if there is a willingness to trust the CPU manufacturer.

Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/Kconfig | 14 ++++++++++++++ drivers/char/random.c | 11 ++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-)

--- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -590,3 +590,17 @@ source "drivers/char/xillybus/Kconfig"

endmenu

+config RANDOM_TRUST_CPU + bool "Trust the CPU manufacturer to initialize Linux's CRNG" + depends on X86 || S390 || PPC + default n + help + Assume that CPU manufacturer (e.g., Intel or AMD for RDSEED or + RDRAND, IBM for the S390 and Power PC architectures) is trustworthy + for the purposes of initializing Linux's CRNG. Since this is not + something that can be independently audited, this amounts to trusting + that CPU manufacturer (perhaps with the insistence or mandate + of a Nation State's intelligence or law enforcement agencies) + has not installed a hidden back door to compromise the CPU's + random number generation facilities. + --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -782,6 +782,7 @@ static void invalidate_batched_entropy(v static void crng_initialize(struct crng_state *crng) { int i; + int arch_init = 1; unsigned long rv;

memcpy(&crng->state[0], "expand 32-byte k", 16); @@ -792,10 +793,18 @@ static void crng_initialize(struct crng_ _get_random_bytes(&crng->state[4], sizeof(__u32) * 12); for (i = 4; i < 16; i++) { if (!arch_get_random_seed_long(&rv) && - !arch_get_random_long(&rv)) + !arch_get_random_long(&rv)) { rv = random_get_entropy(); + arch_init = 0; + } crng->state[i] ^= rv; } +#ifdef CONFIG_RANDOM_TRUST_CPU + if (arch_init) { + crng_init = 2; + pr_notice("random: crng done (trusting CPU's manufacturer)\n"); + } +#endif crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1; }

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 9a47249d444d344051c7c0e909fad0e88515a5c2 upstream.

It is very useful to be able to know whether or not get_random_bytes_wait / wait_for_random_bytes is going to block or not, or whether plain get_random_bytes is going to return good randomness or bad randomness.

The particular use case is for mitigating certain attacks in WireGuard. A handshake packet arrives and is queued up. Elsewhere a worker thread takes items from the queue and processes them. In replying to these items, it needs to use some random data, and it has to be good random data. If we simply block until we can have good randomness, then it's possible for an attacker to fill the queue up with packets waiting to be processed. Upon realizing the queue is full, WireGuard will detect that it's under a denial of service attack, and behave accordingly. A better approach is just to drop incoming handshake packets if the crng is not yet initialized.

This patch, therefore, makes that information directly accessible.

Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 15 +++++++++++++++ include/linux/random.h | 1 + 2 files changed, 16 insertions(+)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1669,6 +1669,21 @@ int wait_for_random_bytes(void) EXPORT_SYMBOL(wait_for_random_bytes);

/* + * Returns whether or not the urandom pool has been seeded and thus guaranteed + * to supply cryptographically secure random numbers. This applies to: the + * /dev/urandom device, the get_random_bytes function, and the get_random_{u32, + * ,u64,int,long} family of functions. + * + * Returns: true if the urandom pool has been seeded. + * false if the urandom pool has not been seeded. + */ +bool rng_is_initialized(void) +{ + return crng_ready(); +} +EXPORT_SYMBOL(rng_is_initialized); + +/* * Add a callback function that will be invoked when the nonblocking * pool is initialised. * --- a/include/linux/random.h +++ b/include/linux/random.h @@ -36,6 +36,7 @@ extern void add_interrupt_randomness(int

extern void get_random_bytes(void *buf, int nbytes); extern int wait_for_random_bytes(void); +extern bool rng_is_initialized(void); extern int add_random_ready_callback(struct random_ready_callback *rdy); extern void del_random_ready_callback(struct random_ready_callback *rdy); extern int __must_check get_random_bytes_arch(void *buf, int nbytes);

From: Rasmus Villemoes linux@rasmusvillemoes.dk

commit 26e0854ab3310bbeef1ed404a2c87132fc91f8e1 upstream.

Never modified, might as well be put in .rodata.

Signed-off-by: Rasmus Villemoes linux@rasmusvillemoes.dk Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -359,7 +359,7 @@ static int random_write_wakeup_bits = 28 * polynomial which improves the resulting TGFSR polynomial to be * irreducible, which we have made here. */ -static struct poolinfo { +static const struct poolinfo { int poolbitshift, poolwords, poolbytes, poolbits, poolfracbits; #define S(x) ilog2(x)+5, (x), (x)*4, (x)*32, (x) << (ENTROPY_SHIFT+5) int tap1, tap2, tap3, tap4, tap5;

From: Rasmus Villemoes linux@rasmusvillemoes.dk

commit 764ed189c82090c1d85f0e30636156736d8f09a8 upstream.

Since the definition of struct crng_state is private to random.c, and primary_crng is neither declared or used elsewhere, there's no reason for that symbol to have external linkage.

Signed-off-by: Rasmus Villemoes linux@rasmusvillemoes.dk Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -415,7 +415,7 @@ struct crng_state { spinlock_t lock; };

-struct crng_state primary_crng = { +static struct crng_state primary_crng = { .lock = __SPIN_LOCK_UNLOCKED(primary_crng.lock), };

From: Kees Cook keescook@chromium.org

commit d55535232c3dbde9a523a9d10d68670f5fe5dec3 upstream.

Right now rand_initialize() is run as an early_initcall(), but it only depends on timekeeping_init() (for mixing ktime_get_real() into the pools). However, the call to boot_init_stack_canary() for stack canary initialization runs earlier, which triggers a warning at boot:

random: get_random_bytes called from start_kernel+0x357/0x548 with crng_init=0

Instead, this moves rand_initialize() to after timekeeping_init(), and moves canary initialization here as well.

Note that this warning may still remain for machines that do not have UEFI RNG support (which initializes the RNG pools during setup_arch()), or for x86 machines without RDRAND (or booting without "random.trust=on" or CONFIG_RANDOM_TRUST_CPU=y).

Signed-off-by: Kees Cook keescook@chromium.org Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 5 ++--- include/linux/random.h | 1 + init/main.c | 21 ++++++++++++++------- 3 files changed, 17 insertions(+), 10 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1789,7 +1789,7 @@ EXPORT_SYMBOL(get_random_bytes_arch); * data into the pool to prepare it for use. The pool is not cleared * as that can only decrease the entropy in the pool. */ -static void init_std_data(struct entropy_store *r) +static void __init init_std_data(struct entropy_store *r) { int i; ktime_t now = ktime_get_real(); @@ -1816,7 +1816,7 @@ static void init_std_data(struct entropy * take care not to overwrite the precious per platform data * we were given. */ -static int rand_initialize(void) +int __init rand_initialize(void) { init_std_data(&input_pool); init_std_data(&blocking_pool); @@ -1828,7 +1828,6 @@ static int rand_initialize(void) } return 0; } -early_initcall(rand_initialize);

#ifdef CONFIG_BLOCK void rand_initialize_disk(struct gendisk *disk) --- a/include/linux/random.h +++ b/include/linux/random.h @@ -36,6 +36,7 @@ extern void add_interrupt_randomness(int

extern void get_random_bytes(void *buf, int nbytes); extern int wait_for_random_bytes(void); +extern int __init rand_initialize(void); extern bool rng_is_initialized(void); extern int add_random_ready_callback(struct random_ready_callback *rdy); extern void del_random_ready_callback(struct random_ready_callback *rdy); --- a/init/main.c +++ b/init/main.c @@ -531,13 +531,6 @@ asmlinkage __visible void __init start_k page_address_init(); pr_notice("%s", linux_banner); setup_arch(&command_line); - /* - * Set up the the initial canary and entropy after arch - * and after adding latent and command line entropy. - */ - add_latent_entropy(); - add_device_randomness(command_line, strlen(command_line)); - boot_init_stack_canary(); mm_init_cpumask(&init_mm); setup_command_line(command_line); setup_nr_cpu_ids(); @@ -614,6 +607,20 @@ asmlinkage __visible void __init start_k hrtimers_init(); softirq_init(); timekeeping_init(); + + /* + * For best initial stack canary entropy, prepare it after: + * - setup_arch() for any UEFI RNG entropy and boot cmdline access + * - timekeeping_init() for ktime entropy used in rand_initialize() + * - rand_initialize() to get any arch-specific entropy like RDRAND + * - add_latent_entropy() to get any latent entropy + * - adding command line entropy + */ + rand_initialize(); + add_latent_entropy(); + add_device_randomness(command_line, strlen(command_line)); + boot_init_stack_canary(); + time_init(); sched_clock_postinit(); printk_safe_init();

From: Vasily Gorbik gor@linux.ibm.com

commit 7e756f423af808b6571fed3144747db2ef7fa1c5 upstream.

Some architectures set up CFLAGS for linux decompressor phase from scratch and do not include GCC_PLUGINS_CFLAGS. Since "latent_entropy" variable declaration is generated by the plugin code itself including linux/random.h in decompressor code then would cause a build error. E.g. on s390:

In file included from ./include/linux/net.h:22, from ./include/linux/skbuff.h:29, from ./include/linux/if_ether.h:23, from ./arch/s390/include/asm/diag.h:12, from arch/s390/boot/startup.c:8: ./include/linux/random.h: In function 'add_latent_entropy': ./include/linux/random.h:26:39: error: 'latent_entropy' undeclared (first use in this function); did you mean 'add_latent_entropy'? 26 | add_device_randomness((const void *)&latent_entropy, | ^~~~~~~~~~~~~~ | add_latent_entropy ./include/linux/random.h:26:39: note: each undeclared identifier is reported only once for each function it appears in

The build error is triggered by commit a80313ff91ab ("s390/kernel: introduce .dma sections") which made it into 5.2 merge window.

To address that avoid using CONFIG_GCC_PLUGIN_LATENT_ENTROPY in favour of LATENT_ENTROPY_PLUGIN definition which is defined as a part of gcc plugins cflags and hence reflect more accurately when gcc plugin is active. Besides that it is also used for similar purpose in linux/compiler-gcc.h for latent_entropy attribute definition.

Signed-off-by: Vasily Gorbik gor@linux.ibm.com Acked-by: Kees Cook keescook@chromium.org Signed-off-by: Martin Schwidefsky schwidefsky@de.ibm.com Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- include/linux/random.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

--- a/include/linux/random.h +++ b/include/linux/random.h @@ -20,7 +20,7 @@ struct random_ready_callback {

extern void add_device_randomness(const void *, unsigned int);

-#if defined(CONFIG_GCC_PLUGIN_LATENT_ENTROPY) && !defined(__CHECKER__) +#if defined(LATENT_ENTROPY_PLUGIN) && !defined(__CHECKER__) static inline void add_latent_entropy(void) { add_device_randomness((const void *)&latent_entropy,

From: Stephen Boyd swboyd@chromium.org

commit ff296293b3538d19278a7f7cd1f3aa600ad9164c upstream.

The kthread calling this function is freezable after commit 03a3bb7ae631 ("hwrng: core - Freeze khwrng thread during suspend") is applied. Unfortunately, this function uses wait_event_interruptible() but doesn't check for the kthread being woken up by the fake freezer signal. When a user suspends the system, this kthread will wake up and if it fails the entropy size check it will immediately go back to sleep and not go into the freezer. Eventually, suspend will fail because the task never froze and a warning message like this may appear:

PM: suspend entry (deep) Filesystems sync: 0.000 seconds Freezing user space processes ... (elapsed 0.001 seconds) done. OOM killer disabled. Freezing remaining freezable tasks ... Freezing of tasks failed after 20.003 seconds (1 tasks refusing to freeze, wq_busy=0): hwrng R running task 0 289 2 0x00000020 [<c08c64c4>] (__schedule) from [<c08c6a10>] (schedule+0x3c/0xc0) [<c08c6a10>] (schedule) from [<c05dbd8c>] (add_hwgenerator_randomness+0xb0/0x100) [<c05dbd8c>] (add_hwgenerator_randomness) from [<bf1803c8>] (hwrng_fillfn+0xc0/0x14c [rng_core]) [<bf1803c8>] (hwrng_fillfn [rng_core]) from [<c015abec>] (kthread+0x134/0x148) [<c015abec>] (kthread) from [<c01010e8>] (ret_from_fork+0x14/0x2c)

Check for a freezer signal here and skip adding any randomness if the task wakes up because it was frozen. This should make the kthread freeze properly and suspend work again.

Fixes: 03a3bb7ae631 ("hwrng: core - Freeze khwrng thread during suspend") Reported-by: Keerthy j-keerthy@ti.com Tested-by: Keerthy j-keerthy@ti.com Signed-off-by: Stephen Boyd swboyd@chromium.org Signed-off-by: Herbert Xu herbert@gondor.apana.org.au Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -2420,6 +2420,7 @@ void add_hwgenerator_randomness(const ch size_t entropy) { struct entropy_store *poolp = &input_pool; + bool frozen = false;

if (unlikely(crng_init == 0)) { crng_fast_load(buffer, count); @@ -2430,9 +2431,12 @@ void add_hwgenerator_randomness(const ch * We'll be woken up again once below random_write_wakeup_thresh, * or when the calling thread is about to terminate. */ - wait_event_interruptible(random_write_wait, kthread_should_stop() || + wait_event_interruptible(random_write_wait, + kthread_freezable_should_stop(&frozen) || ENTROPY_BITS(&input_pool) <= random_write_wakeup_bits); - mix_pool_bytes(poolp, buffer, count); - credit_entropy_bits(poolp, entropy); + if (!frozen) { + mix_pool_bytes(poolp, buffer, count); + credit_entropy_bits(poolp, entropy); + } } EXPORT_SYMBOL_GPL(add_hwgenerator_randomness);

From: Stephen Boyd swboyd@chromium.org

commit 59b569480dc8bb9dce57cdff133853a842dfd805 upstream.

Sebastian reports that after commit ff296293b353 ("random: Support freezable kthreads in add_hwgenerator_randomness()") we can call might_sleep() when the task state is TASK_INTERRUPTIBLE (state=1). This leads to the following warning.

do not call blocking ops when !TASK_RUNNING; state=1 set at [<00000000349d1489>] prepare_to_wait_event+0x5a/0x180 WARNING: CPU: 0 PID: 828 at kernel/sched/core.c:6741 __might_sleep+0x6f/0x80 Modules linked in:

CPU: 0 PID: 828 Comm: hwrng Not tainted 5.3.0-rc7-next-20190903+ #46 RIP: 0010:__might_sleep+0x6f/0x80

Call Trace: kthread_freezable_should_stop+0x1b/0x60 add_hwgenerator_randomness+0xdd/0x130 hwrng_fillfn+0xbf/0x120 kthread+0x10c/0x140 ret_from_fork+0x27/0x50

We shouldn't call kthread_freezable_should_stop() from deep within the wait_event code because the task state is still set as TASK_INTERRUPTIBLE instead of TASK_RUNNING and kthread_freezable_should_stop() will try to call into the freezer with the task in the wrong state. Use wait_event_freezable() instead so that it calls schedule() in the right place and tries to enter the freezer when the task state is TASK_RUNNING instead.

Reported-by: Sebastian Andrzej Siewior bigeasy@linutronix.de Tested-by: Sebastian Andrzej Siewior bigeasy@linutronix.de Cc: Keerthy j-keerthy@ti.com Fixes: ff296293b353 ("random: Support freezable kthreads in add_hwgenerator_randomness()") Signed-off-by: Stephen Boyd swboyd@chromium.org Signed-off-by: Herbert Xu herbert@gondor.apana.org.au Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -327,6 +327,7 @@ #include <linux/percpu.h> #include <linux/cryptohash.h> #include <linux/fips.h> +#include <linux/freezer.h> #include <linux/ptrace.h> #include <linux/workqueue.h> #include <linux/irq.h> @@ -2420,7 +2421,6 @@ void add_hwgenerator_randomness(const ch size_t entropy) { struct entropy_store *poolp = &input_pool; - bool frozen = false;

if (unlikely(crng_init == 0)) { crng_fast_load(buffer, count); @@ -2431,13 +2431,11 @@ void add_hwgenerator_randomness(const ch * We'll be woken up again once below random_write_wakeup_thresh, * or when the calling thread is about to terminate. */ - wait_event_interruptible(random_write_wait, - kthread_freezable_should_stop(&frozen) || + wait_event_freezable(random_write_wait, + kthread_should_stop() || ENTROPY_BITS(&input_pool) <= random_write_wakeup_bits); - if (!frozen) { - mix_pool_bytes(poolp, buffer, count); - credit_entropy_bits(poolp, entropy); - } + mix_pool_bytes(poolp, buffer, count); + credit_entropy_bits(poolp, entropy); } EXPORT_SYMBOL_GPL(add_hwgenerator_randomness);

From: Herbert Xu herbert@gondor.apana.org.au

commit 08e97aec700aeff54c4847f170e566cbd7e14e81 upstream.

This reverts commit 03a3bb7ae631 ("hwrng: core - Freeze khwrng thread during suspend"), ff296293b353 ("random: Support freezable kthreads in add_hwgenerator_randomness()") and 59b569480dc8 ("random: Use wait_event_freezable() in add_hwgenerator_randomness()").

These patches introduced regressions and we need more time to get them ready for mainline.

Signed-off-by: Herbert Xu herbert@gondor.apana.org.au Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -327,7 +327,6 @@ #include <linux/percpu.h> #include <linux/cryptohash.h> #include <linux/fips.h> -#include <linux/freezer.h> #include <linux/ptrace.h> #include <linux/workqueue.h> #include <linux/irq.h> @@ -2431,8 +2430,7 @@ void add_hwgenerator_randomness(const ch * We'll be woken up again once below random_write_wakeup_thresh, * or when the calling thread is about to terminate. */ - wait_event_freezable(random_write_wait, - kthread_should_stop() || + wait_event_interruptible(random_write_wait, kthread_should_stop() || ENTROPY_BITS(&input_pool) <= random_write_wakeup_bits); mix_pool_bytes(poolp, buffer, count); credit_entropy_bits(poolp, entropy);

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 66d7fb94e4ffe5acc589e0b2b4710aecc1f07a28 upstream.

The C implementation was originally based on Samuel Neves' public domain reference implementation but has since been heavily modified for the kernel. We're able to do compile-time optimizations by moving some scaffolding around the final function into the header file.

Information: https://blake2.net/

Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Samuel Neves sneves@dei.uc.pt Co-developed-by: Samuel Neves sneves@dei.uc.pt [ardb: - move from lib/zinc to lib/crypto - remove simd handling - rewrote selftest for better coverage - use fixed digest length for blake2s_hmac() and rename to blake2s256_hmac() ] Signed-off-by: Ard Biesheuvel ardb@kernel.org Signed-off-by: Herbert Xu herbert@gondor.apana.org.au [Jason: for stable, skip kconfig and wire up directly, and skip the arch hooks; optimized implementations need not be backported.] Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- include/crypto/blake2s.h | 106 ++++++ include/crypto/internal/blake2s.h | 19 + lib/Makefile | 2 lib/crypto/Makefile | 7 lib/crypto/blake2s-generic.c | 111 ++++++ lib/crypto/blake2s-selftest.c | 622 ++++++++++++++++++++++++++++++++++++++ lib/crypto/blake2s.c | 115 +++++++ 7 files changed, 982 insertions(+) create mode 100644 include/crypto/blake2s.h create mode 100644 include/crypto/internal/blake2s.h create mode 100644 lib/crypto/Makefile create mode 100644 lib/crypto/blake2s-generic.c create mode 100644 lib/crypto/blake2s-selftest.c create mode 100644 lib/crypto/blake2s.c

--- /dev/null +++ b/include/crypto/blake2s.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld Jason@zx2c4.com. All Rights Reserved. + */ + +#ifndef BLAKE2S_H +#define BLAKE2S_H + +#include <linux/types.h> +#include <linux/kernel.h> +#include <linux/string.h> + +#include <asm/bug.h> + +enum blake2s_lengths { + BLAKE2S_BLOCK_SIZE = 64, + BLAKE2S_HASH_SIZE = 32, + BLAKE2S_KEY_SIZE = 32, + + BLAKE2S_128_HASH_SIZE = 16, + BLAKE2S_160_HASH_SIZE = 20, + BLAKE2S_224_HASH_SIZE = 28, + BLAKE2S_256_HASH_SIZE = 32, +}; + +struct blake2s_state { + u32 h[8]; + u32 t[2]; + u32 f[2]; + u8 buf[BLAKE2S_BLOCK_SIZE]; + unsigned int buflen; + unsigned int outlen; +}; + +enum blake2s_iv { + BLAKE2S_IV0 = 0x6A09E667UL, + BLAKE2S_IV1 = 0xBB67AE85UL, + BLAKE2S_IV2 = 0x3C6EF372UL, + BLAKE2S_IV3 = 0xA54FF53AUL, + BLAKE2S_IV4 = 0x510E527FUL, + BLAKE2S_IV5 = 0x9B05688CUL, + BLAKE2S_IV6 = 0x1F83D9ABUL, + BLAKE2S_IV7 = 0x5BE0CD19UL, +}; + +void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen); +void blake2s_final(struct blake2s_state *state, u8 *out); + +static inline void blake2s_init_param(struct blake2s_state *state, + const u32 param) +{ + *state = (struct blake2s_state){{ + BLAKE2S_IV0 ^ param, + BLAKE2S_IV1, + BLAKE2S_IV2, + BLAKE2S_IV3, + BLAKE2S_IV4, + BLAKE2S_IV5, + BLAKE2S_IV6, + BLAKE2S_IV7, + }}; +} + +static inline void blake2s_init(struct blake2s_state *state, + const size_t outlen) +{ + blake2s_init_param(state, 0x01010000 | outlen); + state->outlen = outlen; +} + +static inline void blake2s_init_key(struct blake2s_state *state, + const size_t outlen, const void *key, + const size_t keylen) +{ + WARN_ON(IS_ENABLED(DEBUG) && (!outlen || outlen > BLAKE2S_HASH_SIZE || + !key || !keylen || keylen > BLAKE2S_KEY_SIZE)); + + blake2s_init_param(state, 0x01010000 | keylen << 8 | outlen); + memcpy(state->buf, key, keylen); + state->buflen = BLAKE2S_BLOCK_SIZE; + state->outlen = outlen; +} + +static inline void blake2s(u8 *out, const u8 *in, const u8 *key, + const size_t outlen, const size_t inlen, + const size_t keylen) +{ + struct blake2s_state state; + + WARN_ON(IS_ENABLED(DEBUG) && ((!in && inlen > 0) || !out || !outlen || + outlen > BLAKE2S_HASH_SIZE || keylen > BLAKE2S_KEY_SIZE || + (!key && keylen))); + + if (keylen) + blake2s_init_key(&state, outlen, key, keylen); + else + blake2s_init(&state, outlen); + + blake2s_update(&state, in, inlen); + blake2s_final(&state, out); +} + +void blake2s256_hmac(u8 *out, const u8 *in, const u8 *key, const size_t inlen, + const size_t keylen); + +#endif /* BLAKE2S_H */ --- /dev/null +++ b/include/crypto/internal/blake2s.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ + +#ifndef BLAKE2S_INTERNAL_H +#define BLAKE2S_INTERNAL_H + +#include <crypto/blake2s.h> + +void blake2s_compress_generic(struct blake2s_state *state,const u8 *block, + size_t nblocks, const u32 inc); + +void blake2s_compress_arch(struct blake2s_state *state,const u8 *block, + size_t nblocks, const u32 inc); + +static inline void blake2s_set_lastblock(struct blake2s_state *state) +{ + state->f[0] = -1; +} + +#endif /* BLAKE2S_INTERNAL_H */ --- a/lib/Makefile +++ b/lib/Makefile @@ -262,3 +262,5 @@ CFLAGS_ubsan.o := $(call cc-option, -fno obj-$(CONFIG_SBITMAP) += sbitmap.o

obj-$(CONFIG_PARMAN) += parman.o + +obj-y += crypto/ --- /dev/null +++ b/lib/crypto/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-y += libblake2s.o +libblake2s-y += blake2s.o blake2s-generic.o +ifneq ($(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS),y) +libblake2s-y += blake2s-selftest.o +endif --- /dev/null +++ b/lib/crypto/blake2s-generic.c @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld Jason@zx2c4.com. All Rights Reserved. + * + * This is an implementation of the BLAKE2s hash and PRF functions. + * + * Information: https://blake2.net/ + * + */ + +#include <crypto/internal/blake2s.h> +#include <linux/types.h> +#include <linux/string.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/bug.h> +#include <asm/unaligned.h> + +static const u8 blake2s_sigma[10][16] = { + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 }, + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 }, + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 }, + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 }, + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 }, + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 }, + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 }, + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 }, +}; + +static inline void blake2s_increment_counter(struct blake2s_state *state, + const u32 inc) +{ + state->t[0] += inc; + state->t[1] += (state->t[0] < inc); +} + +void blake2s_compress_generic(struct blake2s_state *state,const u8 *block, + size_t nblocks, const u32 inc) +{ + u32 m[16]; + u32 v[16]; + int i; + + WARN_ON(IS_ENABLED(DEBUG) && + (nblocks > 1 && inc != BLAKE2S_BLOCK_SIZE)); + + while (nblocks > 0) { + blake2s_increment_counter(state, inc); + memcpy(m, block, BLAKE2S_BLOCK_SIZE); + le32_to_cpu_array(m, ARRAY_SIZE(m)); + memcpy(v, state->h, 32); + v[ 8] = BLAKE2S_IV0; + v[ 9] = BLAKE2S_IV1; + v[10] = BLAKE2S_IV2; + v[11] = BLAKE2S_IV3; + v[12] = BLAKE2S_IV4 ^ state->t[0]; + v[13] = BLAKE2S_IV5 ^ state->t[1]; + v[14] = BLAKE2S_IV6 ^ state->f[0]; + v[15] = BLAKE2S_IV7 ^ state->f[1]; + +#define G(r, i, a, b, c, d) do { \ + a += b + m[blake2s_sigma[r][2 * i + 0]]; \ + d = ror32(d ^ a, 16); \ + c += d; \ + b = ror32(b ^ c, 12); \ + a += b + m[blake2s_sigma[r][2 * i + 1]]; \ + d = ror32(d ^ a, 8); \ + c += d; \ + b = ror32(b ^ c, 7); \ +} while (0) + +#define ROUND(r) do { \ + G(r, 0, v[0], v[ 4], v[ 8], v[12]); \ + G(r, 1, v[1], v[ 5], v[ 9], v[13]); \ + G(r, 2, v[2], v[ 6], v[10], v[14]); \ + G(r, 3, v[3], v[ 7], v[11], v[15]); \ + G(r, 4, v[0], v[ 5], v[10], v[15]); \ + G(r, 5, v[1], v[ 6], v[11], v[12]); \ + G(r, 6, v[2], v[ 7], v[ 8], v[13]); \ + G(r, 7, v[3], v[ 4], v[ 9], v[14]); \ +} while (0) + ROUND(0); + ROUND(1); + ROUND(2); + ROUND(3); + ROUND(4); + ROUND(5); + ROUND(6); + ROUND(7); + ROUND(8); + ROUND(9); + +#undef G +#undef ROUND + + for (i = 0; i < 8; ++i) + state->h[i] ^= v[i] ^ v[i + 8]; + + block += BLAKE2S_BLOCK_SIZE; + --nblocks; + } +} + +EXPORT_SYMBOL(blake2s_compress_generic); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("BLAKE2s hash function"); +MODULE_AUTHOR("Jason A. Donenfeld Jason@zx2c4.com"); --- /dev/null +++ b/lib/crypto/blake2s-selftest.c @@ -0,0 +1,622 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld Jason@zx2c4.com. All Rights Reserved. + */ + +#include <crypto/blake2s.h> +#include <linux/string.h> + +/* + * blake2s_testvecs[] generated with the program below (using libb2-dev and + * libssl-dev [OpenSSL]) + * + * #include <blake2.h> + * #include <stdint.h> + * #include <stdio.h> + * + * #include <openssl/evp.h> + * #include <openssl/hmac.h> + * + * #define BLAKE2S_TESTVEC_COUNT 256 + * + * static void print_vec(const uint8_t vec[], int len) + * { + * int i; + * + * printf(" { "); + * for (i = 0; i < len; i++) { + * if (i && (i % 12) == 0) + * printf("\n "); + * printf("0x%02x, ", vec[i]); + * } + * printf("},\n"); + * } + * + * int main(void) + * { + * uint8_t key[BLAKE2S_KEYBYTES]; + * uint8_t buf[BLAKE2S_TESTVEC_COUNT]; + * uint8_t hash[BLAKE2S_OUTBYTES]; + * int i, j; + * + * key[0] = key[1] = 1; + * for (i = 2; i < BLAKE2S_KEYBYTES; ++i) + * key[i] = key[i - 2] + key[i - 1]; + * + * for (i = 0; i < BLAKE2S_TESTVEC_COUNT; ++i) + * buf[i] = (uint8_t)i; + * + * printf("static const u8 blake2s_testvecs[][BLAKE2S_HASH_SIZE] __initconst = {\n"); + * + * for (i = 0; i < BLAKE2S_TESTVEC_COUNT; ++i) { + * int outlen = 1 + i % BLAKE2S_OUTBYTES; + * int keylen = (13 * i) % (BLAKE2S_KEYBYTES + 1); + * + * blake2s(hash, buf, key + BLAKE2S_KEYBYTES - keylen, outlen, i, + * keylen); + * print_vec(hash, outlen); + * } + * printf("};\n\n"); + * + * printf("static const u8 blake2s_hmac_testvecs[][BLAKE2S_HASH_SIZE] __initconst = {\n"); + * + * HMAC(EVP_blake2s256(), key, sizeof(key), buf, sizeof(buf), hash, NULL); + * print_vec(hash, BLAKE2S_OUTBYTES); + * + * HMAC(EVP_blake2s256(), buf, sizeof(buf), key, sizeof(key), hash, NULL); + * print_vec(hash, BLAKE2S_OUTBYTES); + * + * printf("};\n"); + * + * return 0; + *} + */ +static const u8 blake2s_testvecs[][BLAKE2S_HASH_SIZE] __initconst = { + { 0xa1, }, + { 0x7c, 0x89, }, + { 0x74, 0x0e, 0xd4, }, + { 0x47, 0x0c, 0x21, 0x15, }, + { 0x18, 0xd6, 0x9c, 0xa6, 0xc4, }, + { 0x13, 0x5d, 0x16, 0x63, 0x2e, 0xf9, }, + { 0x2c, 0xb5, 0x04, 0xb7, 0x99, 0xe2, 0x73, }, + { 0x9a, 0x0f, 0xd2, 0x39, 0xd6, 0x68, 0x1b, 0x92, }, + { 0xc8, 0xde, 0x7a, 0xea, 0x2f, 0xf4, 0xd2, 0xe3, 0x2b, }, + { 0x5b, 0xf9, 0x43, 0x52, 0x0c, 0x12, 0xba, 0xb5, 0x93, 0x9f, }, + { 0xc6, 0x2c, 0x4e, 0x80, 0xfc, 0x32, 0x5b, 0x33, 0xb8, 0xb8, 0x0a, }, + { 0xa7, 0x5c, 0xfd, 0x3a, 0xcc, 0xbf, 0x90, 0xca, 0xb7, 0x97, 0xde, 0xd8, }, + { 0x66, 0xca, 0x3c, 0xc4, 0x19, 0xef, 0x92, 0x66, 0x3f, 0x21, 0x8f, 0xda, + 0xb7, }, + { 0xba, 0xe5, 0xbb, 0x30, 0x25, 0x94, 0x6d, 0xc3, 0x89, 0x09, 0xc4, 0x25, + 0x52, 0x3e, }, + { 0xa2, 0xef, 0x0e, 0x52, 0x0b, 0x5f, 0xa2, 0x01, 0x6d, 0x0a, 0x25, 0xbc, + 0x57, 0xe2, 0x27, }, + { 0x4f, 0xe0, 0xf9, 0x52, 0x12, 0xda, 0x84, 0xb7, 0xab, 0xae, 0xb0, 0xa6, + 0x47, 0x2a, 0xc7, 0xf5, }, + { 0x56, 0xe7, 0xa8, 0x1c, 0x4c, 0xca, 0xed, 0x90, 0x31, 0xec, 0x87, 0x43, + 0xe7, 0x72, 0x08, 0xec, 0xbe, }, + { 0x7e, 0xdf, 0x80, 0x1c, 0x93, 0x33, 0xfd, 0x53, 0x44, 0xba, 0xfd, 0x96, + 0xe1, 0xbb, 0xb5, 0x65, 0xa5, 0x00, }, + { 0xec, 0x6b, 0xed, 0xf7, 0x7b, 0x62, 0x1d, 0x7d, 0xf4, 0x82, 0xf3, 0x1e, + 0x18, 0xff, 0x2b, 0xc4, 0x06, 0x20, 0x2a, }, + { 0x74, 0x98, 0xd7, 0x68, 0x63, 0xed, 0x87, 0xe4, 0x5d, 0x8d, 0x9e, 0x1d, + 0xfd, 0x2a, 0xbb, 0x86, 0xac, 0xe9, 0x2a, 0x89, }, + { 0x89, 0xc3, 0x88, 0xce, 0x2b, 0x33, 0x1e, 0x10, 0xd1, 0x37, 0x20, 0x86, + 0x28, 0x43, 0x70, 0xd9, 0xfb, 0x96, 0xd9, 0xb5, 0xd3, }, + { 0xcb, 0x56, 0x74, 0x41, 0x8d, 0x80, 0x01, 0x9a, 0x6b, 0x38, 0xe1, 0x41, + 0xad, 0x9c, 0x62, 0x74, 0xce, 0x35, 0xd5, 0x6c, 0x89, 0x6e, }, + { 0x79, 0xaf, 0x94, 0x59, 0x99, 0x26, 0xe1, 0xc9, 0x34, 0xfe, 0x7c, 0x22, + 0xf7, 0x43, 0xd7, 0x65, 0xd4, 0x48, 0x18, 0xac, 0x3d, 0xfd, 0x93, }, + { 0x85, 0x0d, 0xff, 0xb8, 0x3e, 0x87, 0x41, 0xb0, 0x95, 0xd3, 0x3d, 0x00, + 0x47, 0x55, 0x9e, 0xd2, 0x69, 0xea, 0xbf, 0xe9, 0x7a, 0x2d, 0x61, 0x45, }, + { 0x03, 0xe0, 0x85, 0xec, 0x54, 0xb5, 0x16, 0x53, 0xa8, 0xc4, 0x71, 0xe9, + 0x6a, 0xe7, 0xcb, 0xc4, 0x15, 0x02, 0xfc, 0x34, 0xa4, 0xa4, 0x28, 0x13, + 0xd1, }, + { 0xe3, 0x34, 0x4b, 0xe1, 0xd0, 0x4b, 0x55, 0x61, 0x8f, 0xc0, 0x24, 0x05, + 0xe6, 0xe0, 0x3d, 0x70, 0x24, 0x4d, 0xda, 0xb8, 0x91, 0x05, 0x29, 0x07, + 0x01, 0x3e, }, + { 0x61, 0xff, 0x01, 0x72, 0xb1, 0x4d, 0xf6, 0xfe, 0xd1, 0xd1, 0x08, 0x74, + 0xe6, 0x91, 0x44, 0xeb, 0x61, 0xda, 0x40, 0xaf, 0xfc, 0x8c, 0x91, 0x6b, + 0xec, 0x13, 0xed, }, + { 0xd4, 0x40, 0xd2, 0xa0, 0x7f, 0xc1, 0x58, 0x0c, 0x85, 0xa0, 0x86, 0xc7, + 0x86, 0xb9, 0x61, 0xc9, 0xea, 0x19, 0x86, 0x1f, 0xab, 0x07, 0xce, 0x37, + 0x72, 0x67, 0x09, 0xfc, }, + { 0x9e, 0xf8, 0x18, 0x67, 0x93, 0x10, 0x9b, 0x39, 0x75, 0xe8, 0x8b, 0x38, + 0x82, 0x7d, 0xb8, 0xb7, 0xa5, 0xaf, 0xe6, 0x6a, 0x22, 0x5e, 0x1f, 0x9c, + 0x95, 0x29, 0x19, 0xf2, 0x4b, }, + { 0xc8, 0x62, 0x25, 0xf5, 0x98, 0xc9, 0xea, 0xe5, 0x29, 0x3a, 0xd3, 0x22, + 0xeb, 0xeb, 0x07, 0x7c, 0x15, 0x07, 0xee, 0x15, 0x61, 0xbb, 0x05, 0x30, + 0x99, 0x7f, 0x11, 0xf6, 0x0a, 0x1d, }, + { 0x68, 0x70, 0xf7, 0x90, 0xa1, 0x8b, 0x1f, 0x0f, 0xbb, 0xce, 0xd2, 0x0e, + 0x33, 0x1f, 0x7f, 0xa9, 0x78, 0xa8, 0xa6, 0x81, 0x66, 0xab, 0x8d, 0xcd, + 0x58, 0x55, 0x3a, 0x0b, 0x7a, 0xdb, 0xb5, }, + { 0xdd, 0x35, 0xd2, 0xb4, 0xf6, 0xc7, 0xea, 0xab, 0x64, 0x24, 0x4e, 0xfe, + 0xe5, 0x3d, 0x4e, 0x95, 0x8b, 0x6d, 0x6c, 0xbc, 0xb0, 0xf8, 0x88, 0x61, + 0x09, 0xb7, 0x78, 0xa3, 0x31, 0xfe, 0xd9, 0x2f, }, + { 0x0a, }, + { 0x6e, 0xd4, }, + { 0x64, 0xe9, 0xd1, }, + { 0x30, 0xdd, 0x71, 0xef, }, + { 0x11, 0xb5, 0x0c, 0x87, 0xc9, }, + { 0x06, 0x1c, 0x6d, 0x04, 0x82, 0xd0, }, + { 0x5c, 0x42, 0x0b, 0xee, 0xc5, 0x9c, 0xb2, }, + { 0xe8, 0x29, 0xd6, 0xb4, 0x5d, 0xf7, 0x2b, 0x93, }, + { 0x18, 0xca, 0x27, 0x72, 0x43, 0x39, 0x16, 0xbc, 0x6a, }, + { 0x39, 0x8f, 0xfd, 0x64, 0xf5, 0x57, 0x23, 0xb0, 0x45, 0xf8, }, + { 0xbb, 0x3a, 0x78, 0x6b, 0x02, 0x1d, 0x0b, 0x16, 0xe3, 0xb2, 0x9a, }, + { 0xb8, 0xb4, 0x0b, 0xe5, 0xd4, 0x1d, 0x0d, 0x85, 0x49, 0x91, 0x35, 0xfa, }, + { 0x6d, 0x48, 0x2a, 0x0c, 0x42, 0x08, 0xbd, 0xa9, 0x78, 0x6f, 0x18, 0xaf, + 0xe2, }, + { 0x10, 0x45, 0xd4, 0x58, 0x88, 0xec, 0x4e, 0x1e, 0xf6, 0x14, 0x92, 0x64, + 0x7e, 0xb0, }, + { 0x8b, 0x0b, 0x95, 0xee, 0x92, 0xc6, 0x3b, 0x91, 0xf1, 0x1e, 0xeb, 0x51, + 0x98, 0x0a, 0x8d, }, + { 0xa3, 0x50, 0x4d, 0xa5, 0x1d, 0x03, 0x68, 0xe9, 0x57, 0x78, 0xd6, 0x04, + 0xf1, 0xc3, 0x94, 0xd8, }, + { 0xb8, 0x66, 0x6e, 0xdd, 0x46, 0x15, 0xae, 0x3d, 0x83, 0x7e, 0xcf, 0xe7, + 0x2c, 0xe8, 0x8f, 0xc7, 0x34, }, + { 0x2e, 0xc0, 0x1f, 0x29, 0xea, 0xf6, 0xb9, 0xe2, 0xc2, 0x93, 0xeb, 0x41, + 0x0d, 0xf0, 0x0a, 0x13, 0x0e, 0xa2, }, + { 0x71, 0xb8, 0x33, 0xa9, 0x1b, 0xac, 0xf1, 0xb5, 0x42, 0x8f, 0x5e, 0x81, + 0x34, 0x43, 0xb7, 0xa4, 0x18, 0x5c, 0x47, }, + { 0xda, 0x45, 0xb8, 0x2e, 0x82, 0x1e, 0xc0, 0x59, 0x77, 0x9d, 0xfa, 0xb4, + 0x1c, 0x5e, 0xa0, 0x2b, 0x33, 0x96, 0x5a, 0x58, }, + { 0xe3, 0x09, 0x05, 0xa9, 0xeb, 0x48, 0x13, 0xad, 0x71, 0x88, 0x81, 0x9a, + 0x3e, 0x2c, 0xe1, 0x23, 0x99, 0x13, 0x35, 0x9f, 0xb5, }, + { 0xb7, 0x86, 0x2d, 0x16, 0xe1, 0x04, 0x00, 0x47, 0x47, 0x61, 0x31, 0xfb, + 0x14, 0xac, 0xd8, 0xe9, 0xe3, 0x49, 0xbd, 0xf7, 0x9c, 0x3f, }, + { 0x7f, 0xd9, 0x95, 0xa8, 0xa7, 0xa0, 0xcc, 0xba, 0xef, 0xb1, 0x0a, 0xa9, + 0x21, 0x62, 0x08, 0x0f, 0x1b, 0xff, 0x7b, 0x9d, 0xae, 0xb2, 0x95, }, + { 0x85, 0x99, 0xea, 0x33, 0xe0, 0x56, 0xff, 0x13, 0xc6, 0x61, 0x8c, 0xf9, + 0x57, 0x05, 0x03, 0x11, 0xf9, 0xfb, 0x3a, 0xf7, 0xce, 0xbb, 0x52, 0x30, }, + { 0xb2, 0x72, 0x9c, 0xf8, 0x77, 0x4e, 0x8f, 0x6b, 0x01, 0x6c, 0xff, 0x4e, + 0x4f, 0x02, 0xd2, 0xbc, 0xeb, 0x51, 0x28, 0x99, 0x50, 0xab, 0xc4, 0x42, + 0xe3, }, + { 0x8b, 0x0a, 0xb5, 0x90, 0x8f, 0xf5, 0x7b, 0xdd, 0xba, 0x47, 0x37, 0xc9, + 0x2a, 0xd5, 0x4b, 0x25, 0x08, 0x8b, 0x02, 0x17, 0xa7, 0x9e, 0x6b, 0x6e, + 0xe3, 0x90, }, + { 0x90, 0xdd, 0xf7, 0x75, 0xa7, 0xa3, 0x99, 0x5e, 0x5b, 0x7d, 0x75, 0xc3, + 0x39, 0x6b, 0xa0, 0xe2, 0x44, 0x53, 0xb1, 0x9e, 0xc8, 0xf1, 0x77, 0x10, + 0x58, 0x06, 0x9a, }, + { 0x99, 0x52, 0xf0, 0x49, 0xa8, 0x8c, 0xec, 0xa6, 0x97, 0x32, 0x13, 0xb5, + 0xf7, 0xa3, 0x8e, 0xfb, 0x4b, 0x59, 0x31, 0x3d, 0x01, 0x59, 0x98, 0x5d, + 0x53, 0x03, 0x1a, 0x39, }, + { 0x9f, 0xe0, 0xc2, 0xe5, 0x5d, 0x93, 0xd6, 0x9b, 0x47, 0x8f, 0x9b, 0xe0, + 0x26, 0x35, 0x84, 0x20, 0x1d, 0xc5, 0x53, 0x10, 0x0f, 0x22, 0xb9, 0xb5, + 0xd4, 0x36, 0xb1, 0xac, 0x73, }, + { 0x30, 0x32, 0x20, 0x3b, 0x10, 0x28, 0xec, 0x1f, 0x4f, 0x9b, 0x47, 0x59, + 0xeb, 0x7b, 0xee, 0x45, 0xfb, 0x0c, 0x49, 0xd8, 0x3d, 0x69, 0xbd, 0x90, + 0x2c, 0xf0, 0x9e, 0x8d, 0xbf, 0xd5, }, + { 0x2a, 0x37, 0x73, 0x7f, 0xf9, 0x96, 0x19, 0xaa, 0x25, 0xd8, 0x13, 0x28, + 0x01, 0x29, 0x89, 0xdf, 0x6e, 0x0c, 0x9b, 0x43, 0x44, 0x51, 0xe9, 0x75, + 0x26, 0x0c, 0xb7, 0x87, 0x66, 0x0b, 0x5f, }, + { 0x23, 0xdf, 0x96, 0x68, 0x91, 0x86, 0xd0, 0x93, 0x55, 0x33, 0x24, 0xf6, + 0xba, 0x08, 0x75, 0x5b, 0x59, 0x11, 0x69, 0xb8, 0xb9, 0xe5, 0x2c, 0x77, + 0x02, 0xf6, 0x47, 0xee, 0x81, 0xdd, 0xb9, 0x06, }, + { 0x9d, }, + { 0x9d, 0x7d, }, + { 0xfd, 0xc3, 0xda, }, + { 0xe8, 0x82, 0xcd, 0x21, }, + { 0xc3, 0x1d, 0x42, 0x4c, 0x74, }, + { 0xe9, 0xda, 0xf1, 0xa2, 0xe5, 0x7c, }, + { 0x52, 0xb8, 0x6f, 0x81, 0x5c, 0x3a, 0x4c, }, + { 0x5b, 0x39, 0x26, 0xfc, 0x92, 0x5e, 0xe0, 0x49, }, + { 0x59, 0xe4, 0x7c, 0x93, 0x1c, 0xf9, 0x28, 0x93, 0xde, }, + { 0xde, 0xdf, 0xb2, 0x43, 0x61, 0x0b, 0x86, 0x16, 0x4c, 0x2e, }, + { 0x14, 0x8f, 0x75, 0x51, 0xaf, 0xb9, 0xee, 0x51, 0x5a, 0xae, 0x23, }, + { 0x43, 0x5f, 0x50, 0xd5, 0x70, 0xb0, 0x5b, 0x87, 0xf5, 0xd9, 0xb3, 0x6d, }, + { 0x66, 0x0a, 0x64, 0x93, 0x79, 0x71, 0x94, 0x40, 0xb7, 0x68, 0x2d, 0xd3, + 0x63, }, + { 0x15, 0x00, 0xc4, 0x0c, 0x7d, 0x1b, 0x10, 0xa9, 0x73, 0x1b, 0x90, 0x6f, + 0xe6, 0xa9, }, + { 0x34, 0x75, 0xf3, 0x86, 0x8f, 0x56, 0xcf, 0x2a, 0x0a, 0xf2, 0x62, 0x0a, + 0xf6, 0x0e, 0x20, }, + { 0xb1, 0xde, 0xc9, 0xf5, 0xdb, 0xf3, 0x2f, 0x4c, 0xd6, 0x41, 0x7d, 0x39, + 0x18, 0x3e, 0xc7, 0xc3, }, + { 0xc5, 0x89, 0xb2, 0xf8, 0xb8, 0xc0, 0xa3, 0xb9, 0x3b, 0x10, 0x6d, 0x7c, + 0x92, 0xfc, 0x7f, 0x34, 0x41, }, + { 0xc4, 0xd8, 0xef, 0xba, 0xef, 0xd2, 0xaa, 0xc5, 0x6c, 0x8e, 0x3e, 0xbb, + 0x12, 0xfc, 0x0f, 0x72, 0xbf, 0x0f, }, + { 0xdd, 0x91, 0xd1, 0x15, 0x9e, 0x7d, 0xf8, 0xc1, 0xb9, 0x14, 0x63, 0x96, + 0xb5, 0xcb, 0x83, 0x1d, 0x35, 0x1c, 0xec, }, + { 0xa9, 0xf8, 0x52, 0xc9, 0x67, 0x76, 0x2b, 0xad, 0xfb, 0xd8, 0x3a, 0xa6, + 0x74, 0x02, 0xae, 0xb8, 0x25, 0x2c, 0x63, 0x49, }, + { 0x77, 0x1f, 0x66, 0x70, 0xfd, 0x50, 0x29, 0xaa, 0xeb, 0xdc, 0xee, 0xba, + 0x75, 0x98, 0xdc, 0x93, 0x12, 0x3f, 0xdc, 0x7c, 0x38, }, + { 0xe2, 0xe1, 0x89, 0x5c, 0x37, 0x38, 0x6a, 0xa3, 0x40, 0xac, 0x3f, 0xb0, + 0xca, 0xfc, 0xa7, 0xf3, 0xea, 0xf9, 0x0f, 0x5d, 0x8e, 0x39, }, + { 0x0f, 0x67, 0xc8, 0x38, 0x01, 0xb1, 0xb7, 0xb8, 0xa2, 0xe7, 0x0a, 0x6d, + 0xd2, 0x63, 0x69, 0x9e, 0xcc, 0xf0, 0xf2, 0xbe, 0x9b, 0x98, 0xdd, }, + { 0x13, 0xe1, 0x36, 0x30, 0xfe, 0xc6, 0x01, 0x8a, 0xa1, 0x63, 0x96, 0x59, + 0xc2, 0xa9, 0x68, 0x3f, 0x58, 0xd4, 0x19, 0x0c, 0x40, 0xf3, 0xde, 0x02, }, + { 0xa3, 0x9e, 0xce, 0xda, 0x42, 0xee, 0x8c, 0x6c, 0x5a, 0x7d, 0xdc, 0x89, + 0x02, 0x77, 0xdd, 0xe7, 0x95, 0xbb, 0xff, 0x0d, 0xa4, 0xb5, 0x38, 0x1e, + 0xaf, }, + { 0x9a, 0xf6, 0xb5, 0x9a, 0x4f, 0xa9, 0x4f, 0x2c, 0x35, 0x3c, 0x24, 0xdc, + 0x97, 0x6f, 0xd9, 0xa1, 0x7d, 0x1a, 0x85, 0x0b, 0xf5, 0xda, 0x2e, 0xe7, + 0xb1, 0x1d, }, + { 0x84, 0x1e, 0x8e, 0x3d, 0x45, 0xa5, 0xf2, 0x27, 0xf3, 0x31, 0xfe, 0xb9, + 0xfb, 0xc5, 0x45, 0x99, 0x99, 0xdd, 0x93, 0x43, 0x02, 0xee, 0x58, 0xaf, + 0xee, 0x6a, 0xbe, }, + { 0x07, 0x2f, 0xc0, 0xa2, 0x04, 0xc4, 0xab, 0x7c, 0x26, 0xbb, 0xa8, 0xd8, + 0xe3, 0x1c, 0x75, 0x15, 0x64, 0x5d, 0x02, 0x6a, 0xf0, 0x86, 0xe9, 0xcd, + 0x5c, 0xef, 0xa3, 0x25, }, + { 0x2f, 0x3b, 0x1f, 0xb5, 0x91, 0x8f, 0x86, 0xe0, 0xdc, 0x31, 0x48, 0xb6, + 0xa1, 0x8c, 0xfd, 0x75, 0xbb, 0x7d, 0x3d, 0xc1, 0xf0, 0x10, 0x9a, 0xd8, + 0x4b, 0x0e, 0xe3, 0x94, 0x9f, }, + { 0x29, 0xbb, 0x8f, 0x6c, 0xd1, 0xf2, 0xb6, 0xaf, 0xe5, 0xe3, 0x2d, 0xdc, + 0x6f, 0xa4, 0x53, 0x88, 0xd8, 0xcf, 0x4d, 0x45, 0x42, 0x62, 0xdb, 0xdf, + 0xf8, 0x45, 0xc2, 0x13, 0xec, 0x35, }, + { 0x06, 0x3c, 0xe3, 0x2c, 0x15, 0xc6, 0x43, 0x03, 0x81, 0xfb, 0x08, 0x76, + 0x33, 0xcb, 0x02, 0xc1, 0xba, 0x33, 0xe5, 0xe0, 0xd1, 0x92, 0xa8, 0x46, + 0x28, 0x3f, 0x3e, 0x9d, 0x2c, 0x44, 0x54, }, + { 0xea, 0xbb, 0x96, 0xf8, 0xd1, 0x8b, 0x04, 0x11, 0x40, 0x78, 0x42, 0x02, + 0x19, 0xd1, 0xbc, 0x65, 0x92, 0xd3, 0xc3, 0xd6, 0xd9, 0x19, 0xe7, 0xc3, + 0x40, 0x97, 0xbd, 0xd4, 0xed, 0xfa, 0x5e, 0x28, }, + { 0x02, }, + { 0x52, 0xa8, }, + { 0x38, 0x25, 0x0d, }, + { 0xe3, 0x04, 0xd4, 0x92, }, + { 0x97, 0xdb, 0xf7, 0x81, 0xca, }, + { 0x8a, 0x56, 0x9d, 0x62, 0x56, 0xcc, }, + { 0xa1, 0x8e, 0x3c, 0x72, 0x8f, 0x63, 0x03, }, + { 0xf7, 0xf3, 0x39, 0x09, 0x0a, 0xa1, 0xbb, 0x23, }, + { 0x6b, 0x03, 0xc0, 0xe9, 0xd9, 0x83, 0x05, 0x22, 0x01, }, + { 0x1b, 0x4b, 0xf5, 0xd6, 0x4f, 0x05, 0x75, 0x91, 0x4c, 0x7f, }, + { 0x4c, 0x8c, 0x25, 0x20, 0x21, 0xcb, 0xc2, 0x4b, 0x3a, 0x5b, 0x8d, }, + { 0x56, 0xe2, 0x77, 0xa0, 0xb6, 0x9f, 0x81, 0xec, 0x83, 0x75, 0xc4, 0xf9, }, + { 0x71, 0x70, 0x0f, 0xad, 0x4d, 0x35, 0x81, 0x9d, 0x88, 0x69, 0xf9, 0xaa, + 0xd3, }, + { 0x50, 0x6e, 0x86, 0x6e, 0x43, 0xc0, 0xc2, 0x44, 0xc2, 0xe2, 0xa0, 0x1c, + 0xb7, 0x9a, }, + { 0xe4, 0x7e, 0x72, 0xc6, 0x12, 0x8e, 0x7c, 0xfc, 0xbd, 0xe2, 0x08, 0x31, + 0x3d, 0x47, 0x3d, }, + { 0x08, 0x97, 0x5b, 0x80, 0xae, 0xc4, 0x1d, 0x50, 0x77, 0xdf, 0x1f, 0xd0, + 0x24, 0xf0, 0x17, 0xc0, }, + { 0x01, 0xb6, 0x29, 0xf4, 0xaf, 0x78, 0x5f, 0xb6, 0x91, 0xdd, 0x76, 0x76, + 0xd2, 0xfd, 0x0c, 0x47, 0x40, }, + { 0xa1, 0xd8, 0x09, 0x97, 0x7a, 0xa6, 0xc8, 0x94, 0xf6, 0x91, 0x7b, 0xae, + 0x2b, 0x9f, 0x0d, 0x83, 0x48, 0xf7, }, + { 0x12, 0xd5, 0x53, 0x7d, 0x9a, 0xb0, 0xbe, 0xd9, 0xed, 0xe9, 0x9e, 0xee, + 0x61, 0x5b, 0x42, 0xf2, 0xc0, 0x73, 0xc0, }, + { 0xd5, 0x77, 0xd6, 0x5c, 0x6e, 0xa5, 0x69, 0x2b, 0x3b, 0x8c, 0xd6, 0x7d, + 0x1d, 0xbe, 0x2c, 0xa1, 0x02, 0x21, 0xcd, 0x29, }, + { 0xa4, 0x98, 0x80, 0xca, 0x22, 0xcf, 0x6a, 0xab, 0x5e, 0x40, 0x0d, 0x61, + 0x08, 0x21, 0xef, 0xc0, 0x6c, 0x52, 0xb4, 0xb0, 0x53, }, + { 0xbf, 0xaf, 0x8f, 0x3b, 0x7a, 0x97, 0x33, 0xe5, 0xca, 0x07, 0x37, 0xfd, + 0x15, 0xdf, 0xce, 0x26, 0x2a, 0xb1, 0xa7, 0x0b, 0xb3, 0xac, }, + { 0x16, 0x22, 0xe1, 0xbc, 0x99, 0x4e, 0x01, 0xf0, 0xfa, 0xff, 0x8f, 0xa5, + 0x0c, 0x61, 0xb0, 0xad, 0xcc, 0xb1, 0xe1, 0x21, 0x46, 0xfa, 0x2e, }, + { 0x11, 0x5b, 0x0b, 0x2b, 0xe6, 0x14, 0xc1, 0xd5, 0x4d, 0x71, 0x5e, 0x17, + 0xea, 0x23, 0xdd, 0x6c, 0xbd, 0x1d, 0xbe, 0x12, 0x1b, 0xee, 0x4c, 0x1a, }, + { 0x40, 0x88, 0x22, 0xf3, 0x20, 0x6c, 0xed, 0xe1, 0x36, 0x34, 0x62, 0x2c, + 0x98, 0x83, 0x52, 0xe2, 0x25, 0xee, 0xe9, 0xf5, 0xe1, 0x17, 0xf0, 0x5c, + 0xae, }, + { 0xc3, 0x76, 0x37, 0xde, 0x95, 0x8c, 0xca, 0x2b, 0x0c, 0x23, 0xe7, 0xb5, + 0x38, 0x70, 0x61, 0xcc, 0xff, 0xd3, 0x95, 0x7b, 0xf3, 0xff, 0x1f, 0x9d, + 0x59, 0x00, }, + { 0x0c, 0x19, 0x52, 0x05, 0x22, 0x53, 0xcb, 0x48, 0xd7, 0x10, 0x0e, 0x7e, + 0x14, 0x69, 0xb5, 0xa2, 0x92, 0x43, 0xa3, 0x9e, 0x4b, 0x8f, 0x51, 0x2c, + 0x5a, 0x2c, 0x3b, }, + { 0xe1, 0x9d, 0x70, 0x70, 0x28, 0xec, 0x86, 0x40, 0x55, 0x33, 0x56, 0xda, + 0x88, 0xca, 0xee, 0xc8, 0x6a, 0x20, 0xb1, 0xe5, 0x3d, 0x57, 0xf8, 0x3c, + 0x10, 0x07, 0x2a, 0xc4, }, + { 0x0b, 0xae, 0xf1, 0xc4, 0x79, 0xee, 0x1b, 0x3d, 0x27, 0x35, 0x8d, 0x14, + 0xd6, 0xae, 0x4e, 0x3c, 0xe9, 0x53, 0x50, 0xb5, 0xcc, 0x0c, 0xf7, 0xdf, + 0xee, 0xa1, 0x74, 0xd6, 0x71, }, + { 0xe6, 0xa4, 0xf4, 0x99, 0x98, 0xb9, 0x80, 0xea, 0x96, 0x7f, 0x4f, 0x33, + 0xcf, 0x74, 0x25, 0x6f, 0x17, 0x6c, 0xbf, 0xf5, 0x5c, 0x38, 0xd0, 0xff, + 0x96, 0xcb, 0x13, 0xf9, 0xdf, 0xfd, }, + { 0xbe, 0x92, 0xeb, 0xba, 0x44, 0x2c, 0x24, 0x74, 0xd4, 0x03, 0x27, 0x3c, + 0x5d, 0x5b, 0x03, 0x30, 0x87, 0x63, 0x69, 0xe0, 0xb8, 0x94, 0xf4, 0x44, + 0x7e, 0xad, 0xcd, 0x20, 0x12, 0x16, 0x79, }, + { 0x30, 0xf1, 0xc4, 0x8e, 0x05, 0x90, 0x2a, 0x97, 0x63, 0x94, 0x46, 0xff, + 0xce, 0xd8, 0x67, 0xa7, 0xac, 0x33, 0x8c, 0x95, 0xb7, 0xcd, 0xa3, 0x23, + 0x98, 0x9d, 0x76, 0x6c, 0x9d, 0xa8, 0xd6, 0x8a, }, + { 0xbe, }, + { 0x17, 0x6c, }, + { 0x1a, 0x42, 0x4f, }, + { 0xba, 0xaf, 0xb7, 0x65, }, + { 0xc2, 0x63, 0x43, 0x6a, 0xea, }, + { 0xe4, 0x4d, 0xad, 0xf2, 0x0b, 0x02, }, + { 0x04, 0xc7, 0xc4, 0x7f, 0xa9, 0x2b, 0xce, }, + { 0x66, 0xf6, 0x67, 0xcb, 0x03, 0x53, 0xc8, 0xf1, }, + { 0x56, 0xa3, 0x60, 0x78, 0xc9, 0x5f, 0x70, 0x1b, 0x5e, }, + { 0x99, 0xff, 0x81, 0x7c, 0x13, 0x3c, 0x29, 0x79, 0x4b, 0x65, }, + { 0x51, 0x10, 0x50, 0x93, 0x01, 0x93, 0xb7, 0x01, 0xc9, 0x18, 0xb7, }, + { 0x8e, 0x3c, 0x42, 0x1e, 0x5e, 0x7d, 0xc1, 0x50, 0x70, 0x1f, 0x00, 0x98, }, + { 0x5f, 0xd9, 0x9b, 0xc8, 0xd7, 0xb2, 0x72, 0x62, 0x1a, 0x1e, 0xba, 0x92, + 0xe9, }, + { 0x70, 0x2b, 0xba, 0xfe, 0xad, 0x5d, 0x96, 0x3f, 0x27, 0xc2, 0x41, 0x6d, + 0xc4, 0xb3, }, + { 0xae, 0xe0, 0xd5, 0xd4, 0xc7, 0xae, 0x15, 0x5e, 0xdc, 0xdd, 0x33, 0x60, + 0xd7, 0xd3, 0x5e, }, + { 0x79, 0x8e, 0xbc, 0x9e, 0x20, 0xb9, 0x19, 0x4b, 0x63, 0x80, 0xf3, 0x16, + 0xaf, 0x39, 0xbd, 0x92, }, + { 0xc2, 0x0e, 0x85, 0xa0, 0x0b, 0x9a, 0xb0, 0xec, 0xde, 0x38, 0xd3, 0x10, + 0xd9, 0xa7, 0x66, 0x27, 0xcf, }, + { 0x0e, 0x3b, 0x75, 0x80, 0x67, 0x14, 0x0c, 0x02, 0x90, 0xd6, 0xb3, 0x02, + 0x81, 0xf6, 0xa6, 0x87, 0xce, 0x58, }, + { 0x79, 0xb5, 0xe9, 0x5d, 0x52, 0x4d, 0xf7, 0x59, 0xf4, 0x2e, 0x27, 0xdd, + 0xb3, 0xed, 0x57, 0x5b, 0x82, 0xea, 0x6f, }, + { 0xa2, 0x97, 0xf5, 0x80, 0x02, 0x3d, 0xde, 0xa3, 0xf9, 0xf6, 0xab, 0xe3, + 0x57, 0x63, 0x7b, 0x9b, 0x10, 0x42, 0x6f, 0xf2, }, + { 0x12, 0x7a, 0xfc, 0xb7, 0x67, 0x06, 0x0c, 0x78, 0x1a, 0xfe, 0x88, 0x4f, + 0xc6, 0xac, 0x52, 0x96, 0x64, 0x28, 0x97, 0x84, 0x06, }, + { 0xc5, 0x04, 0x44, 0x6b, 0xb2, 0xa5, 0xa4, 0x66, 0xe1, 0x76, 0xa2, 0x51, + 0xf9, 0x59, 0x69, 0x97, 0x56, 0x0b, 0xbf, 0x50, 0xb3, 0x34, }, + { 0x21, 0x32, 0x6b, 0x42, 0xb5, 0xed, 0x71, 0x8d, 0xf7, 0x5a, 0x35, 0xe3, + 0x90, 0xe2, 0xee, 0xaa, 0x89, 0xf6, 0xc9, 0x9c, 0x4d, 0x73, 0xf4, }, + { 0x4c, 0xa6, 0x09, 0xf4, 0x48, 0xe7, 0x46, 0xbc, 0x49, 0xfc, 0xe5, 0xda, + 0xd1, 0x87, 0x13, 0x17, 0x4c, 0x59, 0x71, 0x26, 0x5b, 0x2c, 0x42, 0xb7, }, + { 0x13, 0x63, 0xf3, 0x40, 0x02, 0xe5, 0xa3, 0x3a, 0x5e, 0x8e, 0xf8, 0xb6, + 0x8a, 0x49, 0x60, 0x76, 0x34, 0x72, 0x94, 0x73, 0xf6, 0xd9, 0x21, 0x6a, + 0x26, }, + { 0xdf, 0x75, 0x16, 0x10, 0x1b, 0x5e, 0x81, 0xc3, 0xc8, 0xde, 0x34, 0x24, + 0xb0, 0x98, 0xeb, 0x1b, 0x8f, 0xa1, 0x9b, 0x05, 0xee, 0xa5, 0xe9, 0x35, + 0xf4, 0x1d, }, + { 0xcd, 0x21, 0x93, 0x6e, 0x5b, 0xa0, 0x26, 0x2b, 0x21, 0x0e, 0xa0, 0xb9, + 0x1c, 0xb5, 0xbb, 0xb8, 0xf8, 0x1e, 0xff, 0x5c, 0xa8, 0xf9, 0x39, 0x46, + 0x4e, 0x29, 0x26, }, + { 0x73, 0x7f, 0x0e, 0x3b, 0x0b, 0x5c, 0xf9, 0x60, 0xaa, 0x88, 0xa1, 0x09, + 0xb1, 0x5d, 0x38, 0x7b, 0x86, 0x8f, 0x13, 0x7a, 0x8d, 0x72, 0x7a, 0x98, + 0x1a, 0x5b, 0xff, 0xc9, }, + { 0xd3, 0x3c, 0x61, 0x71, 0x44, 0x7e, 0x31, 0x74, 0x98, 0x9d, 0x9a, 0xd2, + 0x27, 0xf3, 0x46, 0x43, 0x42, 0x51, 0xd0, 0x5f, 0xe9, 0x1c, 0x5c, 0x69, + 0xbf, 0xf6, 0xbe, 0x3c, 0x40, }, + { 0x31, 0x99, 0x31, 0x9f, 0xaa, 0x43, 0x2e, 0x77, 0x3e, 0x74, 0x26, 0x31, + 0x5e, 0x61, 0xf1, 0x87, 0xe2, 0xeb, 0x9b, 0xcd, 0xd0, 0x3a, 0xee, 0x20, + 0x7e, 0x10, 0x0a, 0x0b, 0x7e, 0xfa, }, + { 0xa4, 0x27, 0x80, 0x67, 0x81, 0x2a, 0xa7, 0x62, 0xf7, 0x6e, 0xda, 0xd4, + 0x5c, 0x39, 0x74, 0xad, 0x7e, 0xbe, 0xad, 0xa5, 0x84, 0x7f, 0xa9, 0x30, + 0x5d, 0xdb, 0xe2, 0x05, 0x43, 0xf7, 0x1b, }, + { 0x0b, 0x37, 0xd8, 0x02, 0xe1, 0x83, 0xd6, 0x80, 0xf2, 0x35, 0xc2, 0xb0, + 0x37, 0xef, 0xef, 0x5e, 0x43, 0x93, 0xf0, 0x49, 0x45, 0x0a, 0xef, 0xb5, + 0x76, 0x70, 0x12, 0x44, 0xc4, 0xdb, 0xf5, 0x7a, }, + { 0x1f, }, + { 0x82, 0x60, }, + { 0xcc, 0xe3, 0x08, }, + { 0x56, 0x17, 0xe4, 0x59, }, + { 0xe2, 0xd7, 0x9e, 0xc4, 0x4c, }, + { 0xb2, 0xad, 0xd3, 0x78, 0x58, 0x5a, }, + { 0xce, 0x43, 0xb4, 0x02, 0x96, 0xab, 0x3c, }, + { 0xe6, 0x05, 0x1a, 0x73, 0x22, 0x32, 0xbb, 0x77, }, + { 0x23, 0xe7, 0xda, 0xfe, 0x2c, 0xef, 0x8c, 0x22, 0xec, }, + { 0xe9, 0x8e, 0x55, 0x38, 0xd1, 0xd7, 0x35, 0x23, 0x98, 0xc7, }, + { 0xb5, 0x81, 0x1a, 0xe5, 0xb5, 0xa5, 0xd9, 0x4d, 0xca, 0x41, 0xe7, }, + { 0x41, 0x16, 0x16, 0x95, 0x8d, 0x9e, 0x0c, 0xea, 0x8c, 0x71, 0x9a, 0xc1, }, + { 0x7c, 0x33, 0xc0, 0xa4, 0x00, 0x62, 0xea, 0x60, 0x67, 0xe4, 0x20, 0xbc, + 0x5b, }, + { 0xdb, 0xb1, 0xdc, 0xfd, 0x08, 0xc0, 0xde, 0x82, 0xd1, 0xde, 0x38, 0xc0, + 0x90, 0x48, }, + { 0x37, 0x18, 0x2e, 0x0d, 0x61, 0xaa, 0x61, 0xd7, 0x86, 0x20, 0x16, 0x60, + 0x04, 0xd9, 0xd5, }, + { 0xb0, 0xcf, 0x2c, 0x4c, 0x5e, 0x5b, 0x4f, 0x2a, 0x23, 0x25, 0x58, 0x47, + 0xe5, 0x31, 0x06, 0x70, }, + { 0x91, 0xa0, 0xa3, 0x86, 0x4e, 0xe0, 0x72, 0x38, 0x06, 0x67, 0x59, 0x5c, + 0x70, 0x25, 0xdb, 0x33, 0x27, }, + { 0x44, 0x58, 0x66, 0xb8, 0x58, 0xc7, 0x13, 0xed, 0x4c, 0xc0, 0xf4, 0x9a, + 0x1e, 0x67, 0x75, 0x33, 0xb6, 0xb8, }, + { 0x7f, 0x98, 0x4a, 0x8e, 0x50, 0xa2, 0x5c, 0xcd, 0x59, 0xde, 0x72, 0xb3, + 0x9d, 0xc3, 0x09, 0x8a, 0xab, 0x56, 0xf1, }, + { 0x80, 0x96, 0x49, 0x1a, 0x59, 0xa2, 0xc5, 0xd5, 0xa7, 0x20, 0x8a, 0xb7, + 0x27, 0x62, 0x84, 0x43, 0xc6, 0xe1, 0x1b, 0x5d, }, + { 0x6b, 0xb7, 0x2b, 0x26, 0x62, 0x14, 0x70, 0x19, 0x3d, 0x4d, 0xac, 0xac, + 0x63, 0x58, 0x5e, 0x94, 0xb5, 0xb7, 0xe8, 0xe8, 0xa2, }, + { 0x20, 0xa8, 0xc0, 0xfd, 0x63, 0x3d, 0x6e, 0x98, 0xcf, 0x0c, 0x49, 0x98, + 0xe4, 0x5a, 0xfe, 0x8c, 0xaa, 0x70, 0x82, 0x1c, 0x7b, 0x74, }, + { 0xc8, 0xe8, 0xdd, 0xdf, 0x69, 0x30, 0x01, 0xc2, 0x0f, 0x7e, 0x2f, 0x11, + 0xcc, 0x3e, 0x17, 0xa5, 0x69, 0x40, 0x3f, 0x0e, 0x79, 0x7f, 0xcf, }, + { 0xdb, 0x61, 0xc0, 0xe2, 0x2e, 0x49, 0x07, 0x31, 0x1d, 0x91, 0x42, 0x8a, + 0xfc, 0x5e, 0xd3, 0xf8, 0x56, 0x1f, 0x2b, 0x73, 0xfd, 0x9f, 0xb2, 0x8e, }, + { 0x0c, 0x89, 0x55, 0x0c, 0x1f, 0x59, 0x2c, 0x9d, 0x1b, 0x29, 0x1d, 0x41, + 0x1d, 0xe6, 0x47, 0x8f, 0x8c, 0x2b, 0xea, 0x8f, 0xf0, 0xff, 0x21, 0x70, + 0x88, }, + { 0x12, 0x18, 0x95, 0xa6, 0x59, 0xb1, 0x31, 0x24, 0x45, 0x67, 0x55, 0xa4, + 0x1a, 0x2d, 0x48, 0x67, 0x1b, 0x43, 0x88, 0x2d, 0x8e, 0xa0, 0x70, 0xb3, + 0xc6, 0xbb, }, + { 0xe7, 0xb1, 0x1d, 0xb2, 0x76, 0x4d, 0x68, 0x68, 0x68, 0x23, 0x02, 0x55, + 0x3a, 0xe2, 0xe5, 0xd5, 0x4b, 0x43, 0xf9, 0x34, 0x77, 0x5c, 0xa1, 0xf5, + 0x55, 0xfd, 0x4f, }, + { 0x8c, 0x87, 0x5a, 0x08, 0x3a, 0x73, 0xad, 0x61, 0xe1, 0xe7, 0x99, 0x7e, + 0xf0, 0x5d, 0xe9, 0x5d, 0x16, 0x43, 0x80, 0x2f, 0xd0, 0x66, 0x34, 0xe2, + 0x42, 0x64, 0x3b, 0x1a, }, + { 0x39, 0xc1, 0x99, 0xcf, 0x22, 0xbf, 0x16, 0x8f, 0x9f, 0x80, 0x7f, 0x95, + 0x0a, 0x05, 0x67, 0x27, 0xe7, 0x15, 0xdf, 0x9d, 0xb2, 0xfe, 0x1c, 0xb5, + 0x1d, 0x60, 0x8f, 0x8a, 0x1d, }, + { 0x9b, 0x6e, 0x08, 0x09, 0x06, 0x73, 0xab, 0x68, 0x02, 0x62, 0x1a, 0xe4, + 0xd4, 0xdf, 0xc7, 0x02, 0x4c, 0x6a, 0x5f, 0xfd, 0x23, 0xac, 0xae, 0x6d, + 0x43, 0xa4, 0x7a, 0x50, 0x60, 0x3c, }, + { 0x1d, 0xb4, 0xc6, 0xe1, 0xb1, 0x4b, 0xe3, 0xf2, 0xe2, 0x1a, 0x73, 0x1b, + 0xa0, 0x92, 0xa7, 0xf5, 0xff, 0x8f, 0x8b, 0x5d, 0xdf, 0xa8, 0x04, 0xb3, + 0xb0, 0xf7, 0xcc, 0x12, 0xfa, 0x35, 0x46, }, + { 0x49, 0x45, 0x97, 0x11, 0x0f, 0x1c, 0x60, 0x8e, 0xe8, 0x47, 0x30, 0xcf, + 0x60, 0xa8, 0x71, 0xc5, 0x1b, 0xe9, 0x39, 0x4d, 0x49, 0xb6, 0x12, 0x1f, + 0x24, 0xab, 0x37, 0xff, 0x83, 0xc2, 0xe1, 0x3a, }, + { 0x60, }, + { 0x24, 0x26, }, + { 0x47, 0xeb, 0xc9, }, + { 0x4a, 0xd0, 0xbc, 0xf0, }, + { 0x8e, 0x2b, 0xc9, 0x85, 0x3c, }, + { 0xa2, 0x07, 0x15, 0xb8, 0x12, 0x74, }, + { 0x0f, 0xdb, 0x5b, 0x33, 0x69, 0xfe, 0x4b, }, + { 0xa2, 0x86, 0x54, 0xf4, 0xfd, 0xb2, 0xd4, 0xe6, }, + { 0xbb, 0x84, 0x78, 0x49, 0x27, 0x8e, 0x61, 0xda, 0x60, }, + { 0x04, 0xc3, 0xcd, 0xaa, 0x8f, 0xa7, 0x03, 0xc9, 0xf9, 0xb6, }, + { 0xf8, 0x27, 0x1d, 0x61, 0xdc, 0x21, 0x42, 0xdd, 0xad, 0x92, 0x40, }, + { 0x12, 0x87, 0xdf, 0xc2, 0x41, 0x45, 0x5a, 0x36, 0x48, 0x5b, 0x51, 0x2b, }, + { 0xbb, 0x37, 0x5d, 0x1f, 0xf1, 0x68, 0x7a, 0xc4, 0xa5, 0xd2, 0xa4, 0x91, + 0x8d, }, + { 0x5b, 0x27, 0xd1, 0x04, 0x54, 0x52, 0x9f, 0xa3, 0x47, 0x86, 0x33, 0x33, + 0xbf, 0xa0, }, + { 0xcf, 0x04, 0xea, 0xf8, 0x03, 0x2a, 0x43, 0xff, 0xa6, 0x68, 0x21, 0x4c, + 0xd5, 0x4b, 0xed, }, + { 0xaf, 0xb8, 0xbc, 0x63, 0x0f, 0x18, 0x4d, 0xe2, 0x7a, 0xdd, 0x46, 0x44, + 0xc8, 0x24, 0x0a, 0xb7, }, + { 0x3e, 0xdc, 0x36, 0xe4, 0x89, 0xb1, 0xfa, 0xc6, 0x40, 0x93, 0x2e, 0x75, + 0xb2, 0x15, 0xd1, 0xb1, 0x10, }, + { 0x6c, 0xd8, 0x20, 0x3b, 0x82, 0x79, 0xf9, 0xc8, 0xbc, 0x9d, 0xe0, 0x35, + 0xbe, 0x1b, 0x49, 0x1a, 0xbc, 0x3a, }, + { 0x78, 0x65, 0x2c, 0xbe, 0x35, 0x67, 0xdc, 0x78, 0xd4, 0x41, 0xf6, 0xc9, + 0xde, 0xde, 0x1f, 0x18, 0x13, 0x31, 0x11, }, + { 0x8a, 0x7f, 0xb1, 0x33, 0x8f, 0x0c, 0x3c, 0x0a, 0x06, 0x61, 0xf0, 0x47, + 0x29, 0x1b, 0x29, 0xbc, 0x1c, 0x47, 0xef, 0x7a, }, + { 0x65, 0x91, 0xf1, 0xe6, 0xb3, 0x96, 0xd3, 0x8c, 0xc2, 0x4a, 0x59, 0x35, + 0x72, 0x8e, 0x0b, 0x9a, 0x87, 0xca, 0x34, 0x7b, 0x63, }, + { 0x5f, 0x08, 0x87, 0x80, 0x56, 0x25, 0x89, 0x77, 0x61, 0x8c, 0x64, 0xa1, + 0x59, 0x6d, 0x59, 0x62, 0xe8, 0x4a, 0xc8, 0x58, 0x99, 0xd1, }, + { 0x23, 0x87, 0x1d, 0xed, 0x6f, 0xf2, 0x91, 0x90, 0xe2, 0xfe, 0x43, 0x21, + 0xaf, 0x97, 0xc6, 0xbc, 0xd7, 0x15, 0xc7, 0x2d, 0x08, 0x77, 0x91, }, + { 0x90, 0x47, 0x9a, 0x9e, 0x3a, 0xdf, 0xf3, 0xc9, 0x4c, 0x1e, 0xa7, 0xd4, + 0x6a, 0x32, 0x90, 0xfe, 0xb7, 0xb6, 0x7b, 0xfa, 0x96, 0x61, 0xfb, 0xa4, }, + { 0xb1, 0x67, 0x60, 0x45, 0xb0, 0x96, 0xc5, 0x15, 0x9f, 0x4d, 0x26, 0xd7, + 0x9d, 0xf1, 0xf5, 0x6d, 0x21, 0x00, 0x94, 0x31, 0x64, 0x94, 0xd3, 0xa7, + 0xd3, }, + { 0x02, 0x3e, 0xaf, 0xf3, 0x79, 0x73, 0xa5, 0xf5, 0xcc, 0x7a, 0x7f, 0xfb, + 0x79, 0x2b, 0x85, 0x8c, 0x88, 0x72, 0x06, 0xbe, 0xfe, 0xaf, 0xc1, 0x16, + 0xa6, 0xd6, }, + { 0x2a, 0xb0, 0x1a, 0xe5, 0xaa, 0x6e, 0xb3, 0xae, 0x53, 0x85, 0x33, 0x80, + 0x75, 0xae, 0x30, 0xe6, 0xb8, 0x72, 0x42, 0xf6, 0x25, 0x4f, 0x38, 0x88, + 0x55, 0xd1, 0xa9, }, + { 0x90, 0xd8, 0x0c, 0xc0, 0x93, 0x4b, 0x4f, 0x9e, 0x65, 0x6c, 0xa1, 0x54, + 0xa6, 0xf6, 0x6e, 0xca, 0xd2, 0xbb, 0x7e, 0x6a, 0x1c, 0xd3, 0xce, 0x46, + 0xef, 0xb0, 0x00, 0x8d, }, + { 0xed, 0x9c, 0x49, 0xcd, 0xc2, 0xde, 0x38, 0x0e, 0xe9, 0x98, 0x6c, 0xc8, + 0x90, 0x9e, 0x3c, 0xd4, 0xd3, 0xeb, 0x88, 0x32, 0xc7, 0x28, 0xe3, 0x94, + 0x1c, 0x9f, 0x8b, 0xf3, 0xcb, }, + { 0xac, 0xe7, 0x92, 0x16, 0xb4, 0x14, 0xa0, 0xe4, 0x04, 0x79, 0xa2, 0xf4, + 0x31, 0xe6, 0x0c, 0x26, 0xdc, 0xbf, 0x2f, 0x69, 0x1b, 0x55, 0x94, 0x67, + 0xda, 0x0c, 0xd7, 0x32, 0x1f, 0xef, }, + { 0x68, 0x63, 0x85, 0x57, 0x95, 0x9e, 0x42, 0x27, 0x41, 0x43, 0x42, 0x02, + 0xa5, 0x78, 0xa7, 0xc6, 0x43, 0xc1, 0x6a, 0xba, 0x70, 0x80, 0xcd, 0x04, + 0xb6, 0x78, 0x76, 0x29, 0xf3, 0xe8, 0xa0, }, + { 0xe6, 0xac, 0x8d, 0x9d, 0xf0, 0xc0, 0xf7, 0xf7, 0xe3, 0x3e, 0x4e, 0x28, + 0x0f, 0x59, 0xb2, 0x67, 0x9e, 0x84, 0x34, 0x42, 0x96, 0x30, 0x2b, 0xca, + 0x49, 0xb6, 0xc5, 0x9a, 0x84, 0x59, 0xa7, 0x81, }, + { 0x7e, }, + { 0x1e, 0x21, }, + { 0x26, 0xd3, 0xdd, }, + { 0x2c, 0xd4, 0xb3, 0x3d, }, + { 0x86, 0x7b, 0x76, 0x3c, 0xf0, }, + { 0x12, 0xc3, 0x70, 0x1d, 0x55, 0x18, }, + { 0x96, 0xc2, 0xbd, 0x61, 0x55, 0xf4, 0x24, }, + { 0x20, 0x51, 0xf7, 0x86, 0x58, 0x8f, 0x07, 0x2a, }, + { 0x93, 0x15, 0xa8, 0x1d, 0xda, 0x97, 0xee, 0x0e, 0x6c, }, + { 0x39, 0x93, 0xdf, 0xd5, 0x0e, 0xca, 0xdc, 0x7a, 0x92, 0xce, }, + { 0x60, 0xd5, 0xfd, 0xf5, 0x1b, 0x26, 0x82, 0x26, 0x73, 0x02, 0xbc, }, + { 0x98, 0xf2, 0x34, 0xe1, 0xf5, 0xfb, 0x00, 0xac, 0x10, 0x4a, 0x38, 0x9f, }, + { 0xda, 0x3a, 0x92, 0x8a, 0xd0, 0xcd, 0x12, 0xcd, 0x15, 0xbb, 0xab, 0x77, + 0x66, }, + { 0xa2, 0x92, 0x1a, 0xe5, 0xca, 0x0c, 0x30, 0x75, 0xeb, 0xaf, 0x00, 0x31, + 0x55, 0x66, }, + { 0x06, 0xea, 0xfd, 0x3e, 0x86, 0x38, 0x62, 0x4e, 0xa9, 0x12, 0xa4, 0x12, + 0x43, 0xbf, 0xa1, }, + { 0xe4, 0x71, 0x7b, 0x94, 0xdb, 0xa0, 0xd2, 0xff, 0x9b, 0xeb, 0xad, 0x8e, + 0x95, 0x8a, 0xc5, 0xed, }, + { 0x25, 0x5a, 0x77, 0x71, 0x41, 0x0e, 0x7a, 0xe9, 0xed, 0x0c, 0x10, 0xef, + 0xf6, 0x2b, 0x3a, 0xba, 0x60, }, + { 0xee, 0xe2, 0xa3, 0x67, 0x64, 0x1d, 0xc6, 0x04, 0xc4, 0xe1, 0x68, 0xd2, + 0x6e, 0xd2, 0x91, 0x75, 0x53, 0x07, }, + { 0xe0, 0xf6, 0x4d, 0x8f, 0x68, 0xfc, 0x06, 0x7e, 0x18, 0x79, 0x7f, 0x2b, + 0x6d, 0xef, 0x46, 0x7f, 0xab, 0xb2, 0xad, }, + { 0x3d, 0x35, 0x88, 0x9f, 0x2e, 0xcf, 0x96, 0x45, 0x07, 0x60, 0x71, 0x94, + 0x00, 0x8d, 0xbf, 0xf4, 0xef, 0x46, 0x2e, 0x3c, }, + { 0x43, 0xcf, 0x98, 0xf7, 0x2d, 0xf4, 0x17, 0xe7, 0x8c, 0x05, 0x2d, 0x9b, + 0x24, 0xfb, 0x4d, 0xea, 0x4a, 0xec, 0x01, 0x25, 0x29, }, + { 0x8e, 0x73, 0x9a, 0x78, 0x11, 0xfe, 0x48, 0xa0, 0x3b, 0x1a, 0x26, 0xdf, + 0x25, 0xe9, 0x59, 0x1c, 0x70, 0x07, 0x9f, 0xdc, 0xa0, 0xa6, }, + { 0xe8, 0x47, 0x71, 0xc7, 0x3e, 0xdf, 0xb5, 0x13, 0xb9, 0x85, 0x13, 0xa8, + 0x54, 0x47, 0x6e, 0x59, 0x96, 0x09, 0x13, 0x5f, 0x82, 0x16, 0x0b, }, + { 0xfb, 0xc0, 0x8c, 0x03, 0x21, 0xb3, 0xc4, 0xb5, 0x43, 0x32, 0x6c, 0xea, + 0x7f, 0xa8, 0x43, 0x91, 0xe8, 0x4e, 0x3f, 0xbf, 0x45, 0x58, 0x6a, 0xa3, }, + { 0x55, 0xf8, 0xf3, 0x00, 0x76, 0x09, 0xef, 0x69, 0x5d, 0xd2, 0x8a, 0xf2, + 0x65, 0xc3, 0xcb, 0x9b, 0x43, 0xfd, 0xb1, 0x7e, 0x7f, 0xa1, 0x94, 0xb0, + 0xd7, }, + { 0xaa, 0x13, 0xc1, 0x51, 0x40, 0x6d, 0x8d, 0x4c, 0x0a, 0x95, 0x64, 0x7b, + 0xd1, 0x96, 0xb6, 0x56, 0xb4, 0x5b, 0xcf, 0xd6, 0xd9, 0x15, 0x97, 0xdd, + 0xb6, 0xef, }, + { 0xaf, 0xb7, 0x36, 0xb0, 0x04, 0xdb, 0xd7, 0x9c, 0x9a, 0x44, 0xc4, 0xf6, + 0x1f, 0x12, 0x21, 0x2d, 0x59, 0x30, 0x54, 0xab, 0x27, 0x61, 0xa3, 0x57, + 0xef, 0xf8, 0x53, }, + { 0x97, 0x34, 0x45, 0x3e, 0xce, 0x7c, 0x35, 0xa2, 0xda, 0x9f, 0x4b, 0x46, + 0x6c, 0x11, 0x67, 0xff, 0x2f, 0x76, 0x58, 0x15, 0x71, 0xfa, 0x44, 0x89, + 0x89, 0xfd, 0xf7, 0x99, }, + { 0x1f, 0xb1, 0x62, 0xeb, 0x83, 0xc5, 0x9c, 0x89, 0xf9, 0x2c, 0xd2, 0x03, + 0x61, 0xbc, 0xbb, 0xa5, 0x74, 0x0e, 0x9b, 0x7e, 0x82, 0x3e, 0x70, 0x0a, + 0xa9, 0x8f, 0x2b, 0x59, 0xfb, }, + { 0xf8, 0xca, 0x5e, 0x3a, 0x4f, 0x9e, 0x10, 0x69, 0x10, 0xd5, 0x4c, 0xeb, + 0x1a, 0x0f, 0x3c, 0x6a, 0x98, 0xf5, 0xb0, 0x97, 0x5b, 0x37, 0x2f, 0x0d, + 0xbd, 0x42, 0x4b, 0x69, 0xa1, 0x82, }, + { 0x12, 0x8c, 0x6d, 0x52, 0x08, 0xef, 0x74, 0xb2, 0xe6, 0xaa, 0xd3, 0xb0, + 0x26, 0xb0, 0xd9, 0x94, 0xb6, 0x11, 0x45, 0x0e, 0x36, 0x71, 0x14, 0x2d, + 0x41, 0x8c, 0x21, 0x53, 0x31, 0xe9, 0x68, }, + { 0xee, 0xea, 0x0d, 0x89, 0x47, 0x7e, 0x72, 0xd1, 0xd8, 0xce, 0x58, 0x4c, + 0x94, 0x1f, 0x0d, 0x51, 0x08, 0xa3, 0xb6, 0x3d, 0xe7, 0x82, 0x46, 0x92, + 0xd6, 0x98, 0x6b, 0x07, 0x10, 0x65, 0x52, 0x65, }, +}; + +static const u8 blake2s_hmac_testvecs[][BLAKE2S_HASH_SIZE] __initconst = { + { 0xce, 0xe1, 0x57, 0x69, 0x82, 0xdc, 0xbf, 0x43, 0xad, 0x56, 0x4c, 0x70, + 0xed, 0x68, 0x16, 0x96, 0xcf, 0xa4, 0x73, 0xe8, 0xe8, 0xfc, 0x32, 0x79, + 0x08, 0x0a, 0x75, 0x82, 0xda, 0x3f, 0x05, 0x11, }, + { 0x77, 0x2f, 0x0c, 0x71, 0x41, 0xf4, 0x4b, 0x2b, 0xb3, 0xc6, 0xb6, 0xf9, + 0x60, 0xde, 0xe4, 0x52, 0x38, 0x66, 0xe8, 0xbf, 0x9b, 0x96, 0xc4, 0x9f, + 0x60, 0xd9, 0x24, 0x37, 0x99, 0xd6, 0xec, 0x31, }, +}; + +bool __init blake2s_selftest(void) +{ + u8 key[BLAKE2S_KEY_SIZE]; + u8 buf[ARRAY_SIZE(blake2s_testvecs)]; + u8 hash[BLAKE2S_HASH_SIZE]; + struct blake2s_state state; + bool success = true; + int i, l; + + key[0] = key[1] = 1; + for (i = 2; i < sizeof(key); ++i) + key[i] = key[i - 2] + key[i - 1]; + + for (i = 0; i < sizeof(buf); ++i) + buf[i] = (u8)i; + + for (i = l = 0; i < ARRAY_SIZE(blake2s_testvecs); l = (l + 37) % ++i) { + int outlen = 1 + i % BLAKE2S_HASH_SIZE; + int keylen = (13 * i) % (BLAKE2S_KEY_SIZE + 1); + + blake2s(hash, buf, key + BLAKE2S_KEY_SIZE - keylen, outlen, i, + keylen); + if (memcmp(hash, blake2s_testvecs[i], outlen)) { + pr_err("blake2s self-test %d: FAIL\n", i + 1); + success = false; + } + + if (!keylen) + blake2s_init(&state, outlen); + else + blake2s_init_key(&state, outlen, + key + BLAKE2S_KEY_SIZE - keylen, + keylen); + + blake2s_update(&state, buf, l); + blake2s_update(&state, buf + l, i - l); + blake2s_final(&state, hash); + if (memcmp(hash, blake2s_testvecs[i], outlen)) { + pr_err("blake2s init/update/final self-test %d: FAIL\n", + i + 1); + success = false; + } + } + + if (success) { + blake2s256_hmac(hash, buf, key, sizeof(buf), sizeof(key)); + success &= !memcmp(hash, blake2s_hmac_testvecs[0], BLAKE2S_HASH_SIZE); + + blake2s256_hmac(hash, key, buf, sizeof(key), sizeof(buf)); + success &= !memcmp(hash, blake2s_hmac_testvecs[1], BLAKE2S_HASH_SIZE); + + if (!success) + pr_err("blake2s256_hmac self-test: FAIL\n"); + } + + return success; +} --- /dev/null +++ b/lib/crypto/blake2s.c @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld Jason@zx2c4.com. All Rights Reserved. + * + * This is an implementation of the BLAKE2s hash and PRF functions. + * + * Information: https://blake2.net/ + * + */ + +#include <crypto/internal/blake2s.h> +#include <linux/types.h> +#include <linux/string.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/bug.h> +#include <asm/unaligned.h> + +bool blake2s_selftest(void); + +void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen) +{ + const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen; + + if (unlikely(!inlen)) + return; + if (inlen > fill) { + memcpy(state->buf + state->buflen, in, fill); + blake2s_compress_generic(state, state->buf, 1, + BLAKE2S_BLOCK_SIZE); + state->buflen = 0; + in += fill; + inlen -= fill; + } + if (inlen > BLAKE2S_BLOCK_SIZE) { + const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE); + /* Hash one less (full) block than strictly possible */ + blake2s_compress_generic(state, in, nblocks - 1, + BLAKE2S_BLOCK_SIZE); + in += BLAKE2S_BLOCK_SIZE * (nblocks - 1); + inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1); + } + memcpy(state->buf + state->buflen, in, inlen); + state->buflen += inlen; +} +EXPORT_SYMBOL(blake2s_update); + +void blake2s_final(struct blake2s_state *state, u8 *out) +{ + WARN_ON(IS_ENABLED(DEBUG) && !out); + blake2s_set_lastblock(state); + memset(state->buf + state->buflen, 0, + BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */ + blake2s_compress_generic(state, state->buf, 1, state->buflen); + cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); + memcpy(out, state->h, state->outlen); + memzero_explicit(state, sizeof(*state)); +} +EXPORT_SYMBOL(blake2s_final); + +void blake2s256_hmac(u8 *out, const u8 *in, const u8 *key, const size_t inlen, + const size_t keylen) +{ + struct blake2s_state state; + u8 x_key[BLAKE2S_BLOCK_SIZE] __aligned(__alignof__(u32)) = { 0 }; + u8 i_hash[BLAKE2S_HASH_SIZE] __aligned(__alignof__(u32)); + int i; + + if (keylen > BLAKE2S_BLOCK_SIZE) { + blake2s_init(&state, BLAKE2S_HASH_SIZE); + blake2s_update(&state, key, keylen); + blake2s_final(&state, x_key); + } else + memcpy(x_key, key, keylen); + + for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i) + x_key[i] ^= 0x36; + + blake2s_init(&state, BLAKE2S_HASH_SIZE); + blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE); + blake2s_update(&state, in, inlen); + blake2s_final(&state, i_hash); + + for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i) + x_key[i] ^= 0x5c ^ 0x36; + + blake2s_init(&state, BLAKE2S_HASH_SIZE); + blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE); + blake2s_update(&state, i_hash, BLAKE2S_HASH_SIZE); + blake2s_final(&state, i_hash); + + memcpy(out, i_hash, BLAKE2S_HASH_SIZE); + memzero_explicit(x_key, BLAKE2S_BLOCK_SIZE); + memzero_explicit(i_hash, BLAKE2S_HASH_SIZE); +} +EXPORT_SYMBOL(blake2s256_hmac); + +static int __init mod_init(void) +{ + if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && + WARN_ON(!blake2s_selftest())) + return -ENODEV; + return 0; +} + +static void __exit mod_exit(void) +{ +} + +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("BLAKE2s hash function"); +MODULE_AUTHOR("Jason A. Donenfeld Jason@zx2c4.com");

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 9a1536b093bb5bf60689021275fd24d513bb8db0 upstream.

With SHA-1 no longer being used for anything performance oriented, and also soon to be phased out entirely, we can make up for the space added by unrolled BLAKE2s by simply re-rolling SHA-1. Since SHA-1 is so much more complex, re-rolling it more or less takes care of the code size added by BLAKE2s. And eventually, hopefully we'll see SHA-1 removed entirely from most small kernel builds.

Cc: Herbert Xu herbert@gondor.apana.org.au Cc: Ard Biesheuvel ardb@kernel.org Tested-by: Geert Uytterhoeven geert@linux-m68k.org Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- lib/sha1.c | 95 ++++++++----------------------------------------------------- 1 file changed, 14 insertions(+), 81 deletions(-)

--- a/lib/sha1.c +++ b/lib/sha1.c @@ -10,6 +10,7 @@ #include <linux/export.h> #include <linux/bitops.h> #include <linux/cryptohash.h> +#include <linux/string.h> #include <asm/unaligned.h>

/* @@ -55,7 +56,8 @@ #define SHA_ROUND(t, input, fn, constant, A, B, C, D, E) do { \ __u32 TEMP = input(t); setW(t, TEMP); \ E += TEMP + rol32(A,5) + (fn) + (constant); \ - B = ror32(B, 2); } while (0) + B = ror32(B, 2); \ + TEMP = E; E = D; D = C; C = B; B = A; A = TEMP; } while (0)

#define T_0_15(t, A, B, C, D, E) SHA_ROUND(t, SHA_SRC, (((C^D)&B)^D) , 0x5a827999, A, B, C, D, E ) #define T_16_19(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, (((C^D)&B)^D) , 0x5a827999, A, B, C, D, E ) @@ -82,6 +84,7 @@ void sha_transform(__u32 *digest, const char *data, __u32 *array) { __u32 A, B, C, D, E; + unsigned int i = 0;

A = digest[0]; B = digest[1]; @@ -90,94 +93,24 @@ void sha_transform(__u32 *digest, const E = digest[4];

/* Round 1 - iterations 0-16 take their input from 'data' */ - T_0_15( 0, A, B, C, D, E); - T_0_15( 1, E, A, B, C, D); - T_0_15( 2, D, E, A, B, C); - T_0_15( 3, C, D, E, A, B); - T_0_15( 4, B, C, D, E, A); - T_0_15( 5, A, B, C, D, E); - T_0_15( 6, E, A, B, C, D); - T_0_15( 7, D, E, A, B, C); - T_0_15( 8, C, D, E, A, B); - T_0_15( 9, B, C, D, E, A); - T_0_15(10, A, B, C, D, E); - T_0_15(11, E, A, B, C, D); - T_0_15(12, D, E, A, B, C); - T_0_15(13, C, D, E, A, B); - T_0_15(14, B, C, D, E, A); - T_0_15(15, A, B, C, D, E); + for (; i < 16; ++i) + T_0_15(i, A, B, C, D, E);

/* Round 1 - tail. Input from 512-bit mixing array */ - T_16_19(16, E, A, B, C, D); - T_16_19(17, D, E, A, B, C); - T_16_19(18, C, D, E, A, B); - T_16_19(19, B, C, D, E, A); + for (; i < 20; ++i) + T_16_19(i, A, B, C, D, E);

/* Round 2 */ - T_20_39(20, A, B, C, D, E); - T_20_39(21, E, A, B, C, D); - T_20_39(22, D, E, A, B, C); - T_20_39(23, C, D, E, A, B); - T_20_39(24, B, C, D, E, A); - T_20_39(25, A, B, C, D, E); - T_20_39(26, E, A, B, C, D); - T_20_39(27, D, E, A, B, C); - T_20_39(28, C, D, E, A, B); - T_20_39(29, B, C, D, E, A); - T_20_39(30, A, B, C, D, E); - T_20_39(31, E, A, B, C, D); - T_20_39(32, D, E, A, B, C); - T_20_39(33, C, D, E, A, B); - T_20_39(34, B, C, D, E, A); - T_20_39(35, A, B, C, D, E); - T_20_39(36, E, A, B, C, D); - T_20_39(37, D, E, A, B, C); - T_20_39(38, C, D, E, A, B); - T_20_39(39, B, C, D, E, A); + for (; i < 40; ++i) + T_20_39(i, A, B, C, D, E);

/* Round 3 */ - T_40_59(40, A, B, C, D, E); - T_40_59(41, E, A, B, C, D); - T_40_59(42, D, E, A, B, C); - T_40_59(43, C, D, E, A, B); - T_40_59(44, B, C, D, E, A); - T_40_59(45, A, B, C, D, E); - T_40_59(46, E, A, B, C, D); - T_40_59(47, D, E, A, B, C); - T_40_59(48, C, D, E, A, B); - T_40_59(49, B, C, D, E, A); - T_40_59(50, A, B, C, D, E); - T_40_59(51, E, A, B, C, D); - T_40_59(52, D, E, A, B, C); - T_40_59(53, C, D, E, A, B); - T_40_59(54, B, C, D, E, A); - T_40_59(55, A, B, C, D, E); - T_40_59(56, E, A, B, C, D); - T_40_59(57, D, E, A, B, C); - T_40_59(58, C, D, E, A, B); - T_40_59(59, B, C, D, E, A); + for (; i < 60; ++i) + T_40_59(i, A, B, C, D, E);

/* Round 4 */ - T_60_79(60, A, B, C, D, E); - T_60_79(61, E, A, B, C, D); - T_60_79(62, D, E, A, B, C); - T_60_79(63, C, D, E, A, B); - T_60_79(64, B, C, D, E, A); - T_60_79(65, A, B, C, D, E); - T_60_79(66, E, A, B, C, D); - T_60_79(67, D, E, A, B, C); - T_60_79(68, C, D, E, A, B); - T_60_79(69, B, C, D, E, A); - T_60_79(70, A, B, C, D, E); - T_60_79(71, E, A, B, C, D); - T_60_79(72, D, E, A, B, C); - T_60_79(73, C, D, E, A, B); - T_60_79(74, B, C, D, E, A); - T_60_79(75, A, B, C, D, E); - T_60_79(76, E, A, B, C, D); - T_60_79(77, D, E, A, B, C); - T_60_79(78, C, D, E, A, B); - T_60_79(79, B, C, D, E, A); + for (; i < 80; ++i) + T_60_79(i, A, B, C, D, E);

digest[0] += A; digest[1] += B;

From: Andy Lutomirski luto@kernel.org

commit c6f1deb158789abba02a7eba600747843eeb3a57 upstream.

/dev/random and getrandom() never warn. Split the meat of urandom_read() into urandom_read_nowarn() and leave the warning code in urandom_read().

This has no effect on kernel behavior, but it makes subsequent patches more straightforward. It also makes the fact that getrandom() never warns more obvious.

Signed-off-by: Andy Lutomirski luto@kernel.org Link: https://lore.kernel.org/r/c87ab200588de746431d9f916501ef11e5242b13.157708852... Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1962,11 +1962,22 @@ random_read(struct file *file, char __us }

static ssize_t +urandom_read_nowarn(struct file *file, char __user *buf, size_t nbytes, + loff_t *ppos) +{ + int ret; + + nbytes = min_t(size_t, nbytes, INT_MAX >> (ENTROPY_SHIFT + 3)); + ret = extract_crng_user(buf, nbytes); + trace_urandom_read(8 * nbytes, 0, ENTROPY_BITS(&input_pool)); + return ret; +} + +static ssize_t urandom_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) { unsigned long flags; static int maxwarn = 10; - int ret;

if (!crng_ready() && maxwarn > 0) { maxwarn--; @@ -1978,10 +1989,8 @@ urandom_read(struct file *file, char __u crng_init_cnt = 0; spin_unlock_irqrestore(&primary_crng.lock, flags); } - nbytes = min_t(size_t, nbytes, INT_MAX >> (ENTROPY_SHIFT + 3)); - ret = extract_crng_user(buf, nbytes); - trace_urandom_read(8 * nbytes, 0, ENTROPY_BITS(&input_pool)); - return ret; + + return urandom_read_nowarn(file, buf, nbytes, ppos); }

static unsigned int @@ -2141,7 +2150,7 @@ SYSCALL_DEFINE3(getrandom, char __user * if (unlikely(ret)) return ret; } - return urandom_read(NULL, buf, count, NULL); + return urandom_read_nowarn(NULL, buf, count, NULL); }

/********************************************************************

From: Andy Lutomirski luto@kernel.org

commit 48446f198f9adcb499b30332488dfd5bc3f176f6 upstream.

The separate blocking pool is going away. Start by ignoring GRND_RANDOM in getentropy(2).

This should not materially break any API. Any code that worked without this change should work at least as well with this change.

Signed-off-by: Andy Lutomirski luto@kernel.org Link: https://lore.kernel.org/r/705c5a091b63cc5da70c99304bb97e0109be0a26.157708852... Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 3 --- include/uapi/linux/random.h | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -2147,9 +2147,6 @@ SYSCALL_DEFINE3(getrandom, char __user * if (count > INT_MAX) count = INT_MAX;

- if (flags & GRND_RANDOM) - return _random_read(flags & GRND_NONBLOCK, buf, count); - if (!(flags & GRND_INSECURE) && !crng_ready()) { if (flags & GRND_NONBLOCK) return -EAGAIN; --- a/include/uapi/linux/random.h +++ b/include/uapi/linux/random.h @@ -48,7 +48,7 @@ struct rand_pool_info { * Flags for getrandom(2) * * GRND_NONBLOCK Don't block and return EAGAIN instead - * GRND_RANDOM Use the /dev/random pool instead of /dev/urandom + * GRND_RANDOM No effect * GRND_INSECURE Return non-cryptographic random bytes */ #define GRND_NONBLOCK 0x0001

From: Sergey Senozhatsky sergey.senozhatsky.work@gmail.com

[ Upstream commit 1b710b1b10eff9d46666064ea25f079f70bc67a8 ]

Sergey didn't like the locking order,

uart_port->lock -> tty_port->lock

uart_write (uart_port->lock) __uart_start pl011_start_tx pl011_tx_chars uart_write_wakeup tty_port_tty_wakeup tty_port_default tty_port_tty_get (tty_port->lock)

but those code is so old, and I have no clue how to de-couple it after checking other locks in the splat. There is an onging effort to make all printk() as deferred, so until that happens, workaround it for now as a short-term fix.

LTP: starting iogen01 (export LTPROOT; rwtest -N iogen01 -i 120s -s read,write -Da -Dv -n 2 500b:$TMPDIR/doio.f1.$$ 1000b:$TMPDIR/doio.f2.$$) WARNING: possible circular locking dependency detected ------------------------------------------------------ doio/49441 is trying to acquire lock: ffff008b7cff7290 (&(&zone->lock)->rlock){..-.}, at: rmqueue+0x138/0x2050

but task is already holding lock: 60ff000822352818 (&pool->lock/1){-.-.}, at: start_flush_work+0xd8/0x3f0

which lock already depends on the new lock.

the existing dependency chain (in reverse order) is:

-> #4 (&pool->lock/1){-.-.}: lock_acquire+0x320/0x360 _raw_spin_lock+0x64/0x80 __queue_work+0x4b4/0xa10 queue_work_on+0xac/0x11c tty_schedule_flip+0x84/0xbc tty_flip_buffer_push+0x1c/0x28 pty_write+0x98/0xd0 n_tty_write+0x450/0x60c tty_write+0x338/0x474 __vfs_write+0x88/0x214 vfs_write+0x12c/0x1a4 redirected_tty_write+0x90/0xdc do_loop_readv_writev+0x140/0x180 do_iter_write+0xe0/0x10c vfs_writev+0x134/0x1cc do_writev+0xbc/0x130 __arm64_sys_writev+0x58/0x8c el0_svc_handler+0x170/0x240 el0_sync_handler+0x150/0x250 el0_sync+0x164/0x180

-> #3 (&(&port->lock)->rlock){-.-.}: lock_acquire+0x320/0x360 _raw_spin_lock_irqsave+0x7c/0x9c tty_port_tty_get+0x24/0x60 tty_port_default_wakeup+0x1c/0x3c tty_port_tty_wakeup+0x34/0x40 uart_write_wakeup+0x28/0x44 pl011_tx_chars+0x1b8/0x270 pl011_start_tx+0x24/0x70 __uart_start+0x5c/0x68 uart_write+0x164/0x1c8 do_output_char+0x33c/0x348 n_tty_write+0x4bc/0x60c tty_write+0x338/0x474 redirected_tty_write+0xc0/0xdc do_loop_readv_writev+0x140/0x180 do_iter_write+0xe0/0x10c vfs_writev+0x134/0x1cc do_writev+0xbc/0x130 __arm64_sys_writev+0x58/0x8c el0_svc_handler+0x170/0x240 el0_sync_handler+0x150/0x250 el0_sync+0x164/0x180

-> #2 (&port_lock_key){-.-.}: lock_acquire+0x320/0x360 _raw_spin_lock+0x64/0x80 pl011_console_write+0xec/0x2cc console_unlock+0x794/0x96c vprintk_emit+0x260/0x31c vprintk_default+0x54/0x7c vprintk_func+0x218/0x254 printk+0x7c/0xa4 register_console+0x734/0x7b0 uart_add_one_port+0x734/0x834 pl011_register_port+0x6c/0xac sbsa_uart_probe+0x234/0x2ec platform_drv_probe+0xd4/0x124 really_probe+0x250/0x71c driver_probe_device+0xb4/0x200 __device_attach_driver+0xd8/0x188 bus_for_each_drv+0xbc/0x110 __device_attach+0x120/0x220 device_initial_probe+0x20/0x2c bus_probe_device+0x54/0x100 device_add+0xae8/0xc2c platform_device_add+0x278/0x3b8 platform_device_register_full+0x238/0x2ac acpi_create_platform_device+0x2dc/0x3a8 acpi_bus_attach+0x390/0x3cc acpi_bus_attach+0x108/0x3cc acpi_bus_attach+0x108/0x3cc acpi_bus_attach+0x108/0x3cc acpi_bus_scan+0x7c/0xb0 acpi_scan_init+0xe4/0x304 acpi_init+0x100/0x114 do_one_initcall+0x348/0x6a0 do_initcall_level+0x190/0x1fc do_basic_setup+0x34/0x4c kernel_init_freeable+0x19c/0x260 kernel_init+0x18/0x338 ret_from_fork+0x10/0x18

-> #1 (console_owner){-...}: lock_acquire+0x320/0x360 console_lock_spinning_enable+0x6c/0x7c console_unlock+0x4f8/0x96c vprintk_emit+0x260/0x31c vprintk_default+0x54/0x7c vprintk_func+0x218/0x254 printk+0x7c/0xa4 get_random_u64+0x1c4/0x1dc shuffle_pick_tail+0x40/0xac __free_one_page+0x424/0x710 free_one_page+0x70/0x120 __free_pages_ok+0x61c/0xa94 __free_pages_core+0x1bc/0x294 memblock_free_pages+0x38/0x48 __free_pages_memory+0xcc/0xfc __free_memory_core+0x70/0x78 free_low_memory_core_early+0x148/0x18c memblock_free_all+0x18/0x54 mem_init+0xb4/0x17c mm_init+0x14/0x38 start_kernel+0x19c/0x530

-> #0 (&(&zone->lock)->rlock){..-.}: validate_chain+0xf6c/0x2e2c __lock_acquire+0x868/0xc2c lock_acquire+0x320/0x360 _raw_spin_lock+0x64/0x80 rmqueue+0x138/0x2050 get_page_from_freelist+0x474/0x688 __alloc_pages_nodemask+0x3b4/0x18dc alloc_pages_current+0xd0/0xe0 alloc_slab_page+0x2b4/0x5e0 new_slab+0xc8/0x6bc ___slab_alloc+0x3b8/0x640 kmem_cache_alloc+0x4b4/0x588 __debug_object_init+0x778/0x8b4 debug_object_init_on_stack+0x40/0x50 start_flush_work+0x16c/0x3f0 __flush_work+0xb8/0x124 flush_work+0x20/0x30 xlog_cil_force_lsn+0x88/0x204 [xfs] xfs_log_force_lsn+0x128/0x1b8 [xfs] xfs_file_fsync+0x3c4/0x488 [xfs] vfs_fsync_range+0xb0/0xd0 generic_write_sync+0x80/0xa0 [xfs] xfs_file_buffered_aio_write+0x66c/0x6e4 [xfs] xfs_file_write_iter+0x1a0/0x218 [xfs] __vfs_write+0x1cc/0x214 vfs_write+0x12c/0x1a4 ksys_write+0xb0/0x120 __arm64_sys_write+0x54/0x88 el0_svc_handler+0x170/0x240 el0_sync_handler+0x150/0x250 el0_sync+0x164/0x180

other info that might help us debug this:

Chain exists of: &(&zone->lock)->rlock --> &(&port->lock)->rlock --> &pool->lock/1

Possible unsafe locking scenario:

CPU0 CPU1 ---- ---- lock(&pool->lock/1); lock(&(&port->lock)->rlock); lock(&pool->lock/1); lock(&(&zone->lock)->rlock);

*** DEADLOCK ***

4 locks held by doio/49441: #0: a0ff00886fc27408 (sb_writers#8){.+.+}, at: vfs_write+0x118/0x1a4 #1: 8fff00080810dfe0 (&xfs_nondir_ilock_class){++++}, at: xfs_ilock+0x2a8/0x300 [xfs] #2: ffff9000129f2390 (rcu_read_lock){....}, at: rcu_lock_acquire+0x8/0x38 #3: 60ff000822352818 (&pool->lock/1){-.-.}, at: start_flush_work+0xd8/0x3f0

stack backtrace: CPU: 48 PID: 49441 Comm: doio Tainted: G W Hardware name: HPE Apollo 70 /C01_APACHE_MB , BIOS L50_5.13_1.11 06/18/2019 Call trace: dump_backtrace+0x0/0x248 show_stack+0x20/0x2c dump_stack+0xe8/0x150 print_circular_bug+0x368/0x380 check_noncircular+0x28c/0x294 validate_chain+0xf6c/0x2e2c __lock_acquire+0x868/0xc2c lock_acquire+0x320/0x360 _raw_spin_lock+0x64/0x80 rmqueue+0x138/0x2050 get_page_from_freelist+0x474/0x688 __alloc_pages_nodemask+0x3b4/0x18dc alloc_pages_current+0xd0/0xe0 alloc_slab_page+0x2b4/0x5e0 new_slab+0xc8/0x6bc ___slab_alloc+0x3b8/0x640 kmem_cache_alloc+0x4b4/0x588 __debug_object_init+0x778/0x8b4 debug_object_init_on_stack+0x40/0x50 start_flush_work+0x16c/0x3f0 __flush_work+0xb8/0x124 flush_work+0x20/0x30 xlog_cil_force_lsn+0x88/0x204 [xfs] xfs_log_force_lsn+0x128/0x1b8 [xfs] xfs_file_fsync+0x3c4/0x488 [xfs] vfs_fsync_range+0xb0/0xd0 generic_write_sync+0x80/0xa0 [xfs] xfs_file_buffered_aio_write+0x66c/0x6e4 [xfs] xfs_file_write_iter+0x1a0/0x218 [xfs] __vfs_write+0x1cc/0x214 vfs_write+0x12c/0x1a4 ksys_write+0xb0/0x120 __arm64_sys_write+0x54/0x88 el0_svc_handler+0x170/0x240 el0_sync_handler+0x150/0x250 el0_sync+0x164/0x180

Reviewed-by: Sergey Senozhatsky sergey.senozhatsky.work@gmail.com Signed-off-by: Qian Cai cai@lca.pw Link: https://lore.kernel.org/r/1573679785-21068-1-git-send-email-cai@lca.pw Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Sasha Levin sashal@kernel.org Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1687,8 +1687,9 @@ static void _warn_unseeded_randomness(co print_once = true; #endif if (__ratelimit(&unseeded_warning)) - pr_notice("random: %s called from %pS with crng_init=%d\n", - func_name, caller, crng_init); + printk_deferred(KERN_NOTICE "random: %s called from %pS " + "with crng_init=%d\n", func_name, caller, + crng_init); }

/*

From: Andy Lutomirski luto@kernel.org

commit 90ea1c6436d26e62496616fb5891e00819ff4849 upstream.

There is no longer any interface to read data from the blocking pool, so remove it.

This enables quite a bit of code deletion, much of which will be done in subsequent patches.

Signed-off-by: Andy Lutomirski luto@kernel.org Link: https://lore.kernel.org/r/511225a224bf0a291149d3c0b8b45393cd03ab96.157708852... Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 106 -------------------------------------------------- 1 file changed, 106 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -470,7 +470,6 @@ static const struct poolinfo { /* * Static global variables */ -static DECLARE_WAIT_QUEUE_HEAD(random_read_wait); static DECLARE_WAIT_QUEUE_HEAD(random_write_wait); static struct fasync_struct *fasync;

@@ -532,7 +531,6 @@ struct entropy_store { __u32 *pool; const char *name; struct entropy_store *pull; - struct work_struct push_work;

/* read-write data: */ unsigned long last_pulled; @@ -551,9 +549,7 @@ static ssize_t _extract_entropy(struct e size_t nbytes, int fips);

static void crng_reseed(struct crng_state *crng, struct entropy_store *r); -static void push_to_pool(struct work_struct *work); static __u32 input_pool_data[INPUT_POOL_WORDS] __latent_entropy; -static __u32 blocking_pool_data[OUTPUT_POOL_WORDS] __latent_entropy;

static struct entropy_store input_pool = { .poolinfo = &poolinfo_table[0], @@ -562,16 +558,6 @@ static struct entropy_store input_pool = .pool = input_pool_data };

-static struct entropy_store blocking_pool = { - .poolinfo = &poolinfo_table[1], - .name = "blocking", - .pull = &input_pool, - .lock = __SPIN_LOCK_UNLOCKED(blocking_pool.lock), - .pool = blocking_pool_data, - .push_work = __WORK_INITIALIZER(blocking_pool.push_work, - push_to_pool), -}; - static __u32 const twist_table[8] = { 0x00000000, 0x3b6e20c8, 0x76dc4190, 0x4db26158, 0xedb88320, 0xd6d6a3e8, 0x9b64c2b0, 0xa00ae278 }; @@ -767,15 +753,11 @@ retry: entropy_count = 0; } else if (entropy_count > pool_size) entropy_count = pool_size; - if ((r == &blocking_pool) && !r->initialized && - (entropy_count >> ENTROPY_SHIFT) > 128) - has_initialized = 1; if (cmpxchg(&r->entropy_count, orig, entropy_count) != orig) goto retry;

if (has_initialized) { r->initialized = 1; - wake_up_interruptible(&random_read_wait); kill_fasync(&fasync, SIGIO, POLL_IN); }

@@ -784,7 +766,6 @@ retry:

if (r == &input_pool) { int entropy_bits = entropy_count >> ENTROPY_SHIFT; - struct entropy_store *other = &blocking_pool;

if (crng_init < 2) { if (entropy_bits < 128) @@ -792,27 +773,6 @@ retry: crng_reseed(&primary_crng, r); entropy_bits = r->entropy_count >> ENTROPY_SHIFT; } - - /* initialize the blocking pool if necessary */ - if (entropy_bits >= random_read_wakeup_bits && - !other->initialized) { - schedule_work(&other->push_work); - return; - } - - /* should we wake readers? */ - if (entropy_bits >= random_read_wakeup_bits && - wq_has_sleeper(&random_read_wait)) { - wake_up_interruptible(&random_read_wait); - } - /* If the input pool is getting full, and the blocking - * pool has room, send some entropy to the blocking - * pool. - */ - if (!work_pending(&other->push_work) && - (ENTROPY_BITS(r) > 6 * r->poolinfo->poolbytes) && - (ENTROPY_BITS(other) <= 6 * other->poolinfo->poolbytes)) - schedule_work(&other->push_work); } }

@@ -1439,22 +1399,6 @@ static void _xfer_secondary_pool(struct }

/* - * Used as a workqueue function so that when the input pool is getting - * full, we can "spill over" some entropy to the output pools. That - * way the output pools can store some of the excess entropy instead - * of letting it go to waste. - */ -static void push_to_pool(struct work_struct *work) -{ - struct entropy_store *r = container_of(work, struct entropy_store, - push_work); - BUG_ON(!r); - _xfer_secondary_pool(r, random_read_wakeup_bits/8); - trace_push_to_pool(r->name, r->entropy_count >> ENTROPY_SHIFT, - r->pull->entropy_count >> ENTROPY_SHIFT); -} - -/* * This function decides how many bytes to actually take from the * given pool, and also debits the entropy count accordingly. */ @@ -1632,54 +1576,6 @@ static ssize_t extract_entropy(struct en return _extract_entropy(r, buf, nbytes, fips_enabled); }

-/* - * This function extracts randomness from the "entropy pool", and - * returns it in a userspace buffer. - */ -static ssize_t extract_entropy_user(struct entropy_store *r, void __user *buf, - size_t nbytes) -{ - ssize_t ret = 0, i; - __u8 tmp[EXTRACT_SIZE]; - int large_request = (nbytes > 256); - - trace_extract_entropy_user(r->name, nbytes, ENTROPY_BITS(r), _RET_IP_); - if (!r->initialized && r->pull) { - xfer_secondary_pool(r, ENTROPY_BITS(r->pull)/8); - if (!r->initialized) - return 0; - } - xfer_secondary_pool(r, nbytes); - nbytes = account(r, nbytes, 0, 0); - - while (nbytes) { - if (large_request && need_resched()) { - if (signal_pending(current)) { - if (ret == 0) - ret = -ERESTARTSYS; - break; - } - schedule(); - } - - extract_buf(r, tmp); - i = min_t(int, nbytes, EXTRACT_SIZE); - if (copy_to_user(buf, tmp, i)) { - ret = -EFAULT; - break; - } - - nbytes -= i; - buf += i; - ret += i; - } - - /* Wipe data just returned from memory */ - memzero_explicit(tmp, sizeof(tmp)); - - return ret; -} - #define warn_unseeded_randomness(previous) \ _warn_unseeded_randomness(__func__, (void *) _RET_IP_, (previous))

@@ -1910,7 +1806,6 @@ static void __init init_std_data(struct int __init rand_initialize(void) { init_std_data(&input_pool); - init_std_data(&blocking_pool); if (crng_need_final_init) crng_finalize_init(&primary_crng); crng_initialize(&primary_crng); @@ -2081,7 +1976,6 @@ static long random_ioctl(struct file *f, if (!capable(CAP_SYS_ADMIN)) return -EPERM; input_pool.entropy_count = 0; - blocking_pool.entropy_count = 0; return 0; case RNDRESEEDCRNG: if (!capable(CAP_SYS_ADMIN))

From: Andy Lutomirski luto@kernel.org

commit c95ea0c69ffda19381c116db2be23c7e654dac98 upstream.

It has no effect any more, so remove it. We can revert this if there is some user code that expects to be able to set this sysctl.

Signed-off-by: Andy Lutomirski luto@kernel.org Link: https://lore.kernel.org/r/a74ed2cf0b5a5451428a246a9239f5bc4e29358f.157708852... Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -370,12 +370,6 @@ #define ENTROPY_BITS(r) ((r)->entropy_count >> ENTROPY_SHIFT)

/* - * The minimum number of bits of entropy before we wake up a read on - * /dev/random. Should be enough to do a significant reseed. - */ -static int random_read_wakeup_bits = 64; - -/* * If the entropy count falls under this number of bits, then we * should wake up processes which are selecting or polling on write * access to /dev/random. @@ -2010,8 +2004,7 @@ SYSCALL_DEFINE3(getrandom, char __user *

#include <linux/sysctl.h>

-static int min_read_thresh = 8, min_write_thresh; -static int max_read_thresh = OUTPUT_POOL_WORDS * 32; +static int min_write_thresh; static int max_write_thresh = INPUT_POOL_WORDS * 32; static int random_min_urandom_seed = 60; static char sysctl_bootid[16]; @@ -2087,15 +2080,6 @@ struct ctl_table random_table[] = { .data = &input_pool.entropy_count, }, { - .procname = "read_wakeup_threshold", - .data = &random_read_wakeup_bits, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = &min_read_thresh, - .extra2 = &max_read_thresh, - }, - { .procname = "write_wakeup_threshold", .data = &random_write_wakeup_bits, .maxlen = sizeof(int),

From: Yangtao Li tiny.windzz@gmail.com

commit 12faac30d157970fdbfa171bbeb1fb88350303b1 upstream.

Signed-off-by: Yangtao Li tiny.windzz@gmail.com Link: https://lore.kernel.org/r/20190607182517.28266-2-tiny.windzz@gmail.com Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -762,7 +762,7 @@ retry: if (entropy_bits < 128) return; crng_reseed(&primary_crng, r); - entropy_bits = r->entropy_count >> ENTROPY_SHIFT; + entropy_bits = ENTROPY_BITS(r); } } } @@ -1394,8 +1394,7 @@ retry: goto retry;

trace_debit_entropy(r->name, 8 * ibytes); - if (ibytes && - (r->entropy_count >> ENTROPY_SHIFT) < random_write_wakeup_bits) { + if (ibytes && ENTROPY_BITS(r) < random_write_wakeup_bits) { wake_up_interruptible(&random_write_wait); kill_fasync(&fasync, SIGIO, POLL_OUT); }

From: Yangtao Li tiny.windzz@gmail.com

commit 727d499a6f4f29b6abdb635032f5e53e5905aedb upstream.

s/entimate/estimate

Signed-off-by: Yangtao Li tiny.windzz@gmail.com Link: https://lore.kernel.org/r/20190607182517.28266-4-tiny.windzz@gmail.com Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1211,7 +1211,7 @@ static void add_timer_randomness(struct /* * delta is now minimum absolute delta. * Round down by 1 bit on general principles, - * and limit entropy entimate to 12 bits. + * and limit entropy estimate to 12 bits. */ credit_entropy_bits(r, min_t(int, fls(delta>>1), 11)); }

From: Mark Rutland mark.rutland@arm.com

commit 5cbe0f13b51ac2fb2fd55902cff8d0077fc084c0 upstream.

Currently crng_initialize() is used for both the primary CRNG and secondary CRNGs. While we wish to share common logic, we need to do a number of additional things for the primary CRNG, and this would be easier to deal with were these handled in separate functions.

This patch splits crng_initialize() into crng_initialize_primary() and crng_initialize_secondary(), with common logic factored out into a crng_init_try_arch() helper.

There should be no functional change as a result of this patch.

Signed-off-by: Mark Rutland mark.rutland@arm.com Cc: Mark Brown broonie@kernel.org Cc: Theodore Ts'o tytso@mit.edu Link: https://lore.kernel.org/r/20200210130015.17664-2-mark.rutland@arm.com Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -782,27 +782,39 @@ static int __init parse_trust_cpu(char * } early_param("random.trust_cpu", parse_trust_cpu);

-static void crng_initialize(struct crng_state *crng) +static bool crng_init_try_arch(struct crng_state *crng) { int i; - int arch_init = 1; + bool arch_init = true; unsigned long rv;

- memcpy(&crng->state[0], "expand 32-byte k", 16); - if (crng == &primary_crng) - _extract_entropy(&input_pool, &crng->state[4], - sizeof(__u32) * 12, 0); - else - _get_random_bytes(&crng->state[4], sizeof(__u32) * 12); for (i = 4; i < 16; i++) { if (!arch_get_random_seed_long(&rv) && !arch_get_random_long(&rv)) { rv = random_get_entropy(); - arch_init = 0; + arch_init = false; } crng->state[i] ^= rv; } - if (trust_cpu && arch_init) { + + return arch_init; +} + +static void crng_initialize_secondary(struct crng_state *crng) +{ + memcpy(&crng->state[0], "expand 32-byte k", 16); + _get_random_bytes(&crng->state[4], sizeof(__u32) * 12); + crng_init_try_arch(crng); + crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1; +} + +static void __init crng_initialize_primary(struct crng_state *crng) +{ + memcpy(&crng->state[0], "expand 32-byte k", 16); + _extract_entropy(&input_pool, &crng->state[4], sizeof(__u32) * 12, 0); + if (crng_init_try_arch(crng) && trust_cpu) { + invalidate_batched_entropy(); + numa_crng_init(); crng_init = 2; pr_notice("crng done (trusting CPU's manufacturer)\n"); } @@ -851,7 +863,7 @@ static void do_numa_crng_init(struct wor crng = kmalloc_node(sizeof(struct crng_state), GFP_KERNEL | __GFP_NOFAIL, i); spin_lock_init(&crng->lock); - crng_initialize(crng); + crng_initialize_secondary(crng); pool[i] = crng; } /* pairs with READ_ONCE() in select_crng() */ @@ -1729,7 +1741,7 @@ int __init rand_initialize(void) init_std_data(&input_pool); if (crng_need_final_init) crng_finalize_init(&primary_crng); - crng_initialize(&primary_crng); + crng_initialize_primary(&primary_crng); crng_global_init_time = jiffies; if (ratelimit_disable) { urandom_warning.interval = 0;

From: Richard Henderson richard.henderson@linaro.org

commit 5f2ed7f5b99b54389b74e53309677831ac9cb9d7 upstream.

Use the expansion of these macros directly in arch_get_random_*.

These symbols are currently part of the generic archrandom.h interface, but are currently unused and can be removed.

Signed-off-by: Richard Henderson rth@twiddle.net Signed-off-by: Mark Brown broonie@kernel.org Link: https://lore.kernel.org/r/20200110145422.49141-2-broonie@kernel.org Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- arch/x86/include/asm/archrandom.h | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-)

--- a/arch/x86/include/asm/archrandom.h +++ b/arch/x86/include/asm/archrandom.h @@ -86,10 +86,6 @@ static inline bool rdseed_int(unsigned i return ok; }

-/* Conditional execution based on CPU type */ -#define arch_has_random() static_cpu_has(X86_FEATURE_RDRAND) -#define arch_has_random_seed() static_cpu_has(X86_FEATURE_RDSEED) - /* * These are the generic interfaces; they must not be declared if the * stubs in <linux/random.h> are to be invoked, @@ -99,22 +95,22 @@ static inline bool rdseed_int(unsigned i

static inline bool arch_get_random_long(unsigned long *v) { - return arch_has_random() ? rdrand_long(v) : false; + return static_cpu_has(X86_FEATURE_RDRAND) ? rdrand_long(v) : false; }

static inline bool arch_get_random_int(unsigned int *v) { - return arch_has_random() ? rdrand_int(v) : false; + return static_cpu_has(X86_FEATURE_RDRAND) ? rdrand_int(v) : false; }

static inline bool arch_get_random_seed_long(unsigned long *v) { - return arch_has_random_seed() ? rdseed_long(v) : false; + return static_cpu_has(X86_FEATURE_RDSEED) ? rdseed_long(v) : false; }

static inline bool arch_get_random_seed_int(unsigned int *v) { - return arch_has_random_seed() ? rdseed_int(v) : false; + return static_cpu_has(X86_FEATURE_RDSEED) ? rdseed_int(v) : false; }

extern void x86_init_rdrand(struct cpuinfo_x86 *c);

From: Richard Henderson richard.henderson@linaro.org

commit 5e054c820f59bbb9714d5767f5f476581c309ca8 upstream.

These symbols are currently part of the generic archrandom.h interface, but are currently unused and can be removed.

Signed-off-by: Richard Henderson rth@twiddle.net Signed-off-by: Mark Brown broonie@kernel.org Link: https://lore.kernel.org/r/20200110145422.49141-4-broonie@kernel.org Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- arch/s390/include/asm/archrandom.h | 12 ------------ 1 file changed, 12 deletions(-)

--- a/arch/s390/include/asm/archrandom.h +++ b/arch/s390/include/asm/archrandom.h @@ -26,18 +26,6 @@ static void s390_arch_random_generate(u8 atomic64_add(nbytes, &s390_arch_random_counter); }

-static inline bool arch_has_random(void) -{ - if (static_branch_likely(&s390_arch_random_available)) - return true; - return false; -} - -static inline bool arch_has_random_seed(void) -{ - return arch_has_random(); -} - static inline bool arch_get_random_long(unsigned long *v) { if (static_branch_likely(&s390_arch_random_available)) {

From: Richard Henderson richard.henderson@linaro.org

commit 66f5ae899ada79c0e9a3d8d954f93a72344cd350 upstream.

Keep the generic fallback versions in sync with the other architecture specific implementations and use the proper name for false.

Suggested-by: Ard Biesheuvel ardb@kernel.org Signed-off-by: Richard Henderson rth@twiddle.net Signed-off-by: Mark Brown broonie@kernel.org Link: https://lore.kernel.org/r/20200110145422.49141-6-broonie@kernel.org Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- include/linux/random.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)

--- a/include/linux/random.h +++ b/include/linux/random.h @@ -120,19 +120,19 @@ unsigned long randomize_page(unsigned lo #else static inline bool arch_get_random_long(unsigned long *v) { - return 0; + return false; } static inline bool arch_get_random_int(unsigned int *v) { - return 0; + return false; } static inline bool arch_get_random_seed_long(unsigned long *v) { - return 0; + return false; } static inline bool arch_get_random_seed_int(unsigned int *v) { - return 0; + return false; } #endif

From: Richard Henderson richard.henderson@linaro.org

commit 98dcfce69729f9ce0fb14f96a39bbdba21429597 upstream.

The generic interface uses bool not int; match that.

Reviewed-by: Ard Biesheuvel ardb@kernel.org Signed-off-by: Richard Henderson rth@twiddle.net Signed-off-by: Mark Brown broonie@kernel.org Link: https://lore.kernel.org/r/20200110145422.49141-9-broonie@kernel.org Signed-off-by: Theodore Ts'o tytso@mit.edu Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- arch/powerpc/include/asm/archrandom.h | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-)

--- a/arch/powerpc/include/asm/archrandom.h +++ b/arch/powerpc/include/asm/archrandom.h @@ -6,27 +6,28 @@

#include <asm/machdep.h>

-static inline int arch_get_random_long(unsigned long *v) +static inline bool arch_get_random_long(unsigned long *v) { - return 0; + return false; }

-static inline int arch_get_random_int(unsigned int *v) +static inline bool arch_get_random_int(unsigned int *v) { - return 0; + return false; }

-static inline int arch_get_random_seed_long(unsigned long *v) +static inline bool arch_get_random_seed_long(unsigned long *v) { if (ppc_md.get_random_seed) return ppc_md.get_random_seed(v);

- return 0; + return false; } -static inline int arch_get_random_seed_int(unsigned int *v) + +static inline bool arch_get_random_seed_int(unsigned int *v) { unsigned long val; - int rc; + bool rc;

rc = arch_get_random_seed_long(&val); if (rc)

From: Ard Biesheuvel ardb@kernel.org

commit 390596c9959c2a4f5b456df339f0604df3d55fe0 upstream.

When reseeding the CRNG periodically, arch_get_random_seed_long() is called to obtain entropy from an architecture specific source if one is implemented. In most cases, these are special instructions, but in some cases, such as on ARM, we may want to back this using firmware calls, which are considerably more expensive.

Another call to arch_get_random_seed_long() exists in the CRNG driver, in add_interrupt_randomness(), which collects entropy by capturing inter-interrupt timing and relying on interrupt jitter to provide random bits. This is done by keeping a per-CPU state, and mixing in the IRQ number, the cycle counter and the return address every time an interrupt is taken, and mixing this per-CPU state into the entropy pool every 64 invocations, or at least once per second. The entropy that is gathered this way is credited as 1 bit of entropy. Every time this happens, arch_get_random_seed_long() is invoked, and the result is mixed in as well, and also credited with 1 bit of entropy.

This means that arch_get_random_seed_long() is called at least once per second on every CPU, which seems excessive, and doesn't really scale, especially in a virtualization scenario where CPUs may be oversubscribed: in cases where arch_get_random_seed_long() is backed by an instruction that actually goes back to a shared hardware entropy source (such as RNDRRS on ARM), we will end up hitting it hundreds of times per second.

So let's drop the call to arch_get_random_seed_long() from add_interrupt_randomness(), and instead, rely on crng_reseed() to call the arch hook to get random seed material from the platform.

Signed-off-by: Ard Biesheuvel ardb@kernel.org Reviewed-by: Andre Przywara andre.przywara@arm.com Tested-by: Andre Przywara andre.przywara@arm.com Reviewed-by: Eric Biggers ebiggers@google.com Acked-by: Marc Zyngier maz@kernel.org Reviewed-by: Jason A. Donenfeld Jason@zx2c4.com Link: https://lore.kernel.org/r/20201105152944.16953-1-ardb@kernel.org Signed-off-by: Will Deacon will@kernel.org Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1279,8 +1279,6 @@ void add_interrupt_randomness(int irq, i cycles_t cycles = random_get_entropy(); __u32 c_high, j_high; __u64 ip; - unsigned long seed; - int credit = 0;

if (cycles == 0) cycles = get_reg(fast_pool, regs); @@ -1316,23 +1314,12 @@ void add_interrupt_randomness(int irq, i

fast_pool->last = now; __mix_pool_bytes(r, &fast_pool->pool, sizeof(fast_pool->pool)); - - /* - * If we have architectural seed generator, produce a seed and - * add it to the pool. For the sake of paranoia don't let the - * architectural seed generator dominate the input from the - * interrupt noise. - */ - if (arch_get_random_seed_long(&seed)) { - __mix_pool_bytes(r, &seed, sizeof(seed)); - credit = 1; - } spin_unlock(&r->lock);

fast_pool->count = 0;

/* award one bit for the contents of the fast pool */ - credit_entropy_bits(r, credit + 1); + credit_entropy_bits(r, 1); } EXPORT_SYMBOL_GPL(add_interrupt_randomness);

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 58e1100fdc5990b0cc0d4beaf2562a92e621ac7d upstream.

random.c is a bit understaffed, and folks want more prompt reviews. I've got the crypto background and the interest to do these reviews, and have authored parts of the file already.

Cc: Theodore Ts'o tytso@mit.edu Cc: Greg Kroah-Hartman gregkh@linuxfoundation.org Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Linus Torvalds torvalds@linux-foundation.org Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+)

--- a/MAINTAINERS +++ b/MAINTAINERS @@ -11307,6 +11307,7 @@ F: drivers/block/brd.c

RANDOM NUMBER DRIVER M: "Theodore Ts'o" tytso@mit.edu +M: Jason A. Donenfeld Jason@zx2c4.com S: Maintained F: drivers/char/random.c

From: Eric Biggers ebiggers@google.com

commit 8786841bc2020f7f2513a6c74e64912f07b9c0dc upstream.

Use the full path in the include guards for the BLAKE2s headers to avoid ambiguity and to match the convention for most files in include/crypto/.

Signed-off-by: Eric Biggers ebiggers@google.com Acked-by: Ard Biesheuvel ardb@kernel.org Signed-off-by: Herbert Xu herbert@gondor.apana.org.au Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- include/crypto/blake2s.h | 6 +++--- include/crypto/internal/blake2s.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-)

--- a/include/crypto/blake2s.h +++ b/include/crypto/blake2s.h @@ -3,8 +3,8 @@ * Copyright (C) 2015-2019 Jason A. Donenfeld Jason@zx2c4.com. All Rights Reserved. */

-#ifndef BLAKE2S_H -#define BLAKE2S_H +#ifndef _CRYPTO_BLAKE2S_H +#define _CRYPTO_BLAKE2S_H

#include <linux/bug.h> #include <linux/types.h> @@ -99,4 +99,4 @@ static inline void blake2s(u8 *out, cons blake2s_final(&state, out); }

-#endif /* BLAKE2S_H */ +#endif /* _CRYPTO_BLAKE2S_H */ --- a/include/crypto/internal/blake2s.h +++ b/include/crypto/internal/blake2s.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 OR MIT */

-#ifndef BLAKE2S_INTERNAL_H -#define BLAKE2S_INTERNAL_H +#ifndef _CRYPTO_INTERNAL_BLAKE2S_H +#define _CRYPTO_INTERNAL_BLAKE2S_H

#include <crypto/blake2s.h>

@@ -16,4 +16,4 @@ static inline void blake2s_set_lastblock state->f[0] = -1; }

-#endif /* BLAKE2S_INTERNAL_H */ +#endif /* _CRYPTO_INTERNAL_BLAKE2S_H */

From: Sebastian Andrzej Siewior bigeasy@linutronix.de

commit 703f7066f40599c290babdb79dd61319264987e9 upstream.

Since commit ee3e00e9e7101 ("random: use registers from interrupted code for CPU's w/o a cycle counter")

the irq_flags argument is no longer used.

Remove unused irq_flags.

Cc: Borislav Petkov bp@alien8.de Cc: Dave Hansen dave.hansen@linux.intel.com Cc: Dexuan Cui decui@microsoft.com Cc: H. Peter Anvin hpa@zytor.com Cc: Haiyang Zhang haiyangz@microsoft.com Cc: Ingo Molnar mingo@redhat.com Cc: K. Y. Srinivasan kys@microsoft.com Cc: Stephen Hemminger sthemmin@microsoft.com Cc: Thomas Gleixner tglx@linutronix.de Cc: Wei Liu wei.liu@kernel.org Cc: linux-hyperv@vger.kernel.org Cc: x86@kernel.org Signed-off-by: Sebastian Andrzej Siewior bigeasy@linutronix.de Acked-by: Wei Liu wei.liu@kernel.org Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 4 ++-- drivers/hv/vmbus_drv.c | 2 +- include/linux/random.h | 2 +- kernel/irq/handle.c | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -200,7 +200,7 @@ * void add_device_randomness(const void *buf, unsigned int size); * void add_input_randomness(unsigned int type, unsigned int code, * unsigned int value); - * void add_interrupt_randomness(int irq, int irq_flags); + * void add_interrupt_randomness(int irq); * void add_disk_randomness(struct gendisk *disk); * void add_hwgenerator_randomness(const char *buffer, size_t count, * size_t entropy); @@ -1271,7 +1271,7 @@ static __u32 get_reg(struct fast_pool *f return *ptr; }

-void add_interrupt_randomness(int irq, int irq_flags) +void add_interrupt_randomness(int irq) { struct entropy_store *r; struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness); --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1064,7 +1064,7 @@ static void vmbus_isr(void) tasklet_schedule(&hv_cpu->msg_dpc); }

- add_interrupt_randomness(HYPERVISOR_CALLBACK_VECTOR, 0); + add_interrupt_randomness(HYPERVISOR_CALLBACK_VECTOR); }

--- a/include/linux/random.h +++ b/include/linux/random.h @@ -35,7 +35,7 @@ static inline void add_latent_entropy(vo

extern void add_input_randomness(unsigned int type, unsigned int code, unsigned int value) __latent_entropy; -extern void add_interrupt_randomness(int irq, int irq_flags) __latent_entropy; +extern void add_interrupt_randomness(int irq) __latent_entropy;

extern void get_random_bytes(void *buf, int nbytes); extern int wait_for_random_bytes(void); --- a/kernel/irq/handle.c +++ b/kernel/irq/handle.c @@ -186,7 +186,7 @@ irqreturn_t handle_irq_event_percpu(stru

retval = __handle_irq_event_percpu(desc, &flags);

- add_interrupt_randomness(desc->irq_data.irq, flags); + add_interrupt_randomness(desc->irq_data.irq);

if (!noirqdebug) note_interrupt(desc, retval);

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 0d9488ffbf2faddebc6bac055bfa6c93b94056a3 upstream.

By using `char` instead of `unsigned char`, certain platforms will sign extend the byte when `w = rol32(*bytes++, input_rotate)` is called, meaning that bit 7 is overrepresented when mixing. This isn't a real problem (unless the mixer itself is already broken) since it's still invertible, but it's not quite correct either. Fix this by using an explicit unsigned type.

Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -549,7 +549,7 @@ static void _mix_pool_bytes(struct entro unsigned long i, tap1, tap2, tap3, tap4, tap5; int input_rotate; int wordmask = r->poolinfo->poolwords - 1; - const char *bytes = in; + const unsigned char *bytes = in; __u32 w;

tap1 = r->poolinfo->tap1;

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 57826feeedb63b091f807ba8325d736775d39afd upstream.

If we're trusting bootloader randomness, crng_fast_load() is called by add_hwgenerator_randomness(), which sets us to crng_init==1. However, usually it is only called once for an initial 64-byte push, so bootloader entropy will not mix any bytes into the input pool. So it's conceivable that crng_init==1 when crng_initialize_primary() is called later, but then the input pool is empty. When that happens, the crng state key will be overwritten with extracted output from the empty input pool. That's bad.

In contrast, if we're not trusting bootloader randomness, we call crng_slow_load() *and* we call mix_pool_bytes(), so that later crng_initialize_primary() isn't drawing on nothing.

In order to prevent crng_initialize_primary() from extracting an empty pool, have the trusted bootloader case mirror that of the untrusted bootloader case, mixing the input into the pool.

[linux@dominikbrodowski.net: rewrite commit message] Signed-off-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -2234,8 +2234,12 @@ void add_hwgenerator_randomness(const ch struct entropy_store *poolp = &input_pool;

if (unlikely(crng_init == 0)) { - crng_fast_load(buffer, count); - return; + size_t ret = crng_fast_load(buffer, count); + mix_pool_bytes(poolp, buffer, ret); + count -= ret; + buffer += ret; + if (!count || crng_init == 0) + return; }

/* Suspend writing if we're above the trickle threshold.

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 7b87324112df2e1f9b395217361626362dcfb9fb upstream.

Rather than an awkward combination of ifdefs and __maybe_unused, we can ensure more source gets parsed, regardless of the configuration, by using IS_ENABLED for the CONFIG_NUMA conditional code. This makes things cleaner and easier to follow.

I've confirmed that on !CONFIG_NUMA, we don't wind up with excess code by accident; the generated object file is the same.

Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -761,7 +761,6 @@ static int credit_entropy_bits_safe(stru

static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait);

-#ifdef CONFIG_NUMA /* * Hack to deal with crazy userspace progams when they are all trying * to access /dev/urandom in parallel. The programs are almost @@ -769,7 +768,6 @@ static DECLARE_WAIT_QUEUE_HEAD(crng_init * their brain damage. */ static struct crng_state **crng_node_pool __read_mostly; -#endif

static void invalidate_batched_entropy(void);

@@ -816,7 +814,7 @@ static bool __init crng_init_try_arch_ea return arch_init; }

-static void __maybe_unused crng_initialize_secondary(struct crng_state *crng) +static void crng_initialize_secondary(struct crng_state *crng) { memcpy(&crng->state[0], "expand 32-byte k", 16); _get_random_bytes(&crng->state[4], sizeof(__u32) * 12); @@ -867,7 +865,6 @@ static void crng_finalize_init(struct cr } }

-#ifdef CONFIG_NUMA static void do_numa_crng_init(struct work_struct *work) { int i; @@ -894,29 +891,24 @@ static DECLARE_WORK(numa_crng_init_work,

static void numa_crng_init(void) { - schedule_work(&numa_crng_init_work); + if (IS_ENABLED(CONFIG_NUMA)) + schedule_work(&numa_crng_init_work); }

static struct crng_state *select_crng(void) { - struct crng_state **pool; - int nid = numa_node_id(); - - /* pairs with cmpxchg_release() in do_numa_crng_init() */ - pool = READ_ONCE(crng_node_pool); - if (pool && pool[nid]) - return pool[nid]; - - return &primary_crng; -} -#else -static void numa_crng_init(void) {} + if (IS_ENABLED(CONFIG_NUMA)) { + struct crng_state **pool; + int nid = numa_node_id(); + + /* pairs with cmpxchg_release() in do_numa_crng_init() */ + pool = READ_ONCE(crng_node_pool); + if (pool && pool[nid]) + return pool[nid]; + }

-static struct crng_state *select_crng(void) -{ return &primary_crng; } -#endif

/* * crng_fast_load() can be called by code in the interrupt service

From: Dominik Brodowski linux@dominikbrodowski.net

commit 96562f286884e2db89c74215b199a1084b5fb7f7 upstream.

Previously, the ChaCha constants for the primary pool were only initialized in crng_initialize_primary(), called by rand_initialize(). However, some randomness is actually extracted from the primary pool beforehand, e.g. by kmem_cache_create(). Therefore, statically initialize the ChaCha constants for the primary pool.

Cc: Herbert Xu herbert@gondor.apana.org.au Cc: "David S. Miller" davem@davemloft.net Cc: linux-crypto@vger.kernel.org Signed-off-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 5 ++++- include/crypto/chacha20.h | 15 +++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -458,6 +458,10 @@ struct crng_state {

static struct crng_state primary_crng = { .lock = __SPIN_LOCK_UNLOCKED(primary_crng.lock), + .state[0] = CHACHA_CONSTANT_EXPA, + .state[1] = CHACHA_CONSTANT_ND_3, + .state[2] = CHACHA_CONSTANT_2_BY, + .state[3] = CHACHA_CONSTANT_TE_K, };

/* @@ -824,7 +828,6 @@ static void crng_initialize_secondary(st

static void __init crng_initialize_primary(struct crng_state *crng) { - chacha_init_consts(crng->state); _extract_entropy(&input_pool, &crng->state[4], sizeof(__u32) * 12, 0); if (crng_init_try_arch_early(crng) && trust_cpu && crng_init < 2) { invalidate_batched_entropy(); --- a/include/crypto/chacha20.h +++ b/include/crypto/chacha20.h @@ -25,12 +25,19 @@ int crypto_chacha20_setkey(struct crypto unsigned int keysize); int crypto_chacha20_crypt(struct skcipher_request *req);

+enum chacha_constants { /* expand 32-byte k */ + CHACHA_CONSTANT_EXPA = 0x61707865U, + CHACHA_CONSTANT_ND_3 = 0x3320646eU, + CHACHA_CONSTANT_2_BY = 0x79622d32U, + CHACHA_CONSTANT_TE_K = 0x6b206574U +}; + static inline void chacha_init_consts(u32 *state) { - state[0] = 0x61707865; /* "expa" */ - state[1] = 0x3320646e; /* "nd 3" */ - state[2] = 0x79622d32; /* "2-by" */ - state[3] = 0x6b206574; /* "te k" */ + state[0] = CHACHA_CONSTANT_EXPA; + state[1] = CHACHA_CONSTANT_ND_3; + state[2] = CHACHA_CONSTANT_2_BY; + state[3] = CHACHA_CONSTANT_TE_K; }

#endif

From: Jann Horn jannh@google.com

commit 6c8e11e08a5b74bb8a5cdd5cbc1e5143df0fba72 upstream.

At the moment, urandom_read() (used for /dev/urandom) resets crng_init_cnt to zero when it is called at crng_init<2. This is inconsistent: We do it for /dev/urandom reads, but not for the equivalent getrandom(GRND_INSECURE).

(And worse, as Jason pointed out, we're only doing this as long as maxwarn>0.)

crng_init_cnt is only read in crng_fast_load(); it is relevant at crng_init==0 for determining when to switch to crng_init==1 (and where in the RNG state array to write).

As far as I understand:

- crng_init==0 means "we have nothing, we might just be returning the same exact numbers on every boot on every machine, we don't even have non-cryptographic randomness; we should shove every bit of entropy we can get into the RNG immediately" - crng_init==1 means "well we have something, it might not be cryptographic, but at least we're not gonna return the same data every time or whatever, it's probably good enough for TCP and ASLR and stuff; we now have time to build up actual cryptographic entropy in the input pool" - crng_init==2 means "this is supposed to be cryptographically secure now, but we'll keep adding more entropy just to be sure".

The current code means that if someone is pulling data from /dev/urandom fast enough at crng_init==0, we'll keep resetting crng_init_cnt, and we'll never make forward progress to crng_init==1. It seems to be intended to prevent an attacker from bruteforcing the contents of small individual RNG inputs on the way from crng_init==0 to crng_init==1, but that's misguided; crng_init==1 isn't supposed to provide proper cryptographic security anyway, RNG users who care about getting secure RNG output have to wait until crng_init==2.

This code was inconsistent, and it probably made things worse - just get rid of it.

Signed-off-by: Jann Horn jannh@google.com Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 4 ---- 1 file changed, 4 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1770,7 +1770,6 @@ urandom_read_nowarn(struct file *file, c static ssize_t urandom_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) { - unsigned long flags; static int maxwarn = 10;

if (!crng_ready() && maxwarn > 0) { @@ -1778,9 +1777,6 @@ urandom_read(struct file *file, char __u if (__ratelimit(&urandom_warning)) pr_notice("%s: uninitialized urandom read (%zd bytes read)\n", current->comm, nbytes); - spin_lock_irqsave(&primary_crng.lock, flags); - crng_init_cnt = 0; - spin_unlock_irqrestore(&primary_crng.lock, flags); }

return urandom_read_nowarn(file, buf, nbytes, ppos);

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 91ec0fe138f107232cb36bc6112211db37cb5306 upstream.

Now that we're only using one polynomial, we can cleanup its representation into constants, instead of passing around pointers dynamically to select different polynomials. This improves the codegen and makes the code a bit more straightforward.

Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 67 ++++++++++++++++++++++---------------------------- 1 file changed, 30 insertions(+), 37 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -431,14 +431,20 @@ static int random_write_wakeup_bits = 28 * polynomial which improves the resulting TGFSR polynomial to be * irreducible, which we have made here. */ -static const struct poolinfo { - int poolbitshift, poolwords, poolbytes, poolfracbits; -#define S(x) ilog2(x)+5, (x), (x)*4, (x) << (ENTROPY_SHIFT+5) - int tap1, tap2, tap3, tap4, tap5; -} poolinfo_table[] = { - /* was: x^128 + x^103 + x^76 + x^51 +x^25 + x + 1 */ +enum poolinfo { + POOL_WORDS = 128, + POOL_WORDMASK = POOL_WORDS - 1, + POOL_BYTES = POOL_WORDS * sizeof(u32), + POOL_BITS = POOL_BYTES * 8, + POOL_BITSHIFT = ilog2(POOL_WORDS) + 5, + POOL_FRACBITS = POOL_WORDS << (ENTROPY_SHIFT + 5), + /* x^128 + x^104 + x^76 + x^51 +x^25 + x + 1 */ - { S(128), 104, 76, 51, 25, 1 }, + POOL_TAP1 = 104, + POOL_TAP2 = 76, + POOL_TAP3 = 51, + POOL_TAP4 = 25, + POOL_TAP5 = 1 };

/* @@ -505,7 +511,6 @@ MODULE_PARM_DESC(ratelimit_disable, "Dis struct entropy_store; struct entropy_store { /* read-only data: */ - const struct poolinfo *poolinfo; __u32 *pool; const char *name;

@@ -527,7 +532,6 @@ static void crng_reseed(struct crng_stat static __u32 input_pool_data[INPUT_POOL_WORDS] __latent_entropy;

static struct entropy_store input_pool = { - .poolinfo = &poolinfo_table[0], .name = "input", .lock = __SPIN_LOCK_UNLOCKED(input_pool.lock), .pool = input_pool_data @@ -550,33 +554,26 @@ static __u32 const twist_table[8] = { static void _mix_pool_bytes(struct entropy_store *r, const void *in, int nbytes) { - unsigned long i, tap1, tap2, tap3, tap4, tap5; + unsigned long i; int input_rotate; - int wordmask = r->poolinfo->poolwords - 1; const unsigned char *bytes = in; __u32 w;

- tap1 = r->poolinfo->tap1; - tap2 = r->poolinfo->tap2; - tap3 = r->poolinfo->tap3; - tap4 = r->poolinfo->tap4; - tap5 = r->poolinfo->tap5; - input_rotate = r->input_rotate; i = r->add_ptr;

/* mix one byte at a time to simplify size handling and churn faster */ while (nbytes--) { w = rol32(*bytes++, input_rotate); - i = (i - 1) & wordmask; + i = (i - 1) & POOL_WORDMASK;

/* XOR in the various taps */ w ^= r->pool[i]; - w ^= r->pool[(i + tap1) & wordmask]; - w ^= r->pool[(i + tap2) & wordmask]; - w ^= r->pool[(i + tap3) & wordmask]; - w ^= r->pool[(i + tap4) & wordmask]; - w ^= r->pool[(i + tap5) & wordmask]; + w ^= r->pool[(i + POOL_TAP1) & POOL_WORDMASK]; + w ^= r->pool[(i + POOL_TAP2) & POOL_WORDMASK]; + w ^= r->pool[(i + POOL_TAP3) & POOL_WORDMASK]; + w ^= r->pool[(i + POOL_TAP4) & POOL_WORDMASK]; + w ^= r->pool[(i + POOL_TAP5) & POOL_WORDMASK];

/* Mix the result back in with a twist */ r->pool[i] = (w >> 3) ^ twist_table[w & 7]; @@ -674,7 +671,6 @@ static void process_random_ready_list(vo static void credit_entropy_bits(struct entropy_store *r, int nbits) { int entropy_count, orig; - const int pool_size = r->poolinfo->poolfracbits; int nfrac = nbits << ENTROPY_SHIFT;

if (!nbits) @@ -708,25 +704,25 @@ retry: * turns no matter how large nbits is. */ int pnfrac = nfrac; - const int s = r->poolinfo->poolbitshift + ENTROPY_SHIFT + 2; + const int s = POOL_BITSHIFT + ENTROPY_SHIFT + 2; /* The +2 corresponds to the /4 in the denominator */

do { - unsigned int anfrac = min(pnfrac, pool_size/2); + unsigned int anfrac = min(pnfrac, POOL_FRACBITS/2); unsigned int add = - ((pool_size - entropy_count)*anfrac*3) >> s; + ((POOL_FRACBITS - entropy_count)*anfrac*3) >> s;

entropy_count += add; pnfrac -= anfrac; - } while (unlikely(entropy_count < pool_size-2 && pnfrac)); + } while (unlikely(entropy_count < POOL_FRACBITS-2 && pnfrac)); }

if (WARN_ON(entropy_count < 0)) { pr_warn("negative entropy/overflow: pool %s count %d\n", r->name, entropy_count); entropy_count = 0; - } else if (entropy_count > pool_size) - entropy_count = pool_size; + } else if (entropy_count > POOL_FRACBITS) + entropy_count = POOL_FRACBITS; if (cmpxchg(&r->entropy_count, orig, entropy_count) != orig) goto retry;

@@ -743,13 +739,11 @@ retry:

static int credit_entropy_bits_safe(struct entropy_store *r, int nbits) { - const int nbits_max = r->poolinfo->poolwords * 32; - if (nbits < 0) return -EINVAL;

/* Cap the value to avoid overflows */ - nbits = min(nbits, nbits_max); + nbits = min(nbits, POOL_BITS);

credit_entropy_bits(r, nbits); return 0; @@ -1342,7 +1336,7 @@ static size_t account(struct entropy_sto int entropy_count, orig, have_bytes; size_t ibytes, nfrac;

- BUG_ON(r->entropy_count > r->poolinfo->poolfracbits); + BUG_ON(r->entropy_count > POOL_FRACBITS);

/* Can we pull enough? */ retry: @@ -1408,8 +1402,7 @@ static void extract_buf(struct entropy_s

/* Generate a hash across the pool */ spin_lock_irqsave(&r->lock, flags); - blake2s_update(&state, (const u8 *)r->pool, - r->poolinfo->poolwords * sizeof(*r->pool)); + blake2s_update(&state, (const u8 *)r->pool, POOL_BYTES); blake2s_final(&state, hash); /* final zeros out state */

/* @@ -1705,7 +1698,7 @@ static void __init init_std_data(struct unsigned long rv;

mix_pool_bytes(r, &now, sizeof(now)); - for (i = r->poolinfo->poolbytes; i > 0; i -= sizeof(rv)) { + for (i = POOL_BYTES; i > 0; i -= sizeof(rv)) { if (!arch_get_random_seed_long(&rv) && !arch_get_random_long(&rv)) rv = random_get_entropy();

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit d38bb0853589c939573ea50e9cb64f733e0e273d upstream.

Rather than using the userspace type, __uXX, switch to using uXX. And rather than using variously chosen `char *` or `unsigned char *`, use `u8 *` uniformly for things that aren't strings, in the case where we are doing byte-by-byte traversal.

Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 114 +++++++++++++++++++++++++------------------------- 1 file changed, 57 insertions(+), 57 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -457,7 +457,7 @@ static DEFINE_SPINLOCK(random_ready_list static LIST_HEAD(random_ready_list);

struct crng_state { - __u32 state[16]; + u32 state[16]; unsigned long init_time; spinlock_t lock; }; @@ -484,10 +484,9 @@ static bool crng_need_final_init = false static int crng_init_cnt = 0; static unsigned long crng_global_init_time = 0; #define CRNG_INIT_CNT_THRESH (2*CHACHA20_KEY_SIZE) -static void _extract_crng(struct crng_state *crng, - __u8 out[CHACHA20_BLOCK_SIZE]); +static void _extract_crng(struct crng_state *crng, u8 out[CHACHA20_BLOCK_SIZE]); static void _crng_backtrack_protect(struct crng_state *crng, - __u8 tmp[CHACHA20_BLOCK_SIZE], int used); + u8 tmp[CHACHA20_BLOCK_SIZE], int used); static void process_random_ready_list(void); static void _get_random_bytes(void *buf, int nbytes);

@@ -511,16 +510,16 @@ MODULE_PARM_DESC(ratelimit_disable, "Dis struct entropy_store; struct entropy_store { /* read-only data: */ - __u32 *pool; + u32 *pool; const char *name;

/* read-write data: */ spinlock_t lock; - unsigned short add_ptr; - unsigned short input_rotate; + u16 add_ptr; + u16 input_rotate; int entropy_count; unsigned int last_data_init:1; - __u8 last_data[EXTRACT_SIZE]; + u8 last_data[EXTRACT_SIZE]; };

static ssize_t extract_entropy(struct entropy_store *r, void *buf, @@ -529,7 +528,7 @@ static ssize_t _extract_entropy(struct e size_t nbytes, int fips);

static void crng_reseed(struct crng_state *crng, struct entropy_store *r); -static __u32 input_pool_data[INPUT_POOL_WORDS] __latent_entropy; +static u32 input_pool_data[INPUT_POOL_WORDS] __latent_entropy;

static struct entropy_store input_pool = { .name = "input", @@ -537,7 +536,7 @@ static struct entropy_store input_pool = .pool = input_pool_data };

-static __u32 const twist_table[8] = { +static u32 const twist_table[8] = { 0x00000000, 0x3b6e20c8, 0x76dc4190, 0x4db26158, 0xedb88320, 0xd6d6a3e8, 0x9b64c2b0, 0xa00ae278 };

@@ -556,8 +555,8 @@ static void _mix_pool_bytes(struct entro { unsigned long i; int input_rotate; - const unsigned char *bytes = in; - __u32 w; + const u8 *bytes = in; + u32 w;

input_rotate = r->input_rotate; i = r->add_ptr; @@ -610,10 +609,10 @@ static void mix_pool_bytes(struct entrop }

struct fast_pool { - __u32 pool[4]; + u32 pool[4]; unsigned long last; - unsigned short reg_idx; - unsigned char count; + u16 reg_idx; + u8 count; };

/* @@ -623,8 +622,8 @@ struct fast_pool { */ static void fast_mix(struct fast_pool *f) { - __u32 a = f->pool[0], b = f->pool[1]; - __u32 c = f->pool[2], d = f->pool[3]; + u32 a = f->pool[0], b = f->pool[1]; + u32 c = f->pool[2], d = f->pool[3];

a += b; c += d; b = rol32(b, 6); d = rol32(d, 27); @@ -815,14 +814,14 @@ static bool __init crng_init_try_arch_ea static void crng_initialize_secondary(struct crng_state *crng) { chacha_init_consts(crng->state); - _get_random_bytes(&crng->state[4], sizeof(__u32) * 12); + _get_random_bytes(&crng->state[4], sizeof(u32) * 12); crng_init_try_arch(crng); crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1; }

static void __init crng_initialize_primary(struct crng_state *crng) { - _extract_entropy(&input_pool, &crng->state[4], sizeof(__u32) * 12, 0); + _extract_entropy(&input_pool, &crng->state[4], sizeof(u32) * 12, 0); if (crng_init_try_arch_early(crng) && trust_cpu && crng_init < 2) { invalidate_batched_entropy(); numa_crng_init(); @@ -909,12 +908,14 @@ static struct crng_state *select_crng(vo

/* * crng_fast_load() can be called by code in the interrupt service - * path. So we can't afford to dilly-dally. + * path. So we can't afford to dilly-dally. Returns the number of + * bytes processed from cp. */ -static int crng_fast_load(const char *cp, size_t len) +static size_t crng_fast_load(const u8 *cp, size_t len) { unsigned long flags; - char *p; + u8 *p; + size_t ret = 0;

if (!spin_trylock_irqsave(&primary_crng.lock, flags)) return 0; @@ -922,10 +923,10 @@ static int crng_fast_load(const char *cp spin_unlock_irqrestore(&primary_crng.lock, flags); return 0; } - p = (unsigned char *) &primary_crng.state[4]; + p = (u8 *) &primary_crng.state[4]; while (len > 0 && crng_init_cnt < CRNG_INIT_CNT_THRESH) { p[crng_init_cnt % CHACHA20_KEY_SIZE] ^= *cp; - cp++; crng_init_cnt++; len--; + cp++; crng_init_cnt++; len--; ret++; } spin_unlock_irqrestore(&primary_crng.lock, flags); if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) { @@ -933,7 +934,7 @@ static int crng_fast_load(const char *cp crng_init = 1; pr_notice("fast init done\n"); } - return 1; + return ret; }

/* @@ -950,14 +951,14 @@ static int crng_fast_load(const char *cp * like a fixed DMI table (for example), which might very well be * unique to the machine, but is otherwise unvarying. */ -static int crng_slow_load(const char *cp, size_t len) +static int crng_slow_load(const u8 *cp, size_t len) { unsigned long flags; - static unsigned char lfsr = 1; - unsigned char tmp; - unsigned i, max = CHACHA20_KEY_SIZE; - const char * src_buf = cp; - char * dest_buf = (char *) &primary_crng.state[4]; + static u8 lfsr = 1; + u8 tmp; + unsigned int i, max = CHACHA20_KEY_SIZE; + const u8 * src_buf = cp; + u8 * dest_buf = (u8 *) &primary_crng.state[4];

if (!spin_trylock_irqsave(&primary_crng.lock, flags)) return 0; @@ -986,8 +987,8 @@ static void crng_reseed(struct crng_stat unsigned long flags; int i, num; union { - __u8 block[CHACHA20_BLOCK_SIZE]; - __u32 key[8]; + u8 block[CHACHA20_BLOCK_SIZE]; + u32 key[8]; } buf;

if (r) { @@ -1014,7 +1015,7 @@ static void crng_reseed(struct crng_stat }

static void _extract_crng(struct crng_state *crng, - __u8 out[CHACHA20_BLOCK_SIZE]) + u8 out[CHACHA20_BLOCK_SIZE]) { unsigned long flags, init_time;

@@ -1032,7 +1033,7 @@ static void _extract_crng(struct crng_st spin_unlock_irqrestore(&crng->lock, flags); }

-static void extract_crng(__u8 out[CHACHA20_BLOCK_SIZE]) +static void extract_crng(u8 out[CHACHA20_BLOCK_SIZE]) { _extract_crng(select_crng(), out); } @@ -1042,26 +1043,26 @@ static void extract_crng(__u8 out[CHACHA * enough) to mutate the CRNG key to provide backtracking protection. */ static void _crng_backtrack_protect(struct crng_state *crng, - __u8 tmp[CHACHA20_BLOCK_SIZE], int used) + u8 tmp[CHACHA20_BLOCK_SIZE], int used) { unsigned long flags; - __u32 *s, *d; + u32 *s, *d; int i;

- used = round_up(used, sizeof(__u32)); + used = round_up(used, sizeof(u32)); if (used + CHACHA20_KEY_SIZE > CHACHA20_BLOCK_SIZE) { extract_crng(tmp); used = 0; } spin_lock_irqsave(&crng->lock, flags); - s = (__u32 *) &tmp[used]; + s = (u32 *) &tmp[used]; d = &crng->state[4]; for (i=0; i < 8; i++) *d++ ^= *s++; spin_unlock_irqrestore(&crng->lock, flags); }

-static void crng_backtrack_protect(__u8 tmp[CHACHA20_BLOCK_SIZE], int used) +static void crng_backtrack_protect(u8 tmp[CHACHA20_BLOCK_SIZE], int used) { _crng_backtrack_protect(select_crng(), tmp, used); } @@ -1069,7 +1070,7 @@ static void crng_backtrack_protect(__u8 static ssize_t extract_crng_user(void __user *buf, size_t nbytes) { ssize_t ret = 0, i = CHACHA20_BLOCK_SIZE; - __u8 tmp[CHACHA20_BLOCK_SIZE] __aligned(4); + u8 tmp[CHACHA20_BLOCK_SIZE] __aligned(4); int large_request = (nbytes > 256);

while (nbytes) { @@ -1157,8 +1158,8 @@ static void add_timer_randomness(struct struct entropy_store *r; struct { long jiffies; - unsigned cycles; - unsigned num; + unsigned int cycles; + unsigned int num; } sample; long delta, delta2, delta3;

@@ -1240,15 +1241,15 @@ static void add_interrupt_bench(cycles_t #define add_interrupt_bench(x) #endif

-static __u32 get_reg(struct fast_pool *f, struct pt_regs *regs) +static u32 get_reg(struct fast_pool *f, struct pt_regs *regs) { - __u32 *ptr = (__u32 *) regs; + u32 *ptr = (u32 *) regs; unsigned int idx;

if (regs == NULL) return 0; idx = READ_ONCE(f->reg_idx); - if (idx >= sizeof(struct pt_regs) / sizeof(__u32)) + if (idx >= sizeof(struct pt_regs) / sizeof(u32)) idx = 0; ptr += idx++; WRITE_ONCE(f->reg_idx, idx); @@ -1262,8 +1263,8 @@ void add_interrupt_randomness(int irq) struct pt_regs *regs = get_irq_regs(); unsigned long now = jiffies; cycles_t cycles = random_get_entropy(); - __u32 c_high, j_high; - __u64 ip; + u32 c_high, j_high; + u64 ip;

if (cycles == 0) cycles = get_reg(fast_pool, regs); @@ -1281,8 +1282,7 @@ void add_interrupt_randomness(int irq)

if (unlikely(crng_init == 0)) { if ((fast_pool->count >= 64) && - crng_fast_load((char *) fast_pool->pool, - sizeof(fast_pool->pool))) { + crng_fast_load((u8 *)fast_pool->pool, sizeof(fast_pool->pool)) > 0) { fast_pool->count = 0; fast_pool->last = now; } @@ -1379,7 +1379,7 @@ retry: * * Note: we assume that .poolwords is a multiple of 16 words. */ -static void extract_buf(struct entropy_store *r, __u8 *out) +static void extract_buf(struct entropy_store *r, u8 *out) { struct blake2s_state state __aligned(__alignof__(unsigned long)); u8 hash[BLAKE2S_HASH_SIZE]; @@ -1429,7 +1429,7 @@ static ssize_t _extract_entropy(struct e size_t nbytes, int fips) { ssize_t ret = 0, i; - __u8 tmp[EXTRACT_SIZE]; + u8 tmp[EXTRACT_SIZE]; unsigned long flags;

while (nbytes) { @@ -1467,7 +1467,7 @@ static ssize_t _extract_entropy(struct e static ssize_t extract_entropy(struct entropy_store *r, void *buf, size_t nbytes, int min, int reserved) { - __u8 tmp[EXTRACT_SIZE]; + u8 tmp[EXTRACT_SIZE]; unsigned long flags;

/* if last_data isn't primed, we need EXTRACT_SIZE extra bytes */ @@ -1529,7 +1529,7 @@ static void _warn_unseeded_randomness(co */ static void _get_random_bytes(void *buf, int nbytes) { - __u8 tmp[CHACHA20_BLOCK_SIZE] __aligned(4); + u8 tmp[CHACHA20_BLOCK_SIZE] __aligned(4);

trace_get_random_bytes(nbytes, _RET_IP_);

@@ -1663,7 +1663,7 @@ EXPORT_SYMBOL(del_random_ready_callback) int __must_check get_random_bytes_arch(void *buf, int nbytes) { int left = nbytes; - char *p = buf; + u8 *p = buf;

trace_get_random_bytes_arch(left, _RET_IP_); while (left) { @@ -1805,7 +1805,7 @@ static int write_pool(struct entropy_store *r, const char __user *buffer, size_t count) { size_t bytes; - __u32 t, buf[16]; + u32 t, buf[16]; const char __user *p = buffer;

while (count > 0) { @@ -1815,7 +1815,7 @@ write_pool(struct entropy_store *r, cons if (copy_from_user(&buf, p, bytes)) return -EFAULT;

- for (b = bytes ; b > 0 ; b -= sizeof(__u32), i++) { + for (b = bytes; b > 0; b -= sizeof(u32), i++) { if (!arch_get_random_int(&t)) break; buf[i] ^= t;

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 8b2d953b91e7f60200c24067ab17b77cc7bfd0d4 upstream.

This argument is always set to zero, as a result of us not caring about keeping a certain amount reserved in the pool these days. So just remove it and cleanup the function signatures.

Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -519,7 +519,7 @@ struct entropy_store { };

static ssize_t extract_entropy(struct entropy_store *r, void *buf, - size_t nbytes, int min, int rsvd); + size_t nbytes, int min); static ssize_t _extract_entropy(struct entropy_store *r, void *buf, size_t nbytes);

@@ -988,7 +988,7 @@ static void crng_reseed(struct crng_stat } buf;

if (r) { - num = extract_entropy(r, &buf, 32, 16, 0); + num = extract_entropy(r, &buf, 32, 16); if (num == 0) return; } else { @@ -1326,8 +1326,7 @@ EXPORT_SYMBOL_GPL(add_disk_randomness); * This function decides how many bytes to actually take from the * given pool, and also debits the entropy count accordingly. */ -static size_t account(struct entropy_store *r, size_t nbytes, int min, - int reserved) +static size_t account(struct entropy_store *r, size_t nbytes, int min) { int entropy_count, orig, have_bytes; size_t ibytes, nfrac; @@ -1341,7 +1340,7 @@ retry: /* never pull more than available */ have_bytes = entropy_count >> (ENTROPY_SHIFT + 3);

- if ((have_bytes -= reserved) < 0) + if (have_bytes < 0) have_bytes = 0; ibytes = min_t(size_t, ibytes, have_bytes); if (ibytes < min) @@ -1447,15 +1446,13 @@ static ssize_t _extract_entropy(struct e * returns it in a buffer. * * The min parameter specifies the minimum amount we can pull before - * failing to avoid races that defeat catastrophic reseeding while the - * reserved parameter indicates how much entropy we must leave in the - * pool after each pull to avoid starving other readers. + * failing to avoid races that defeat catastrophic reseeding. */ static ssize_t extract_entropy(struct entropy_store *r, void *buf, - size_t nbytes, int min, int reserved) + size_t nbytes, int min) { trace_extract_entropy(r->name, nbytes, ENTROPY_BITS(r), _RET_IP_); - nbytes = account(r, nbytes, min, reserved); + nbytes = account(r, nbytes, min); return _extract_entropy(r, buf, nbytes); }

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 90ed1e67e896cc8040a523f8428fc02f9b164394 upstream.

Originally, the RNG used several pools, so having things abstracted out over a generic entropy_store object made sense. These days, there's only one input pool, and then an uneven mix of usage via the abstraction and usage via &input_pool. Rather than this uneasy mixture, just get rid of the abstraction entirely and have things always use the global. This simplifies the code and makes reading it a bit easier.

Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 219 ++++++++++++++++++------------------------ include/trace/events/random.h | 56 ++++------ 2 files changed, 117 insertions(+), 158 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -375,7 +375,7 @@ * credit_entropy_bits() needs to be 64 bits wide. */ #define ENTROPY_SHIFT 3 -#define ENTROPY_BITS(r) ((r)->entropy_count >> ENTROPY_SHIFT) +#define ENTROPY_BITS() (input_pool.entropy_count >> ENTROPY_SHIFT)

/* * If the entropy count falls under this number of bits, then we @@ -505,33 +505,27 @@ MODULE_PARM_DESC(ratelimit_disable, "Dis * **********************************************************************/

-struct entropy_store; -struct entropy_store { +static u32 input_pool_data[INPUT_POOL_WORDS] __latent_entropy; + +static struct { /* read-only data: */ u32 *pool; - const char *name;

/* read-write data: */ spinlock_t lock; u16 add_ptr; u16 input_rotate; int entropy_count; -}; - -static ssize_t extract_entropy(struct entropy_store *r, void *buf, - size_t nbytes, int min); -static ssize_t _extract_entropy(struct entropy_store *r, void *buf, - size_t nbytes); - -static void crng_reseed(struct crng_state *crng, struct entropy_store *r); -static u32 input_pool_data[INPUT_POOL_WORDS] __latent_entropy; - -static struct entropy_store input_pool = { - .name = "input", +} input_pool = { .lock = __SPIN_LOCK_UNLOCKED(input_pool.lock), .pool = input_pool_data };

+static ssize_t extract_entropy(void *buf, size_t nbytes, int min); +static ssize_t _extract_entropy(void *buf, size_t nbytes); + +static void crng_reseed(struct crng_state *crng, bool use_input_pool); + static u32 const twist_table[8] = { 0x00000000, 0x3b6e20c8, 0x76dc4190, 0x4db26158, 0xedb88320, 0xd6d6a3e8, 0x9b64c2b0, 0xa00ae278 }; @@ -546,16 +540,15 @@ static u32 const twist_table[8] = { * it's cheap to do so and helps slightly in the expected case where * the entropy is concentrated in the low-order bits. */ -static void _mix_pool_bytes(struct entropy_store *r, const void *in, - int nbytes) +static void _mix_pool_bytes(const void *in, int nbytes) { unsigned long i; int input_rotate; const u8 *bytes = in; u32 w;

- input_rotate = r->input_rotate; - i = r->add_ptr; + input_rotate = input_pool.input_rotate; + i = input_pool.add_ptr;

/* mix one byte at a time to simplify size handling and churn faster */ while (nbytes--) { @@ -563,15 +556,15 @@ static void _mix_pool_bytes(struct entro i = (i - 1) & POOL_WORDMASK;

/* XOR in the various taps */ - w ^= r->pool[i]; - w ^= r->pool[(i + POOL_TAP1) & POOL_WORDMASK]; - w ^= r->pool[(i + POOL_TAP2) & POOL_WORDMASK]; - w ^= r->pool[(i + POOL_TAP3) & POOL_WORDMASK]; - w ^= r->pool[(i + POOL_TAP4) & POOL_WORDMASK]; - w ^= r->pool[(i + POOL_TAP5) & POOL_WORDMASK]; + w ^= input_pool.pool[i]; + w ^= input_pool.pool[(i + POOL_TAP1) & POOL_WORDMASK]; + w ^= input_pool.pool[(i + POOL_TAP2) & POOL_WORDMASK]; + w ^= input_pool.pool[(i + POOL_TAP3) & POOL_WORDMASK]; + w ^= input_pool.pool[(i + POOL_TAP4) & POOL_WORDMASK]; + w ^= input_pool.pool[(i + POOL_TAP5) & POOL_WORDMASK];

/* Mix the result back in with a twist */ - r->pool[i] = (w >> 3) ^ twist_table[w & 7]; + input_pool.pool[i] = (w >> 3) ^ twist_table[w & 7];

/* * Normally, we add 7 bits of rotation to the pool. @@ -582,26 +575,24 @@ static void _mix_pool_bytes(struct entro input_rotate = (input_rotate + (i ? 7 : 14)) & 31; }

- r->input_rotate = input_rotate; - r->add_ptr = i; + input_pool.input_rotate = input_rotate; + input_pool.add_ptr = i; }

-static void __mix_pool_bytes(struct entropy_store *r, const void *in, - int nbytes) +static void __mix_pool_bytes(const void *in, int nbytes) { - trace_mix_pool_bytes_nolock(r->name, nbytes, _RET_IP_); - _mix_pool_bytes(r, in, nbytes); + trace_mix_pool_bytes_nolock(nbytes, _RET_IP_); + _mix_pool_bytes(in, nbytes); }

-static void mix_pool_bytes(struct entropy_store *r, const void *in, - int nbytes) +static void mix_pool_bytes(const void *in, int nbytes) { unsigned long flags;

- trace_mix_pool_bytes(r->name, nbytes, _RET_IP_); - spin_lock_irqsave(&r->lock, flags); - _mix_pool_bytes(r, in, nbytes); - spin_unlock_irqrestore(&r->lock, flags); + trace_mix_pool_bytes(nbytes, _RET_IP_); + spin_lock_irqsave(&input_pool.lock, flags); + _mix_pool_bytes(in, nbytes); + spin_unlock_irqrestore(&input_pool.lock, flags); }

struct fast_pool { @@ -663,16 +654,16 @@ static void process_random_ready_list(vo * Use credit_entropy_bits_safe() if the value comes from userspace * or otherwise should be checked for extreme values. */ -static void credit_entropy_bits(struct entropy_store *r, int nbits) +static void credit_entropy_bits(int nbits) { - int entropy_count, orig; + int entropy_count, entropy_bits, orig; int nfrac = nbits << ENTROPY_SHIFT;

if (!nbits) return;

retry: - entropy_count = orig = ACCESS_ONCE(r->entropy_count); + entropy_count = orig = READ_ONCE(input_pool.entropy_count); if (nfrac < 0) { /* Debit */ entropy_count += nfrac; @@ -713,26 +704,21 @@ retry: }

if (WARN_ON(entropy_count < 0)) { - pr_warn("negative entropy/overflow: pool %s count %d\n", - r->name, entropy_count); + pr_warn("negative entropy/overflow: count %d\n", entropy_count); entropy_count = 0; } else if (entropy_count > POOL_FRACBITS) entropy_count = POOL_FRACBITS; - if (cmpxchg(&r->entropy_count, orig, entropy_count) != orig) + if (cmpxchg(&input_pool.entropy_count, orig, entropy_count) != orig) goto retry;

- trace_credit_entropy_bits(r->name, nbits, - entropy_count >> ENTROPY_SHIFT, _RET_IP_); + trace_credit_entropy_bits(nbits, entropy_count >> ENTROPY_SHIFT, _RET_IP_);

- if (r == &input_pool) { - int entropy_bits = entropy_count >> ENTROPY_SHIFT; - - if (crng_init < 2 && entropy_bits >= 128) - crng_reseed(&primary_crng, r); - } + entropy_bits = entropy_count >> ENTROPY_SHIFT; + if (crng_init < 2 && entropy_bits >= 128) + crng_reseed(&primary_crng, true); }

-static int credit_entropy_bits_safe(struct entropy_store *r, int nbits) +static int credit_entropy_bits_safe(int nbits) { if (nbits < 0) return -EINVAL; @@ -740,7 +726,7 @@ static int credit_entropy_bits_safe(stru /* Cap the value to avoid overflows */ nbits = min(nbits, POOL_BITS);

- credit_entropy_bits(r, nbits); + credit_entropy_bits(nbits); return 0; }

@@ -817,7 +803,7 @@ static void crng_initialize_secondary(st

static void __init crng_initialize_primary(struct crng_state *crng) { - _extract_entropy(&input_pool, &crng->state[4], sizeof(u32) * 12); + _extract_entropy(&crng->state[4], sizeof(u32) * 12); if (crng_init_try_arch_early(crng) && trust_cpu && crng_init < 2) { invalidate_batched_entropy(); numa_crng_init(); @@ -978,7 +964,7 @@ static int crng_slow_load(const u8 *cp, return 1; }

-static void crng_reseed(struct crng_state *crng, struct entropy_store *r) +static void crng_reseed(struct crng_state *crng, bool use_input_pool) { unsigned long flags; int i, num; @@ -987,8 +973,8 @@ static void crng_reseed(struct crng_stat u32 key[8]; } buf;

- if (r) { - num = extract_entropy(r, &buf, 32, 16); + if (use_input_pool) { + num = extract_entropy(&buf, 32, 16); if (num == 0) return; } else { @@ -1019,8 +1005,7 @@ static void _extract_crng(struct crng_st init_time = READ_ONCE(crng->init_time); if (time_after(READ_ONCE(crng_global_init_time), init_time) || time_after(jiffies, init_time + CRNG_RESEED_INTERVAL)) - crng_reseed(crng, crng == &primary_crng ? - &input_pool : NULL); + crng_reseed(crng, crng == &primary_crng); } spin_lock_irqsave(&crng->lock, flags); chacha20_block(&crng->state[0], out); @@ -1131,8 +1116,8 @@ void add_device_randomness(const void *b

trace_add_device_randomness(size, _RET_IP_); spin_lock_irqsave(&input_pool.lock, flags); - _mix_pool_bytes(&input_pool, buf, size); - _mix_pool_bytes(&input_pool, &time, sizeof(time)); + _mix_pool_bytes(buf, size); + _mix_pool_bytes(&time, sizeof(time)); spin_unlock_irqrestore(&input_pool.lock, flags); } EXPORT_SYMBOL(add_device_randomness); @@ -1151,7 +1136,6 @@ static struct timer_rand_state input_tim */ static void add_timer_randomness(struct timer_rand_state *state, unsigned num) { - struct entropy_store *r; struct { long jiffies; unsigned int cycles; @@ -1162,8 +1146,7 @@ static void add_timer_randomness(struct sample.jiffies = jiffies; sample.cycles = random_get_entropy(); sample.num = num; - r = &input_pool; - mix_pool_bytes(r, &sample, sizeof(sample)); + mix_pool_bytes(&sample, sizeof(sample));

/* * Calculate number of bits of randomness we probably added. @@ -1195,7 +1178,7 @@ static void add_timer_randomness(struct * Round down by 1 bit on general principles, * and limit entropy estimate to 12 bits. */ - credit_entropy_bits(r, min_t(int, fls(delta>>1), 11)); + credit_entropy_bits(min_t(int, fls(delta>>1), 11)); }

void add_input_randomness(unsigned int type, unsigned int code, @@ -1210,7 +1193,7 @@ void add_input_randomness(unsigned int t last_value = value; add_timer_randomness(&input_timer_state, (type << 4) ^ code ^ (code >> 4) ^ value); - trace_add_input_randomness(ENTROPY_BITS(&input_pool)); + trace_add_input_randomness(ENTROPY_BITS()); } EXPORT_SYMBOL_GPL(add_input_randomness);

@@ -1254,7 +1237,6 @@ static u32 get_reg(struct fast_pool *f,

void add_interrupt_randomness(int irq) { - struct entropy_store *r; struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness); struct pt_regs *regs = get_irq_regs(); unsigned long now = jiffies; @@ -1289,18 +1271,17 @@ void add_interrupt_randomness(int irq) !time_after(now, fast_pool->last + HZ)) return;

- r = &input_pool; - if (!spin_trylock(&r->lock)) + if (!spin_trylock(&input_pool.lock)) return;

fast_pool->last = now; - __mix_pool_bytes(r, &fast_pool->pool, sizeof(fast_pool->pool)); - spin_unlock(&r->lock); + __mix_pool_bytes(&fast_pool->pool, sizeof(fast_pool->pool)); + spin_unlock(&input_pool.lock);

fast_pool->count = 0;

/* award one bit for the contents of the fast pool */ - credit_entropy_bits(r, 1); + credit_entropy_bits(1); } EXPORT_SYMBOL_GPL(add_interrupt_randomness);

@@ -1311,7 +1292,7 @@ void add_disk_randomness(struct gendisk return; /* first major is 1, so we get >= 0x200 here */ add_timer_randomness(disk->random, 0x100 + disk_devt(disk)); - trace_add_disk_randomness(disk_devt(disk), ENTROPY_BITS(&input_pool)); + trace_add_disk_randomness(disk_devt(disk), ENTROPY_BITS()); } EXPORT_SYMBOL_GPL(add_disk_randomness); #endif @@ -1326,16 +1307,16 @@ EXPORT_SYMBOL_GPL(add_disk_randomness); * This function decides how many bytes to actually take from the * given pool, and also debits the entropy count accordingly. */ -static size_t account(struct entropy_store *r, size_t nbytes, int min) +static size_t account(size_t nbytes, int min) { int entropy_count, orig, have_bytes; size_t ibytes, nfrac;

- BUG_ON(r->entropy_count > POOL_FRACBITS); + BUG_ON(input_pool.entropy_count > POOL_FRACBITS);

/* Can we pull enough? */ retry: - entropy_count = orig = ACCESS_ONCE(r->entropy_count); + entropy_count = orig = READ_ONCE(input_pool.entropy_count); ibytes = nbytes; /* never pull more than available */ have_bytes = entropy_count >> (ENTROPY_SHIFT + 3); @@ -1347,8 +1328,7 @@ retry: ibytes = 0;

if (WARN_ON(entropy_count < 0)) { - pr_warn("negative entropy count: pool %s count %d\n", - r->name, entropy_count); + pr_warn("negative entropy count: count %d\n", entropy_count); entropy_count = 0; } nfrac = ibytes << (ENTROPY_SHIFT + 3); @@ -1357,11 +1337,11 @@ retry: else entropy_count = 0;

- if (cmpxchg(&r->entropy_count, orig, entropy_count) != orig) + if (cmpxchg(&input_pool.entropy_count, orig, entropy_count) != orig) goto retry;

- trace_debit_entropy(r->name, 8 * ibytes); - if (ibytes && ENTROPY_BITS(r) < random_write_wakeup_bits) { + trace_debit_entropy(8 * ibytes); + if (ibytes && ENTROPY_BITS() < random_write_wakeup_bits) { wake_up_interruptible(&random_write_wait); kill_fasync(&fasync, SIGIO, POLL_OUT); } @@ -1374,7 +1354,7 @@ retry: * * Note: we assume that .poolwords is a multiple of 16 words. */ -static void extract_buf(struct entropy_store *r, u8 *out) +static void extract_buf(u8 *out) { struct blake2s_state state __aligned(__alignof__(unsigned long)); u8 hash[BLAKE2S_HASH_SIZE]; @@ -1396,8 +1376,8 @@ static void extract_buf(struct entropy_s }

/* Generate a hash across the pool */ - spin_lock_irqsave(&r->lock, flags); - blake2s_update(&state, (const u8 *)r->pool, POOL_BYTES); + spin_lock_irqsave(&input_pool.lock, flags); + blake2s_update(&state, (const u8 *)input_pool.pool, POOL_BYTES); blake2s_final(&state, hash); /* final zeros out state */

/* @@ -1409,8 +1389,8 @@ static void extract_buf(struct entropy_s * brute-forcing the feedback as hard as brute-forcing the * hash. */ - __mix_pool_bytes(r, hash, sizeof(hash)); - spin_unlock_irqrestore(&r->lock, flags); + __mix_pool_bytes(hash, sizeof(hash)); + spin_unlock_irqrestore(&input_pool.lock, flags);

/* Note that EXTRACT_SIZE is half of hash size here, because above * we've dumped the full length back into mixer. By reducing the @@ -1420,14 +1400,13 @@ static void extract_buf(struct entropy_s memzero_explicit(hash, sizeof(hash)); }

-static ssize_t _extract_entropy(struct entropy_store *r, void *buf, - size_t nbytes) +static ssize_t _extract_entropy(void *buf, size_t nbytes) { ssize_t ret = 0, i; u8 tmp[EXTRACT_SIZE];

while (nbytes) { - extract_buf(r, tmp); + extract_buf(tmp); i = min_t(int, nbytes, EXTRACT_SIZE); memcpy(buf, tmp, i); nbytes -= i; @@ -1448,12 +1427,11 @@ static ssize_t _extract_entropy(struct e * The min parameter specifies the minimum amount we can pull before * failing to avoid races that defeat catastrophic reseeding. */ -static ssize_t extract_entropy(struct entropy_store *r, void *buf, - size_t nbytes, int min) +static ssize_t extract_entropy(void *buf, size_t nbytes, int min) { - trace_extract_entropy(r->name, nbytes, ENTROPY_BITS(r), _RET_IP_); - nbytes = account(r, nbytes, min); - return _extract_entropy(r, buf, nbytes); + trace_extract_entropy(nbytes, ENTROPY_BITS(), _RET_IP_); + nbytes = account(nbytes, min); + return _extract_entropy(buf, nbytes); }

#define warn_unseeded_randomness(previous) \ @@ -1538,7 +1516,7 @@ EXPORT_SYMBOL(get_random_bytes); */ static void entropy_timer(unsigned long data) { - credit_entropy_bits(&input_pool, 1); + credit_entropy_bits(1); }

/* @@ -1562,14 +1540,14 @@ static void try_to_generate_entropy(void while (!crng_ready()) { if (!timer_pending(&stack.timer)) mod_timer(&stack.timer, jiffies+1); - mix_pool_bytes(&input_pool, &stack.now, sizeof(stack.now)); + mix_pool_bytes(&stack.now, sizeof(stack.now)); schedule(); stack.now = random_get_entropy(); }

del_timer_sync(&stack.timer); destroy_timer_on_stack(&stack.timer); - mix_pool_bytes(&input_pool, &stack.now, sizeof(stack.now)); + mix_pool_bytes(&stack.now, sizeof(stack.now)); }

/* @@ -1710,26 +1688,24 @@ EXPORT_SYMBOL(get_random_bytes_arch); /* * init_std_data - initialize pool with system data * - * @r: pool to initialize - * * This function clears the pool's entropy count and mixes some system * data into the pool to prepare it for use. The pool is not cleared * as that can only decrease the entropy in the pool. */ -static void __init init_std_data(struct entropy_store *r) +static void __init init_std_data(void) { int i; ktime_t now = ktime_get_real(); unsigned long rv;

- mix_pool_bytes(r, &now, sizeof(now)); + mix_pool_bytes(&now, sizeof(now)); for (i = POOL_BYTES; i > 0; i -= sizeof(rv)) { if (!arch_get_random_seed_long(&rv) && !arch_get_random_long(&rv)) rv = random_get_entropy(); - mix_pool_bytes(r, &rv, sizeof(rv)); + mix_pool_bytes(&rv, sizeof(rv)); } - mix_pool_bytes(r, utsname(), sizeof(*(utsname()))); + mix_pool_bytes(utsname(), sizeof(*(utsname()))); }

/* @@ -1744,7 +1720,7 @@ static void __init init_std_data(struct */ int __init rand_initialize(void) { - init_std_data(&input_pool); + init_std_data(); if (crng_need_final_init) crng_finalize_init(&primary_crng); crng_initialize_primary(&primary_crng); @@ -1781,7 +1757,7 @@ urandom_read_nowarn(struct file *file, c

nbytes = min_t(size_t, nbytes, INT_MAX >> (ENTROPY_SHIFT + 3)); ret = extract_crng_user(buf, nbytes); - trace_urandom_read(8 * nbytes, 0, ENTROPY_BITS(&input_pool)); + trace_urandom_read(8 * nbytes, 0, ENTROPY_BITS()); return ret; }

@@ -1821,13 +1797,13 @@ random_poll(struct file *file, poll_tabl mask = 0; if (crng_ready()) mask |= POLLIN | POLLRDNORM; - if (ENTROPY_BITS(&input_pool) < random_write_wakeup_bits) + if (ENTROPY_BITS() < random_write_wakeup_bits) mask |= POLLOUT | POLLWRNORM; return mask; }

static int -write_pool(struct entropy_store *r, const char __user *buffer, size_t count) +write_pool(const char __user *buffer, size_t count) { size_t bytes; u32 t, buf[16]; @@ -1849,7 +1825,7 @@ write_pool(struct entropy_store *r, cons count -= bytes; p += bytes;

- mix_pool_bytes(r, buf, bytes); + mix_pool_bytes(buf, bytes); cond_resched(); }

@@ -1861,7 +1837,7 @@ static ssize_t random_write(struct file { size_t ret;

- ret = write_pool(&input_pool, buffer, count); + ret = write_pool(buffer, count); if (ret) return ret;

@@ -1877,7 +1853,7 @@ static long random_ioctl(struct file *f, switch (cmd) { case RNDGETENTCNT: /* inherently racy, no point locking */ - ent_count = ENTROPY_BITS(&input_pool); + ent_count = ENTROPY_BITS(); if (put_user(ent_count, p)) return -EFAULT; return 0; @@ -1886,7 +1862,7 @@ static long random_ioctl(struct file *f, return -EPERM; if (get_user(ent_count, p)) return -EFAULT; - return credit_entropy_bits_safe(&input_pool, ent_count); + return credit_entropy_bits_safe(ent_count); case RNDADDENTROPY: if (!capable(CAP_SYS_ADMIN)) return -EPERM; @@ -1896,11 +1872,10 @@ static long random_ioctl(struct file *f, return -EINVAL; if (get_user(size, p++)) return -EFAULT; - retval = write_pool(&input_pool, (const char __user *)p, - size); + retval = write_pool((const char __user *)p, size); if (retval < 0) return retval; - return credit_entropy_bits_safe(&input_pool, ent_count); + return credit_entropy_bits_safe(ent_count); case RNDZAPENTCNT: case RNDCLEARPOOL: /* @@ -1916,7 +1891,7 @@ static long random_ioctl(struct file *f, return -EPERM; if (crng_init < 2) return -ENODATA; - crng_reseed(&primary_crng, &input_pool); + crng_reseed(&primary_crng, true); WRITE_ONCE(crng_global_init_time, jiffies - 1); return 0; default: @@ -2238,11 +2213,9 @@ randomize_page(unsigned long start, unsi void add_hwgenerator_randomness(const char *buffer, size_t count, size_t entropy) { - struct entropy_store *poolp = &input_pool; - if (unlikely(crng_init == 0)) { size_t ret = crng_fast_load(buffer, count); - mix_pool_bytes(poolp, buffer, ret); + mix_pool_bytes(buffer, ret); count -= ret; buffer += ret; if (!count || crng_init == 0) @@ -2255,9 +2228,9 @@ void add_hwgenerator_randomness(const ch */ wait_event_interruptible(random_write_wait, !system_wq || kthread_should_stop() || - ENTROPY_BITS(&input_pool) <= random_write_wakeup_bits); - mix_pool_bytes(poolp, buffer, count); - credit_entropy_bits(poolp, entropy); + ENTROPY_BITS() <= random_write_wakeup_bits); + mix_pool_bytes(buffer, count); + credit_entropy_bits(entropy); } EXPORT_SYMBOL_GPL(add_hwgenerator_randomness);

--- a/include/trace/events/random.h +++ b/include/trace/events/random.h @@ -28,80 +28,71 @@ TRACE_EVENT(add_device_randomness, );

DECLARE_EVENT_CLASS(random__mix_pool_bytes, - TP_PROTO(const char *pool_name, int bytes, unsigned long IP), + TP_PROTO(int bytes, unsigned long IP),

- TP_ARGS(pool_name, bytes, IP), + TP_ARGS(bytes, IP),

TP_STRUCT__entry( - __field( const char *, pool_name ) __field( int, bytes ) __field(unsigned long, IP ) ),

TP_fast_assign( - __entry->pool_name = pool_name; __entry->bytes = bytes; __entry->IP = IP; ),

- TP_printk("%s pool: bytes %d caller %pS", - __entry->pool_name, __entry->bytes, (void *)__entry->IP) + TP_printk("input pool: bytes %d caller %pS", + __entry->bytes, (void *)__entry->IP) );

DEFINE_EVENT(random__mix_pool_bytes, mix_pool_bytes, - TP_PROTO(const char *pool_name, int bytes, unsigned long IP), + TP_PROTO(int bytes, unsigned long IP),

- TP_ARGS(pool_name, bytes, IP) + TP_ARGS(bytes, IP) );

DEFINE_EVENT(random__mix_pool_bytes, mix_pool_bytes_nolock, - TP_PROTO(const char *pool_name, int bytes, unsigned long IP), + TP_PROTO(int bytes, unsigned long IP),

- TP_ARGS(pool_name, bytes, IP) + TP_ARGS(bytes, IP) );

TRACE_EVENT(credit_entropy_bits, - TP_PROTO(const char *pool_name, int bits, int entropy_count, - unsigned long IP), + TP_PROTO(int bits, int entropy_count, unsigned long IP),

- TP_ARGS(pool_name, bits, entropy_count, IP), + TP_ARGS(bits, entropy_count, IP),

TP_STRUCT__entry( - __field( const char *, pool_name ) __field( int, bits ) __field( int, entropy_count ) __field(unsigned long, IP ) ),

TP_fast_assign( - __entry->pool_name = pool_name; __entry->bits = bits; __entry->entropy_count = entropy_count; __entry->IP = IP; ),

- TP_printk("%s pool: bits %d entropy_count %d caller %pS", - __entry->pool_name, __entry->bits, - __entry->entropy_count, (void *)__entry->IP) + TP_printk("input pool: bits %d entropy_count %d caller %pS", + __entry->bits, __entry->entropy_count, (void *)__entry->IP) );

TRACE_EVENT(debit_entropy, - TP_PROTO(const char *pool_name, int debit_bits), + TP_PROTO(int debit_bits),

- TP_ARGS(pool_name, debit_bits), + TP_ARGS( debit_bits),

TP_STRUCT__entry( - __field( const char *, pool_name ) __field( int, debit_bits ) ),

TP_fast_assign( - __entry->pool_name = pool_name; __entry->debit_bits = debit_bits; ),

- TP_printk("%s: debit_bits %d", __entry->pool_name, - __entry->debit_bits) + TP_printk("input pool: debit_bits %d", __entry->debit_bits) );

TRACE_EVENT(add_input_randomness, @@ -170,36 +161,31 @@ DEFINE_EVENT(random__get_random_bytes, g );

DECLARE_EVENT_CLASS(random__extract_entropy, - TP_PROTO(const char *pool_name, int nbytes, int entropy_count, - unsigned long IP), + TP_PROTO(int nbytes, int entropy_count, unsigned long IP),

- TP_ARGS(pool_name, nbytes, entropy_count, IP), + TP_ARGS(nbytes, entropy_count, IP),

TP_STRUCT__entry( - __field( const char *, pool_name ) __field( int, nbytes ) __field( int, entropy_count ) __field(unsigned long, IP ) ),

TP_fast_assign( - __entry->pool_name = pool_name; __entry->nbytes = nbytes; __entry->entropy_count = entropy_count; __entry->IP = IP; ),

- TP_printk("%s pool: nbytes %d entropy_count %d caller %pS", - __entry->pool_name, __entry->nbytes, __entry->entropy_count, - (void *)__entry->IP) + TP_printk("input pool: nbytes %d entropy_count %d caller %pS", + __entry->nbytes, __entry->entropy_count, (void *)__entry->IP) );

DEFINE_EVENT(random__extract_entropy, extract_entropy, - TP_PROTO(const char *pool_name, int nbytes, int entropy_count, - unsigned long IP), + TP_PROTO(int nbytes, int entropy_count, unsigned long IP),

- TP_ARGS(pool_name, nbytes, entropy_count, IP) + TP_ARGS(nbytes, entropy_count, IP) );

TRACE_EVENT(urandom_read,

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 5b87adf30f1464477169a1d653e9baf8c012bbfe upstream.

We already had the POOL_* constants, so deduplicate the older INPUT_POOL ones. As well, fold EXTRACT_SIZE into the poolinfo enum, since it's related.

Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -359,13 +359,6 @@ /* #define ADD_INTERRUPT_BENCH */

/* - * Configuration information - */ -#define INPUT_POOL_SHIFT 12 -#define INPUT_POOL_WORDS (1 << (INPUT_POOL_SHIFT-5)) -#define EXTRACT_SIZE (BLAKE2S_HASH_SIZE / 2) - -/* * To allow fractional bits to be tracked, the entropy_count field is * denominated in units of 1/8th bits. * @@ -440,7 +433,9 @@ enum poolinfo { POOL_TAP2 = 76, POOL_TAP3 = 51, POOL_TAP4 = 25, - POOL_TAP5 = 1 + POOL_TAP5 = 1, + + EXTRACT_SIZE = BLAKE2S_HASH_SIZE / 2 };

/* @@ -503,7 +498,7 @@ MODULE_PARM_DESC(ratelimit_disable, "Dis * **********************************************************************/

-static u32 input_pool_data[INPUT_POOL_WORDS] __latent_entropy; +static u32 input_pool_data[POOL_WORDS] __latent_entropy;

static struct { /* read-only data: */ @@ -1958,7 +1953,7 @@ SYSCALL_DEFINE3(getrandom, char __user * #include <linux/sysctl.h>

static int min_write_thresh; -static int max_write_thresh = INPUT_POOL_WORDS * 32; +static int max_write_thresh = POOL_BITS; static int random_min_urandom_seed = 60; static char sysctl_bootid[16];

@@ -2015,7 +2010,7 @@ static int proc_do_entropy(struct ctl_ta return proc_dointvec(&fake_table, write, buffer, lenp, ppos); }

-static int sysctl_poolsize = INPUT_POOL_WORDS * 32; +static int sysctl_poolsize = POOL_BITS; extern struct ctl_table random_table[]; struct ctl_table random_table[] = { {

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 18263c4e8e62f7329f38f5eadc568751242ca89c upstream.

The entropy estimator is calculated in terms of 1/8 bits, which means there are various constants where things are shifted by 3. Move these into our pool info enum with the other relevant constants. While we're at it, move an English assertion about sizes into a proper BUILD_BUG_ON so that the compiler can ensure this invariant.

Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -359,16 +359,6 @@ /* #define ADD_INTERRUPT_BENCH */

/* - * To allow fractional bits to be tracked, the entropy_count field is - * denominated in units of 1/8th bits. - * - * 2*(POOL_ENTROPY_SHIFT + poolbitshift) must <= 31, or the multiply in - * credit_entropy_bits() needs to be 64 bits wide. - */ -#define POOL_ENTROPY_SHIFT 3 -#define POOL_ENTROPY_BITS() (input_pool.entropy_count >> POOL_ENTROPY_SHIFT) - -/* * If the entropy count falls under this number of bits, then we * should wake up processes which are selecting or polling on write * access to /dev/random. @@ -425,8 +415,13 @@ enum poolinfo { POOL_WORDMASK = POOL_WORDS - 1, POOL_BYTES = POOL_WORDS * sizeof(u32), POOL_BITS = POOL_BYTES * 8, - POOL_BITSHIFT = ilog2(POOL_WORDS) + 5, - POOL_FRACBITS = POOL_WORDS << (POOL_ENTROPY_SHIFT + 5), + POOL_BITSHIFT = ilog2(POOL_BITS), + + /* To allow fractional bits to be tracked, the entropy_count field is + * denominated in units of 1/8th bits. */ + POOL_ENTROPY_SHIFT = 3, +#define POOL_ENTROPY_BITS() (input_pool.entropy_count >> POOL_ENTROPY_SHIFT) + POOL_FRACBITS = POOL_BITS << POOL_ENTROPY_SHIFT,

/* x^128 + x^104 + x^76 + x^51 +x^25 + x + 1 */ POOL_TAP1 = 104, @@ -652,6 +647,9 @@ static void credit_entropy_bits(int nbit int entropy_count, entropy_bits, orig; int nfrac = nbits << POOL_ENTROPY_SHIFT;

+ /* Ensure that the multiplication can avoid being 64 bits wide. */ + BUILD_BUG_ON(2 * (POOL_ENTROPY_SHIFT + POOL_BITSHIFT) > 31); + if (!nbits) return;

@@ -687,13 +685,13 @@ retry: /* The +2 corresponds to the /4 in the denominator */

do { - unsigned int anfrac = min(pnfrac, POOL_FRACBITS/2); + unsigned int anfrac = min(pnfrac, POOL_FRACBITS / 2); unsigned int add = - ((POOL_FRACBITS - entropy_count)*anfrac*3) >> s; + ((POOL_FRACBITS - entropy_count) * anfrac * 3) >> s;

entropy_count += add; pnfrac -= anfrac; - } while (unlikely(entropy_count < POOL_FRACBITS-2 && pnfrac)); + } while (unlikely(entropy_count < POOL_FRACBITS - 2 && pnfrac)); }

if (WARN_ON(entropy_count < 0)) {

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit a254a0e4093fce8c832414a83940736067eed515 upstream.

Now that have_bytes is never modified, we can simplify this function. First, we move the check for negative entropy_count to be first. That ensures that subsequent reads of this will be non-negative. Then, have_bytes and ibytes can be folded into their one use site in the min_t() function.

Suggested-by: Dominik Brodowski linux@dominikbrodowski.net Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1292,7 +1292,7 @@ EXPORT_SYMBOL_GPL(add_disk_randomness); */ static size_t account(size_t nbytes, int min) { - int entropy_count, orig, have_bytes; + int entropy_count, orig; size_t ibytes, nfrac;

BUG_ON(input_pool.entropy_count > POOL_FRACBITS); @@ -1300,20 +1300,15 @@ static size_t account(size_t nbytes, int /* Can we pull enough? */ retry: entropy_count = orig = READ_ONCE(input_pool.entropy_count); - ibytes = nbytes; - /* never pull more than available */ - have_bytes = entropy_count >> (POOL_ENTROPY_SHIFT + 3); - - if (have_bytes < 0) - have_bytes = 0; - ibytes = min_t(size_t, ibytes, have_bytes); - if (ibytes < min) - ibytes = 0; - if (WARN_ON(entropy_count < 0)) { pr_warn("negative entropy count: count %d\n", entropy_count); entropy_count = 0; } + + /* never pull more than available */ + ibytes = min_t(size_t, nbytes, entropy_count >> (POOL_ENTROPY_SHIFT + 3)); + if (ibytes < min) + ibytes = 0; nfrac = ibytes << (POOL_ENTROPY_SHIFT + 3); if ((size_t)entropy_count > nfrac) entropy_count -= nfrac;

From: Dominik Brodowski linux@dominikbrodowski.net

commit ebf7606388732ecf2821ca21087e9446cb4a5b57 upstream.

Both crng_initialize_primary() and crng_init_try_arch_early() are only called for the primary_pool. Accessing it directly instead of through a function parameter simplifies the code.

Signed-off-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -761,7 +761,7 @@ static bool crng_init_try_arch(struct cr return arch_init; }

-static bool __init crng_init_try_arch_early(struct crng_state *crng) +static bool __init crng_init_try_arch_early(void) { int i; bool arch_init = true; @@ -773,7 +773,7 @@ static bool __init crng_init_try_arch_ea rv = random_get_entropy(); arch_init = false; } - crng->state[i] ^= rv; + primary_crng.state[i] ^= rv; }

return arch_init; @@ -787,16 +787,16 @@ static void crng_initialize_secondary(st crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1; }

-static void __init crng_initialize_primary(struct crng_state *crng) +static void __init crng_initialize_primary(void) { - _extract_entropy(&crng->state[4], sizeof(u32) * 12); - if (crng_init_try_arch_early(crng) && trust_cpu && crng_init < 2) { + _extract_entropy(&primary_crng.state[4], sizeof(u32) * 12); + if (crng_init_try_arch_early() && trust_cpu && crng_init < 2) { invalidate_batched_entropy(); numa_crng_init(); crng_init = 2; pr_notice("crng init done (trusting CPU's manufacturer)\n"); } - crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1; + primary_crng.init_time = jiffies - CRNG_RESEED_INTERVAL - 1; }

static void crng_finalize_init(struct crng_state *crng) @@ -1697,7 +1697,7 @@ int __init rand_initialize(void) init_std_data(); if (crng_need_final_init) crng_finalize_init(&primary_crng); - crng_initialize_primary(&primary_crng); + crng_initialize_primary(); crng_global_init_time = jiffies; if (ratelimit_disable) { urandom_warning.interval = 0;

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 6e8ec2552c7d13991148e551e3325a624d73fac6 upstream.

The current 4096-bit LFSR used for entropy collection had a few desirable attributes for the context in which it was created. For example, the state was huge, which meant that /dev/random would be able to output quite a bit of accumulated entropy before blocking. It was also, in its time, quite fast at accumulating entropy byte-by-byte, which matters given the varying contexts in which mix_pool_bytes() is called. And its diffusion was relatively high, which meant that changes would ripple across several words of state rather quickly.

However, it also suffers from a few security vulnerabilities. In particular, inputs learned by an attacker can be undone, but moreover, if the state of the pool leaks, its contents can be controlled and entirely zeroed out. I've demonstrated this attack with this SMT2 script, https://xn--4db.cc/5o9xO8pb, which Boolector/CaDiCal solves in a matter of seconds on a single core of my laptop, resulting in little proof of concept C demonstrators such as https://xn--4db.cc/jCkvvIaH/c.

For basically all recent formal models of RNGs, these attacks represent a significant cryptographic flaw. But how does this manifest practically? If an attacker has access to the system to such a degree that he can learn the internal state of the RNG, arguably there are other lower hanging vulnerabilities -- side-channel, infoleak, or otherwise -- that might have higher priority. On the other hand, seed files are frequently used on systems that have a hard time generating much entropy on their own, and these seed files, being files, often leak or are duplicated and distributed accidentally, or are even seeded over the Internet intentionally, where their contents might be recorded or tampered with. Seen this way, an otherwise quasi-implausible vulnerability is a bit more practical than initially thought.

Another aspect of the current mix_pool_bytes() function is that, while its performance was arguably competitive for the time in which it was created, it's no longer considered so. This patch improves performance significantly: on a high-end CPU, an i7-11850H, it improves performance of mix_pool_bytes() by 225%, and on a low-end CPU, a Cortex-A7, it improves performance by 103%.

This commit replaces the LFSR of mix_pool_bytes() with a straight- forward cryptographic hash function, BLAKE2s, which is already in use for pool extraction. Universal hashing with a secret seed was considered too, something along the lines of https://eprint.iacr.org/2013/338, but the requirement for a secret seed makes for a chicken & egg problem. Instead we go with a formally proven scheme using a computational hash function, described in sections 5.1, 6.4, and B.1.8 of https://eprint.iacr.org/2019/198.

BLAKE2s outputs 256 bits, which should give us an appropriate amount of min-entropy accumulation, and a wide enough margin of collision resistance against active attacks. mix_pool_bytes() becomes a simple call to blake2s_update(), for accumulation, while the extraction step becomes a blake2s_final() to generate a seed, with which we can then do a HKDF-like or BLAKE2X-like expansion, the first part of which we fold back as an init key for subsequent blake2s_update()s, and the rest we produce to the caller. This then is provided to our CRNG like usual. In that expansion step, we make opportunistic use of 32 bytes of RDRAND output, just as before. We also always reseed the crng with 32 bytes, unconditionally, or not at all, rather than sometimes with 16 as before, as we don't win anything by limiting beyond the 16 byte threshold.

Going for a hash function as an entropy collector is a conservative, proven approach. The result of all this is a much simpler and much less bespoke construction than what's there now, which not only plugs a vulnerability but also improves performance considerably.

Cc: Theodore Ts'o tytso@mit.edu Cc: Dominik Brodowski linux@dominikbrodowski.net Reviewed-by: Eric Biggers ebiggers@google.com Reviewed-by: Greg Kroah-Hartman gregkh@linuxfoundation.org Reviewed-by: Jean-Philippe Aumasson jeanphilippe.aumasson@gmail.com Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 304 +++++++++----------------------------------------- 1 file changed, 55 insertions(+), 249 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -42,61 +42,6 @@ */

/* - * (now, with legal B.S. out of the way.....) - * - * This routine gathers environmental noise from device drivers, etc., - * and returns good random numbers, suitable for cryptographic use. - * Besides the obvious cryptographic uses, these numbers are also good - * for seeding TCP sequence numbers, and other places where it is - * desirable to have numbers which are not only random, but hard to - * predict by an attacker. - * - * Theory of operation - * =================== - * - * Computers are very predictable devices. Hence it is extremely hard - * to produce truly random numbers on a computer --- as opposed to - * pseudo-random numbers, which can easily generated by using a - * algorithm. Unfortunately, it is very easy for attackers to guess - * the sequence of pseudo-random number generators, and for some - * applications this is not acceptable. So instead, we must try to - * gather "environmental noise" from the computer's environment, which - * must be hard for outside attackers to observe, and use that to - * generate random numbers. In a Unix environment, this is best done - * from inside the kernel. - * - * Sources of randomness from the environment include inter-keyboard - * timings, inter-interrupt timings from some interrupts, and other - * events which are both (a) non-deterministic and (b) hard for an - * outside observer to measure. Randomness from these sources are - * added to an "entropy pool", which is mixed using a CRC-like function. - * This is not cryptographically strong, but it is adequate assuming - * the randomness is not chosen maliciously, and it is fast enough that - * the overhead of doing it on every interrupt is very reasonable. - * As random bytes are mixed into the entropy pool, the routines keep - * an *estimate* of how many bits of randomness have been stored into - * the random number generator's internal state. - * - * When random bytes are desired, they are obtained by taking the BLAKE2s - * hash of the contents of the "entropy pool". The BLAKE2s hash avoids - * exposing the internal state of the entropy pool. It is believed to - * be computationally infeasible to derive any useful information - * about the input of BLAKE2s from its output. Even if it is possible to - * analyze BLAKE2s in some clever way, as long as the amount of data - * returned from the generator is less than the inherent entropy in - * the pool, the output data is totally unpredictable. For this - * reason, the routine decreases its internal estimate of how many - * bits of "true randomness" are contained in the entropy pool as it - * outputs random numbers. - * - * If this estimate goes to zero, the routine can still generate - * random numbers; however, an attacker may (at least in theory) be - * able to infer the future output of the generator from prior - * outputs. This requires successful cryptanalysis of BLAKE2s, which is - * not believed to be feasible, but there is a remote possibility. - * Nonetheless, these numbers should be useful for the vast majority - * of purposes. - * * Exported interfaces ---- output * =============================== * @@ -298,23 +243,6 @@ * * mknod /dev/random c 1 8 * mknod /dev/urandom c 1 9 - * - * Acknowledgements: - * ================= - * - * Ideas for constructing this random number generator were derived - * from Pretty Good Privacy's random number generator, and from private - * discussions with Phil Karn. Colin Plumb provided a faster random - * number generator, which speed up the mixing function of the entropy - * pool, taken from PGPfone. Dale Worley has also contributed many - * useful ideas and suggestions to improve this driver. - * - * Any flaws in the design are solely my responsibility, and should - * not be attributed to the Phil, Colin, or any of authors of PGP. - * - * Further background information on this topic may be obtained from - * RFC 1750, "Randomness Recommendations for Security", by Donald - * Eastlake, Steve Crocker, and Jeff Schiller. */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -358,79 +286,15 @@

/* #define ADD_INTERRUPT_BENCH */

-/* - * If the entropy count falls under this number of bits, then we - * should wake up processes which are selecting or polling on write - * access to /dev/random. - */ -static int random_write_wakeup_bits = 28 * (1 << 5); - -/* - * Originally, we used a primitive polynomial of degree .poolwords - * over GF(2). The taps for various sizes are defined below. They - * were chosen to be evenly spaced except for the last tap, which is 1 - * to get the twisting happening as fast as possible. - * - * For the purposes of better mixing, we use the CRC-32 polynomial as - * well to make a (modified) twisted Generalized Feedback Shift - * Register. (See M. Matsumoto & Y. Kurita, 1992. Twisted GFSR - * generators. ACM Transactions on Modeling and Computer Simulation - * 2(3):179-194. Also see M. Matsumoto & Y. Kurita, 1994. Twisted - * GFSR generators II. ACM Transactions on Modeling and Computer - * Simulation 4:254-266) - * - * Thanks to Colin Plumb for suggesting this. - * - * The mixing operation is much less sensitive than the output hash, - * where we use BLAKE2s. All that we want of mixing operation is that - * it be a good non-cryptographic hash; i.e. it not produce collisions - * when fed "random" data of the sort we expect to see. As long as - * the pool state differs for different inputs, we have preserved the - * input entropy and done a good job. The fact that an intelligent - * attacker can construct inputs that will produce controlled - * alterations to the pool's state is not important because we don't - * consider such inputs to contribute any randomness. The only - * property we need with respect to them is that the attacker can't - * increase his/her knowledge of the pool's state. Since all - * additions are reversible (knowing the final state and the input, - * you can reconstruct the initial state), if an attacker has any - * uncertainty about the initial state, he/she can only shuffle that - * uncertainty about, but never cause any collisions (which would - * decrease the uncertainty). - * - * Our mixing functions were analyzed by Lacharme, Roeck, Strubel, and - * Videau in their paper, "The Linux Pseudorandom Number Generator - * Revisited" (see: http://eprint.iacr.org/2012/251.pdf). In their - * paper, they point out that we are not using a true Twisted GFSR, - * since Matsumoto & Kurita used a trinomial feedback polynomial (that - * is, with only three taps, instead of the six that we are using). - * As a result, the resulting polynomial is neither primitive nor - * irreducible, and hence does not have a maximal period over - * GF(2**32). They suggest a slight change to the generator - * polynomial which improves the resulting TGFSR polynomial to be - * irreducible, which we have made here. - */ enum poolinfo { - POOL_WORDS = 128, - POOL_WORDMASK = POOL_WORDS - 1, - POOL_BYTES = POOL_WORDS * sizeof(u32), - POOL_BITS = POOL_BYTES * 8, + POOL_BITS = BLAKE2S_HASH_SIZE * 8, POOL_BITSHIFT = ilog2(POOL_BITS),

/* To allow fractional bits to be tracked, the entropy_count field is * denominated in units of 1/8th bits. */ POOL_ENTROPY_SHIFT = 3, #define POOL_ENTROPY_BITS() (input_pool.entropy_count >> POOL_ENTROPY_SHIFT) - POOL_FRACBITS = POOL_BITS << POOL_ENTROPY_SHIFT, - - /* x^128 + x^104 + x^76 + x^51 +x^25 + x + 1 */ - POOL_TAP1 = 104, - POOL_TAP2 = 76, - POOL_TAP3 = 51, - POOL_TAP4 = 25, - POOL_TAP5 = 1, - - EXTRACT_SIZE = BLAKE2S_HASH_SIZE / 2 + POOL_FRACBITS = POOL_BITS << POOL_ENTROPY_SHIFT };

/* @@ -438,6 +302,12 @@ enum poolinfo { */ static DECLARE_WAIT_QUEUE_HEAD(random_write_wait); static struct fasync_struct *fasync; +/* + * If the entropy count falls under this number of bits, then we + * should wake up processes which are selecting or polling on write + * access to /dev/random. + */ +static int random_write_wakeup_bits = POOL_BITS * 3 / 4;

static DEFINE_SPINLOCK(random_ready_list_lock); static LIST_HEAD(random_ready_list); @@ -493,73 +363,31 @@ MODULE_PARM_DESC(ratelimit_disable, "Dis * **********************************************************************/

-static u32 input_pool_data[POOL_WORDS] __latent_entropy; - static struct { + struct blake2s_state hash; spinlock_t lock; - u16 add_ptr; - u16 input_rotate; int entropy_count; } input_pool = { + .hash.h = { BLAKE2S_IV0 ^ (0x01010000 | BLAKE2S_HASH_SIZE), + BLAKE2S_IV1, BLAKE2S_IV2, BLAKE2S_IV3, BLAKE2S_IV4, + BLAKE2S_IV5, BLAKE2S_IV6, BLAKE2S_IV7 }, + .hash.outlen = BLAKE2S_HASH_SIZE, .lock = __SPIN_LOCK_UNLOCKED(input_pool.lock), };

-static ssize_t extract_entropy(void *buf, size_t nbytes, int min); -static ssize_t _extract_entropy(void *buf, size_t nbytes); +static bool extract_entropy(void *buf, size_t nbytes, int min); +static void _extract_entropy(void *buf, size_t nbytes);

static void crng_reseed(struct crng_state *crng, bool use_input_pool);

-static const u32 twist_table[8] = { - 0x00000000, 0x3b6e20c8, 0x76dc4190, 0x4db26158, - 0xedb88320, 0xd6d6a3e8, 0x9b64c2b0, 0xa00ae278 }; - /* * This function adds bytes into the entropy "pool". It does not * update the entropy estimate. The caller should call * credit_entropy_bits if this is appropriate. - * - * The pool is stirred with a primitive polynomial of the appropriate - * degree, and then twisted. We twist by three bits at a time because - * it's cheap to do so and helps slightly in the expected case where - * the entropy is concentrated in the low-order bits. */ static void _mix_pool_bytes(const void *in, int nbytes) { - unsigned long i; - int input_rotate; - const u8 *bytes = in; - u32 w; - - input_rotate = input_pool.input_rotate; - i = input_pool.add_ptr; - - /* mix one byte at a time to simplify size handling and churn faster */ - while (nbytes--) { - w = rol32(*bytes++, input_rotate); - i = (i - 1) & POOL_WORDMASK; - - /* XOR in the various taps */ - w ^= input_pool_data[i]; - w ^= input_pool_data[(i + POOL_TAP1) & POOL_WORDMASK]; - w ^= input_pool_data[(i + POOL_TAP2) & POOL_WORDMASK]; - w ^= input_pool_data[(i + POOL_TAP3) & POOL_WORDMASK]; - w ^= input_pool_data[(i + POOL_TAP4) & POOL_WORDMASK]; - w ^= input_pool_data[(i + POOL_TAP5) & POOL_WORDMASK]; - - /* Mix the result back in with a twist */ - input_pool_data[i] = (w >> 3) ^ twist_table[w & 7]; - - /* - * Normally, we add 7 bits of rotation to the pool. - * At the beginning of the pool, add an extra 7 bits - * rotation, so that successive passes spread the - * input bits across the pool evenly. - */ - input_rotate = (input_rotate + (i ? 7 : 14)) & 31; - } - - input_pool.input_rotate = input_rotate; - input_pool.add_ptr = i; + blake2s_update(&input_pool.hash, in, nbytes); }

static void __mix_pool_bytes(const void *in, int nbytes) @@ -952,15 +780,14 @@ static int crng_slow_load(const u8 *cp, static void crng_reseed(struct crng_state *crng, bool use_input_pool) { unsigned long flags; - int i, num; + int i; union { u8 block[CHACHA20_BLOCK_SIZE]; u32 key[8]; } buf;

if (use_input_pool) { - num = extract_entropy(&buf, 32, 16); - if (num == 0) + if (!extract_entropy(&buf, 32, 16)) return; } else { _extract_crng(&primary_crng, buf.block); @@ -1328,74 +1155,48 @@ retry: }

/* - * This function does the actual extraction for extract_entropy. - * - * Note: we assume that .poolwords is a multiple of 16 words. + * This is an HKDF-like construction for using the hashed collected entropy + * as a PRF key, that's then expanded block-by-block. */ -static void extract_buf(u8 *out) +static void _extract_entropy(void *buf, size_t nbytes) { - struct blake2s_state state __aligned(__alignof__(unsigned long)); - u8 hash[BLAKE2S_HASH_SIZE]; - unsigned long *salt; unsigned long flags; - - blake2s_init(&state, sizeof(hash)); - - /* - * If we have an architectural hardware random number - * generator, use it for BLAKE2's salt & personal fields. - */ - for (salt = (unsigned long *)&state.h[4]; - salt < (unsigned long *)&state.h[8]; ++salt) { - unsigned long v; - if (!arch_get_random_long(&v)) - break; - *salt ^= v; + u8 seed[BLAKE2S_HASH_SIZE], next_key[BLAKE2S_HASH_SIZE]; + struct { + unsigned long rdrand[32 / sizeof(long)]; + size_t counter; + } block; + size_t i; + + for (i = 0; i < ARRAY_SIZE(block.rdrand); ++i) { + if (!arch_get_random_long(&block.rdrand[i])) + block.rdrand[i] = random_get_entropy(); }

- /* Generate a hash across the pool */ spin_lock_irqsave(&input_pool.lock, flags); - blake2s_update(&state, (const u8 *)input_pool_data, POOL_BYTES); - blake2s_final(&state, hash); /* final zeros out state */

- /* - * We mix the hash back into the pool to prevent backtracking - * attacks (where the attacker knows the state of the pool - * plus the current outputs, and attempts to find previous - * outputs), unless the hash function can be inverted. By - * mixing at least a hash worth of hash data back, we make - * brute-forcing the feedback as hard as brute-forcing the - * hash. - */ - __mix_pool_bytes(hash, sizeof(hash)); - spin_unlock_irqrestore(&input_pool.lock, flags); + /* seed = HASHPRF(last_key, entropy_input) */ + blake2s_final(&input_pool.hash, seed);

- /* Note that EXTRACT_SIZE is half of hash size here, because above - * we've dumped the full length back into mixer. By reducing the - * amount that we emit, we retain a level of forward secrecy. - */ - memcpy(out, hash, EXTRACT_SIZE); - memzero_explicit(hash, sizeof(hash)); -} + /* next_key = HASHPRF(seed, RDRAND || 0) */ + block.counter = 0; + blake2s(next_key, (u8 *)&block, seed, sizeof(next_key), sizeof(block), sizeof(seed)); + blake2s_init_key(&input_pool.hash, BLAKE2S_HASH_SIZE, next_key, sizeof(next_key));

-static ssize_t _extract_entropy(void *buf, size_t nbytes) -{ - ssize_t ret = 0, i; - u8 tmp[EXTRACT_SIZE]; + spin_unlock_irqrestore(&input_pool.lock, flags); + memzero_explicit(next_key, sizeof(next_key));

while (nbytes) { - extract_buf(tmp); - i = min_t(int, nbytes, EXTRACT_SIZE); - memcpy(buf, tmp, i); + i = min_t(size_t, nbytes, BLAKE2S_HASH_SIZE); + /* output = HASHPRF(seed, RDRAND || ++counter) */ + ++block.counter; + blake2s(buf, (u8 *)&block, seed, i, sizeof(block), sizeof(seed)); nbytes -= i; buf += i; - ret += i; }

- /* Wipe data just returned from memory */ - memzero_explicit(tmp, sizeof(tmp)); - - return ret; + memzero_explicit(seed, sizeof(seed)); + memzero_explicit(&block, sizeof(block)); }

/* @@ -1403,13 +1204,18 @@ static ssize_t _extract_entropy(void *bu * returns it in a buffer. * * The min parameter specifies the minimum amount we can pull before - * failing to avoid races that defeat catastrophic reseeding. + * failing to avoid races that defeat catastrophic reseeding. If we + * have less than min entropy available, we return false and buf is + * not filled. */ -static ssize_t extract_entropy(void *buf, size_t nbytes, int min) +static bool extract_entropy(void *buf, size_t nbytes, int min) { trace_extract_entropy(nbytes, POOL_ENTROPY_BITS(), _RET_IP_); - nbytes = account(nbytes, min); - return _extract_entropy(buf, nbytes); + if (account(nbytes, min)) { + _extract_entropy(buf, nbytes); + return true; + } + return false; }

#define warn_unseeded_randomness(previous) \ @@ -1673,7 +1479,7 @@ static void __init init_std_data(void) unsigned long rv;

mix_pool_bytes(&now, sizeof(now)); - for (i = POOL_BYTES; i > 0; i -= sizeof(rv)) { + for (i = BLAKE2S_BLOCK_SIZE; i > 0; i -= sizeof(rv)) { if (!arch_get_random_seed_long(&rv) && !arch_get_random_long(&rv)) rv = random_get_entropy();

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit c570449094844527577c5c914140222cb1893e3f upstream.

30e37ec516ae ("random: account for entropy loss due to overwrites") assumed that adding new entropy to the LFSR pool probabilistically cancelled out old entropy there, so entropy was credited asymptotically, approximating Shannon entropy of independent sources (rather than a stronger min-entropy notion) using 1/8th fractional bits and replacing a constant 2-2/√𝑒 term (~0.786938) with 3/4 (0.75) to slightly underestimate it. This wasn't superb, but it was perhaps better than nothing, so that's what was done. Which entropy specifically was being cancelled out and how much precisely each time is hard to tell, though as I showed with the attack code in my previous commit, a motivated adversary with sufficient information can actually cancel out everything.

Since we're no longer using an LFSR for entropy accumulation, this probabilistic cancellation is no longer relevant. Rather, we're now using a computational hash function as the accumulator and we've switched to working in the random oracle model, from which we can now revisit the question of min-entropy accumulation, which is done in detail in https://eprint.iacr.org/2019/198.

Consider a long input bit string that is built by concatenating various smaller independent input bit strings. Each one of these inputs has a designated min-entropy, which is what we're passing to credit_entropy_bits(h). When we pass the concatenation of these to a random oracle, it means that an adversary trying to receive back the same reply as us would need to become certain about each part of the concatenated bit string we passed in, which means becoming certain about all of those h values. That means we can estimate the accumulation by simply adding up the h values in calls to credit_entropy_bits(h); there's no probabilistic cancellation at play like there was said to be for the LFSR. Incidentally, this is also what other entropy accumulators based on computational hash functions do as well.

So this commit replaces credit_entropy_bits(h) with essentially `total = min(POOL_BITS, total + h)`, done with a cmpxchg loop as before.

What if we're wrong and the above is nonsense? It's not, but let's assume we don't want the actual _behavior_ of the code to change much. Currently that behavior is not extracting from the input pool until it has 128 bits of entropy in it. With the old algorithm, we'd hit that magic 128 number after roughly 256 calls to credit_entropy_bits(1). So, we can retain more or less the old behavior by waiting to extract from the input pool until it hits 256 bits of entropy using the new code. For people concerned about this change, it means that there's not that much practical behavioral change. And for folks actually trying to model the behavior rigorously, it means that we have an even higher margin against attacks.

Cc: Theodore Ts'o tytso@mit.edu Cc: Dominik Brodowski linux@dominikbrodowski.net Cc: Greg Kroah-Hartman gregkh@linuxfoundation.org Reviewed-by: Eric Biggers ebiggers@google.com Reviewed-by: Jean-Philippe Aumasson jeanphilippe.aumasson@gmail.com Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 114 ++++++++------------------------------------------ 1 file changed, 20 insertions(+), 94 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -286,17 +286,9 @@

/* #define ADD_INTERRUPT_BENCH */

-enum poolinfo { +enum { POOL_BITS = BLAKE2S_HASH_SIZE * 8, - POOL_BITSHIFT = ilog2(POOL_BITS), - POOL_MIN_BITS = POOL_BITS / 2, - - /* To allow fractional bits to be tracked, the entropy_count field is - * denominated in units of 1/8th bits. */ - POOL_ENTROPY_SHIFT = 3, -#define POOL_ENTROPY_BITS() (input_pool.entropy_count >> POOL_ENTROPY_SHIFT) - POOL_FRACBITS = POOL_BITS << POOL_ENTROPY_SHIFT, - POOL_MIN_FRACBITS = POOL_MIN_BITS << POOL_ENTROPY_SHIFT + POOL_MIN_BITS = POOL_BITS /* No point in settling for less. */ };

/* @@ -309,7 +301,7 @@ static struct fasync_struct *fasync; * should wake up processes which are selecting or polling on write * access to /dev/random. */ -static int random_write_wakeup_bits = POOL_BITS * 3 / 4; +static int random_write_wakeup_bits = POOL_MIN_BITS;

static DEFINE_SPINLOCK(random_ready_list_lock); static LIST_HEAD(random_ready_list); @@ -469,66 +461,18 @@ static void process_random_ready_list(vo static void credit_entropy_bits(int nbits) { int entropy_count, orig; - int nfrac = nbits << POOL_ENTROPY_SHIFT; - - /* Ensure that the multiplication can avoid being 64 bits wide. */ - BUILD_BUG_ON(2 * (POOL_ENTROPY_SHIFT + POOL_BITSHIFT) > 31);

if (!nbits) return;

-retry: - entropy_count = orig = READ_ONCE(input_pool.entropy_count); - if (nfrac < 0) { - /* Debit */ - entropy_count += nfrac; - } else { - /* - * Credit: we have to account for the possibility of - * overwriting already present entropy. Even in the - * ideal case of pure Shannon entropy, new contributions - * approach the full value asymptotically: - * - * entropy <- entropy + (pool_size - entropy) * - * (1 - exp(-add_entropy/pool_size)) - * - * For add_entropy <= pool_size/2 then - * (1 - exp(-add_entropy/pool_size)) >= - * (add_entropy/pool_size)*0.7869... - * so we can approximate the exponential with - * 3/4*add_entropy/pool_size and still be on the - * safe side by adding at most pool_size/2 at a time. - * - * The use of pool_size-2 in the while statement is to - * prevent rounding artifacts from making the loop - * arbitrarily long; this limits the loop to log2(pool_size)*2 - * turns no matter how large nbits is. - */ - int pnfrac = nfrac; - const int s = POOL_BITSHIFT + POOL_ENTROPY_SHIFT + 2; - /* The +2 corresponds to the /4 in the denominator */ - - do { - unsigned int anfrac = min(pnfrac, POOL_FRACBITS / 2); - unsigned int add = - ((POOL_FRACBITS - entropy_count) * anfrac * 3) >> s; - - entropy_count += add; - pnfrac -= anfrac; - } while (unlikely(entropy_count < POOL_FRACBITS - 2 && pnfrac)); - } - - if (WARN_ON(entropy_count < 0)) { - pr_warn("negative entropy/overflow: count %d\n", entropy_count); - entropy_count = 0; - } else if (entropy_count > POOL_FRACBITS) - entropy_count = POOL_FRACBITS; - if (cmpxchg(&input_pool.entropy_count, orig, entropy_count) != orig) - goto retry; + do { + orig = READ_ONCE(input_pool.entropy_count); + entropy_count = min(POOL_BITS, orig + nbits); + } while (cmpxchg(&input_pool.entropy_count, orig, entropy_count) != orig);

- trace_credit_entropy_bits(nbits, entropy_count >> POOL_ENTROPY_SHIFT, _RET_IP_); + trace_credit_entropy_bits(nbits, entropy_count, _RET_IP_);

- if (crng_init < 2 && entropy_count >= POOL_MIN_FRACBITS) + if (crng_init < 2 && entropy_count >= POOL_MIN_BITS) crng_reseed(&primary_crng, true); }

@@ -790,7 +734,7 @@ static void crng_reseed(struct crng_stat int entropy_count; do { entropy_count = READ_ONCE(input_pool.entropy_count); - if (entropy_count < POOL_MIN_FRACBITS) + if (entropy_count < POOL_MIN_BITS) return; } while (cmpxchg(&input_pool.entropy_count, entropy_count, 0) != entropy_count); extract_entropy(buf.key, sizeof(buf.key)); @@ -1013,7 +957,7 @@ void add_input_randomness(unsigned int t last_value = value; add_timer_randomness(&input_timer_state, (type << 4) ^ code ^ (code >> 4) ^ value); - trace_add_input_randomness(POOL_ENTROPY_BITS()); + trace_add_input_randomness(input_pool.entropy_count); } EXPORT_SYMBOL_GPL(add_input_randomness);

@@ -1111,7 +1055,7 @@ void add_disk_randomness(struct gendisk return; /* first major is 1, so we get >= 0x200 here */ add_timer_randomness(disk->random, 0x100 + disk_devt(disk)); - trace_add_disk_randomness(disk_devt(disk), POOL_ENTROPY_BITS()); + trace_add_disk_randomness(disk_devt(disk), input_pool.entropy_count); } EXPORT_SYMBOL_GPL(add_disk_randomness); #endif @@ -1136,7 +1080,7 @@ static void extract_entropy(void *buf, s } block; size_t i;

- trace_extract_entropy(nbytes, POOL_ENTROPY_BITS()); + trace_extract_entropy(nbytes, input_pool.entropy_count);

for (i = 0; i < ARRAY_SIZE(block.rdrand); ++i) { if (!arch_get_random_long(&block.rdrand[i])) @@ -1485,9 +1429,9 @@ static ssize_t urandom_read_nowarn(struc { int ret;

- nbytes = min_t(size_t, nbytes, INT_MAX >> (POOL_ENTROPY_SHIFT + 3)); + nbytes = min_t(size_t, nbytes, INT_MAX >> 6); ret = extract_crng_user(buf, nbytes); - trace_urandom_read(8 * nbytes, 0, POOL_ENTROPY_BITS()); + trace_urandom_read(8 * nbytes, 0, input_pool.entropy_count); return ret; }

@@ -1526,7 +1470,7 @@ static unsigned int random_poll(struct f mask = 0; if (crng_ready()) mask |= POLLIN | POLLRDNORM; - if (POOL_ENTROPY_BITS() < random_write_wakeup_bits) + if (input_pool.entropy_count < random_write_wakeup_bits) mask |= POLLOUT | POLLWRNORM; return mask; } @@ -1581,8 +1525,7 @@ static long random_ioctl(struct file *f, switch (cmd) { case RNDGETENTCNT: /* inherently racy, no point locking */ - ent_count = POOL_ENTROPY_BITS(); - if (put_user(ent_count, p)) + if (put_user(input_pool.entropy_count, p)) return -EFAULT; return 0; case RNDADDTOENTCNT: @@ -1728,23 +1671,6 @@ static int proc_do_uuid(struct ctl_table return proc_dostring(&fake_table, write, buffer, lenp, ppos); }

-/* - * Return entropy available scaled to integral bits - */ -static int proc_do_entropy(struct ctl_table *table, int write, - void __user *buffer, size_t *lenp, loff_t *ppos) -{ - struct ctl_table fake_table; - int entropy_count; - - entropy_count = *(int *)table->data >> POOL_ENTROPY_SHIFT; - - fake_table.data = &entropy_count; - fake_table.maxlen = sizeof(entropy_count); - - return proc_dointvec(&fake_table, write, buffer, lenp, ppos); -} - static int sysctl_poolsize = POOL_BITS; extern struct ctl_table random_table[]; struct ctl_table random_table[] = { @@ -1757,10 +1683,10 @@ struct ctl_table random_table[] = { }, { .procname = "entropy_avail", + .data = &input_pool.entropy_count, .maxlen = sizeof(int), .mode = 0444, - .proc_handler = proc_do_entropy, - .data = &input_pool.entropy_count, + .proc_handler = proc_dointvec, }, { .procname = "write_wakeup_threshold", @@ -1956,7 +1882,7 @@ void add_hwgenerator_randomness(const ch */ wait_event_interruptible_timeout(random_write_wait, !system_wq || kthread_should_stop() || - POOL_ENTROPY_BITS() <= random_write_wakeup_bits, + input_pool.entropy_count <= random_write_wakeup_bits, CRNG_RESEED_INTERVAL); mix_pool_bytes(buffer, count); credit_entropy_bits(entropy);

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit a49c010e61e1938be851f5e49ac219d49b704103 upstream.

This is called from various hwgenerator drivers, so rather than having one "safe" version for userspace and one "unsafe" version for the kernel, just make everything safe; the checks are cheap and sensible to have anyway.

Reported-by: Sultan Alsawaf sultan@kerneltoast.com Reviewed-by: Eric Biggers ebiggers@google.com Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -447,18 +447,15 @@ static void process_random_ready_list(vo spin_unlock_irqrestore(&random_ready_list_lock, flags); }

-/* - * Credit (or debit) the entropy store with n bits of entropy. - * Use credit_entropy_bits_safe() if the value comes from userspace - * or otherwise should be checked for extreme values. - */ static void credit_entropy_bits(int nbits) { int entropy_count, orig;

- if (!nbits) + if (nbits <= 0) return;

+ nbits = min(nbits, POOL_BITS); + do { orig = READ_ONCE(input_pool.entropy_count); entropy_count = min(POOL_BITS, orig + nbits); @@ -470,18 +467,6 @@ static void credit_entropy_bits(int nbit crng_reseed(&primary_crng, true); }

-static int credit_entropy_bits_safe(int nbits) -{ - if (nbits < 0) - return -EINVAL; - - /* Cap the value to avoid overflows */ - nbits = min(nbits, POOL_BITS); - - credit_entropy_bits(nbits); - return 0; -} - /********************************************************************* * * CRNG using CHACHA20 @@ -1525,7 +1510,10 @@ static long random_ioctl(struct file *f, return -EPERM; if (get_user(ent_count, p)) return -EFAULT; - return credit_entropy_bits_safe(ent_count); + if (ent_count < 0) + return -EINVAL; + credit_entropy_bits(ent_count); + return 0; case RNDADDENTROPY: if (!capable(CAP_SYS_ADMIN)) return -EPERM; @@ -1538,7 +1526,8 @@ static long random_ioctl(struct file *f, retval = write_pool((const char __user *)p, size); if (retval < 0) return retval; - return credit_entropy_bits_safe(ent_count); + credit_entropy_bits(ent_count); + return 0; case RNDZAPENTCNT: case RNDCLEARPOOL: /*

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 77760fd7f7ae3dfd03668204e708d1568d75447d upstream.

Rather than use spinlocks to protect batched entropy, we can instead disable interrupts locally, since we're dealing with per-cpu data, and manage resets with a basic generation counter. At the same time, we can't quite do this on PREEMPT_RT, where we still want spinlocks-as- mutexes semantics. So we use a local_lock_t, which provides the right behavior for each. Because this is a per-cpu lock, that generation counter is still doing the necessary CPU-to-CPU communication.

This should improve performance a bit. It will also fix the linked splat that Jonathan received with a PROVE_RAW_LOCK_NESTING=y.

Reviewed-by: Sebastian Andrzej Siewior bigeasy@linutronix.de Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Reviewed-by: Eric Biggers ebiggers@google.com Suggested-by: Andy Lutomirski luto@kernel.org Reported-by: Jonathan Neuschäfer j.neuschaefer@gmx.net Tested-by: Jonathan Neuschäfer j.neuschaefer@gmx.net Link: https://lore.kernel.org/lkml/YfMa0QgsjCVdRAvJ@latitude/ Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 58 +++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 31 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1718,13 +1718,15 @@ struct ctl_table random_table[] = { }; #endif /* CONFIG_SYSCTL */

+static atomic_t batch_generation = ATOMIC_INIT(0); + struct batched_entropy { union { u64 entropy_u64[CHACHA20_BLOCK_SIZE / sizeof(u64)]; u32 entropy_u32[CHACHA20_BLOCK_SIZE / sizeof(u32)]; }; unsigned int position; - spinlock_t batch_lock; + int generation; };

/* @@ -1735,9 +1737,7 @@ struct batched_entropy { * wait_for_random_bytes() should be called and return 0 at least once at any * point prior. */ -static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = { - .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u64.lock), -}; +static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64);

u64 get_random_u64(void) { @@ -1745,67 +1745,63 @@ u64 get_random_u64(void) unsigned long flags; struct batched_entropy *batch; static void *previous; + int next_gen;

warn_unseeded_randomness(&previous);

+ local_irq_save(flags); batch = raw_cpu_ptr(&batched_entropy_u64); - spin_lock_irqsave(&batch->batch_lock, flags); - if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) { + + next_gen = atomic_read(&batch_generation); + if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0 || + next_gen != batch->generation) { extract_crng((u8 *)batch->entropy_u64); batch->position = 0; + batch->generation = next_gen; } + ret = batch->entropy_u64[batch->position++]; - spin_unlock_irqrestore(&batch->batch_lock, flags); + local_irq_restore(flags); return ret; } EXPORT_SYMBOL(get_random_u64);

-static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = { - .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u32.lock), -}; +static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32); + u32 get_random_u32(void) { u32 ret; unsigned long flags; struct batched_entropy *batch; static void *previous; + int next_gen;

warn_unseeded_randomness(&previous);

+ local_irq_save(flags); batch = raw_cpu_ptr(&batched_entropy_u32); - spin_lock_irqsave(&batch->batch_lock, flags); - if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) { + + next_gen = atomic_read(&batch_generation); + if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0 || + next_gen != batch->generation) { extract_crng((u8 *)batch->entropy_u32); batch->position = 0; + batch->generation = next_gen; } + ret = batch->entropy_u32[batch->position++]; - spin_unlock_irqrestore(&batch->batch_lock, flags); + local_irq_restore(flags); return ret; } EXPORT_SYMBOL(get_random_u32);

/* It's important to invalidate all potential batched entropy that might * be stored before the crng is initialized, which we can do lazily by - * simply resetting the counter to zero so that it's re-extracted on the - * next usage. */ + * bumping the generation counter. + */ static void invalidate_batched_entropy(void) { - int cpu; - unsigned long flags; - - for_each_possible_cpu(cpu) { - struct batched_entropy *batched_entropy; - - batched_entropy = per_cpu_ptr(&batched_entropy_u32, cpu); - spin_lock_irqsave(&batched_entropy->batch_lock, flags); - batched_entropy->position = 0; - spin_unlock(&batched_entropy->batch_lock); - - batched_entropy = per_cpu_ptr(&batched_entropy_u64, cpu); - spin_lock(&batched_entropy->batch_lock); - batched_entropy->position = 0; - spin_unlock_irqrestore(&batched_entropy->batch_lock, flags); - } + atomic_inc(&batch_generation); }

/**

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 28f425e573e906a4c15f8392cc2b1561ef448595 upstream.

When /dev/random was directly connected with entropy extraction, without any expansion stage, extract_buf() was called for every 10 bytes of data read from /dev/random. For that reason, RDRAND was used rather than RDSEED. At the same time, crng_reseed() was still only called every 5 minutes, so there RDSEED made sense.

Those olden days were also a time when the entropy collector did not use a cryptographic hash function, which meant most bets were off in terms of real preimage resistance. For that reason too it didn't matter _that_ much whether RDSEED was mixed in before or after entropy extraction; both choices were sort of bad.

But now we have a cryptographic hash function at work, and with that we get real preimage resistance. We also now only call extract_entropy() every 5 minutes, rather than every 10 bytes. This allows us to do two important things.

First, we can switch to using RDSEED in extract_entropy(), as Dominik suggested. Second, we can ensure that RDSEED input always goes into the cryptographic hash function with other things before being used directly. This eliminates a category of attacks in which the CPU knows the current state of the crng and knows that we're going to xor RDSEED into it, and so it computes a malicious RDSEED. By going through our hash function, it would require the CPU to compute a preimage on the fly, which isn't going to happen.

Cc: Theodore Ts'o tytso@mit.edu Reviewed-by: Eric Biggers ebiggers@google.com Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Suggested-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 246 ++++++++++++-------------------------------------- 1 file changed, 62 insertions(+), 184 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -323,14 +323,11 @@ static struct crng_state primary_crng = * its value (from 0->1->2). */ static int crng_init = 0; -static bool crng_need_final_init = false; #define crng_ready() (likely(crng_init > 1)) static int crng_init_cnt = 0; -static unsigned long crng_global_init_time = 0; #define CRNG_INIT_CNT_THRESH (2 * CHACHA20_KEY_SIZE) -static void _extract_crng(struct crng_state *crng, u8 out[CHACHA20_BLOCK_SIZE]); -static void _crng_backtrack_protect(struct crng_state *crng, - u8 tmp[CHACHA20_BLOCK_SIZE], int used); +static void extract_crng(u8 out[CHACHA20_BLOCK_SIZE]); +static void crng_backtrack_protect(u8 tmp[CHACHA20_BLOCK_SIZE], int used); static void process_random_ready_list(void); static void _get_random_bytes(void *buf, int nbytes);

@@ -365,7 +362,7 @@ static struct {

static void extract_entropy(void *buf, size_t nbytes);

-static void crng_reseed(struct crng_state *crng); +static void crng_reseed(void);

/* * This function adds bytes into the entropy "pool". It does not @@ -464,7 +461,7 @@ static void credit_entropy_bits(int nbit trace_credit_entropy_bits(nbits, entropy_count, _RET_IP_);

if (crng_init < 2 && entropy_count >= POOL_MIN_BITS) - crng_reseed(&primary_crng); + crng_reseed(); }

/********************************************************************* @@ -477,14 +474,6 @@ static void credit_entropy_bits(int nbit

static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait);

-/* - * Hack to deal with crazy userspace progams when they are all trying - * to access /dev/urandom in parallel. The programs are almost - * certainly doing something terribly wrong, but we'll work around - * their brain damage. - */ -static struct crng_state **crng_node_pool __read_mostly; - static void invalidate_batched_entropy(void);

static bool trust_cpu __ro_after_init = IS_ENABLED(CONFIG_RANDOM_TRUST_CPU); @@ -494,24 +483,6 @@ static int __init parse_trust_cpu(char * } early_param("random.trust_cpu", parse_trust_cpu);

-static bool crng_init_try_arch(struct crng_state *crng) -{ - int i; - bool arch_init = true; - unsigned long rv; - - for (i = 4; i < 16; i++) { - if (!arch_get_random_seed_long(&rv) && - !arch_get_random_long(&rv)) { - rv = random_get_entropy(); - arch_init = false; - } - crng->state[i] ^= rv; - } - - return arch_init; -} - static bool __init crng_init_try_arch_early(void) { int i; @@ -530,100 +501,17 @@ static bool __init crng_init_try_arch_ea return arch_init; }

-static void crng_initialize_secondary(struct crng_state *crng) -{ - chacha_init_consts(crng->state); - _get_random_bytes(&crng->state[4], sizeof(u32) * 12); - crng_init_try_arch(crng); - crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1; -} - -static void __init crng_initialize_primary(void) +static void __init crng_initialize(void) { extract_entropy(&primary_crng.state[4], sizeof(u32) * 12); if (crng_init_try_arch_early() && trust_cpu && crng_init < 2) { invalidate_batched_entropy(); - numa_crng_init(); crng_init = 2; pr_notice("crng init done (trusting CPU's manufacturer)\n"); } primary_crng.init_time = jiffies - CRNG_RESEED_INTERVAL - 1; }

-static void crng_finalize_init(void) -{ - if (!system_wq) { - /* We can't call numa_crng_init until we have workqueues, - * so mark this for processing later. */ - crng_need_final_init = true; - return; - } - - invalidate_batched_entropy(); - numa_crng_init(); - crng_init = 2; - crng_need_final_init = false; - process_random_ready_list(); - wake_up_interruptible(&crng_init_wait); - kill_fasync(&fasync, SIGIO, POLL_IN); - pr_notice("crng init done\n"); - if (unseeded_warning.missed) { - pr_notice("%d get_random_xx warning(s) missed due to ratelimiting\n", - unseeded_warning.missed); - unseeded_warning.missed = 0; - } - if (urandom_warning.missed) { - pr_notice("%d urandom warning(s) missed due to ratelimiting\n", - urandom_warning.missed); - urandom_warning.missed = 0; - } -} - -static void do_numa_crng_init(struct work_struct *work) -{ - int i; - struct crng_state *crng; - struct crng_state **pool; - - pool = kcalloc(nr_node_ids, sizeof(*pool), GFP_KERNEL | __GFP_NOFAIL); - for_each_online_node(i) { - crng = kmalloc_node(sizeof(struct crng_state), - GFP_KERNEL | __GFP_NOFAIL, i); - spin_lock_init(&crng->lock); - crng_initialize_secondary(crng); - pool[i] = crng; - } - /* pairs with READ_ONCE() in select_crng() */ - if (cmpxchg_release(&crng_node_pool, NULL, pool) != NULL) { - for_each_node(i) - kfree(pool[i]); - kfree(pool); - } -} - -static DECLARE_WORK(numa_crng_init_work, do_numa_crng_init); - -static void numa_crng_init(void) -{ - if (IS_ENABLED(CONFIG_NUMA)) - schedule_work(&numa_crng_init_work); -} - -static struct crng_state *select_crng(void) -{ - if (IS_ENABLED(CONFIG_NUMA)) { - struct crng_state **pool; - int nid = numa_node_id(); - - /* pairs with cmpxchg_release() in do_numa_crng_init() */ - pool = READ_ONCE(crng_node_pool); - if (pool && pool[nid]) - return pool[nid]; - } - - return &primary_crng; -} - /* * crng_fast_load() can be called by code in the interrupt service * path. So we can't afford to dilly-dally. Returns the number of @@ -701,73 +589,71 @@ static int crng_slow_load(const u8 *cp, return 1; }

-static void crng_reseed(struct crng_state *crng) +static void crng_reseed(void) { unsigned long flags; - int i; + int i, entropy_count; union { u8 block[CHACHA20_BLOCK_SIZE]; u32 key[8]; } buf;

- if (crng == &primary_crng) { - int entropy_count; - do { - entropy_count = READ_ONCE(input_pool.entropy_count); - if (entropy_count < POOL_MIN_BITS) - return; - } while (cmpxchg(&input_pool.entropy_count, entropy_count, 0) != entropy_count); - extract_entropy(buf.key, sizeof(buf.key)); - wake_up_interruptible(&random_write_wait); - kill_fasync(&fasync, SIGIO, POLL_OUT); - } else { - _extract_crng(&primary_crng, buf.block); - _crng_backtrack_protect(&primary_crng, buf.block, - CHACHA20_KEY_SIZE); - } - spin_lock_irqsave(&crng->lock, flags); - for (i = 0; i < 8; i++) { - unsigned long rv; - if (!arch_get_random_seed_long(&rv) && - !arch_get_random_long(&rv)) - rv = random_get_entropy(); - crng->state[i + 4] ^= buf.key[i] ^ rv; - } + do { + entropy_count = READ_ONCE(input_pool.entropy_count); + if (entropy_count < POOL_MIN_BITS) + return; + } while (cmpxchg(&input_pool.entropy_count, entropy_count, 0) != entropy_count); + extract_entropy(buf.key, sizeof(buf.key)); + wake_up_interruptible(&random_write_wait); + kill_fasync(&fasync, SIGIO, POLL_OUT); + + spin_lock_irqsave(&primary_crng.lock, flags); + for (i = 0; i < 8; i++) + primary_crng.state[i + 4] ^= buf.key[i]; memzero_explicit(&buf, sizeof(buf)); - WRITE_ONCE(crng->init_time, jiffies); - spin_unlock_irqrestore(&crng->lock, flags); - if (crng == &primary_crng && crng_init < 2) - crng_finalize_init(); + WRITE_ONCE(primary_crng.init_time, jiffies); + spin_unlock_irqrestore(&primary_crng.lock, flags); + if (crng_init < 2) { + invalidate_batched_entropy(); + crng_init = 2; + process_random_ready_list(); + wake_up_interruptible(&crng_init_wait); + kill_fasync(&fasync, SIGIO, POLL_IN); + pr_notice("crng init done\n"); + if (unseeded_warning.missed) { + pr_notice("%d get_random_xx warning(s) missed due to ratelimiting\n", + unseeded_warning.missed); + unseeded_warning.missed = 0; + } + if (urandom_warning.missed) { + pr_notice("%d urandom warning(s) missed due to ratelimiting\n", + urandom_warning.missed); + urandom_warning.missed = 0; + } + } }

-static void _extract_crng(struct crng_state *crng, u8 out[CHACHA20_BLOCK_SIZE]) +static void extract_crng(u8 out[CHACHA20_BLOCK_SIZE]) { unsigned long flags, init_time;

if (crng_ready()) { - init_time = READ_ONCE(crng->init_time); - if (time_after(READ_ONCE(crng_global_init_time), init_time) || - time_after(jiffies, init_time + CRNG_RESEED_INTERVAL)) - crng_reseed(crng); - } - spin_lock_irqsave(&crng->lock, flags); - chacha20_block(&crng->state[0], out); - if (crng->state[12] == 0) - crng->state[13]++; - spin_unlock_irqrestore(&crng->lock, flags); -} - -static void extract_crng(u8 out[CHACHA20_BLOCK_SIZE]) -{ - _extract_crng(select_crng(), out); + init_time = READ_ONCE(primary_crng.init_time); + if (time_after(jiffies, init_time + CRNG_RESEED_INTERVAL)) + crng_reseed(); + } + spin_lock_irqsave(&primary_crng.lock, flags); + chacha20_block(&primary_crng.state[0], out); + if (primary_crng.state[12] == 0) + primary_crng.state[13]++; + spin_unlock_irqrestore(&primary_crng.lock, flags); }

/* * Use the leftover bytes from the CRNG block output (if there is * enough) to mutate the CRNG key to provide backtracking protection. */ -static void _crng_backtrack_protect(struct crng_state *crng, - u8 tmp[CHACHA20_BLOCK_SIZE], int used) +static void crng_backtrack_protect(u8 tmp[CHACHA20_BLOCK_SIZE], int used) { unsigned long flags; u32 *s, *d; @@ -778,17 +664,12 @@ static void _crng_backtrack_protect(stru extract_crng(tmp); used = 0; } - spin_lock_irqsave(&crng->lock, flags); + spin_lock_irqsave(&primary_crng.lock, flags); s = (u32 *)&tmp[used]; - d = &crng->state[4]; + d = &primary_crng.state[4]; for (i = 0; i < 8; i++) *d++ ^= *s++; - spin_unlock_irqrestore(&crng->lock, flags); -} - -static void crng_backtrack_protect(u8 tmp[CHACHA20_BLOCK_SIZE], int used) -{ - _crng_backtrack_protect(select_crng(), tmp, used); + spin_unlock_irqrestore(&primary_crng.lock, flags); }

static ssize_t extract_crng_user(void __user *buf, size_t nbytes) @@ -1053,16 +934,17 @@ static void extract_entropy(void *buf, s unsigned long flags; u8 seed[BLAKE2S_HASH_SIZE], next_key[BLAKE2S_HASH_SIZE]; struct { - unsigned long rdrand[32 / sizeof(long)]; + unsigned long rdseed[32 / sizeof(long)]; size_t counter; } block; size_t i;

trace_extract_entropy(nbytes, input_pool.entropy_count);

- for (i = 0; i < ARRAY_SIZE(block.rdrand); ++i) { - if (!arch_get_random_long(&block.rdrand[i])) - block.rdrand[i] = random_get_entropy(); + for (i = 0; i < ARRAY_SIZE(block.rdseed); ++i) { + if (!arch_get_random_seed_long(&block.rdseed[i]) && + !arch_get_random_long(&block.rdseed[i])) + block.rdseed[i] = random_get_entropy(); }

spin_lock_irqsave(&input_pool.lock, flags); @@ -1070,7 +952,7 @@ static void extract_entropy(void *buf, s /* seed = HASHPRF(last_key, entropy_input) */ blake2s_final(&input_pool.hash, seed);

- /* next_key = HASHPRF(seed, RDRAND || 0) */ + /* next_key = HASHPRF(seed, RDSEED || 0) */ block.counter = 0; blake2s(next_key, (u8 *)&block, seed, sizeof(next_key), sizeof(block), sizeof(seed)); blake2s_init_key(&input_pool.hash, BLAKE2S_HASH_SIZE, next_key, sizeof(next_key)); @@ -1080,7 +962,7 @@ static void extract_entropy(void *buf, s

while (nbytes) { i = min_t(size_t, nbytes, BLAKE2S_HASH_SIZE); - /* output = HASHPRF(seed, RDRAND || ++counter) */ + /* output = HASHPRF(seed, RDSEED || ++counter) */ ++block.counter; blake2s(buf, (u8 *)&block, seed, i, sizeof(block), sizeof(seed)); nbytes -= i; @@ -1374,10 +1256,7 @@ static void __init init_std_data(void) int __init rand_initialize(void) { init_std_data(); - if (crng_need_final_init) - crng_finalize_init(); - crng_initialize_primary(); - crng_global_init_time = jiffies; + crng_initialize(); if (ratelimit_disable) { urandom_warning.interval = 0; unseeded_warning.interval = 0; @@ -1547,8 +1426,7 @@ static long random_ioctl(struct file *f, return -EPERM; if (crng_init < 2) return -ENODATA; - crng_reseed(&primary_crng); - WRITE_ONCE(crng_global_init_time, jiffies - 1); + crng_reseed(); return 0; default: return -EINVAL;

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit a02cf3d0dd77244fd5333ac48d78871de459ae6d upstream.

Continuing the reasoning of "random: use RDSEED instead of RDRAND in entropy extraction" from this series, at init time we also don't want to be xoring RDSEED directly into the crng. Instead it's safer to put it into our entropy collector and then re-extract it, so that it goes through a hash function with preimage resistance. As a matter of hygiene, we also order these now so that the RDSEED byte are hashed in first, followed by the bytes that are likely more predictable (e.g. utsname()).

Cc: Theodore Ts'o tytso@mit.edu Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Reviewed-by: Eric Biggers ebiggers@google.com Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1208,24 +1208,18 @@ int __init rand_initialize(void) bool arch_init = true; unsigned long rv;

- mix_pool_bytes(&now, sizeof(now)); for (i = BLAKE2S_BLOCK_SIZE; i > 0; i -= sizeof(rv)) { - if (!arch_get_random_seed_long(&rv) && - !arch_get_random_long(&rv)) - rv = random_get_entropy(); - mix_pool_bytes(&rv, sizeof(rv)); - } - mix_pool_bytes(utsname(), sizeof(*(utsname()))); - - extract_entropy(&primary_crng.state[4], sizeof(u32) * 12); - for (i = 4; i < 16; i++) { if (!arch_get_random_seed_long_early(&rv) && !arch_get_random_long_early(&rv)) { rv = random_get_entropy(); arch_init = false; } - primary_crng.state[i] ^= rv; + mix_pool_bytes(&rv, sizeof(rv)); } + mix_pool_bytes(&now, sizeof(now)); + mix_pool_bytes(utsname(), sizeof(*(utsname()))); + + extract_entropy(&primary_crng.state[4], sizeof(u32) * 12); if (arch_init && trust_cpu && crng_init < 2) { invalidate_batched_entropy(); crng_init = 2;

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit c30c575db4858f0bbe5e315ff2e529c782f33a1f upstream.

During crng_init == 0, we never credit entropy in add_interrupt_ randomness(), but instead dump it directly into the primary_crng. That's fine, except for the fact that we then wind up throwing away that entropy later when we switch to extracting from the input pool and xoring into (and later in this series overwriting) the primary_crng key. The two other early init sites -- add_hwgenerator_randomness()'s use crng_fast_load() and add_device_ randomness()'s use of crng_slow_load() -- always additionally give their inputs to the input pool. But not add_interrupt_randomness().

This commit fixes that shortcoming by calling mix_pool_bytes() after crng_fast_load() in add_interrupt_randomness(). That's partially verboten on PREEMPT_RT, where it implies taking spinlock_t from an IRQ handler. But this also only happens during early boot and then never again after that. Plus it's a trylock so it has the same considerations as calling crng_fast_load(), which we're already using.

Cc: Theodore Ts'o tytso@mit.edu Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Reviewed-by: Eric Biggers ebiggers@google.com Suggested-by: Eric Biggers ebiggers@google.com Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 407 +++++++++++++++++++++++++++++--------------------- 1 file changed, 237 insertions(+), 170 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -67,63 +67,19 @@ * Exported interfaces ---- kernel output * -------------------------------------- * - * The primary kernel interface is + * The primary kernel interfaces are: * * void get_random_bytes(void *buf, int nbytes); - * - * This interface will return the requested number of random bytes, - * and place it in the requested buffer. This is equivalent to a - * read from /dev/urandom. - * - * For less critical applications, there are the functions: - * * u32 get_random_u32() * u64 get_random_u64() * unsigned int get_random_int() * unsigned long get_random_long() * - * These are produced by a cryptographic RNG seeded from get_random_bytes, - * and so do not deplete the entropy pool as much. These are recommended - * for most in-kernel operations *if the result is going to be stored in - * the kernel*. - * - * Specifically, the get_random_int() family do not attempt to do - * "anti-backtracking". If you capture the state of the kernel (e.g. - * by snapshotting the VM), you can figure out previous get_random_int() - * return values. But if the value is stored in the kernel anyway, - * this is not a problem. - * - * It *is* safe to expose get_random_int() output to attackers (e.g. as - * network cookies); given outputs 1..n, it's not feasible to predict - * outputs 0 or n+1. The only concern is an attacker who breaks into - * the kernel later; the get_random_int() engine is not reseeded as - * often as the get_random_bytes() one. - * - * get_random_bytes() is needed for keys that need to stay secret after - * they are erased from the kernel. For example, any key that will - * be wrapped and stored encrypted. And session encryption keys: we'd - * like to know that after the session is closed and the keys erased, - * the plaintext is unrecoverable to someone who recorded the ciphertext. - * - * But for network ports/cookies, stack canaries, PRNG seeds, address - * space layout randomization, session *authentication* keys, or other - * applications where the sensitive data is stored in the kernel in - * plaintext for as long as it's sensitive, the get_random_int() family - * is just fine. - * - * Consider ASLR. We want to keep the address space secret from an - * outside attacker while the process is running, but once the address - * space is torn down, it's of no use to an attacker any more. And it's - * stored in kernel data structures as long as it's alive, so worrying - * about an attacker's ability to extrapolate it from the get_random_int() - * CRNG is silly. - * - * Even some cryptographic keys are safe to generate with get_random_int(). - * In particular, keys for SipHash are generally fine. Here, knowledge - * of the key authorizes you to do something to a kernel object (inject - * packets to a network connection, or flood a hash table), and the - * key is stored with the object being protected. Once it goes away, - * we no longer care if anyone knows the key. + * These interfaces will return the requested number of random bytes + * into the given buffer or as a return value. This is equivalent to a + * read from /dev/urandom. The get_random_{u32,u64,int,long}() family + * of functions may be higher performance for one-off random integers, + * because they do a bit of buffering. * * prandom_u32() * ------------- @@ -300,20 +256,6 @@ static struct fasync_struct *fasync; static DEFINE_SPINLOCK(random_ready_list_lock); static LIST_HEAD(random_ready_list);

-struct crng_state { - u32 state[16]; - unsigned long init_time; - spinlock_t lock; -}; - -static struct crng_state primary_crng = { - .lock = __SPIN_LOCK_UNLOCKED(primary_crng.lock), - .state[0] = CHACHA_CONSTANT_EXPA, - .state[1] = CHACHA_CONSTANT_ND_3, - .state[2] = CHACHA_CONSTANT_2_BY, - .state[3] = CHACHA_CONSTANT_TE_K, -}; - /* * crng_init = 0 --> Uninitialized * 1 --> Initialized @@ -325,9 +267,6 @@ static struct crng_state primary_crng = static int crng_init = 0; #define crng_ready() (likely(crng_init > 1)) static int crng_init_cnt = 0; -#define CRNG_INIT_CNT_THRESH (2 * CHACHA20_KEY_SIZE) -static void extract_crng(u8 out[CHACHA20_BLOCK_SIZE]); -static void crng_backtrack_protect(u8 tmp[CHACHA20_BLOCK_SIZE], int used); static void process_random_ready_list(void); static void _get_random_bytes(void *buf, int nbytes);

@@ -470,7 +409,28 @@ static void credit_entropy_bits(int nbit * *********************************************************************/

-#define CRNG_RESEED_INTERVAL (300 * HZ) +enum { + CRNG_RESEED_INTERVAL = 300 * HZ, + CRNG_INIT_CNT_THRESH = 2 * CHACHA20_KEY_SIZE +}; + +static struct { + u8 key[CHACHA20_KEY_SIZE] __aligned(__alignof__(long)); + unsigned long birth; + unsigned long generation; + spinlock_t lock; +} base_crng = { + .lock = __SPIN_LOCK_UNLOCKED(base_crng.lock) +}; + +struct crng { + u8 key[CHACHA20_KEY_SIZE]; + unsigned long generation; +}; + +static DEFINE_PER_CPU(struct crng, crngs) = { + .generation = ULONG_MAX +};

static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait);

@@ -487,22 +447,22 @@ static size_t crng_fast_load(const u8 *c u8 *p; size_t ret = 0;

- if (!spin_trylock_irqsave(&primary_crng.lock, flags)) + if (!spin_trylock_irqsave(&base_crng.lock, flags)) return 0; if (crng_init != 0) { - spin_unlock_irqrestore(&primary_crng.lock, flags); + spin_unlock_irqrestore(&base_crng.lock, flags); return 0; } - p = (u8 *)&primary_crng.state[4]; + p = base_crng.key; while (len > 0 && crng_init_cnt < CRNG_INIT_CNT_THRESH) { - p[crng_init_cnt % CHACHA20_KEY_SIZE] ^= *cp; + p[crng_init_cnt % sizeof(base_crng.key)] ^= *cp; cp++; crng_init_cnt++; len--; ret++; } if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) { invalidate_batched_entropy(); crng_init = 1; } - spin_unlock_irqrestore(&primary_crng.lock, flags); + spin_unlock_irqrestore(&base_crng.lock, flags); if (crng_init == 1) pr_notice("fast init done\n"); return ret; @@ -527,14 +487,14 @@ static int crng_slow_load(const u8 *cp, unsigned long flags; static u8 lfsr = 1; u8 tmp; - unsigned int i, max = CHACHA20_KEY_SIZE; + unsigned int i, max = sizeof(base_crng.key); const u8 *src_buf = cp; - u8 *dest_buf = (u8 *)&primary_crng.state[4]; + u8 *dest_buf = base_crng.key;

- if (!spin_trylock_irqsave(&primary_crng.lock, flags)) + if (!spin_trylock_irqsave(&base_crng.lock, flags)) return 0; if (crng_init != 0) { - spin_unlock_irqrestore(&primary_crng.lock, flags); + spin_unlock_irqrestore(&base_crng.lock, flags); return 0; } if (len > max) @@ -545,38 +505,50 @@ static int crng_slow_load(const u8 *cp, lfsr >>= 1; if (tmp & 1) lfsr ^= 0xE1; - tmp = dest_buf[i % CHACHA20_KEY_SIZE]; - dest_buf[i % CHACHA20_KEY_SIZE] ^= src_buf[i % len] ^ lfsr; + tmp = dest_buf[i % sizeof(base_crng.key)]; + dest_buf[i % sizeof(base_crng.key)] ^= src_buf[i % len] ^ lfsr; lfsr += (tmp << 3) | (tmp >> 5); } - spin_unlock_irqrestore(&primary_crng.lock, flags); + spin_unlock_irqrestore(&base_crng.lock, flags); return 1; }

static void crng_reseed(void) { unsigned long flags; - int i, entropy_count; - union { - u8 block[CHACHA20_BLOCK_SIZE]; - u32 key[8]; - } buf; + int entropy_count; + unsigned long next_gen; + u8 key[CHACHA20_KEY_SIZE];

+ /* + * First we make sure we have POOL_MIN_BITS of entropy in the pool, + * and then we drain all of it. Only then can we extract a new key. + */ do { entropy_count = READ_ONCE(input_pool.entropy_count); if (entropy_count < POOL_MIN_BITS) return; } while (cmpxchg(&input_pool.entropy_count, entropy_count, 0) != entropy_count); - extract_entropy(buf.key, sizeof(buf.key)); + extract_entropy(key, sizeof(key)); wake_up_interruptible(&random_write_wait); kill_fasync(&fasync, SIGIO, POLL_OUT);

- spin_lock_irqsave(&primary_crng.lock, flags); - for (i = 0; i < 8; i++) - primary_crng.state[i + 4] ^= buf.key[i]; - memzero_explicit(&buf, sizeof(buf)); - WRITE_ONCE(primary_crng.init_time, jiffies); - spin_unlock_irqrestore(&primary_crng.lock, flags); + /* + * We copy the new key into the base_crng, overwriting the old one, + * and update the generation counter. We avoid hitting ULONG_MAX, + * because the per-cpu crngs are initialized to ULONG_MAX, so this + * forces new CPUs that come online to always initialize. + */ + spin_lock_irqsave(&base_crng.lock, flags); + memcpy(base_crng.key, key, sizeof(base_crng.key)); + next_gen = base_crng.generation + 1; + if (next_gen == ULONG_MAX) + ++next_gen; + WRITE_ONCE(base_crng.generation, next_gen); + WRITE_ONCE(base_crng.birth, jiffies); + spin_unlock_irqrestore(&base_crng.lock, flags); + memzero_explicit(key, sizeof(key)); + if (crng_init < 2) { invalidate_batched_entropy(); crng_init = 2; @@ -597,77 +569,143 @@ static void crng_reseed(void) } }

-static void extract_crng(u8 out[CHACHA20_BLOCK_SIZE]) +/* + * The general form here is based on a "fast key erasure RNG" from + * https://blog.cr.yp.to/20170723-random.html. It generates a ChaCha + * block using the provided key, and then immediately overwites that + * key with half the block. It returns the resultant ChaCha state to the + * user, along with the second half of the block containing 32 bytes of + * random data that may be used; random_data_len may not be greater than + * 32. + */ +static void crng_fast_key_erasure(u8 key[CHACHA20_KEY_SIZE], + u32 chacha_state[CHACHA20_BLOCK_SIZE / sizeof(u32)], + u8 *random_data, size_t random_data_len) { - unsigned long flags, init_time; + u8 first_block[CHACHA20_BLOCK_SIZE];

- if (crng_ready()) { - init_time = READ_ONCE(primary_crng.init_time); - if (time_after(jiffies, init_time + CRNG_RESEED_INTERVAL)) - crng_reseed(); - } - spin_lock_irqsave(&primary_crng.lock, flags); - chacha20_block(&primary_crng.state[0], out); - if (primary_crng.state[12] == 0) - primary_crng.state[13]++; - spin_unlock_irqrestore(&primary_crng.lock, flags); + BUG_ON(random_data_len > 32); + + chacha_init_consts(chacha_state); + memcpy(&chacha_state[4], key, CHACHA20_KEY_SIZE); + memset(&chacha_state[12], 0, sizeof(u32) * 4); + chacha20_block(chacha_state, first_block); + + memcpy(key, first_block, CHACHA20_KEY_SIZE); + memcpy(random_data, first_block + CHACHA20_KEY_SIZE, random_data_len); + memzero_explicit(first_block, sizeof(first_block)); }

/* - * Use the leftover bytes from the CRNG block output (if there is - * enough) to mutate the CRNG key to provide backtracking protection. + * This function returns a ChaCha state that you may use for generating + * random data. It also returns up to 32 bytes on its own of random data + * that may be used; random_data_len may not be greater than 32. */ -static void crng_backtrack_protect(u8 tmp[CHACHA20_BLOCK_SIZE], int used) +static void crng_make_state(u32 chacha_state[CHACHA20_BLOCK_SIZE / sizeof(u32)], + u8 *random_data, size_t random_data_len) { unsigned long flags; - u32 *s, *d; - int i; + struct crng *crng; + + BUG_ON(random_data_len > 32); + + /* + * For the fast path, we check whether we're ready, unlocked first, and + * then re-check once locked later. In the case where we're really not + * ready, we do fast key erasure with the base_crng directly, because + * this is what crng_{fast,slow}_load mutate during early init. + */ + if (unlikely(!crng_ready())) { + bool ready; + + spin_lock_irqsave(&base_crng.lock, flags); + ready = crng_ready(); + if (!ready) + crng_fast_key_erasure(base_crng.key, chacha_state, + random_data, random_data_len); + spin_unlock_irqrestore(&base_crng.lock, flags); + if (!ready) + return; + } + + /* + * If the base_crng is more than 5 minutes old, we reseed, which + * in turn bumps the generation counter that we check below. + */ + if (unlikely(time_after(jiffies, READ_ONCE(base_crng.birth) + CRNG_RESEED_INTERVAL))) + crng_reseed(); + + local_irq_save(flags); + crng = raw_cpu_ptr(&crngs); + + /* + * If our per-cpu crng is older than the base_crng, then it means + * somebody reseeded the base_crng. In that case, we do fast key + * erasure on the base_crng, and use its output as the new key + * for our per-cpu crng. This brings us up to date with base_crng. + */ + if (unlikely(crng->generation != READ_ONCE(base_crng.generation))) { + spin_lock(&base_crng.lock); + crng_fast_key_erasure(base_crng.key, chacha_state, + crng->key, sizeof(crng->key)); + crng->generation = base_crng.generation; + spin_unlock(&base_crng.lock); + } + + /* + * Finally, when we've made it this far, our per-cpu crng has an up + * to date key, and we can do fast key erasure with it to produce + * some random data and a ChaCha state for the caller. All other + * branches of this function are "unlikely", so most of the time we + * should wind up here immediately. + */ + crng_fast_key_erasure(crng->key, chacha_state, random_data, random_data_len); + local_irq_restore(flags); +} + +static ssize_t get_random_bytes_user(void __user *buf, size_t nbytes) +{ + bool large_request = nbytes > 256; + ssize_t ret = 0, len; + u32 chacha_state[CHACHA20_BLOCK_SIZE / sizeof(u32)]; + u8 output[CHACHA20_BLOCK_SIZE]; + + if (!nbytes) + return 0;

- used = round_up(used, sizeof(u32)); - if (used + CHACHA20_KEY_SIZE > CHACHA20_BLOCK_SIZE) { - extract_crng(tmp); - used = 0; - } - spin_lock_irqsave(&primary_crng.lock, flags); - s = (u32 *)&tmp[used]; - d = &primary_crng.state[4]; - for (i = 0; i < 8; i++) - *d++ ^= *s++; - spin_unlock_irqrestore(&primary_crng.lock, flags); -} - -static ssize_t extract_crng_user(void __user *buf, size_t nbytes) -{ - ssize_t ret = 0, i = CHACHA20_BLOCK_SIZE; - u8 tmp[CHACHA20_BLOCK_SIZE] __aligned(4); - int large_request = (nbytes > 256); + len = min_t(ssize_t, 32, nbytes); + crng_make_state(chacha_state, output, len); + + if (copy_to_user(buf, output, len)) + return -EFAULT; + nbytes -= len; + buf += len; + ret += len;

while (nbytes) { if (large_request && need_resched()) { - if (signal_pending(current)) { - if (ret == 0) - ret = -ERESTARTSYS; + if (signal_pending(current)) break; - } schedule(); }

- extract_crng(tmp); - i = min_t(int, nbytes, CHACHA20_BLOCK_SIZE); - if (copy_to_user(buf, tmp, i)) { + chacha20_block(chacha_state, output); + if (unlikely(chacha_state[12] == 0)) + ++chacha_state[13]; + + len = min_t(ssize_t, nbytes, CHACHA20_BLOCK_SIZE); + if (copy_to_user(buf, output, len)) { ret = -EFAULT; break; }

- nbytes -= i; - buf += i; - ret += i; + nbytes -= len; + buf += len; + ret += len; } - crng_backtrack_protect(tmp, i); - - /* Wipe data just written to memory */ - memzero_explicit(tmp, sizeof(tmp));

+ memzero_explicit(chacha_state, sizeof(chacha_state)); + memzero_explicit(output, sizeof(output)); return ret; }

@@ -850,6 +888,10 @@ void add_interrupt_randomness(int irq) crng_fast_load((u8 *)fast_pool->pool, sizeof(fast_pool->pool)) > 0) { fast_pool->count = 0; fast_pool->last = now; + if (spin_trylock(&input_pool.lock)) { + _mix_pool_bytes(&fast_pool->pool, sizeof(fast_pool->pool)); + spin_unlock(&input_pool.lock); + } } return; } @@ -972,23 +1014,36 @@ static void _warn_unseeded_randomness(co */ static void _get_random_bytes(void *buf, int nbytes) { - u8 tmp[CHACHA20_BLOCK_SIZE] __aligned(4); + u32 chacha_state[CHACHA20_BLOCK_SIZE / sizeof(u32)]; + u8 tmp[CHACHA20_BLOCK_SIZE]; + ssize_t len;

trace_get_random_bytes(nbytes, _RET_IP_);

- while (nbytes >= CHACHA20_BLOCK_SIZE) { - extract_crng(buf); - buf += CHACHA20_BLOCK_SIZE; + if (!nbytes) + return; + + len = min_t(ssize_t, 32, nbytes); + crng_make_state(chacha_state, buf, len); + nbytes -= len; + buf += len; + + while (nbytes) { + if (nbytes < CHACHA20_BLOCK_SIZE) { + chacha20_block(chacha_state, tmp); + memcpy(buf, tmp, nbytes); + memzero_explicit(tmp, sizeof(tmp)); + break; + } + + chacha20_block(chacha_state, buf); + if (unlikely(chacha_state[12] == 0)) + ++chacha_state[13]; nbytes -= CHACHA20_BLOCK_SIZE; + buf += CHACHA20_BLOCK_SIZE; }

- if (nbytes > 0) { - extract_crng(tmp); - memcpy(buf, tmp, nbytes); - crng_backtrack_protect(tmp, nbytes); - } else - crng_backtrack_protect(tmp, CHACHA20_BLOCK_SIZE); - memzero_explicit(tmp, sizeof(tmp)); + memzero_explicit(chacha_state, sizeof(chacha_state)); }

void get_random_bytes(void *buf, int nbytes) @@ -1219,13 +1274,12 @@ int __init rand_initialize(void) mix_pool_bytes(&now, sizeof(now)); mix_pool_bytes(utsname(), sizeof(*(utsname())));

- extract_entropy(&primary_crng.state[4], sizeof(u32) * 12); + extract_entropy(base_crng.key, sizeof(base_crng.key)); if (arch_init && trust_cpu && crng_init < 2) { invalidate_batched_entropy(); crng_init = 2; pr_notice("crng init done (trusting CPU's manufacturer)\n"); } - primary_crng.init_time = jiffies - CRNG_RESEED_INTERVAL - 1;

if (ratelimit_disable) { urandom_warning.interval = 0; @@ -1257,7 +1311,7 @@ static ssize_t urandom_read_nowarn(struc int ret;

nbytes = min_t(size_t, nbytes, INT_MAX >> 6); - ret = extract_crng_user(buf, nbytes); + ret = get_random_bytes_user(buf, nbytes); trace_urandom_read(8 * nbytes, 0, input_pool.entropy_count); return ret; } @@ -1561,8 +1615,15 @@ static atomic_t batch_generation = ATOMI

struct batched_entropy { union { - u64 entropy_u64[CHACHA20_BLOCK_SIZE / sizeof(u64)]; - u32 entropy_u32[CHACHA20_BLOCK_SIZE / sizeof(u32)]; + /* + * We make this 1.5x a ChaCha block, so that we get the + * remaining 32 bytes from fast key erasure, plus one full + * block from the detached ChaCha state. We can increase + * the size of this later if needed so long as we keep the + * formula of (integer_blocks + 0.5) * CHACHA20_BLOCK_SIZE. + */ + u64 entropy_u64[CHACHA20_BLOCK_SIZE * 3 / (2 * sizeof(u64))]; + u32 entropy_u32[CHACHA20_BLOCK_SIZE * 3 / (2 * sizeof(u32))]; }; unsigned int position; int generation; @@ -1570,13 +1631,13 @@ struct batched_entropy {

/* * Get a random word for internal kernel use only. The quality of the random - * number is good as /dev/urandom, but there is no backtrack protection, with - * the goal of being quite fast and not depleting entropy. In order to ensure - * that the randomness provided by this function is okay, the function - * wait_for_random_bytes() should be called and return 0 at least once at any - * point prior. + * number is good as /dev/urandom. In order to ensure that the randomness + * provided by this function is okay, the function wait_for_random_bytes() + * should be called and return 0 at least once at any point prior. */ -static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64); +static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = { + .position = UINT_MAX +};

u64 get_random_u64(void) { @@ -1592,20 +1653,24 @@ u64 get_random_u64(void) batch = raw_cpu_ptr(&batched_entropy_u64);

next_gen = atomic_read(&batch_generation); - if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0 || + if (batch->position >= ARRAY_SIZE(batch->entropy_u64) || next_gen != batch->generation) { - extract_crng((u8 *)batch->entropy_u64); + _get_random_bytes(batch->entropy_u64, sizeof(batch->entropy_u64)); batch->position = 0; batch->generation = next_gen; }

- ret = batch->entropy_u64[batch->position++]; + ret = batch->entropy_u64[batch->position]; + batch->entropy_u64[batch->position] = 0; + ++batch->position; local_irq_restore(flags); return ret; } EXPORT_SYMBOL(get_random_u64);

-static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32); +static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = { + .position = UINT_MAX +};

u32 get_random_u32(void) { @@ -1621,14 +1686,16 @@ u32 get_random_u32(void) batch = raw_cpu_ptr(&batched_entropy_u32);

next_gen = atomic_read(&batch_generation); - if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0 || + if (batch->position >= ARRAY_SIZE(batch->entropy_u32) || next_gen != batch->generation) { - extract_crng((u8 *)batch->entropy_u32); + _get_random_bytes(batch->entropy_u32, sizeof(batch->entropy_u32)); batch->position = 0; batch->generation = next_gen; }

- ret = batch->entropy_u32[batch->position++]; + ret = batch->entropy_u32[batch->position]; + batch->entropy_u32[batch->position] = 0; + ++batch->position; local_irq_restore(flags); return ret; }

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 434537ae54ad37e93555de21b6ac8133d6d773a9 upstream.

In 79a8468747c5 ("random: check for increase of entropy_count because of signed conversion"), a number of checks were added around what values were passed to account(), because account() was doing fancy fixed point fractional arithmetic, and a user had some ability to pass large values directly into it. One of things in that commit was limiting those values to INT_MAX >> 6. The first >> 3 was for bytes to bits, and the next >> 3 was for bits to 1/8 fractional bits.

However, for several years now, urandom reads no longer touch entropy accounting, and so this check serves no purpose. The current flow is:

urandom_read_nowarn()-->get_random_bytes_user()-->chacha20_block()

Of course, we don't want that size_t to be truncated when adding it into the ssize_t. But we arrive at urandom_read_nowarn() in the first place either via ordinary fops, which limits reads to MAX_RW_COUNT, or via getrandom() which limits reads to INT_MAX.

Cc: Theodore Ts'o tytso@mit.edu Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Reviewed-by: Jann Horn jannh@google.com Reviewed-by: Eric Biggers ebiggers@google.com Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1284,9 +1284,8 @@ void rand_initialize_disk(struct gendisk static ssize_t urandom_read_nowarn(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) { - int ret; + ssize_t ret;

- nbytes = min_t(size_t, nbytes, INT_MAX >> 6); ret = get_random_bytes_user(buf, nbytes); trace_urandom_read(nbytes, input_pool.entropy_count); return ret;

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 0791e8b655cc373718f0f58800fdc625a3447ac5 upstream.

Now that we have an explicit base_crng generation counter, we don't need a separate one for batched entropy. Rather, we can just move the generation forward every time we change crng_init state or update the base_crng key.

Cc: Theodore Ts'o tytso@mit.edu Reviewed-by: Eric Biggers ebiggers@google.com Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -428,8 +428,6 @@ static DEFINE_PER_CPU(struct crng, crngs

static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait);

-static void invalidate_batched_entropy(void); - /* * crng_fast_load() can be called by code in the interrupt service * path. So we can't afford to dilly-dally. Returns the number of @@ -452,7 +450,7 @@ static size_t crng_fast_load(const void src++; crng_init_cnt++; len--; ret++; } if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) { - invalidate_batched_entropy(); + ++base_crng.generation; crng_init = 1; } spin_unlock_irqrestore(&base_crng.lock, flags); @@ -529,7 +527,6 @@ static void crng_reseed(void) WRITE_ONCE(base_crng.generation, next_gen); WRITE_ONCE(base_crng.birth, jiffies); if (crng_init < 2) { - invalidate_batched_entropy(); crng_init = 2; finalize_init = true; } @@ -1254,8 +1251,9 @@ int __init rand_initialize(void) mix_pool_bytes(utsname(), sizeof(*(utsname())));

extract_entropy(base_crng.key, sizeof(base_crng.key)); + ++base_crng.generation; + if (arch_init && trust_cpu && crng_init < 2) { - invalidate_batched_entropy(); crng_init = 2; pr_notice("crng init done (trusting CPU's manufacturer)\n"); } @@ -1593,8 +1591,6 @@ struct ctl_table random_table[] = { }; #endif /* CONFIG_SYSCTL */

-static atomic_t batch_generation = ATOMIC_INIT(0); - struct batched_entropy { union { /* @@ -1607,8 +1603,8 @@ struct batched_entropy { u64 entropy_u64[CHACHA20_BLOCK_SIZE * 3 / (2 * sizeof(u64))]; u32 entropy_u32[CHACHA20_BLOCK_SIZE * 3 / (2 * sizeof(u32))]; }; + unsigned long generation; unsigned int position; - int generation; };

/* @@ -1627,14 +1623,14 @@ u64 get_random_u64(void) unsigned long flags; struct batched_entropy *batch; static void *previous; - int next_gen; + unsigned long next_gen;

warn_unseeded_randomness(&previous);

local_irq_save(flags); batch = raw_cpu_ptr(&batched_entropy_u64);

- next_gen = atomic_read(&batch_generation); + next_gen = READ_ONCE(base_crng.generation); if (batch->position >= ARRAY_SIZE(batch->entropy_u64) || next_gen != batch->generation) { _get_random_bytes(batch->entropy_u64, sizeof(batch->entropy_u64)); @@ -1660,14 +1656,14 @@ u32 get_random_u32(void) unsigned long flags; struct batched_entropy *batch; static void *previous; - int next_gen; + unsigned long next_gen;

warn_unseeded_randomness(&previous);

local_irq_save(flags); batch = raw_cpu_ptr(&batched_entropy_u32);

- next_gen = atomic_read(&batch_generation); + next_gen = READ_ONCE(base_crng.generation); if (batch->position >= ARRAY_SIZE(batch->entropy_u32) || next_gen != batch->generation) { _get_random_bytes(batch->entropy_u32, sizeof(batch->entropy_u32)); @@ -1683,15 +1679,6 @@ u32 get_random_u32(void) } EXPORT_SYMBOL(get_random_u32);

-/* It's important to invalidate all potential batched entropy that might - * be stored before the crng is initialized, which we can do lazily by - * bumping the generation counter. - */ -static void invalidate_batched_entropy(void) -{ - atomic_inc(&batch_generation); -} - /** * randomize_page - Generate a random, page aligned address * @start: The smallest acceptable address the caller will take.

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 14c174633f349cb41ea90c2c0aaddac157012f74 upstream.

These explicit tracepoints aren't really used and show sign of aging. It's work to keep these up to date, and before I attempted to keep them up to date, they weren't up to date, which indicates that they're not really used. These days there are better ways of introspecting anyway.

Cc: Theodore Ts'o tytso@mit.edu Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Reviewed-by: Eric Biggers ebiggers@google.com Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 30 ------ include/trace/events/random.h | 195 ------------------------------------------ lib/random32.c | 2 3 files changed, 5 insertions(+), 222 deletions(-) delete mode 100644 include/trace/events/random.h

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -237,9 +237,6 @@ #include <asm/irq_regs.h> #include <asm/io.h>

-#define CREATE_TRACE_POINTS -#include <trace/events/random.h> - enum { POOL_BITS = BLAKE2S_HASH_SIZE * 8, POOL_MIN_BITS = POOL_BITS /* No point in settling for less. */ @@ -315,7 +312,6 @@ static void mix_pool_bytes(const void *i { unsigned long flags;

- trace_mix_pool_bytes(nbytes, _RET_IP_); spin_lock_irqsave(&input_pool.lock, flags); _mix_pool_bytes(in, nbytes); spin_unlock_irqrestore(&input_pool.lock, flags); @@ -389,8 +385,6 @@ static void credit_entropy_bits(size_t n entropy_count = min_t(unsigned int, POOL_BITS, orig + add); } while (cmpxchg(&input_pool.entropy_count, orig, entropy_count) != orig);

- trace_credit_entropy_bits(nbits, entropy_count, _RET_IP_); - if (crng_init < 2 && entropy_count >= POOL_MIN_BITS) crng_reseed(); } @@ -719,7 +713,6 @@ void add_device_randomness(const void *b if (!crng_ready() && size) crng_slow_load(buf, size);

- trace_add_device_randomness(size, _RET_IP_); spin_lock_irqsave(&input_pool.lock, flags); _mix_pool_bytes(buf, size); _mix_pool_bytes(&time, sizeof(time)); @@ -798,7 +791,6 @@ void add_input_randomness(unsigned int t last_value = value; add_timer_randomness(&input_timer_state, (type << 4) ^ code ^ (code >> 4) ^ value); - trace_add_input_randomness(input_pool.entropy_count); } EXPORT_SYMBOL_GPL(add_input_randomness);

@@ -878,7 +870,6 @@ void add_disk_randomness(struct gendisk return; /* first major is 1, so we get >= 0x200 here */ add_timer_randomness(disk->random, 0x100 + disk_devt(disk)); - trace_add_disk_randomness(disk_devt(disk), input_pool.entropy_count); } EXPORT_SYMBOL_GPL(add_disk_randomness); #endif @@ -903,8 +894,6 @@ static void extract_entropy(void *buf, s } block; size_t i;

- trace_extract_entropy(nbytes, input_pool.entropy_count); - for (i = 0; i < ARRAY_SIZE(block.rdseed); ++i) { if (!arch_get_random_seed_long(&block.rdseed[i]) && !arch_get_random_long(&block.rdseed[i])) @@ -976,8 +965,6 @@ static void _get_random_bytes(void *buf, u8 tmp[CHACHA20_BLOCK_SIZE]; size_t len;

- trace_get_random_bytes(nbytes, _RET_IP_); - if (!nbytes) return;

@@ -1174,7 +1161,6 @@ size_t __must_check get_random_bytes_arc size_t left = nbytes; u8 *p = buf;

- trace_get_random_bytes_arch(left, _RET_IP_); while (left) { unsigned long v; size_t chunk = min_t(size_t, left, sizeof(unsigned long)); @@ -1258,16 +1244,6 @@ void rand_initialize_disk(struct gendisk } #endif

-static ssize_t urandom_read_nowarn(struct file *file, char __user *buf, - size_t nbytes, loff_t *ppos) -{ - ssize_t ret; - - ret = get_random_bytes_user(buf, nbytes); - trace_urandom_read(nbytes, input_pool.entropy_count); - return ret; -} - static ssize_t urandom_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) { @@ -1280,7 +1256,7 @@ static ssize_t urandom_read(struct file current->comm, nbytes); }

- return urandom_read_nowarn(file, buf, nbytes, ppos); + return get_random_bytes_user(buf, nbytes); }

static ssize_t random_read(struct file *file, char __user *buf, size_t nbytes, @@ -1291,7 +1267,7 @@ static ssize_t random_read(struct file * ret = wait_for_random_bytes(); if (ret != 0) return ret; - return urandom_read_nowarn(file, buf, nbytes, ppos); + return get_random_bytes_user(buf, nbytes); }

static unsigned int random_poll(struct file *file, poll_table *wait) @@ -1450,7 +1426,7 @@ SYSCALL_DEFINE3(getrandom, char __user * if (unlikely(ret)) return ret; } - return urandom_read_nowarn(NULL, buf, count, NULL); + return get_random_bytes_user(buf, count); }

/******************************************************************** --- a/include/trace/events/random.h +++ /dev/null @@ -1,195 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#undef TRACE_SYSTEM -#define TRACE_SYSTEM random - -#if !defined(_TRACE_RANDOM_H) || defined(TRACE_HEADER_MULTI_READ) -#define _TRACE_RANDOM_H - -#include <linux/writeback.h> -#include <linux/tracepoint.h> - -TRACE_EVENT(add_device_randomness, - TP_PROTO(size_t bytes, unsigned long IP), - - TP_ARGS(bytes, IP), - - TP_STRUCT__entry( - __field(size_t, bytes ) - __field(unsigned long, IP ) - ), - - TP_fast_assign( - __entry->bytes = bytes; - __entry->IP = IP; - ), - - TP_printk("bytes %zu caller %pS", - __entry->bytes, (void *)__entry->IP) -); - -DECLARE_EVENT_CLASS(random__mix_pool_bytes, - TP_PROTO(size_t bytes, unsigned long IP), - - TP_ARGS(bytes, IP), - - TP_STRUCT__entry( - __field(size_t, bytes ) - __field(unsigned long, IP ) - ), - - TP_fast_assign( - __entry->bytes = bytes; - __entry->IP = IP; - ), - - TP_printk("input pool: bytes %zu caller %pS", - __entry->bytes, (void *)__entry->IP) -); - -DEFINE_EVENT(random__mix_pool_bytes, mix_pool_bytes, - TP_PROTO(size_t bytes, unsigned long IP), - - TP_ARGS(bytes, IP) -); - -DEFINE_EVENT(random__mix_pool_bytes, mix_pool_bytes_nolock, - TP_PROTO(int bytes, unsigned long IP), - - TP_ARGS(bytes, IP) -); - -TRACE_EVENT(credit_entropy_bits, - TP_PROTO(size_t bits, size_t entropy_count, unsigned long IP), - - TP_ARGS(bits, entropy_count, IP), - - TP_STRUCT__entry( - __field(size_t, bits ) - __field(size_t, entropy_count ) - __field(unsigned long, IP ) - ), - - TP_fast_assign( - __entry->bits = bits; - __entry->entropy_count = entropy_count; - __entry->IP = IP; - ), - - TP_printk("input pool: bits %zu entropy_count %zu caller %pS", - __entry->bits, __entry->entropy_count, (void *)__entry->IP) -); - -TRACE_EVENT(add_input_randomness, - TP_PROTO(size_t input_bits), - - TP_ARGS(input_bits), - - TP_STRUCT__entry( - __field(size_t, input_bits ) - ), - - TP_fast_assign( - __entry->input_bits = input_bits; - ), - - TP_printk("input_pool_bits %zu", __entry->input_bits) -); - -TRACE_EVENT(add_disk_randomness, - TP_PROTO(dev_t dev, size_t input_bits), - - TP_ARGS(dev, input_bits), - - TP_STRUCT__entry( - __field(dev_t, dev ) - __field(size_t, input_bits ) - ), - - TP_fast_assign( - __entry->dev = dev; - __entry->input_bits = input_bits; - ), - - TP_printk("dev %d,%d input_pool_bits %zu", MAJOR(__entry->dev), - MINOR(__entry->dev), __entry->input_bits) -); - -DECLARE_EVENT_CLASS(random__get_random_bytes, - TP_PROTO(size_t nbytes, unsigned long IP), - - TP_ARGS(nbytes, IP), - - TP_STRUCT__entry( - __field(size_t, nbytes ) - __field(unsigned long, IP ) - ), - - TP_fast_assign( - __entry->nbytes = nbytes; - __entry->IP = IP; - ), - - TP_printk("nbytes %zu caller %pS", __entry->nbytes, (void *)__entry->IP) -); - -DEFINE_EVENT(random__get_random_bytes, get_random_bytes, - TP_PROTO(size_t nbytes, unsigned long IP), - - TP_ARGS(nbytes, IP) -); - -DEFINE_EVENT(random__get_random_bytes, get_random_bytes_arch, - TP_PROTO(size_t nbytes, unsigned long IP), - - TP_ARGS(nbytes, IP) -); - -DECLARE_EVENT_CLASS(random__extract_entropy, - TP_PROTO(size_t nbytes, size_t entropy_count), - - TP_ARGS(nbytes, entropy_count), - - TP_STRUCT__entry( - __field( size_t, nbytes ) - __field( size_t, entropy_count ) - ), - - TP_fast_assign( - __entry->nbytes = nbytes; - __entry->entropy_count = entropy_count; - ), - - TP_printk("input pool: nbytes %zu entropy_count %zu", - __entry->nbytes, __entry->entropy_count) -); - - -DEFINE_EVENT(random__extract_entropy, extract_entropy, - TP_PROTO(size_t nbytes, size_t entropy_count), - - TP_ARGS(nbytes, entropy_count) -); - -TRACE_EVENT(urandom_read, - TP_PROTO(size_t nbytes, size_t entropy_count), - - TP_ARGS(nbytes, entropy_count), - - TP_STRUCT__entry( - __field( size_t, nbytes ) - __field( size_t, entropy_count ) - ), - - TP_fast_assign( - __entry->nbytes = nbytes; - __entry->entropy_count = entropy_count; - ), - - TP_printk("reading: nbytes %zu entropy_count %zu", - __entry->nbytes, __entry->entropy_count) -); - -#endif /* _TRACE_RANDOM_H */ - -/* This part must be outside protection */ -#include <trace/define_trace.h> --- a/lib/random32.c +++ b/lib/random32.c @@ -38,6 +38,8 @@ #include <linux/jiffies.h> #include <linux/random.h> #include <linux/sched.h> +#include <linux/bitops.h> +#include <linux/slab.h> #include <asm/unaligned.h>

/**

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit b2f408fe403800c91a49f6589d95b6759ce1b30b upstream.

In the irq handler, we fill out 16 bytes differently on 32-bit and 64-bit platforms, and for 32-bit vs 64-bit cycle counters, which doesn't always correspond with the bitness of the platform. Whether or not you like this strangeness, it is a matter of fact. But it might not be a fact you well realized until now, because the code that loaded the irq info into 4 32-bit words was quite confusing. Instead, this commit makes everything explicit by having separate (compile-time) branches for 32-bit and 64-bit types.

Cc: Theodore Ts'o tytso@mit.edu Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 49 ++++++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 21 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -283,7 +283,10 @@ static void mix_pool_bytes(const void *i }

struct fast_pool { - u32 pool[4]; + union { + u32 pool32[4]; + u64 pool64[2]; + }; unsigned long last; u16 reg_idx; u8 count; @@ -294,10 +297,10 @@ struct fast_pool { * collector. It's hardcoded for an 128 bit pool and assumes that any * locks that might be needed are taken by the caller. */ -static void fast_mix(struct fast_pool *f) +static void fast_mix(u32 pool[4]) { - u32 a = f->pool[0], b = f->pool[1]; - u32 c = f->pool[2], d = f->pool[3]; + u32 a = pool[0], b = pool[1]; + u32 c = pool[2], d = pool[3];

a += b; c += d; b = rol32(b, 6); d = rol32(d, 27); @@ -315,9 +318,8 @@ static void fast_mix(struct fast_pool *f b = rol32(b, 16); d = rol32(d, 14); d ^= a; b ^= c;

- f->pool[0] = a; f->pool[1] = b; - f->pool[2] = c; f->pool[3] = d; - f->count++; + pool[0] = a; pool[1] = b; + pool[2] = c; pool[3] = d; }

static void process_random_ready_list(void) @@ -782,29 +784,34 @@ void add_interrupt_randomness(int irq) struct pt_regs *regs = get_irq_regs(); unsigned long now = jiffies; cycles_t cycles = random_get_entropy(); - u32 c_high, j_high; - u64 ip;

if (cycles == 0) cycles = get_reg(fast_pool, regs); - c_high = (sizeof(cycles) > 4) ? cycles >> 32 : 0; - j_high = (sizeof(now) > 4) ? now >> 32 : 0; - fast_pool->pool[0] ^= cycles ^ j_high ^ irq; - fast_pool->pool[1] ^= now ^ c_high; - ip = regs ? instruction_pointer(regs) : _RET_IP_; - fast_pool->pool[2] ^= ip; - fast_pool->pool[3] ^= - (sizeof(ip) > 4) ? ip >> 32 : get_reg(fast_pool, regs);

- fast_mix(fast_pool); + if (sizeof(cycles) == 8) + fast_pool->pool64[0] ^= cycles ^ rol64(now, 32) ^ irq; + else { + fast_pool->pool32[0] ^= cycles ^ irq; + fast_pool->pool32[1] ^= now; + } + + if (sizeof(unsigned long) == 8) + fast_pool->pool64[1] ^= regs ? instruction_pointer(regs) : _RET_IP_; + else { + fast_pool->pool32[2] ^= regs ? instruction_pointer(regs) : _RET_IP_; + fast_pool->pool32[3] ^= get_reg(fast_pool, regs); + } + + fast_mix(fast_pool->pool32); + ++fast_pool->count;

if (unlikely(crng_init == 0)) { if (fast_pool->count >= 64 && - crng_fast_load(fast_pool->pool, sizeof(fast_pool->pool)) > 0) { + crng_fast_load(fast_pool->pool32, sizeof(fast_pool->pool32)) > 0) { fast_pool->count = 0; fast_pool->last = now; if (spin_trylock(&input_pool.lock)) { - _mix_pool_bytes(&fast_pool->pool, sizeof(fast_pool->pool)); + _mix_pool_bytes(&fast_pool->pool32, sizeof(fast_pool->pool32)); spin_unlock(&input_pool.lock); } } @@ -818,7 +825,7 @@ void add_interrupt_randomness(int irq) return;

fast_pool->last = now; - _mix_pool_bytes(&fast_pool->pool, sizeof(fast_pool->pool)); + _mix_pool_bytes(&fast_pool->pool32, sizeof(fast_pool->pool32)); spin_unlock(&input_pool.lock);

fast_pool->count = 0;

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 6071a6c0fba2d747742cadcbb3ba26ed756ed73b upstream.

This really adds nothing at all useful.

Cc: Theodore Ts'o tytso@mit.edu Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Reviewed-by: Eric Biggers ebiggers@google.com Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- include/linux/random.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-)

--- a/include/linux/random.h +++ b/include/linux/random.h @@ -1,9 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* - * include/linux/random.h - * - * Include file for the random number generator. - */ + #ifndef _LINUX_RANDOM_H #define _LINUX_RANDOM_H

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 5f1bb112006b104b3e2a1e1b39bbb9b2617581e6 upstream.

This pulls all of the readiness waiting-focused functions into the first labeled section.

No functional changes.

Cc: Theodore Ts'o tytso@mit.edu Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Reviewed-by: Eric Biggers ebiggers@google.com Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 1015 +++++++++++++++++++++++++------------------------- 1 file changed, 527 insertions(+), 488 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -201,126 +201,144 @@ #include <asm/irq_regs.h> #include <asm/io.h>

-enum { - POOL_BITS = BLAKE2S_HASH_SIZE * 8, - POOL_MIN_BITS = POOL_BITS /* No point in settling for less. */ -}; - -/* - * Static global variables - */ -static DECLARE_WAIT_QUEUE_HEAD(random_write_wait); -static struct fasync_struct *fasync; - -static DEFINE_SPINLOCK(random_ready_list_lock); -static LIST_HEAD(random_ready_list); +/********************************************************************* + * + * Initialization and readiness waiting. + * + * Much of the RNG infrastructure is devoted to various dependencies + * being able to wait until the RNG has collected enough entropy and + * is ready for safe consumption. + * + *********************************************************************/

/* * crng_init = 0 --> Uninitialized * 1 --> Initialized * 2 --> Initialized from input_pool * - * crng_init is protected by primary_crng->lock, and only increases + * crng_init is protected by base_crng->lock, and only increases * its value (from 0->1->2). */ static int crng_init = 0; #define crng_ready() (likely(crng_init > 1)) -static int crng_init_cnt = 0; -static void process_random_ready_list(void); -static void _get_random_bytes(void *buf, size_t nbytes); +/* Various types of waiters for crng_init->2 transition. */ +static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait); +static struct fasync_struct *fasync; +static DEFINE_SPINLOCK(random_ready_list_lock); +static LIST_HEAD(random_ready_list);

+/* Control how we warn userspace. */ static struct ratelimit_state unseeded_warning = RATELIMIT_STATE_INIT("warn_unseeded_randomness", HZ, 3); static struct ratelimit_state urandom_warning = RATELIMIT_STATE_INIT("warn_urandom_randomness", HZ, 3); - static int ratelimit_disable __read_mostly; - module_param_named(ratelimit_disable, ratelimit_disable, int, 0644); MODULE_PARM_DESC(ratelimit_disable, "Disable random ratelimit suppression");

-/********************************************************************** - * - * OS independent entropy store. Here are the functions which handle - * storing entropy in an entropy pool. +/* + * Returns whether or not the input pool has been seeded and thus guaranteed + * to supply cryptographically secure random numbers. This applies to: the + * /dev/urandom device, the get_random_bytes function, and the get_random_{u32, + * ,u64,int,long} family of functions. * - **********************************************************************/ - -static struct { - struct blake2s_state hash; - spinlock_t lock; - unsigned int entropy_count; -} input_pool = { - .hash.h = { BLAKE2S_IV0 ^ (0x01010000 | BLAKE2S_HASH_SIZE), - BLAKE2S_IV1, BLAKE2S_IV2, BLAKE2S_IV3, BLAKE2S_IV4, - BLAKE2S_IV5, BLAKE2S_IV6, BLAKE2S_IV7 }, - .hash.outlen = BLAKE2S_HASH_SIZE, - .lock = __SPIN_LOCK_UNLOCKED(input_pool.lock), -}; - -static void extract_entropy(void *buf, size_t nbytes); -static bool drain_entropy(void *buf, size_t nbytes); + * Returns: true if the input pool has been seeded. + * false if the input pool has not been seeded. + */ +bool rng_is_initialized(void) +{ + return crng_ready(); +} +EXPORT_SYMBOL(rng_is_initialized);

-static void crng_reseed(void); +/* Used by wait_for_random_bytes(), and considered an entropy collector, below. */ +static void try_to_generate_entropy(void);

/* - * This function adds bytes into the entropy "pool". It does not - * update the entropy estimate. The caller should call - * credit_entropy_bits if this is appropriate. + * Wait for the input pool to be seeded and thus guaranteed to supply + * cryptographically secure random numbers. This applies to: the /dev/urandom + * device, the get_random_bytes function, and the get_random_{u32,u64,int,long} + * family of functions. Using any of these functions without first calling + * this function forfeits the guarantee of security. + * + * Returns: 0 if the input pool has been seeded. + * -ERESTARTSYS if the function was interrupted by a signal. */ -static void _mix_pool_bytes(const void *in, size_t nbytes) +int wait_for_random_bytes(void) { - blake2s_update(&input_pool.hash, in, nbytes); -} + if (likely(crng_ready())) + return 0;

-static void mix_pool_bytes(const void *in, size_t nbytes) -{ - unsigned long flags; + do { + int ret; + ret = wait_event_interruptible_timeout(crng_init_wait, crng_ready(), HZ); + if (ret) + return ret > 0 ? 0 : ret;

- spin_lock_irqsave(&input_pool.lock, flags); - _mix_pool_bytes(in, nbytes); - spin_unlock_irqrestore(&input_pool.lock, flags); -} + try_to_generate_entropy(); + } while (!crng_ready());

-struct fast_pool { - union { - u32 pool32[4]; - u64 pool64[2]; - }; - unsigned long last; - u16 reg_idx; - u8 count; -}; + return 0; +} +EXPORT_SYMBOL(wait_for_random_bytes);

/* - * This is a fast mixing routine used by the interrupt randomness - * collector. It's hardcoded for an 128 bit pool and assumes that any - * locks that might be needed are taken by the caller. + * Add a callback function that will be invoked when the input + * pool is initialised. + * + * returns: 0 if callback is successfully added + * -EALREADY if pool is already initialised (callback not called) + * -ENOENT if module for callback is not alive */ -static void fast_mix(u32 pool[4]) +int add_random_ready_callback(struct random_ready_callback *rdy) { - u32 a = pool[0], b = pool[1]; - u32 c = pool[2], d = pool[3]; + struct module *owner; + unsigned long flags; + int err = -EALREADY;

- a += b; c += d; - b = rol32(b, 6); d = rol32(d, 27); - d ^= a; b ^= c; + if (crng_ready()) + return err;

- a += b; c += d; - b = rol32(b, 16); d = rol32(d, 14); - d ^= a; b ^= c; + owner = rdy->owner; + if (!try_module_get(owner)) + return -ENOENT;

- a += b; c += d; - b = rol32(b, 6); d = rol32(d, 27); - d ^= a; b ^= c; + spin_lock_irqsave(&random_ready_list_lock, flags); + if (crng_ready()) + goto out;

- a += b; c += d; - b = rol32(b, 16); d = rol32(d, 14); - d ^= a; b ^= c; + owner = NULL;

- pool[0] = a; pool[1] = b; - pool[2] = c; pool[3] = d; + list_add(&rdy->list, &random_ready_list); + err = 0; + +out: + spin_unlock_irqrestore(&random_ready_list_lock, flags); + + module_put(owner); + + return err; +} +EXPORT_SYMBOL(add_random_ready_callback); + +/* + * Delete a previously registered readiness callback function. + */ +void del_random_ready_callback(struct random_ready_callback *rdy) +{ + unsigned long flags; + struct module *owner = NULL; + + spin_lock_irqsave(&random_ready_list_lock, flags); + if (!list_empty(&rdy->list)) { + list_del_init(&rdy->list); + owner = rdy->owner; + } + spin_unlock_irqrestore(&random_ready_list_lock, flags); + + module_put(owner); } +EXPORT_SYMBOL(del_random_ready_callback);

static void process_random_ready_list(void) { @@ -338,27 +356,51 @@ static void process_random_ready_list(vo spin_unlock_irqrestore(&random_ready_list_lock, flags); }

-static void credit_entropy_bits(size_t nbits) +#define warn_unseeded_randomness(previous) \ + _warn_unseeded_randomness(__func__, (void *)_RET_IP_, (previous)) + +static void _warn_unseeded_randomness(const char *func_name, void *caller, void **previous) { - unsigned int entropy_count, orig, add; +#ifdef CONFIG_WARN_ALL_UNSEEDED_RANDOM + const bool print_once = false; +#else + static bool print_once __read_mostly; +#endif

- if (!nbits) + if (print_once || crng_ready() || + (previous && (caller == READ_ONCE(*previous)))) return; - - add = min_t(size_t, nbits, POOL_BITS); - - do { - orig = READ_ONCE(input_pool.entropy_count); - entropy_count = min_t(unsigned int, POOL_BITS, orig + add); - } while (cmpxchg(&input_pool.entropy_count, orig, entropy_count) != orig); - - if (crng_init < 2 && entropy_count >= POOL_MIN_BITS) - crng_reseed(); + WRITE_ONCE(*previous, caller); +#ifndef CONFIG_WARN_ALL_UNSEEDED_RANDOM + print_once = true; +#endif + if (__ratelimit(&unseeded_warning)) + printk_deferred(KERN_NOTICE "random: %s called from %pS with crng_init=%d\n", + func_name, caller, crng_init); }

+ /********************************************************************* * - * CRNG using CHACHA20 + * Fast key erasure RNG, the "crng". + * + * These functions expand entropy from the entropy extractor into + * long streams for external consumption using the "fast key erasure" + * RNG described at https://blog.cr.yp.to/20170723-random.html. + * + * There are a few exported interfaces for use by other drivers: + * + * void get_random_bytes(void *buf, size_t nbytes) + * u32 get_random_u32() + * u64 get_random_u64() + * unsigned int get_random_int() + * unsigned long get_random_long() + * + * These interfaces will return the requested number of random bytes + * into the given buffer or as a return value. This is equivalent to + * a read from /dev/urandom. The integer family of functions may be + * higher performance for one-off random integers, because they do a + * bit of buffering. * *********************************************************************/

@@ -385,72 +427,14 @@ static DEFINE_PER_CPU(struct crng, crngs .generation = ULONG_MAX };

-static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait); +/* Used by crng_reseed() to extract a new seed from the input pool. */ +static bool drain_entropy(void *buf, size_t nbytes);

/* - * crng_fast_load() can be called by code in the interrupt service - * path. So we can't afford to dilly-dally. Returns the number of - * bytes processed from cp. + * This extracts a new crng key from the input pool, but only if there is a + * sufficient amount of entropy available, in order to mitigate bruteforcing + * of newly added bits. */ -static size_t crng_fast_load(const void *cp, size_t len) -{ - unsigned long flags; - const u8 *src = (const u8 *)cp; - size_t ret = 0; - - if (!spin_trylock_irqsave(&base_crng.lock, flags)) - return 0; - if (crng_init != 0) { - spin_unlock_irqrestore(&base_crng.lock, flags); - return 0; - } - while (len > 0 && crng_init_cnt < CRNG_INIT_CNT_THRESH) { - base_crng.key[crng_init_cnt % sizeof(base_crng.key)] ^= *src; - src++; crng_init_cnt++; len--; ret++; - } - if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) { - ++base_crng.generation; - crng_init = 1; - } - spin_unlock_irqrestore(&base_crng.lock, flags); - if (crng_init == 1) - pr_notice("fast init done\n"); - return ret; -} - -/* - * crng_slow_load() is called by add_device_randomness, which has two - * attributes. (1) We can't trust the buffer passed to it is - * guaranteed to be unpredictable (so it might not have any entropy at - * all), and (2) it doesn't have the performance constraints of - * crng_fast_load(). - * - * So, we simply hash the contents in with the current key. Finally, - * we do *not* advance crng_init_cnt since buffer we may get may be - * something like a fixed DMI table (for example), which might very - * well be unique to the machine, but is otherwise unvarying. - */ -static void crng_slow_load(const void *cp, size_t len) -{ - unsigned long flags; - struct blake2s_state hash; - - blake2s_init(&hash, sizeof(base_crng.key)); - - if (!spin_trylock_irqsave(&base_crng.lock, flags)) - return; - if (crng_init != 0) { - spin_unlock_irqrestore(&base_crng.lock, flags); - return; - } - - blake2s_update(&hash, base_crng.key, sizeof(base_crng.key)); - blake2s_update(&hash, cp, len); - blake2s_final(&hash, base_crng.key); - - spin_unlock_irqrestore(&base_crng.lock, flags); -} - static void crng_reseed(void) { unsigned long flags; @@ -500,13 +484,11 @@ static void crng_reseed(void) }

/* - * The general form here is based on a "fast key erasure RNG" from - * https://blog.cr.yp.to/20170723-random.html. It generates a ChaCha - * block using the provided key, and then immediately overwites that - * key with half the block. It returns the resultant ChaCha state to the - * user, along with the second half of the block containing 32 bytes of - * random data that may be used; random_data_len may not be greater than - * 32. + * This generates a ChaCha block using the provided key, and then + * immediately overwites that key with half the block. It returns + * the resultant ChaCha state to the user, along with the second + * half of the block containing 32 bytes of random data that may + * be used; random_data_len may not be greater than 32. */ static void crng_fast_key_erasure(u8 key[CHACHA20_KEY_SIZE], u32 chacha_state[CHACHA20_BLOCK_SIZE / sizeof(u32)], @@ -593,6 +575,126 @@ static void crng_make_state(u32 chacha_s local_irq_restore(flags); }

+/* + * This function is for crng_init == 0 only. + * + * crng_fast_load() can be called by code in the interrupt service + * path. So we can't afford to dilly-dally. Returns the number of + * bytes processed from cp. + */ +static size_t crng_fast_load(const void *cp, size_t len) +{ + static int crng_init_cnt = 0; + unsigned long flags; + const u8 *src = (const u8 *)cp; + size_t ret = 0; + + if (!spin_trylock_irqsave(&base_crng.lock, flags)) + return 0; + if (crng_init != 0) { + spin_unlock_irqrestore(&base_crng.lock, flags); + return 0; + } + while (len > 0 && crng_init_cnt < CRNG_INIT_CNT_THRESH) { + base_crng.key[crng_init_cnt % sizeof(base_crng.key)] ^= *src; + src++; crng_init_cnt++; len--; ret++; + } + if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) { + ++base_crng.generation; + crng_init = 1; + } + spin_unlock_irqrestore(&base_crng.lock, flags); + if (crng_init == 1) + pr_notice("fast init done\n"); + return ret; +} + +/* + * This function is for crng_init == 0 only. + * + * crng_slow_load() is called by add_device_randomness, which has two + * attributes. (1) We can't trust the buffer passed to it is + * guaranteed to be unpredictable (so it might not have any entropy at + * all), and (2) it doesn't have the performance constraints of + * crng_fast_load(). + * + * So, we simply hash the contents in with the current key. Finally, + * we do *not* advance crng_init_cnt since buffer we may get may be + * something like a fixed DMI table (for example), which might very + * well be unique to the machine, but is otherwise unvarying. + */ +static void crng_slow_load(const void *cp, size_t len) +{ + unsigned long flags; + struct blake2s_state hash; + + blake2s_init(&hash, sizeof(base_crng.key)); + + if (!spin_trylock_irqsave(&base_crng.lock, flags)) + return; + if (crng_init != 0) { + spin_unlock_irqrestore(&base_crng.lock, flags); + return; + } + + blake2s_update(&hash, base_crng.key, sizeof(base_crng.key)); + blake2s_update(&hash, cp, len); + blake2s_final(&hash, base_crng.key); + + spin_unlock_irqrestore(&base_crng.lock, flags); +} + +static void _get_random_bytes(void *buf, size_t nbytes) +{ + u32 chacha_state[CHACHA20_BLOCK_SIZE / sizeof(u32)]; + u8 tmp[CHACHA20_BLOCK_SIZE]; + size_t len; + + if (!nbytes) + return; + + len = min_t(size_t, 32, nbytes); + crng_make_state(chacha_state, buf, len); + nbytes -= len; + buf += len; + + while (nbytes) { + if (nbytes < CHACHA20_BLOCK_SIZE) { + chacha20_block(chacha_state, tmp); + memcpy(buf, tmp, nbytes); + memzero_explicit(tmp, sizeof(tmp)); + break; + } + + chacha20_block(chacha_state, buf); + if (unlikely(chacha_state[12] == 0)) + ++chacha_state[13]; + nbytes -= CHACHA20_BLOCK_SIZE; + buf += CHACHA20_BLOCK_SIZE; + } + + memzero_explicit(chacha_state, sizeof(chacha_state)); +} + +/* + * This function is the exported kernel interface. It returns some + * number of good random numbers, suitable for key generation, seeding + * TCP sequence numbers, etc. It does not rely on the hardware random + * number generator. For random bytes direct from the hardware RNG + * (when available), use get_random_bytes_arch(). In order to ensure + * that the randomness provided by this function is okay, the function + * wait_for_random_bytes() should be called and return 0 at least once + * at any point prior. + */ +void get_random_bytes(void *buf, size_t nbytes) +{ + static void *previous; + + warn_unseeded_randomness(&previous); + _get_random_bytes(buf, nbytes); +} +EXPORT_SYMBOL(get_random_bytes); + static ssize_t get_random_bytes_user(void __user *buf, size_t nbytes) { bool large_request = nbytes > 256; @@ -640,6 +742,265 @@ static ssize_t get_random_bytes_user(voi return ret; }

+/* + * Batched entropy returns random integers. The quality of the random + * number is good as /dev/urandom. In order to ensure that the randomness + * provided by this function is okay, the function wait_for_random_bytes() + * should be called and return 0 at least once at any point prior. + */ +struct batched_entropy { + union { + /* + * We make this 1.5x a ChaCha block, so that we get the + * remaining 32 bytes from fast key erasure, plus one full + * block from the detached ChaCha state. We can increase + * the size of this later if needed so long as we keep the + * formula of (integer_blocks + 0.5) * CHACHA20_BLOCK_SIZE. + */ + u64 entropy_u64[CHACHA20_BLOCK_SIZE * 3 / (2 * sizeof(u64))]; + u32 entropy_u32[CHACHA20_BLOCK_SIZE * 3 / (2 * sizeof(u32))]; + }; + unsigned long generation; + unsigned int position; +}; + + +static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = { + .position = UINT_MAX +}; + +u64 get_random_u64(void) +{ + u64 ret; + unsigned long flags; + struct batched_entropy *batch; + static void *previous; + unsigned long next_gen; + + warn_unseeded_randomness(&previous); + + local_irq_save(flags); + batch = raw_cpu_ptr(&batched_entropy_u64); + + next_gen = READ_ONCE(base_crng.generation); + if (batch->position >= ARRAY_SIZE(batch->entropy_u64) || + next_gen != batch->generation) { + _get_random_bytes(batch->entropy_u64, sizeof(batch->entropy_u64)); + batch->position = 0; + batch->generation = next_gen; + } + + ret = batch->entropy_u64[batch->position]; + batch->entropy_u64[batch->position] = 0; + ++batch->position; + local_irq_restore(flags); + return ret; +} +EXPORT_SYMBOL(get_random_u64); + +static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = { + .position = UINT_MAX +}; + +u32 get_random_u32(void) +{ + u32 ret; + unsigned long flags; + struct batched_entropy *batch; + static void *previous; + unsigned long next_gen; + + warn_unseeded_randomness(&previous); + + local_irq_save(flags); + batch = raw_cpu_ptr(&batched_entropy_u32); + + next_gen = READ_ONCE(base_crng.generation); + if (batch->position >= ARRAY_SIZE(batch->entropy_u32) || + next_gen != batch->generation) { + _get_random_bytes(batch->entropy_u32, sizeof(batch->entropy_u32)); + batch->position = 0; + batch->generation = next_gen; + } + + ret = batch->entropy_u32[batch->position]; + batch->entropy_u32[batch->position] = 0; + ++batch->position; + local_irq_restore(flags); + return ret; +} +EXPORT_SYMBOL(get_random_u32); + +/** + * randomize_page - Generate a random, page aligned address + * @start: The smallest acceptable address the caller will take. + * @range: The size of the area, starting at @start, within which the + * random address must fall. + * + * If @start + @range would overflow, @range is capped. + * + * NOTE: Historical use of randomize_range, which this replaces, presumed that + * @start was already page aligned. We now align it regardless. + * + * Return: A page aligned address within [start, start + range). On error, + * @start is returned. + */ +unsigned long randomize_page(unsigned long start, unsigned long range) +{ + if (!PAGE_ALIGNED(start)) { + range -= PAGE_ALIGN(start) - start; + start = PAGE_ALIGN(start); + } + + if (start > ULONG_MAX - range) + range = ULONG_MAX - start; + + range >>= PAGE_SHIFT; + + if (range == 0) + return start; + + return start + (get_random_long() % range << PAGE_SHIFT); +} + +/* + * This function will use the architecture-specific hardware random + * number generator if it is available. It is not recommended for + * use. Use get_random_bytes() instead. It returns the number of + * bytes filled in. + */ +size_t __must_check get_random_bytes_arch(void *buf, size_t nbytes) +{ + size_t left = nbytes; + u8 *p = buf; + + while (left) { + unsigned long v; + size_t chunk = min_t(size_t, left, sizeof(unsigned long)); + + if (!arch_get_random_long(&v)) + break; + + memcpy(p, &v, chunk); + p += chunk; + left -= chunk; + } + + return nbytes - left; +} +EXPORT_SYMBOL(get_random_bytes_arch); + +enum { + POOL_BITS = BLAKE2S_HASH_SIZE * 8, + POOL_MIN_BITS = POOL_BITS /* No point in settling for less. */ +}; + +/* + * Static global variables + */ +static DECLARE_WAIT_QUEUE_HEAD(random_write_wait); + +/********************************************************************** + * + * OS independent entropy store. Here are the functions which handle + * storing entropy in an entropy pool. + * + **********************************************************************/ + +static struct { + struct blake2s_state hash; + spinlock_t lock; + unsigned int entropy_count; +} input_pool = { + .hash.h = { BLAKE2S_IV0 ^ (0x01010000 | BLAKE2S_HASH_SIZE), + BLAKE2S_IV1, BLAKE2S_IV2, BLAKE2S_IV3, BLAKE2S_IV4, + BLAKE2S_IV5, BLAKE2S_IV6, BLAKE2S_IV7 }, + .hash.outlen = BLAKE2S_HASH_SIZE, + .lock = __SPIN_LOCK_UNLOCKED(input_pool.lock), +}; + +static void extract_entropy(void *buf, size_t nbytes); +static bool drain_entropy(void *buf, size_t nbytes); + +static void crng_reseed(void); + +/* + * This function adds bytes into the entropy "pool". It does not + * update the entropy estimate. The caller should call + * credit_entropy_bits if this is appropriate. + */ +static void _mix_pool_bytes(const void *in, size_t nbytes) +{ + blake2s_update(&input_pool.hash, in, nbytes); +} + +static void mix_pool_bytes(const void *in, size_t nbytes) +{ + unsigned long flags; + + spin_lock_irqsave(&input_pool.lock, flags); + _mix_pool_bytes(in, nbytes); + spin_unlock_irqrestore(&input_pool.lock, flags); +} + +struct fast_pool { + union { + u32 pool32[4]; + u64 pool64[2]; + }; + unsigned long last; + u16 reg_idx; + u8 count; +}; + +/* + * This is a fast mixing routine used by the interrupt randomness + * collector. It's hardcoded for an 128 bit pool and assumes that any + * locks that might be needed are taken by the caller. + */ +static void fast_mix(u32 pool[4]) +{ + u32 a = pool[0], b = pool[1]; + u32 c = pool[2], d = pool[3]; + + a += b; c += d; + b = rol32(b, 6); d = rol32(d, 27); + d ^= a; b ^= c; + + a += b; c += d; + b = rol32(b, 16); d = rol32(d, 14); + d ^= a; b ^= c; + + a += b; c += d; + b = rol32(b, 6); d = rol32(d, 27); + d ^= a; b ^= c; + + a += b; c += d; + b = rol32(b, 16); d = rol32(d, 14); + d ^= a; b ^= c; + + pool[0] = a; pool[1] = b; + pool[2] = c; pool[3] = d; +} + +static void credit_entropy_bits(size_t nbits) +{ + unsigned int entropy_count, orig, add; + + if (!nbits) + return; + + add = min_t(size_t, nbits, POOL_BITS); + + do { + orig = READ_ONCE(input_pool.entropy_count); + entropy_count = min_t(unsigned int, POOL_BITS, orig + add); + } while (cmpxchg(&input_pool.entropy_count, orig, entropy_count) != orig); + + if (crng_init < 2 && entropy_count >= POOL_MIN_BITS) + crng_reseed(); +} + /********************************************************************* * * Entropy input management @@ -907,80 +1268,6 @@ static bool drain_entropy(void *buf, siz return true; }

-#define warn_unseeded_randomness(previous) \ - _warn_unseeded_randomness(__func__, (void *)_RET_IP_, (previous)) - -static void _warn_unseeded_randomness(const char *func_name, void *caller, void **previous) -{ -#ifdef CONFIG_WARN_ALL_UNSEEDED_RANDOM - const bool print_once = false; -#else - static bool print_once __read_mostly; -#endif - - if (print_once || crng_ready() || - (previous && (caller == READ_ONCE(*previous)))) - return; - WRITE_ONCE(*previous, caller); -#ifndef CONFIG_WARN_ALL_UNSEEDED_RANDOM - print_once = true; -#endif - if (__ratelimit(&unseeded_warning)) - printk_deferred(KERN_NOTICE "random: %s called from %pS with crng_init=%d\n", - func_name, caller, crng_init); -} - -/* - * This function is the exported kernel interface. It returns some - * number of good random numbers, suitable for key generation, seeding - * TCP sequence numbers, etc. It does not rely on the hardware random - * number generator. For random bytes direct from the hardware RNG - * (when available), use get_random_bytes_arch(). In order to ensure - * that the randomness provided by this function is okay, the function - * wait_for_random_bytes() should be called and return 0 at least once - * at any point prior. - */ -static void _get_random_bytes(void *buf, size_t nbytes) -{ - u32 chacha_state[CHACHA20_BLOCK_SIZE / sizeof(u32)]; - u8 tmp[CHACHA20_BLOCK_SIZE]; - size_t len; - - if (!nbytes) - return; - - len = min_t(size_t, 32, nbytes); - crng_make_state(chacha_state, buf, len); - nbytes -= len; - buf += len; - - while (nbytes) { - if (nbytes < CHACHA20_BLOCK_SIZE) { - chacha20_block(chacha_state, tmp); - memcpy(buf, tmp, nbytes); - memzero_explicit(tmp, sizeof(tmp)); - break; - } - - chacha20_block(chacha_state, buf); - if (unlikely(chacha_state[12] == 0)) - ++chacha_state[13]; - nbytes -= CHACHA20_BLOCK_SIZE; - buf += CHACHA20_BLOCK_SIZE; - } - - memzero_explicit(chacha_state, sizeof(chacha_state)); -} - -void get_random_bytes(void *buf, size_t nbytes) -{ - static void *previous; - - warn_unseeded_randomness(&previous); - _get_random_bytes(buf, nbytes); -} -EXPORT_SYMBOL(get_random_bytes); - /* * Each time the timer fires, we expect that we got an unpredictable * jump in the cycle counter. Even if the timer is running on another @@ -1030,134 +1317,6 @@ static void try_to_generate_entropy(void mix_pool_bytes(&stack.now, sizeof(stack.now)); }

-/* - * Wait for the urandom pool to be seeded and thus guaranteed to supply - * cryptographically secure random numbers. This applies to: the /dev/urandom - * device, the get_random_bytes function, and the get_random_{u32,u64,int,long} - * family of functions. Using any of these functions without first calling - * this function forfeits the guarantee of security. - * - * Returns: 0 if the urandom pool has been seeded. - * -ERESTARTSYS if the function was interrupted by a signal. - */ -int wait_for_random_bytes(void) -{ - if (likely(crng_ready())) - return 0; - - do { - int ret; - ret = wait_event_interruptible_timeout(crng_init_wait, crng_ready(), HZ); - if (ret) - return ret > 0 ? 0 : ret; - - try_to_generate_entropy(); - } while (!crng_ready()); - - return 0; -} -EXPORT_SYMBOL(wait_for_random_bytes); - -/* - * Returns whether or not the urandom pool has been seeded and thus guaranteed - * to supply cryptographically secure random numbers. This applies to: the - * /dev/urandom device, the get_random_bytes function, and the get_random_{u32, - * ,u64,int,long} family of functions. - * - * Returns: true if the urandom pool has been seeded. - * false if the urandom pool has not been seeded. - */ -bool rng_is_initialized(void) -{ - return crng_ready(); -} -EXPORT_SYMBOL(rng_is_initialized); - -/* - * Add a callback function that will be invoked when the nonblocking - * pool is initialised. - * - * returns: 0 if callback is successfully added - * -EALREADY if pool is already initialised (callback not called) - * -ENOENT if module for callback is not alive - */ -int add_random_ready_callback(struct random_ready_callback *rdy) -{ - struct module *owner; - unsigned long flags; - int err = -EALREADY; - - if (crng_ready()) - return err; - - owner = rdy->owner; - if (!try_module_get(owner)) - return -ENOENT; - - spin_lock_irqsave(&random_ready_list_lock, flags); - if (crng_ready()) - goto out; - - owner = NULL; - - list_add(&rdy->list, &random_ready_list); - err = 0; - -out: - spin_unlock_irqrestore(&random_ready_list_lock, flags); - - module_put(owner); - - return err; -} -EXPORT_SYMBOL(add_random_ready_callback); - -/* - * Delete a previously registered readiness callback function. - */ -void del_random_ready_callback(struct random_ready_callback *rdy) -{ - unsigned long flags; - struct module *owner = NULL; - - spin_lock_irqsave(&random_ready_list_lock, flags); - if (!list_empty(&rdy->list)) { - list_del_init(&rdy->list); - owner = rdy->owner; - } - spin_unlock_irqrestore(&random_ready_list_lock, flags); - - module_put(owner); -} -EXPORT_SYMBOL(del_random_ready_callback); - -/* - * This function will use the architecture-specific hardware random - * number generator if it is available. It is not recommended for - * use. Use get_random_bytes() instead. It returns the number of - * bytes filled in. - */ -size_t __must_check get_random_bytes_arch(void *buf, size_t nbytes) -{ - size_t left = nbytes; - u8 *p = buf; - - while (left) { - unsigned long v; - size_t chunk = min_t(size_t, left, sizeof(unsigned long)); - - if (!arch_get_random_long(&v)) - break; - - memcpy(p, &v, chunk); - p += chunk; - left -= chunk; - } - - return nbytes - left; -} -EXPORT_SYMBOL(get_random_bytes_arch); - static bool trust_cpu __ro_after_init = IS_ENABLED(CONFIG_RANDOM_TRUST_CPU); static int __init parse_trust_cpu(char *arg) { @@ -1508,126 +1667,6 @@ struct ctl_table random_table[] = { }; #endif /* CONFIG_SYSCTL */

-struct batched_entropy { - union { - /* - * We make this 1.5x a ChaCha block, so that we get the - * remaining 32 bytes from fast key erasure, plus one full - * block from the detached ChaCha state. We can increase - * the size of this later if needed so long as we keep the - * formula of (integer_blocks + 0.5) * CHACHA20_BLOCK_SIZE. - */ - u64 entropy_u64[CHACHA20_BLOCK_SIZE * 3 / (2 * sizeof(u64))]; - u32 entropy_u32[CHACHA20_BLOCK_SIZE * 3 / (2 * sizeof(u32))]; - }; - unsigned long generation; - unsigned int position; -}; - -/* - * Get a random word for internal kernel use only. The quality of the random - * number is good as /dev/urandom. In order to ensure that the randomness - * provided by this function is okay, the function wait_for_random_bytes() - * should be called and return 0 at least once at any point prior. - */ -static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = { - .position = UINT_MAX -}; - -u64 get_random_u64(void) -{ - u64 ret; - unsigned long flags; - struct batched_entropy *batch; - static void *previous; - unsigned long next_gen; - - warn_unseeded_randomness(&previous); - - local_irq_save(flags); - batch = raw_cpu_ptr(&batched_entropy_u64); - - next_gen = READ_ONCE(base_crng.generation); - if (batch->position >= ARRAY_SIZE(batch->entropy_u64) || - next_gen != batch->generation) { - _get_random_bytes(batch->entropy_u64, sizeof(batch->entropy_u64)); - batch->position = 0; - batch->generation = next_gen; - } - - ret = batch->entropy_u64[batch->position]; - batch->entropy_u64[batch->position] = 0; - ++batch->position; - local_irq_restore(flags); - return ret; -} -EXPORT_SYMBOL(get_random_u64); - -static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = { - .position = UINT_MAX -}; - -u32 get_random_u32(void) -{ - u32 ret; - unsigned long flags; - struct batched_entropy *batch; - static void *previous; - unsigned long next_gen; - - warn_unseeded_randomness(&previous); - - local_irq_save(flags); - batch = raw_cpu_ptr(&batched_entropy_u32); - - next_gen = READ_ONCE(base_crng.generation); - if (batch->position >= ARRAY_SIZE(batch->entropy_u32) || - next_gen != batch->generation) { - _get_random_bytes(batch->entropy_u32, sizeof(batch->entropy_u32)); - batch->position = 0; - batch->generation = next_gen; - } - - ret = batch->entropy_u32[batch->position]; - batch->entropy_u32[batch->position] = 0; - ++batch->position; - local_irq_restore(flags); - return ret; -} -EXPORT_SYMBOL(get_random_u32); - -/** - * randomize_page - Generate a random, page aligned address - * @start: The smallest acceptable address the caller will take. - * @range: The size of the area, starting at @start, within which the - * random address must fall. - * - * If @start + @range would overflow, @range is capped. - * - * NOTE: Historical use of randomize_range, which this replaces, presumed that - * @start was already page aligned. We now align it regardless. - * - * Return: A page aligned address within [start, start + range). On error, - * @start is returned. - */ -unsigned long randomize_page(unsigned long start, unsigned long range) -{ - if (!PAGE_ALIGNED(start)) { - range -= PAGE_ALIGN(start) - start; - start = PAGE_ALIGN(start); - } - - if (start > ULONG_MAX - range) - range = ULONG_MAX - start; - - range >>= PAGE_SHIFT; - - if (range == 0) - return start; - - return start + (get_random_long() % range << PAGE_SHIFT); -} - /* Interface for in-kernel drivers of true hardware RNGs. * Those devices may produce endless random bits and will be throttled * when our pool is full.

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 92c653cf14400946f376a29b828d6af7e01f38dd upstream.

This pulls all of the entropy collection-focused functions into the fourth labeled section.

No functional changes.

Cc: Theodore Ts'o tytso@mit.edu Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Reviewed-by: Eric Biggers ebiggers@google.com Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 370 +++++++++++++++++++++++++++----------------------- 1 file changed, 206 insertions(+), 164 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1034,60 +1034,112 @@ static bool drain_entropy(void *buf, siz return true; }

-struct fast_pool { - union { - u32 pool32[4]; - u64 pool64[2]; - }; - unsigned long last; - u16 reg_idx; - u8 count; -}; + +/********************************************************************** + * + * Entropy collection routines. + * + * The following exported functions are used for pushing entropy into + * the above entropy accumulation routines: + * + * void add_device_randomness(const void *buf, size_t size); + * void add_input_randomness(unsigned int type, unsigned int code, + * unsigned int value); + * void add_disk_randomness(struct gendisk *disk); + * void add_hwgenerator_randomness(const void *buffer, size_t count, + * size_t entropy); + * void add_bootloader_randomness(const void *buf, size_t size); + * void add_interrupt_randomness(int irq); + * + * add_device_randomness() adds data to the input pool that + * is likely to differ between two devices (or possibly even per boot). + * This would be things like MAC addresses or serial numbers, or the + * read-out of the RTC. This does *not* credit any actual entropy to + * the pool, but it initializes the pool to different values for devices + * that might otherwise be identical and have very little entropy + * available to them (particularly common in the embedded world). + * + * add_input_randomness() uses the input layer interrupt timing, as well + * as the event type information from the hardware. + * + * add_disk_randomness() uses what amounts to the seek time of block + * layer request events, on a per-disk_devt basis, as input to the + * entropy pool. Note that high-speed solid state drives with very low + * seek times do not make for good sources of entropy, as their seek + * times are usually fairly consistent. + * + * The above two routines try to estimate how many bits of entropy + * to credit. They do this by keeping track of the first and second + * order deltas of the event timings. + * + * add_hwgenerator_randomness() is for true hardware RNGs, and will credit + * entropy as specified by the caller. If the entropy pool is full it will + * block until more entropy is needed. + * + * add_bootloader_randomness() is the same as add_hwgenerator_randomness() or + * add_device_randomness(), depending on whether or not the configuration + * option CONFIG_RANDOM_TRUST_BOOTLOADER is set. + * + * add_interrupt_randomness() uses the interrupt timing as random + * inputs to the entropy pool. Using the cycle counters and the irq source + * as inputs, it feeds the input pool roughly once a second or after 64 + * interrupts, crediting 1 bit of entropy for whichever comes first. + * + **********************************************************************/ + +static bool trust_cpu __ro_after_init = IS_ENABLED(CONFIG_RANDOM_TRUST_CPU); +static int __init parse_trust_cpu(char *arg) +{ + return kstrtobool(arg, &trust_cpu); +} +early_param("random.trust_cpu", parse_trust_cpu);

/* - * This is a fast mixing routine used by the interrupt randomness - * collector. It's hardcoded for an 128 bit pool and assumes that any - * locks that might be needed are taken by the caller. + * The first collection of entropy occurs at system boot while interrupts + * are still turned off. Here we push in RDSEED, a timestamp, and utsname(). + * Depending on the above configuration knob, RDSEED may be considered + * sufficient for initialization. Note that much earlier setup may already + * have pushed entropy into the input pool by the time we get here. */ -static void fast_mix(u32 pool[4]) +int __init rand_initialize(void) { - u32 a = pool[0], b = pool[1]; - u32 c = pool[2], d = pool[3]; - - a += b; c += d; - b = rol32(b, 6); d = rol32(d, 27); - d ^= a; b ^= c; + size_t i; + ktime_t now = ktime_get_real(); + bool arch_init = true; + unsigned long rv;

- a += b; c += d; - b = rol32(b, 16); d = rol32(d, 14); - d ^= a; b ^= c; + for (i = 0; i < BLAKE2S_BLOCK_SIZE; i += sizeof(rv)) { + if (!arch_get_random_seed_long_early(&rv) && + !arch_get_random_long_early(&rv)) { + rv = random_get_entropy(); + arch_init = false; + } + mix_pool_bytes(&rv, sizeof(rv)); + } + mix_pool_bytes(&now, sizeof(now)); + mix_pool_bytes(utsname(), sizeof(*(utsname())));

- a += b; c += d; - b = rol32(b, 6); d = rol32(d, 27); - d ^= a; b ^= c; + extract_entropy(base_crng.key, sizeof(base_crng.key)); + ++base_crng.generation;

- a += b; c += d; - b = rol32(b, 16); d = rol32(d, 14); - d ^= a; b ^= c; + if (arch_init && trust_cpu && crng_init < 2) { + crng_init = 2; + pr_notice("crng init done (trusting CPU's manufacturer)\n"); + }

- pool[0] = a; pool[1] = b; - pool[2] = c; pool[3] = d; + if (ratelimit_disable) { + urandom_warning.interval = 0; + unseeded_warning.interval = 0; + } + return 0; }

-/********************************************************************* - * - * Entropy input management - * - *********************************************************************/ - /* There is one of these per entropy source */ struct timer_rand_state { cycles_t last_time; long last_delta, last_delta2; };

-#define INIT_TIMER_RAND_STATE { INITIAL_JIFFIES, }; - /* * Add device- or boot-specific data to the input pool to help * initialize it. @@ -1111,8 +1163,6 @@ void add_device_randomness(const void *b } EXPORT_SYMBOL(add_device_randomness);

-static struct timer_rand_state input_timer_state = INIT_TIMER_RAND_STATE; - /* * This function adds entropy to the entropy "pool" by using timing * delays. It uses the timer_rand_state structure to make an estimate @@ -1174,8 +1224,9 @@ void add_input_randomness(unsigned int t unsigned int value) { static unsigned char last_value; + static struct timer_rand_state input_timer_state = { INITIAL_JIFFIES };

- /* ignore autorepeat and the like */ + /* Ignore autorepeat and the like. */ if (value == last_value) return;

@@ -1185,6 +1236,119 @@ void add_input_randomness(unsigned int t } EXPORT_SYMBOL_GPL(add_input_randomness);

+#ifdef CONFIG_BLOCK +void add_disk_randomness(struct gendisk *disk) +{ + if (!disk || !disk->random) + return; + /* First major is 1, so we get >= 0x200 here. */ + add_timer_randomness(disk->random, 0x100 + disk_devt(disk)); +} +EXPORT_SYMBOL_GPL(add_disk_randomness); + +void rand_initialize_disk(struct gendisk *disk) +{ + struct timer_rand_state *state; + + /* + * If kzalloc returns null, we just won't use that entropy + * source. + */ + state = kzalloc(sizeof(struct timer_rand_state), GFP_KERNEL); + if (state) { + state->last_time = INITIAL_JIFFIES; + disk->random = state; + } +} +#endif + +/* + * Interface for in-kernel drivers of true hardware RNGs. + * Those devices may produce endless random bits and will be throttled + * when our pool is full. + */ +void add_hwgenerator_randomness(const void *buffer, size_t count, + size_t entropy) +{ + if (unlikely(crng_init == 0)) { + size_t ret = crng_fast_load(buffer, count); + mix_pool_bytes(buffer, ret); + count -= ret; + buffer += ret; + if (!count || crng_init == 0) + return; + } + + /* + * Throttle writing if we're above the trickle threshold. + * We'll be woken up again once below POOL_MIN_BITS, when + * the calling thread is about to terminate, or once + * CRNG_RESEED_INTERVAL has elapsed. + */ + wait_event_interruptible_timeout(random_write_wait, + !system_wq || kthread_should_stop() || + input_pool.entropy_count < POOL_MIN_BITS, + CRNG_RESEED_INTERVAL); + mix_pool_bytes(buffer, count); + credit_entropy_bits(entropy); +} +EXPORT_SYMBOL_GPL(add_hwgenerator_randomness); + +/* + * Handle random seed passed by bootloader. + * If the seed is trustworthy, it would be regarded as hardware RNGs. Otherwise + * it would be regarded as device data. + * The decision is controlled by CONFIG_RANDOM_TRUST_BOOTLOADER. + */ +void add_bootloader_randomness(const void *buf, size_t size) +{ + if (IS_ENABLED(CONFIG_RANDOM_TRUST_BOOTLOADER)) + add_hwgenerator_randomness(buf, size, size * 8); + else + add_device_randomness(buf, size); +} +EXPORT_SYMBOL_GPL(add_bootloader_randomness); + +struct fast_pool { + union { + u32 pool32[4]; + u64 pool64[2]; + }; + unsigned long last; + u16 reg_idx; + u8 count; +}; + +/* + * This is a fast mixing routine used by the interrupt randomness + * collector. It's hardcoded for an 128 bit pool and assumes that any + * locks that might be needed are taken by the caller. + */ +static void fast_mix(u32 pool[4]) +{ + u32 a = pool[0], b = pool[1]; + u32 c = pool[2], d = pool[3]; + + a += b; c += d; + b = rol32(b, 6); d = rol32(d, 27); + d ^= a; b ^= c; + + a += b; c += d; + b = rol32(b, 16); d = rol32(d, 14); + d ^= a; b ^= c; + + a += b; c += d; + b = rol32(b, 6); d = rol32(d, 27); + d ^= a; b ^= c; + + a += b; c += d; + b = rol32(b, 16); d = rol32(d, 14); + d ^= a; b ^= c; + + pool[0] = a; pool[1] = b; + pool[2] = c; pool[3] = d; +} + static DEFINE_PER_CPU(struct fast_pool, irq_randomness);

static u32 get_reg(struct fast_pool *f, struct pt_regs *regs) @@ -1254,22 +1418,11 @@ void add_interrupt_randomness(int irq)

fast_pool->count = 0;

- /* award one bit for the contents of the fast pool */ + /* Award one bit for the contents of the fast pool. */ credit_entropy_bits(1); } EXPORT_SYMBOL_GPL(add_interrupt_randomness);

-#ifdef CONFIG_BLOCK -void add_disk_randomness(struct gendisk *disk) -{ - if (!disk || !disk->random) - return; - /* first major is 1, so we get >= 0x200 here */ - add_timer_randomness(disk->random, 0x100 + disk_devt(disk)); -} -EXPORT_SYMBOL_GPL(add_disk_randomness); -#endif - /* * Each time the timer fires, we expect that we got an unpredictable * jump in the cycle counter. Even if the timer is running on another @@ -1319,73 +1472,6 @@ static void try_to_generate_entropy(void mix_pool_bytes(&stack.now, sizeof(stack.now)); }

-static bool trust_cpu __ro_after_init = IS_ENABLED(CONFIG_RANDOM_TRUST_CPU); -static int __init parse_trust_cpu(char *arg) -{ - return kstrtobool(arg, &trust_cpu); -} -early_param("random.trust_cpu", parse_trust_cpu); - -/* - * Note that setup_arch() may call add_device_randomness() - * long before we get here. This allows seeding of the pools - * with some platform dependent data very early in the boot - * process. But it limits our options here. We must use - * statically allocated structures that already have all - * initializations complete at compile time. We should also - * take care not to overwrite the precious per platform data - * we were given. - */ -int __init rand_initialize(void) -{ - size_t i; - ktime_t now = ktime_get_real(); - bool arch_init = true; - unsigned long rv; - - for (i = 0; i < BLAKE2S_BLOCK_SIZE; i += sizeof(rv)) { - if (!arch_get_random_seed_long_early(&rv) && - !arch_get_random_long_early(&rv)) { - rv = random_get_entropy(); - arch_init = false; - } - mix_pool_bytes(&rv, sizeof(rv)); - } - mix_pool_bytes(&now, sizeof(now)); - mix_pool_bytes(utsname(), sizeof(*(utsname()))); - - extract_entropy(base_crng.key, sizeof(base_crng.key)); - ++base_crng.generation; - - if (arch_init && trust_cpu && crng_init < 2) { - crng_init = 2; - pr_notice("crng init done (trusting CPU's manufacturer)\n"); - } - - if (ratelimit_disable) { - urandom_warning.interval = 0; - unseeded_warning.interval = 0; - } - return 0; -} - -#ifdef CONFIG_BLOCK -void rand_initialize_disk(struct gendisk *disk) -{ - struct timer_rand_state *state; - - /* - * If kzalloc returns null, we just won't use that entropy - * source. - */ - state = kzalloc(sizeof(struct timer_rand_state), GFP_KERNEL); - if (state) { - state->last_time = INITIAL_JIFFIES; - disk->random = state; - } -} -#endif - static ssize_t urandom_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) { @@ -1668,47 +1754,3 @@ struct ctl_table random_table[] = { { } }; #endif /* CONFIG_SYSCTL */ - -/* Interface for in-kernel drivers of true hardware RNGs. - * Those devices may produce endless random bits and will be throttled - * when our pool is full. - */ -void add_hwgenerator_randomness(const void *buffer, size_t count, - size_t entropy) -{ - if (unlikely(crng_init == 0)) { - size_t ret = crng_fast_load(buffer, count); - mix_pool_bytes(buffer, ret); - count -= ret; - buffer += ret; - if (!count || crng_init == 0) - return; - } - - /* Throttle writing if we're above the trickle threshold. - * We'll be woken up again once below POOL_MIN_BITS, when - * the calling thread is about to terminate, or once - * CRNG_RESEED_INTERVAL has elapsed. - */ - wait_event_interruptible_timeout(random_write_wait, - !system_wq || kthread_should_stop() || - input_pool.entropy_count < POOL_MIN_BITS, - CRNG_RESEED_INTERVAL); - mix_pool_bytes(buffer, count); - credit_entropy_bits(entropy); -} -EXPORT_SYMBOL_GPL(add_hwgenerator_randomness); - -/* Handle random seed passed by bootloader. - * If the seed is trustworthy, it would be regarded as hardware RNGs. Otherwise - * it would be regarded as device data. - * The decision is controlled by CONFIG_RANDOM_TRUST_BOOTLOADER. - */ -void add_bootloader_randomness(const void *buf, size_t size) -{ - if (IS_ENABLED(CONFIG_RANDOM_TRUST_BOOTLOADER)) - add_hwgenerator_randomness(buf, size, size * 8); - else - add_device_randomness(buf, size); -} -EXPORT_SYMBOL_GPL(add_bootloader_randomness);

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 0deff3c43206c24e746b1410f11125707ad3040e upstream.

This pulls all of the sysctl-focused functions into the sixth labeled section.

No functional changes.

Cc: Theodore Ts'o tytso@mit.edu Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1686,9 +1686,34 @@ const struct file_operations urandom_fop .llseek = noop_llseek, };

+ /******************************************************************** * - * Sysctl interface + * Sysctl interface. + * + * These are partly unused legacy knobs with dummy values to not break + * userspace and partly still useful things. They are usually accessible + * in /proc/sys/kernel/random/ and are as follows: + * + * - boot_id - a UUID representing the current boot. + * + * - uuid - a random UUID, different each time the file is read. + * + * - poolsize - the number of bits of entropy that the input pool can + * hold, tied to the POOL_BITS constant. + * + * - entropy_avail - the number of bits of entropy currently in the + * input pool. Always <= poolsize. + * + * - write_wakeup_threshold - the amount of entropy in the input pool + * below which write polls to /dev/random will unblock, requesting + * more entropy, tied to the POOL_MIN_BITS constant. It is writable + * to avoid breaking old userspaces, but writing to it does not + * change any behavior of the RNG. + * + * - urandom_min_reseed_secs - fixed to the meaningless value "60". + * It is writable to avoid breaking old userspaces, but writing + * to it does not change any behavior of the RNG. * ********************************************************************/

@@ -1696,8 +1721,8 @@ const struct file_operations urandom_fop

#include <linux/sysctl.h>

-static int random_min_urandom_seed = 60; -static int random_write_wakeup_bits = POOL_MIN_BITS; +static int sysctl_random_min_urandom_seed = 60; +static int sysctl_random_write_wakeup_bits = POOL_MIN_BITS; static int sysctl_poolsize = POOL_BITS; static char sysctl_bootid[16];

@@ -1755,14 +1780,14 @@ struct ctl_table random_table[] = { }, { .procname = "write_wakeup_threshold", - .data = &random_write_wakeup_bits, + .data = &sysctl_random_write_wakeup_bits, .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec, }, { .procname = "urandom_min_reseed_secs", - .data = &random_min_urandom_seed, + .data = &sysctl_random_min_urandom_seed, .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec,

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 58340f8e952b613e0ead0bed58b97b05bf4743c5 upstream.

On PREEMPT_RT, it's problematic to take spinlocks from hard irq handlers. We can fix this by deferring to a workqueue the dumping of the fast pool into the input pool.

We accomplish this with some careful rules on fast_pool->count:

- When it's incremented to >= 64, we schedule the work. - If the top bit is set, we never schedule the work, even if >= 64. - The worker is responsible for setting it back to 0 when it's done.

There are two small issues around using workqueues for this purpose that we work around.

The first issue is that mix_interrupt_randomness() might be migrated to another CPU during CPU hotplug. This issue is rectified by checking that it hasn't been migrated (after disabling irqs). If it has been migrated, then we set the count to zero, so that when the CPU comes online again, it can requeue the work. As part of this, we switch to using an atomic_t, so that the increment in the irq handler doesn't wipe out the zeroing if the CPU comes back online while this worker is running.

The second issue is that, though relatively minor in effect, we probably want to make sure we get a consistent view of the pool onto the stack, in case it's interrupted by an irq while reading. To do this, we don't reenable irqs until after the copy. There are only 18 instructions between the cli and sti, so this is a pretty tiny window.

Cc: Thomas Gleixner tglx@linutronix.de Cc: Peter Zijlstra peterz@infradead.org Cc: Theodore Ts'o tytso@mit.edu Cc: Jonathan Neuschäfer j.neuschaefer@gmx.net Acked-by: Sebastian Andrzej Siewior bigeasy@linutronix.de Reviewed-by: Sultan Alsawaf sultan@kerneltoast.com Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 63 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 14 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1173,9 +1173,10 @@ struct fast_pool { u32 pool32[4]; u64 pool64[2]; }; + struct work_struct mix; unsigned long last; + atomic_t count; u16 reg_idx; - u8 count; };

/* @@ -1225,12 +1226,49 @@ static u32 get_reg(struct fast_pool *f, return *ptr; }

+static void mix_interrupt_randomness(struct work_struct *work) +{ + struct fast_pool *fast_pool = container_of(work, struct fast_pool, mix); + u32 pool[4]; + + /* Check to see if we're running on the wrong CPU due to hotplug. */ + local_irq_disable(); + if (fast_pool != this_cpu_ptr(&irq_randomness)) { + local_irq_enable(); + /* + * If we are unlucky enough to have been moved to another CPU, + * during CPU hotplug while the CPU was shutdown then we set + * our count to zero atomically so that when the CPU comes + * back online, it can enqueue work again. The _release here + * pairs with the atomic_inc_return_acquire in + * add_interrupt_randomness(). + */ + atomic_set_release(&fast_pool->count, 0); + return; + } + + /* + * Copy the pool to the stack so that the mixer always has a + * consistent view, before we reenable irqs again. + */ + memcpy(pool, fast_pool->pool32, sizeof(pool)); + atomic_set(&fast_pool->count, 0); + fast_pool->last = jiffies; + local_irq_enable(); + + mix_pool_bytes(pool, sizeof(pool)); + credit_entropy_bits(1); + memzero_explicit(pool, sizeof(pool)); +} + void add_interrupt_randomness(int irq) { + enum { MIX_INFLIGHT = 1U << 31 }; struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness); struct pt_regs *regs = get_irq_regs(); unsigned long now = jiffies; cycles_t cycles = random_get_entropy(); + unsigned int new_count;

if (cycles == 0) cycles = get_reg(fast_pool, regs); @@ -1250,12 +1288,13 @@ void add_interrupt_randomness(int irq) }

fast_mix(fast_pool->pool32); - ++fast_pool->count; + /* The _acquire here pairs with the atomic_set_release in mix_interrupt_randomness(). */ + new_count = (unsigned int)atomic_inc_return_acquire(&fast_pool->count);

if (unlikely(crng_init == 0)) { - if (fast_pool->count >= 64 && + if (new_count >= 64 && crng_fast_load(fast_pool->pool32, sizeof(fast_pool->pool32)) > 0) { - fast_pool->count = 0; + atomic_set(&fast_pool->count, 0); fast_pool->last = now; if (spin_trylock(&input_pool.lock)) { _mix_pool_bytes(&fast_pool->pool32, sizeof(fast_pool->pool32)); @@ -1265,20 +1304,16 @@ void add_interrupt_randomness(int irq) return; }

- if ((fast_pool->count < 64) && !time_after(now, fast_pool->last + HZ)) + if (new_count & MIX_INFLIGHT) return;

- if (!spin_trylock(&input_pool.lock)) + if (new_count < 64 && !time_after(now, fast_pool->last + HZ)) return;

- fast_pool->last = now; - _mix_pool_bytes(&fast_pool->pool32, sizeof(fast_pool->pool32)); - spin_unlock(&input_pool.lock); - - fast_pool->count = 0; - - /* Award one bit for the contents of the fast pool. */ - credit_entropy_bits(1); + if (unlikely(!fast_pool->mix.func)) + INIT_WORK(&fast_pool->mix, mix_interrupt_randomness); + atomic_or(MIX_INFLIGHT, &fast_pool->count); + queue_work_on(raw_smp_processor_id(), system_highpri_wq, &fast_pool->mix); } EXPORT_SYMBOL_GPL(add_interrupt_randomness);

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit da792c6d5f59a76c10a310c5d4c93428fd18f996 upstream.

crng_fast_load() and crng_slow_load() have different semantics:

- crng_fast_load() xors and accounts with crng_init_cnt. - crng_slow_load() hashes and doesn't account.

However add_hwgenerator_randomness() can afford to hash (it's called from a kthread), and it should account. Additionally, ones that can afford to hash don't need to take a trylock but can take a normal lock. So, we combine these into one function, crng_pre_init_inject(), which allows us to control these in a uniform way. This will make it simpler later to simplify this all down when the time comes for that.

Cc: Theodore Ts'o tytso@mit.edu Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Reviewed-by: Eric Biggers ebiggers@google.com Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 114 +++++++++++++++++++++++++------------------------- 1 file changed, 59 insertions(+), 55 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -384,7 +384,7 @@ static void crng_make_state(u32 chacha_s * For the fast path, we check whether we're ready, unlocked first, and * then re-check once locked later. In the case where we're really not * ready, we do fast key erasure with the base_crng directly, because - * this is what crng_{fast,slow}_load mutate during early init. + * this is what crng_pre_init_inject() mutates during early init. */ if (unlikely(!crng_ready())) { bool ready; @@ -435,72 +435,75 @@ static void crng_make_state(u32 chacha_s }

/* - * This function is for crng_init == 0 only. + * This function is for crng_init == 0 only. It loads entropy directly + * into the crng's key, without going through the input pool. It is, + * generally speaking, not very safe, but we use this only at early + * boot time when it's better to have something there rather than + * nothing. + * + * There are two paths, a slow one and a fast one. The slow one + * hashes the input along with the current key. The fast one simply + * xors it in, and should only be used from interrupt context. + * + * If account is set, then the crng_init_cnt counter is incremented. + * This shouldn't be set by functions like add_device_randomness(), + * where we can't trust the buffer passed to it is guaranteed to be + * unpredictable (so it might not have any entropy at all). * - * crng_fast_load() can be called by code in the interrupt service - * path. So we can't afford to dilly-dally. Returns the number of - * bytes processed from cp. + * Returns the number of bytes processed from input, which is bounded + * by CRNG_INIT_CNT_THRESH if account is true. */ -static size_t crng_fast_load(const void *cp, size_t len) +static size_t crng_pre_init_inject(const void *input, size_t len, + bool fast, bool account) { static int crng_init_cnt = 0; unsigned long flags; - const u8 *src = (const u8 *)cp; - size_t ret = 0;

- if (!spin_trylock_irqsave(&base_crng.lock, flags)) - return 0; + if (fast) { + if (!spin_trylock_irqsave(&base_crng.lock, flags)) + return 0; + } else { + spin_lock_irqsave(&base_crng.lock, flags); + } + if (crng_init != 0) { spin_unlock_irqrestore(&base_crng.lock, flags); return 0; } - while (len > 0 && crng_init_cnt < CRNG_INIT_CNT_THRESH) { - base_crng.key[crng_init_cnt % sizeof(base_crng.key)] ^= *src; - src++; crng_init_cnt++; len--; ret++; - } - if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) { - ++base_crng.generation; - crng_init = 1; - } - spin_unlock_irqrestore(&base_crng.lock, flags); - if (crng_init == 1) - pr_notice("fast init done\n"); - return ret; -}

-/* - * This function is for crng_init == 0 only. - * - * crng_slow_load() is called by add_device_randomness, which has two - * attributes. (1) We can't trust the buffer passed to it is - * guaranteed to be unpredictable (so it might not have any entropy at - * all), and (2) it doesn't have the performance constraints of - * crng_fast_load(). - * - * So, we simply hash the contents in with the current key. Finally, - * we do *not* advance crng_init_cnt since buffer we may get may be - * something like a fixed DMI table (for example), which might very - * well be unique to the machine, but is otherwise unvarying. - */ -static void crng_slow_load(const void *cp, size_t len) -{ - unsigned long flags; - struct blake2s_state hash; + if (account) + len = min_t(size_t, len, CRNG_INIT_CNT_THRESH - crng_init_cnt);

- blake2s_init(&hash, sizeof(base_crng.key)); - - if (!spin_trylock_irqsave(&base_crng.lock, flags)) - return; - if (crng_init != 0) { - spin_unlock_irqrestore(&base_crng.lock, flags); - return; + if (fast) { + const u8 *src = input; + size_t i; + + for (i = 0; i < len; ++i) + base_crng.key[(crng_init_cnt + i) % + sizeof(base_crng.key)] ^= src[i]; + } else { + struct blake2s_state hash; + + blake2s_init(&hash, sizeof(base_crng.key)); + blake2s_update(&hash, base_crng.key, sizeof(base_crng.key)); + blake2s_update(&hash, input, len); + blake2s_final(&hash, base_crng.key); + } + + if (account) { + crng_init_cnt += len; + if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) { + ++base_crng.generation; + crng_init = 1; + } }

- blake2s_update(&hash, base_crng.key, sizeof(base_crng.key)); - blake2s_update(&hash, cp, len); - blake2s_final(&hash, base_crng.key); - spin_unlock_irqrestore(&base_crng.lock, flags); + + if (crng_init == 1) + pr_notice("fast init done\n"); + + return len; }

static void _get_random_bytes(void *buf, size_t nbytes) @@ -1013,7 +1016,7 @@ void add_device_randomness(const void *b unsigned long flags;

if (!crng_ready() && size) - crng_slow_load(buf, size); + crng_pre_init_inject(buf, size, false, false);

spin_lock_irqsave(&input_pool.lock, flags); _mix_pool_bytes(buf, size); @@ -1130,7 +1133,7 @@ void add_hwgenerator_randomness(const vo size_t entropy) { if (unlikely(crng_init == 0)) { - size_t ret = crng_fast_load(buffer, count); + size_t ret = crng_pre_init_inject(buffer, count, false, true); mix_pool_bytes(buffer, ret); count -= ret; buffer += ret; @@ -1293,7 +1296,8 @@ void add_interrupt_randomness(int irq)

if (unlikely(crng_init == 0)) { if (new_count >= 64 && - crng_fast_load(fast_pool->pool32, sizeof(fast_pool->pool32)) > 0) { + crng_pre_init_inject(fast_pool->pool32, sizeof(fast_pool->pool32), + true, true) > 0) { atomic_set(&fast_pool->count, 0); fast_pool->last = now; if (spin_trylock(&input_pool.lock)) {

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit b777c38239fec5a528e59f55b379e31b1a187524 upstream.

add_hwgenerator_randomness() is a function implemented and documented inside of random.c. It is the way that hardware RNGs push data into it. Therefore, it should be declared in random.h. Otherwise sparse complains with:

random.c:1137:6: warning: symbol 'add_hwgenerator_randomness' was not declared. Should it be static?

The alternative would be to include hw_random.h into random.c, but that wouldn't really be good for anything except slowing down compile time.

Cc: Matt Mackall mpm@selenic.com Cc: Theodore Ts'o tytso@mit.edu Acked-by: Herbert Xu herbert@gondor.apana.org.au Reviewed-by: Eric Biggers ebiggers@google.com Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/hw_random/core.c | 1 + include/linux/hw_random.h | 2 -- include/linux/random.h | 2 ++ 3 files changed, 3 insertions(+), 2 deletions(-)

--- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -15,6 +15,7 @@ #include <linux/err.h> #include <linux/fs.h> #include <linux/hw_random.h> +#include <linux/random.h> #include <linux/kernel.h> #include <linux/kthread.h> #include <linux/sched/signal.h> --- a/include/linux/hw_random.h +++ b/include/linux/hw_random.h @@ -59,7 +59,5 @@ extern int devm_hwrng_register(struct de /** Unregister a Hardware Random Number Generator driver. */ extern void hwrng_unregister(struct hwrng *rng); extern void devm_hwrng_unregister(struct device *dve, struct hwrng *rng); -/** Feed random bits into the pool. */ -extern void add_hwgenerator_randomness(const void *buffer, size_t count, size_t entropy);

#endif /* LINUX_HWRANDOM_H_ */ --- a/include/linux/random.h +++ b/include/linux/random.h @@ -32,6 +32,8 @@ static inline void add_latent_entropy(vo extern void add_input_randomness(unsigned int type, unsigned int code, unsigned int value) __latent_entropy; extern void add_interrupt_randomness(int irq) __latent_entropy; +extern void add_hwgenerator_randomness(const void *buffer, size_t count, + size_t entropy);

extern void get_random_bytes(void *buf, size_t nbytes); extern int wait_for_random_bytes(void);

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit da3951ebdcd1cb1d5c750e08cd05aee7b0c04d9a upstream.

When the interrupt handler does not have a valid cycle counter, it calls get_reg() to read a register from the irq stack, in round-robin. Currently it does this assuming that registers are 32-bit. This is _probably_ the case, and probably all platforms without cycle counters are in fact 32-bit platforms. But maybe not, and either way, it's not quite correct. This commit fixes that to deal with `unsigned long` rather than `u32`.

Cc: Theodore Ts'o tytso@mit.edu Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1256,15 +1256,15 @@ int random_online_cpu(unsigned int cpu) } #endif

-static u32 get_reg(struct fast_pool *f, struct pt_regs *regs) +static unsigned long get_reg(struct fast_pool *f, struct pt_regs *regs) { - u32 *ptr = (u32 *)regs; + unsigned long *ptr = (unsigned long *)regs; unsigned int idx;

if (regs == NULL) return 0; idx = READ_ONCE(f->reg_idx); - if (idx >= sizeof(struct pt_regs) / sizeof(u32)) + if (idx >= sizeof(struct pt_regs) / sizeof(unsigned long)) idx = 0; ptr += idx++; WRITE_ONCE(f->reg_idx, idx);

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 64276a9939ff414f2f0db38036cf4e1a0a703394 upstream.

Rather than hard coding various lengths, we can use the right constants. Strings should be `char *` while buffers should be `u8 *`. Rather than have a nonsensical and unused maxlength, just remove it. Finally, use snprintf instead of sprintf, just out of good hygiene.

As well, remove the old comment about returning a binary UUID via the binary sysctl syscall. That syscall was removed from the kernel in 5.5, and actually, the "uuid_strategy" function and related infrastructure for even serving it via the binary sysctl syscall was removed with 894d2491153a ("sysctl drivers: Remove dead binary sysctl support") back in 2.6.33.

Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1654,22 +1654,25 @@ const struct file_operations urandom_fop static int sysctl_random_min_urandom_seed = 60; static int sysctl_random_write_wakeup_bits = POOL_MIN_BITS; static int sysctl_poolsize = POOL_BITS; -static char sysctl_bootid[16]; +static u8 sysctl_bootid[UUID_SIZE];

/* * This function is used to return both the bootid UUID, and random - * UUID. The difference is in whether table->data is NULL; if it is, + * UUID. The difference is in whether table->data is NULL; if it is, * then a new UUID is generated and returned to the user. - * - * If the user accesses this via the proc interface, the UUID will be - * returned as an ASCII string in the standard UUID format; if via the - * sysctl system call, as 16 bytes of binary data. */ static int proc_do_uuid(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos) { - struct ctl_table fake_table; - unsigned char buf[64], tmp_uuid[16], *uuid; + u8 tmp_uuid[UUID_SIZE], *uuid; + char uuid_string[UUID_STRING_LEN + 1]; + struct ctl_table fake_table = { + .data = uuid_string, + .maxlen = UUID_STRING_LEN + }; + + if (write) + return -EPERM;

uuid = table->data; if (!uuid) { @@ -1684,12 +1687,8 @@ static int proc_do_uuid(struct ctl_table spin_unlock(&bootid_spinlock); }

- sprintf(buf, "%pU", uuid); - - fake_table.data = buf; - fake_table.maxlen = sizeof(buf); - - return proc_dostring(&fake_table, write, buffer, lenp, ppos); + snprintf(uuid_string, sizeof(uuid_string), "%pU", uuid); + return proc_dostring(&fake_table, 0, buffer, lenp, ppos); }

extern struct ctl_table random_table[]; @@ -1725,13 +1724,11 @@ struct ctl_table random_table[] = { { .procname = "boot_id", .data = &sysctl_bootid, - .maxlen = 16, .mode = 0444, .proc_handler = proc_do_uuid, }, { .procname = "uuid", - .maxlen = 16, .mode = 0444, .proc_handler = proc_do_uuid, },

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit c2a7de4feb6e09f23af7accc0f882a8fa92e7ae5 upstream.

Taking spinlocks from IRQ context is generally problematic for PREEMPT_RT. That is, in part, why we take trylocks instead. However, a spin_try_lock() is also problematic since another spin_lock() invocation can potentially PI-boost the wrong task, as the spin_try_lock() is invoked from an IRQ-context, so the task on CPU (random task or idle) is not the actual owner.

Additionally, by deferring the crng pre-init loading to the worker, we can use the cryptographic hash function rather than xor, which is perhaps a meaningful difference when considering this data has only been through the relatively weak fast_mix() function.

The biggest downside of this approach is that the pre-init loading is now deferred until later, which means things that need random numbers after interrupts are enabled, but before workqueues are running -- or before this particular worker manages to run -- are going to get into trouble. Hopefully in the real world, this window is rather small, especially since this code won't run until 64 interrupts had occurred.

Cc: Sultan Alsawaf sultan@kerneltoast.com Cc: Thomas Gleixner tglx@linutronix.de Cc: Peter Zijlstra peterz@infradead.org Cc: Eric Biggers ebiggers@kernel.org Cc: Theodore Ts'o tytso@mit.edu Acked-by: Sebastian Andrzej Siewior bigeasy@linutronix.de Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 65 ++++++++++++++------------------------------------ 1 file changed, 19 insertions(+), 46 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -441,10 +441,6 @@ static void crng_make_state(u32 chacha_s * boot time when it's better to have something there rather than * nothing. * - * There are two paths, a slow one and a fast one. The slow one - * hashes the input along with the current key. The fast one simply - * xors it in, and should only be used from interrupt context. - * * If account is set, then the crng_init_cnt counter is incremented. * This shouldn't be set by functions like add_device_randomness(), * where we can't trust the buffer passed to it is guaranteed to be @@ -453,19 +449,15 @@ static void crng_make_state(u32 chacha_s * Returns the number of bytes processed from input, which is bounded * by CRNG_INIT_CNT_THRESH if account is true. */ -static size_t crng_pre_init_inject(const void *input, size_t len, - bool fast, bool account) +static size_t crng_pre_init_inject(const void *input, size_t len, bool account) { static int crng_init_cnt = 0; + struct blake2s_state hash; unsigned long flags;

- if (fast) { - if (!spin_trylock_irqsave(&base_crng.lock, flags)) - return 0; - } else { - spin_lock_irqsave(&base_crng.lock, flags); - } + blake2s_init(&hash, sizeof(base_crng.key));

+ spin_lock_irqsave(&base_crng.lock, flags); if (crng_init != 0) { spin_unlock_irqrestore(&base_crng.lock, flags); return 0; @@ -474,21 +466,9 @@ static size_t crng_pre_init_inject(const if (account) len = min_t(size_t, len, CRNG_INIT_CNT_THRESH - crng_init_cnt);

- if (fast) { - const u8 *src = input; - size_t i; - - for (i = 0; i < len; ++i) - base_crng.key[(crng_init_cnt + i) % - sizeof(base_crng.key)] ^= src[i]; - } else { - struct blake2s_state hash; - - blake2s_init(&hash, sizeof(base_crng.key)); - blake2s_update(&hash, base_crng.key, sizeof(base_crng.key)); - blake2s_update(&hash, input, len); - blake2s_final(&hash, base_crng.key); - } + blake2s_update(&hash, base_crng.key, sizeof(base_crng.key)); + blake2s_update(&hash, input, len); + blake2s_final(&hash, base_crng.key);

if (account) { crng_init_cnt += len; @@ -1029,7 +1009,7 @@ void add_device_randomness(const void *b unsigned long flags, now = jiffies;

if (crng_init == 0 && size) - crng_pre_init_inject(buf, size, false, false); + crng_pre_init_inject(buf, size, false);

spin_lock_irqsave(&input_pool.lock, flags); _mix_pool_bytes(&cycles, sizeof(cycles)); @@ -1150,7 +1130,7 @@ void add_hwgenerator_randomness(const vo size_t entropy) { if (unlikely(crng_init == 0)) { - size_t ret = crng_pre_init_inject(buffer, count, false, true); + size_t ret = crng_pre_init_inject(buffer, count, true); mix_pool_bytes(buffer, ret); count -= ret; buffer += ret; @@ -1290,8 +1270,14 @@ static void mix_interrupt_randomness(str fast_pool->last = jiffies; local_irq_enable();

- mix_pool_bytes(pool, sizeof(pool)); - credit_entropy_bits(1); + if (unlikely(crng_init == 0)) { + crng_pre_init_inject(pool, sizeof(pool), true); + mix_pool_bytes(pool, sizeof(pool)); + } else { + mix_pool_bytes(pool, sizeof(pool)); + credit_entropy_bits(1); + } + memzero_explicit(pool, sizeof(pool)); }

@@ -1324,24 +1310,11 @@ void add_interrupt_randomness(int irq) fast_mix(fast_pool->pool32); new_count = ++fast_pool->count;

- if (unlikely(crng_init == 0)) { - if (new_count >= 64 && - crng_pre_init_inject(fast_pool->pool32, sizeof(fast_pool->pool32), - true, true) > 0) { - fast_pool->count = 0; - fast_pool->last = now; - if (spin_trylock(&input_pool.lock)) { - _mix_pool_bytes(&fast_pool->pool32, sizeof(fast_pool->pool32)); - spin_unlock(&input_pool.lock); - } - } - return; - } - if (new_count & MIX_INFLIGHT) return;

- if (new_count < 64 && !time_after(now, fast_pool->last + HZ)) + if (new_count < 64 && (!time_after(now, fast_pool->last + HZ) || + unlikely(crng_init == 0))) return;

if (unlikely(!fast_pool->mix.func))

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 77553cf8f44863b31da242cf24671d76ddb61597 upstream.

We leave around these old sysctls for compatibility, and we keep them "writable" for compatibility, but even after writing, we should keep reporting the same value. This is consistent with how userspaces tend to use sysctl_random_write_wakeup_bits, writing to it, and then later reading from it and using the value.

Cc: Theodore Ts'o tytso@mit.edu Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1662,6 +1662,13 @@ static int proc_do_uuid(struct ctl_table return proc_dostring(&fake_table, 0, buffer, lenp, ppos); }

+/* The same as proc_dointvec, but writes don't change anything. */ +static int proc_do_rointvec(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + return write ? 0 : proc_dointvec(table, 0, buffer, lenp, ppos); +} + extern struct ctl_table random_table[]; struct ctl_table random_table[] = { { @@ -1683,14 +1690,14 @@ struct ctl_table random_table[] = { .data = &sysctl_random_write_wakeup_bits, .maxlen = sizeof(int), .mode = 0644, - .proc_handler = proc_dointvec, + .proc_handler = proc_do_rointvec, }, { .procname = "urandom_min_reseed_secs", .data = &sysctl_random_min_urandom_seed, .maxlen = sizeof(int), .mode = 0644, - .proc_handler = proc_dointvec, + .proc_handler = proc_do_rointvec, }, { .procname = "boot_id",

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit f5eab0e2db4f881fb2b62b3fdad5b9be673dd7ae upstream.

The current fast_mix() function is a piece of classic mailing list crypto, where it just sort of sprung up by an anonymous author without a lot of real analysis of what precisely it was accomplishing. As an ARX permutation alone, there are some easily searchable differential trails in it, and as a means of preventing malicious interrupts, it completely fails, since it xors new data into the entire state every time. It can't really be analyzed as a random permutation, because it clearly isn't, and it can't be analyzed as an interesting linear algebraic structure either, because it's also not that. There really is very little one can say about it in terms of entropy accumulation. It might diffuse bits, some of the time, maybe, we hope, I guess. But for the most part, it fails to accomplish anything concrete.

As a reminder, the simple goal of add_interrupt_randomness() is to simply accumulate entropy until ~64 interrupts have elapsed, and then dump it into the main input pool, which uses a cryptographic hash.

It would be nice to have something cryptographically strong in the interrupt handler itself, in case a malicious interrupt compromises a per-cpu fast pool within the 64 interrupts / 1 second window, and then inside of that same window somehow can control its return address and cycle counter, even if that's a bit far fetched. However, with a very CPU-limited budget, actually doing that remains an active research project (and perhaps there'll be something useful for Linux to come out of it). And while the abundance of caution would be nice, this isn't *currently* the security model, and we don't yet have a fast enough solution to make it our security model. Plus there's not exactly a pressing need to do that. (And for the avoidance of doubt, the actual cluster of 64 accumulated interrupts still gets dumped into our cryptographically secure input pool.)

So, for now we are going to stick with the existing interrupt security model, which assumes that each cluster of 64 interrupt data samples is mostly non-malicious and not colluding with an infoleaker. With this as our goal, we have a few more choices, simply aiming to accumulate entropy, while discarding the least amount of it.

We know from https://eprint.iacr.org/2019/198 that random oracles, instantiated as computational hash functions, make good entropy accumulators and extractors, which is the justification for using BLAKE2s in the main input pool. As mentioned, we don't have that luxury here, but we also don't have the same security model requirements, because we're assuming that there aren't malicious inputs. A pseudorandom function instance can approximately behave like a random oracle, provided that the key is uniformly random. But since we're not concerned with malicious inputs, we can pick a fixed key, which is not secret, knowing that "nature" won't interact with a sufficiently chosen fixed key by accident. So we pick a PRF with a fixed initial key, and accumulate into it continuously, dumping the result every 64 interrupts into our cryptographically secure input pool.

For this, we make use of SipHash-1-x on 64-bit and HalfSipHash-1-x on 32-bit, which are already in use in the kernel's hsiphash family of functions and achieve the same performance as the function they replace. It would be nice to do two rounds, but we don't exactly have the CPU budget handy for that, and one round alone is already sufficient.

As mentioned, we start with a fixed initial key (zeros is fine), and allow SipHash's symmetry breaking constants to turn that into a useful starting point. Also, since we're dumping the result (or half of it on 64-bit so as to tax our hash function the same amount on all platforms) into the cryptographically secure input pool, there's no point in finalizing SipHash's output, since it'll wind up being finalized by something much stronger. This means that all we need to do is use the ordinary round function word-by-word, as normal SipHash does. Simplified, the flow is as follows:

Initialize:

siphash_state_t state; siphash_init(&state, key={0, 0, 0, 0});

Update (accumulate) on interrupt:

siphash_update(&state, interrupt_data_and_timing);

Dump into input pool after 64 interrupts:

blake2s_update(&input_pool, &state, sizeof(state) / 2);

The result of all of this is that the security model is unchanged from before -- we assume non-malicious inputs -- yet we now implement that model with a stronger argument. I would like to emphasize, again, that the purpose of this commit is to improve the existing design, by making it analyzable, without changing any fundamental assumptions. There may well be value down the road in changing up the existing design, using something cryptographically strong, or simply using a ring buffer of samples rather than having a fast_mix() at all, or changing which and how much data we collect each interrupt so that we can use something linear, or a variety of other ideas. This commit does not invalidate the potential for those in the future.

For example, in the future, if we're able to characterize the data we're collecting on each interrupt, we may be able to inch toward information theoretic accumulators. https://eprint.iacr.org/2021/523 shows that `s = ror32(s, 7) ^ x` and `s = ror64(s, 19) ^ x` make very good accumulators for 2-monotone distributions, which would apply to timestamp counters, like random_get_entropy() or jiffies, but would not apply to our current combination of the two values, or to the various function addresses and register values we mix in. Alternatively, https://eprint.iacr.org/2021/1002 shows that max-period linear functions with no non-trivial invariant subspace make good extractors, used in the form `s = f(s) ^ x`. However, this only works if the input data is both identical and independent, and obviously a collection of address values and counters fails; so it goes with theoretical papers. Future directions here may involve trying to characterize more precisely what we actually need to collect in the interrupt handler, and building something specific around that.

However, as mentioned, the morass of data we're gathering at the interrupt handler presently defies characterization, and so we use SipHash for now, which works well and performs well.

Cc: Theodore Ts'o tytso@mit.edu Cc: Greg Kroah-Hartman gregkh@linuxfoundation.org Reviewed-by: Jean-Philippe Aumasson jeanphilippe.aumasson@gmail.com Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 94 +++++++++++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 39 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1142,48 +1142,51 @@ void add_bootloader_randomness(const voi EXPORT_SYMBOL_GPL(add_bootloader_randomness);

struct fast_pool { - union { - u32 pool32[4]; - u64 pool64[2]; - }; struct work_struct mix; + unsigned long pool[4]; unsigned long last; unsigned int count; u16 reg_idx; };

+static DEFINE_PER_CPU(struct fast_pool, irq_randomness) = { +#ifdef CONFIG_64BIT + /* SipHash constants */ + .pool = { 0x736f6d6570736575UL, 0x646f72616e646f6dUL, + 0x6c7967656e657261UL, 0x7465646279746573UL } +#else + /* HalfSipHash constants */ + .pool = { 0, 0, 0x6c796765U, 0x74656462U } +#endif +}; + /* - * This is a fast mixing routine used by the interrupt randomness - * collector. It's hardcoded for an 128 bit pool and assumes that any - * locks that might be needed are taken by the caller. + * This is [Half]SipHash-1-x, starting from an empty key. Because + * the key is fixed, it assumes that its inputs are non-malicious, + * and therefore this has no security on its own. s represents the + * 128 or 256-bit SipHash state, while v represents a 128-bit input. */ -static void fast_mix(u32 pool[4]) +static void fast_mix(unsigned long s[4], const unsigned long *v) { - u32 a = pool[0], b = pool[1]; - u32 c = pool[2], d = pool[3]; - - a += b; c += d; - b = rol32(b, 6); d = rol32(d, 27); - d ^= a; b ^= c; - - a += b; c += d; - b = rol32(b, 16); d = rol32(d, 14); - d ^= a; b ^= c; - - a += b; c += d; - b = rol32(b, 6); d = rol32(d, 27); - d ^= a; b ^= c; - - a += b; c += d; - b = rol32(b, 16); d = rol32(d, 14); - d ^= a; b ^= c; + size_t i;

- pool[0] = a; pool[1] = b; - pool[2] = c; pool[3] = d; + for (i = 0; i < 16 / sizeof(long); ++i) { + s[3] ^= v[i]; +#ifdef CONFIG_64BIT + s[0] += s[1]; s[1] = rol64(s[1], 13); s[1] ^= s[0]; s[0] = rol64(s[0], 32); + s[2] += s[3]; s[3] = rol64(s[3], 16); s[3] ^= s[2]; + s[0] += s[3]; s[3] = rol64(s[3], 21); s[3] ^= s[0]; + s[2] += s[1]; s[1] = rol64(s[1], 17); s[1] ^= s[2]; s[2] = rol64(s[2], 32); +#else + s[0] += s[1]; s[1] = rol32(s[1], 5); s[1] ^= s[0]; s[0] = rol32(s[0], 16); + s[2] += s[3]; s[3] = rol32(s[3], 8); s[3] ^= s[2]; + s[0] += s[3]; s[3] = rol32(s[3], 7); s[3] ^= s[0]; + s[2] += s[1]; s[1] = rol32(s[1], 13); s[1] ^= s[2]; s[2] = rol32(s[2], 16); +#endif + s[0] ^= v[i]; + } }

-static DEFINE_PER_CPU(struct fast_pool, irq_randomness); - #ifdef CONFIG_SMP /* * This function is called when the CPU has just come online, with @@ -1225,7 +1228,15 @@ static unsigned long get_reg(struct fast static void mix_interrupt_randomness(struct work_struct *work) { struct fast_pool *fast_pool = container_of(work, struct fast_pool, mix); - u32 pool[4]; + /* + * The size of the copied stack pool is explicitly 16 bytes so that we + * tax mix_pool_byte()'s compression function the same amount on all + * platforms. This means on 64-bit we copy half the pool into this, + * while on 32-bit we copy all of it. The entropy is supposed to be + * sufficiently dispersed between bits that in the sponge-like + * half case, on average we don't wind up "losing" some. + */ + u8 pool[16];

/* Check to see if we're running on the wrong CPU due to hotplug. */ local_irq_disable(); @@ -1238,7 +1249,7 @@ static void mix_interrupt_randomness(str * Copy the pool to the stack so that the mixer always has a * consistent view, before we reenable irqs again. */ - memcpy(pool, fast_pool->pool32, sizeof(pool)); + memcpy(pool, fast_pool->pool, sizeof(pool)); fast_pool->count = 0; fast_pool->last = jiffies; local_irq_enable(); @@ -1262,25 +1273,30 @@ void add_interrupt_randomness(int irq) struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness); struct pt_regs *regs = get_irq_regs(); unsigned int new_count; + union { + u32 u32[4]; + u64 u64[2]; + unsigned long longs[16 / sizeof(long)]; + } irq_data;

if (cycles == 0) cycles = get_reg(fast_pool, regs);

if (sizeof(cycles) == 8) - fast_pool->pool64[0] ^= cycles ^ rol64(now, 32) ^ irq; + irq_data.u64[0] = cycles ^ rol64(now, 32) ^ irq; else { - fast_pool->pool32[0] ^= cycles ^ irq; - fast_pool->pool32[1] ^= now; + irq_data.u32[0] = cycles ^ irq; + irq_data.u32[1] = now; }

if (sizeof(unsigned long) == 8) - fast_pool->pool64[1] ^= regs ? instruction_pointer(regs) : _RET_IP_; + irq_data.u64[1] = regs ? instruction_pointer(regs) : _RET_IP_; else { - fast_pool->pool32[2] ^= regs ? instruction_pointer(regs) : _RET_IP_; - fast_pool->pool32[3] ^= get_reg(fast_pool, regs); + irq_data.u32[2] = regs ? instruction_pointer(regs) : _RET_IP_; + irq_data.u32[3] = get_reg(fast_pool, regs); }

- fast_mix(fast_pool->pool32); + fast_mix(fast_pool->pool, irq_data.longs); new_count = ++fast_pool->count;

if (new_count & MIX_INFLIGHT)

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 7a7ff644aeaf071d433caffb3b8ea57354b55bd3 upstream.

In order to chip away at the "premature first" problem, we augment our existing entropy accounting with more frequent reseedings at boot.

The idea is that at boot, we're getting entropy from various places, and we're not very sure which of early boot entropy is good and which isn't. Even when we're crediting the entropy, we're still not totally certain that it's any good. Since boot is the one time (aside from a compromise) that we have zero entropy, it's important that we shepherd entropy into the crng fairly often.

At the same time, we don't want a "premature next" problem, whereby an attacker can brute force individual bits of added entropy. In lieu of going full-on Fortuna (for now), we can pick a simpler strategy of just reseeding more often during the first 5 minutes after boot. This is still bounded by the 256-bit entropy credit requirement, so we'll skip a reseeding if we haven't reached that, but in case entropy /is/ coming in, this ensures that it makes its way into the crng rather rapidly during these early stages.

Ordinarily we reseed if the previous reseeding is 300 seconds old. This commit changes things so that for the first 600 seconds of boot time, we reseed if the previous reseeding is uptime / 2 seconds old. That means that we'll reseed at the very least double the uptime of the previous reseeding.

Cc: Theodore Ts'o tytso@mit.edu Reviewed-by: Eric Biggers ebiggers@google.com Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -336,6 +336,28 @@ static void crng_fast_key_erasure(u8 key }

/* + * Return whether the crng seed is considered to be sufficiently + * old that a reseeding might be attempted. This happens if the last + * reseeding was CRNG_RESEED_INTERVAL ago, or during early boot, at + * an interval proportional to the uptime. + */ +static bool crng_has_old_seed(void) +{ + static bool early_boot = true; + unsigned long interval = CRNG_RESEED_INTERVAL; + + if (unlikely(READ_ONCE(early_boot))) { + time64_t uptime = ktime_get_seconds(); + if (uptime >= CRNG_RESEED_INTERVAL / HZ * 2) + WRITE_ONCE(early_boot, false); + else + interval = max_t(unsigned int, 5 * HZ, + (unsigned int)uptime / 2 * HZ); + } + return time_after(jiffies, READ_ONCE(base_crng.birth) + interval); +} + +/* * This function returns a ChaCha state that you may use for generating * random data. It also returns up to 32 bytes on its own of random data * that may be used; random_data_len may not be greater than 32. @@ -368,10 +390,10 @@ static void crng_make_state(u32 chacha_s }

/* - * If the base_crng is more than 5 minutes old, we reseed, which - * in turn bumps the generation counter that we check below. + * If the base_crng is old enough, we try to reseed, which in turn + * bumps the generation counter that we check below. */ - if (unlikely(time_after(jiffies, READ_ONCE(base_crng.birth) + CRNG_RESEED_INTERVAL))) + if (unlikely(crng_has_old_seed())) crng_reseed();

local_irq_save(flags);

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit af704c856e888fb044b058d731d61b46eeec499d upstream.

At boot time, EFI calls add_bootloader_randomness(), which in turn calls add_hwgenerator_randomness(). Currently add_hwgenerator_randomness() feeds the first 64 bytes of randomness to the "fast init" non-crypto-grade phase. But if add_hwgenerator_randomness() gets called with more than POOL_MIN_BITS of entropy, there's no point in passing it off to the "fast init" stage, since that's enough entropy to bootstrap the real RNG. The "fast init" stage is just there to provide _something_ in the case where we don't have enough entropy to properly bootstrap the RNG. But if we do have enough entropy to bootstrap the RNG, the current logic doesn't serve a purpose. So, in the case where we're passed greater than or equal to POOL_MIN_BITS of entropy, this commit makes us skip the "fast init" phase.

Cc: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1120,7 +1120,7 @@ void rand_initialize_disk(struct gendisk void add_hwgenerator_randomness(const void *buffer, size_t count, size_t entropy) { - if (unlikely(crng_init == 0)) { + if (unlikely(crng_init == 0 && entropy < POOL_MIN_BITS)) { size_t ret = crng_pre_init_inject(buffer, count, true); mix_pool_bytes(buffer, ret); count -= ret;

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit dd7aa36e535797926d8eb311da7151919130139d upstream.

The comment about get_random_{u32,u64}() not invoking reseeding got added in an unrelated commit, that then was recently reverted by 0313bc278dac ("Revert "random: block in /dev/urandom""). So this adds that little comment snippet back, and improves the wording a bit too.

Reviewed-by: Dominik Brodowski linux@dominikbrodowski.net Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -226,9 +226,10 @@ static void _warn_unseeded_randomness(co * * These interfaces will return the requested number of random bytes * into the given buffer or as a return value. This is equivalent to - * a read from /dev/urandom. The integer family of functions may be - * higher performance for one-off random integers, because they do a - * bit of buffering. + * a read from /dev/urandom. The u32, u64, int, and long family of + * functions may be higher performance for one-off random integers, + * because they do a bit of buffering and do not invoke reseeding + * until the buffer is emptied. * *********************************************************************/

From: Jan Varho jan.varho@gmail.com

commit 527a9867af29ff89f278d037db704e0ed50fb666 upstream.

add_hwgenerator_randomness() tries to only use the required amount of input for fast init, but credits all the entropy, rather than a fraction of it. Since it's hard to determine how much entropy is left over out of a non-unformly random sample, either give it all to fast init or credit it, but don't attempt to do both. In the process, we can clean up the injection code to no longer need to return a value.

Signed-off-by: Jan Varho jan.varho@gmail.com [Jason: expanded commit message] Fixes: 73c7733f122e ("random: do not throw away excess input to crng_fast_load") Cc: stable@vger.kernel.org # 5.17+, requires af704c856e88 Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -437,11 +437,8 @@ static void crng_make_state(u32 chacha_s * This shouldn't be set by functions like add_device_randomness(), * where we can't trust the buffer passed to it is guaranteed to be * unpredictable (so it might not have any entropy at all). - * - * Returns the number of bytes processed from input, which is bounded - * by CRNG_INIT_CNT_THRESH if account is true. */ -static size_t crng_pre_init_inject(const void *input, size_t len, bool account) +static void crng_pre_init_inject(const void *input, size_t len, bool account) { static int crng_init_cnt = 0; struct blake2s_state hash; @@ -452,18 +449,15 @@ static size_t crng_pre_init_inject(const spin_lock_irqsave(&base_crng.lock, flags); if (crng_init != 0) { spin_unlock_irqrestore(&base_crng.lock, flags); - return 0; + return; }

- if (account) - len = min_t(size_t, len, CRNG_INIT_CNT_THRESH - crng_init_cnt); - blake2s_update(&hash, base_crng.key, sizeof(base_crng.key)); blake2s_update(&hash, input, len); blake2s_final(&hash, base_crng.key);

if (account) { - crng_init_cnt += len; + crng_init_cnt += min_t(size_t, len, CRNG_INIT_CNT_THRESH - crng_init_cnt); if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) { ++base_crng.generation; crng_init = 1; @@ -474,8 +468,6 @@ static size_t crng_pre_init_inject(const

if (crng_init == 1) pr_notice("fast init done\n"); - - return len; }

static void _get_random_bytes(void *buf, size_t nbytes) @@ -1133,12 +1125,9 @@ void add_hwgenerator_randomness(const vo size_t entropy) { if (unlikely(crng_init == 0 && entropy < POOL_MIN_BITS)) { - size_t ret = crng_pre_init_inject(buffer, count, true); - mix_pool_bytes(buffer, ret); - count -= ret; - buffer += ret; - if (!count || crng_init == 0) - return; + crng_pre_init_inject(buffer, count, true); + mix_pool_bytes(buffer, count); + return; }

/*

From: Jann Horn jannh@google.com

commit 1448769c9cdb69ad65287f4f7ab58bc5f2f5d7ba upstream.

signal_pending() checks TIF_NOTIFY_SIGNAL and TIF_SIGPENDING, which signal that the task should bail out of the syscall when possible. This is a separate concept from need_resched(), which checks TIF_NEED_RESCHED, signaling that the task should preempt.

In particular, with the current code, the signal_pending() bailout probably won't work reliably.

Change this to look like other functions that read lots of data, such as read_zero().

Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Signed-off-by: Jann Horn jannh@google.com Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -549,13 +549,13 @@ static ssize_t get_random_bytes_user(voi }

do { - if (large_request && need_resched()) { + if (large_request) { if (signal_pending(current)) { if (!ret) ret = -ERESTARTSYS; break; } - schedule(); + cond_resched(); }

chacha20_block(chacha_state, output);

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit b0c3e796f24b588b862b61ce235d3c9417dc8983 upstream.

Some implementations were returning type `unsigned long`, while others that fell back to get_cycles() were implicitly returning a `cycles_t` or an untyped constant int literal. That makes for weird and confusing code, and basically all code in the kernel already handled it like it was an `unsigned long`. I recently tried to handle it as the largest type it could be, a `cycles_t`, but doing so doesn't really help with much.

Instead let's just make random_get_entropy() return an unsigned long all the time. This also matches the commonly used `arch_get_random_long()` function, so now RDRAND and RDTSC return the same sized integer, which means one can fallback to the other more gracefully.

Cc: Dominik Brodowski linux@dominikbrodowski.net Cc: Theodore Ts'o tytso@mit.edu Acked-by: Thomas Gleixner tglx@linutronix.de Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- drivers/char/random.c | 20 +++++++------------- include/linux/timex.h | 2 +- 2 files changed, 8 insertions(+), 14 deletions(-)

--- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1010,7 +1010,7 @@ int __init rand_initialize(void) */ void add_device_randomness(const void *buf, size_t size) { - cycles_t cycles = random_get_entropy(); + unsigned long cycles = random_get_entropy(); unsigned long flags, now = jiffies;

if (crng_init == 0 && size) @@ -1041,8 +1041,7 @@ struct timer_rand_state { */ static void add_timer_randomness(struct timer_rand_state *state, unsigned int num) { - cycles_t cycles = random_get_entropy(); - unsigned long flags, now = jiffies; + unsigned long cycles = random_get_entropy(), now = jiffies, flags; long delta, delta2, delta3;

spin_lock_irqsave(&input_pool.lock, flags); @@ -1297,8 +1296,7 @@ static void mix_interrupt_randomness(str void add_interrupt_randomness(int irq) { enum { MIX_INFLIGHT = 1U << 31 }; - cycles_t cycles = random_get_entropy(); - unsigned long now = jiffies; + unsigned long cycles = random_get_entropy(), now = jiffies; struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness); struct pt_regs *regs = get_irq_regs(); unsigned int new_count; @@ -1311,16 +1309,12 @@ void add_interrupt_randomness(int irq) if (cycles == 0) cycles = get_reg(fast_pool, regs);

- if (sizeof(cycles) == 8) + if (sizeof(unsigned long) == 8) { irq_data.u64[0] = cycles ^ rol64(now, 32) ^ irq; - else { + irq_data.u64[1] = regs ? instruction_pointer(regs) : _RET_IP_; + } else { irq_data.u32[0] = cycles ^ irq; irq_data.u32[1] = now; - } - - if (sizeof(unsigned long) == 8) - irq_data.u64[1] = regs ? instruction_pointer(regs) : _RET_IP_; - else { irq_data.u32[2] = regs ? instruction_pointer(regs) : _RET_IP_; irq_data.u32[3] = get_reg(fast_pool, regs); } @@ -1367,7 +1361,7 @@ static void entropy_timer(unsigned long static void try_to_generate_entropy(void) { struct { - cycles_t cycles; + unsigned long cycles; struct timer_list timer; } stack;

--- a/include/linux/timex.h +++ b/include/linux/timex.h @@ -75,7 +75,7 @@ * By default we use get_cycles() for this purpose, but individual * architectures may override this in their asm/timex.h header file. */ -#define random_get_entropy() get_cycles() +#define random_get_entropy() ((unsigned long)get_cycles()) #endif

/*

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 069c4ea6871c18bd368f27756e0f91ffb524a788 upstream.

A semicolon was missing, and the almost-alphabetical-but-not ordering was confusing, so regroup these by category instead.

Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- Documentation/sysctl/kernel.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)

--- a/Documentation/sysctl/kernel.txt +++ b/Documentation/sysctl/kernel.txt @@ -795,6 +795,9 @@ This is a directory, with the following * ``boot_id``: a UUID generated the first time this is retrieved, and unvarying after that;

+* ``uuid``: a UUID generated every time this is retrieved (this can + thus be used to generate UUIDs at will); + * ``entropy_avail``: the pool's entropy count, in bits;

* ``poolsize``: the entropy pool size, in bits; @@ -802,10 +805,7 @@ This is a directory, with the following * ``urandom_min_reseed_secs``: obsolete (used to determine the minimum number of seconds between urandom pool reseeding). This file is writable for compatibility purposes, but writing to it has no effect - on any RNG behavior. - -* ``uuid``: a UUID generated every time this is retrieved (this can - thus be used to generate UUIDs at will); + on any RNG behavior;

* ``write_wakeup_threshold``: when the entropy count drops below this (as a number of bits), processes waiting to write to ``/dev/random``

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 57c0900b91d8891ab43f0e6b464d059fda51d102 upstream.

Itanium defines a get_cycles() function, but it does not do the usual `#define get_cycles get_cycles` dance, making it impossible for generic code to see if an arch-specific function was defined. While the get_cycles() ifdef is not currently used, the following timekeeping patch in this series will depend on the macro existing (or not existing) when defining random_get_entropy().

Cc: Thomas Gleixner tglx@linutronix.de Cc: Arnd Bergmann arnd@arndb.de Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- arch/ia64/include/asm/timex.h | 1 + 1 file changed, 1 insertion(+)

--- a/arch/ia64/include/asm/timex.h +++ b/arch/ia64/include/asm/timex.h @@ -39,6 +39,7 @@ get_cycles (void) ret = ia64_getreg(_IA64_REG_AR_ITC); return ret; } +#define get_cycles get_cycles

extern void ia64_cpu_local_tick (void); extern unsigned long long ia64_native_sched_clock (void);

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 8865bbe6ba1120e67f72201b7003a16202cd42be upstream.

PA-RISC defines a get_cycles() function, but it does not do the usual `#define get_cycles get_cycles` dance, making it impossible for generic code to see if an arch-specific function was defined. While the get_cycles() ifdef is not currently used, the following timekeeping patch in this series will depend on the macro existing (or not existing) when defining random_get_entropy().

Cc: Thomas Gleixner tglx@linutronix.de Cc: Arnd Bergmann arnd@arndb.de Acked-by: Helge Deller deller@gmx.de Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- arch/parisc/include/asm/timex.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)

--- a/arch/parisc/include/asm/timex.h +++ b/arch/parisc/include/asm/timex.h @@ -12,9 +12,10 @@

typedef unsigned long cycles_t;

-static inline cycles_t get_cycles (void) +static inline cycles_t get_cycles(void) { return mfctl(16); } +#define get_cycles get_cycles

#endif

From: "Jason A. Donenfeld" Jason@zx2c4.com

commit 408835832158df0357e18e96da7f2d1ed6b80e7f upstream.

PowerPC defines a get_cycles() function, but it does not do the usual `#define get_cycles get_cycles` dance, making it impossible for generic code to see if an arch-specific function was defined. While the get_cycles() ifdef is not currently used, the following timekeeping patch in this series will depend on the macro existing (or not existing) when defining random_get_entropy().

Cc: Thomas Gleixner tglx@linutronix.de Cc: Arnd Bergmann arnd@arndb.de Cc: Benjamin Herrenschmidt benh@ozlabs.org Cc: Paul Mackerras paulus@samba.org Acked-by: Michael Ellerman mpe@ellerman.id.au Signed-off-by: Jason A. Donenfeld Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org --- arch/powerpc/include/asm/timex.h | 1 + 1 file changed, 1 insertion(+)

--- a/arch/powerpc/include/asm/timex.h +++ b/arch/powerpc/include/asm/timex.h @@ -50,6 +50,7 @@ static inline cycles_t get_cycles(void) return ret; #endif } +#define get_cycles get_cycles

#endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_TIMEX_H */