On Tue, 28 May 2024 07:51:30 +0300 Ilkka Naulapää digirigawa@gmail.com wrote:

yeah, the cache_from_obj tracing bug (without panic) has been displayed quite some time now - maybe even since 6.7.x or so. I could try checking a few versions back for this and try bisecting it if I can find when this started.

OK, so I don't think the commit your last bisect hit is the cause of the bug. It added a delay (via RCU) and is causing the real bug to blow up more.

Can you add this patch to v6.9.2 and hopefully it crashes in a better location that we can find where the mixup happened.

You may need to add the other commit (too if this doesn't trigger.

Thanks,

-- Steve

diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c index 417c840e6403..7af3f696696d 100644 --- a/fs/tracefs/inode.c +++ b/fs/tracefs/inode.c @@ -50,6 +50,7 @@ static struct inode *tracefs_alloc_inode(struct super_block *sb) list_add_rcu(&ti->list, &tracefs_inodes); spin_unlock_irqrestore(&tracefs_inode_lock, flags);

+ ti->magic = 20240823; return &ti->vfs_inode; }

@@ -66,6 +67,7 @@ static void tracefs_free_inode(struct inode *inode) struct tracefs_inode *ti = get_tracefs(inode); unsigned long flags;

+ BUG_ON(ti->magic != 20240823); spin_lock_irqsave(&tracefs_inode_lock, flags); list_del_rcu(&ti->list); spin_unlock_irqrestore(&tracefs_inode_lock, flags); @@ -271,16 +273,6 @@ static const struct inode_operations tracefs_file_inode_operations = { .setattr = tracefs_setattr, };

-struct inode *tracefs_get_inode(struct super_block *sb) -{ - struct inode *inode = new_inode(sb); - if (inode) { - inode->i_ino = get_next_ino(); - simple_inode_init_ts(inode); - } - return inode; -} - struct tracefs_mount_opts { kuid_t uid; kgid_t gid; @@ -448,6 +440,17 @@ static const struct super_operations tracefs_super_operations = { .show_options = tracefs_show_options, };

+struct inode *tracefs_get_inode(struct super_block *sb) +{ + struct inode *inode = new_inode(sb); + BUG_ON(sb->s_op != &tracefs_super_operations); + if (inode) { + inode->i_ino = get_next_ino(); + simple_inode_init_ts(inode); + } + return inode; +} + /* * It would be cleaner if eventfs had its own dentry ops. * diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h index f704d8348357..dda7d2708e30 100644 --- a/fs/tracefs/internal.h +++ b/fs/tracefs/internal.h @@ -16,6 +16,7 @@ struct tracefs_inode { }; /* The below gets initialized with memset_after(ti, 0, vfs_inode) */ struct list_head list; + unsigned long magic; unsigned long flags; void *private; };

applied your patch and here's the output.

--Ilkka

On Wed, May 29, 2024 at 10:48 PM Steven Rostedt rostedt@goodmis.org wrote:

On Wed, 29 May 2024 14:47:57 -0400 Steven Rostedt rostedt@goodmis.org wrote:

...

Let me make a debug patch (that crashes on this issue) for that kernel, and perhaps you could bisect it?

Can you try this on 6.6-rc1 and see if it gives you any other splats?

Hmm, you can switch it to WARN_ON and that way it may not crash the machine, and you can use dmesg to get the output.

Thanks,

-- Steve

diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c index de5b72216b1a..a090495e78c9 100644 --- a/fs/tracefs/inode.c +++ b/fs/tracefs/inode.c @@ -39,13 +39,17 @@ static struct inode *tracefs_alloc_inode(struct super_block *sb) return NULL;

    ti->flags = 0;

•   ti->magic = 20240823;
  
    return &ti->vfs_inode;
  

}

static void tracefs_free_inode(struct inode *inode) {

•   kmem_cache_free(tracefs_inode_cachep, get_tracefs(inode));
  

•   struct tracefs_inode *ti = get_tracefs(inode);
  

•   BUG_ON(ti->magic != 20240823);
  

•   kmem_cache_free(tracefs_inode_cachep, ti);
  

}

static ssize_t default_read_file(struct file *file, char __user *buf, @@ -147,16 +151,6 @@ static const struct inode_operations tracefs_dir_inode_operations = { .rmdir = tracefs_syscall_rmdir, };

-struct inode *tracefs_get_inode(struct super_block *sb) -{

•   struct inode *inode = new_inode(sb);
  

•   if (inode) {
  

•           inode->i_ino = get_next_ino();
  

•           inode->i_atime = inode->i_mtime = inode_set_ctime_current(inode);
  

•   }
  

•   return inode;
  

-}

struct tracefs_mount_opts { kuid_t uid; kgid_t gid; @@ -384,6 +378,7 @@ static void tracefs_dentry_iput(struct dentry *dentry, struct inode *inode) return;

    ti = get_tracefs(inode);

•   BUG_ON(ti->magic != 20240823);
    if (ti && ti->flags & TRACEFS_EVENT_INODE)
            eventfs_set_ef_status_free(dentry);
    iput(inode);
  

@@ -568,6 +563,18 @@ struct dentry *eventfs_end_creating(struct dentry *dentry) return dentry; }

+struct inode *tracefs_get_inode(struct super_block *sb) +{

•   struct inode *inode = new_inode(sb);
  

•   BUG_ON(sb->s_op != &tracefs_super_operations);
  

•   if (inode) {
  

•           inode->i_ino = get_next_ino();
  

•           inode->i_atime = inode->i_mtime = inode_set_ctime_current(inode);
  

•   }
  

•   return inode;
  

+}

/**

• tracefs_create_file - create a file in the tracefs filesystem
• @name: a pointer to a string containing the name of the file to create.

diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h index 69c2b1d87c46..9f6f303a9e58 100644 --- a/fs/tracefs/internal.h +++ b/fs/tracefs/internal.h @@ -9,6 +9,7 @@ enum { struct tracefs_inode { unsigned long flags; void *private;

•   unsigned long           magic;
    struct inode            vfs_inode;
  

};

sorry longer delay, been a bit busy but here is the result from that new patch. Only applied this patch so if the previous one is needed also, let me know and I'll rerun it.

--Ilkka

On Thu, May 30, 2024 at 5:00 PM Steven Rostedt rostedt@goodmis.org wrote:

On Thu, 30 May 2024 16:02:37 +0300 Ilkka Naulapää digirigawa@gmail.com wrote:

...

applied your patch and here's the output.

Unfortunately, it doesn't give me any new information. I added one more BUG on, want to try this? Otherwise, I'm pretty much at a lost. :-/

-- Steve

diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c index de5b72216b1a..a090495e78c9 100644 --- a/fs/tracefs/inode.c +++ b/fs/tracefs/inode.c @@ -39,13 +39,17 @@ static struct inode *tracefs_alloc_inode(struct super_block *sb) return NULL;

    ti->flags = 0;

•   ti->magic = 20240823;
  
    return &ti->vfs_inode;
  

}

static void tracefs_free_inode(struct inode *inode) {

•   kmem_cache_free(tracefs_inode_cachep, get_tracefs(inode));
  

•   struct tracefs_inode *ti = get_tracefs(inode);
  

•   BUG_ON(ti->magic != 20240823);
  

•   kmem_cache_free(tracefs_inode_cachep, ti);
  

}

static ssize_t default_read_file(struct file *file, char __user *buf, @@ -147,16 +151,6 @@ static const struct inode_operations tracefs_dir_inode_operations = { .rmdir = tracefs_syscall_rmdir, };

-struct inode *tracefs_get_inode(struct super_block *sb) -{

•   struct inode *inode = new_inode(sb);
  

•   if (inode) {
  

•           inode->i_ino = get_next_ino();
  

•           inode->i_atime = inode->i_mtime = inode_set_ctime_current(inode);
  

•   }
  

•   return inode;
  

-}

struct tracefs_mount_opts { kuid_t uid; kgid_t gid; @@ -384,6 +378,7 @@ static void tracefs_dentry_iput(struct dentry *dentry, struct inode *inode) return;

    ti = get_tracefs(inode);

•   BUG_ON(ti->magic != 20240823);
    if (ti && ti->flags & TRACEFS_EVENT_INODE)
            eventfs_set_ef_status_free(dentry);
    iput(inode);
  

@@ -568,6 +563,18 @@ struct dentry *eventfs_end_creating(struct dentry *dentry) return dentry; }

+struct inode *tracefs_get_inode(struct super_block *sb) +{

•   struct inode *inode = new_inode(sb);
  

•   BUG_ON(sb->s_op != &tracefs_super_operations);
  

•   if (inode) {
  

•           inode->i_ino = get_next_ino();
  

•           inode->i_atime = inode->i_mtime = inode_set_ctime_current(inode);
  

•   }
  

•   return inode;
  

+}

/**

• tracefs_create_file - create a file in the tracefs filesystem
• @name: a pointer to a string containing the name of the file to create.

diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h index 69c2b1d87c46..9059b8b11bb6 100644 --- a/fs/tracefs/internal.h +++ b/fs/tracefs/internal.h @@ -9,12 +9,15 @@ enum { struct tracefs_inode { unsigned long flags; void *private;

•   unsigned long           magic;
    struct inode            vfs_inode;
  

};

static inline struct tracefs_inode *get_tracefs(const struct inode *inode) {

•   return container_of(inode, struct tracefs_inode, vfs_inode);
  

•   struct tracefs_inode *ti = container_of(inode, struct tracefs_inode, vfs_inode);
  

•   BUG_ON(ti->magic != 20240823);
  

•   return ti;
  

}

struct dentry *tracefs_start_creating(const char *name, struct dentry *parent);

On Wed, 12 Jun 2024 15:36:22 +0200 "Linux regression tracking (Thorsten Leemhuis)" regressions@leemhuis.info wrote:

Hi, Thorsten here, the Linux kernel's regression tracker. Top-posting for once, to make this easily accessible to everyone.

Ilkka or Steven, what happened to this? This thread looks stalled. I also was unsuccessful when looking for other threads related to this report or the culprit. Did it fall through the cracks or am I missing something here?

Honesty, I have no idea where the bug is. I can't reproduce it. These patches I sent would check all the places that add to the list to make sure the proper trace_inode was being added, and the output shows that they are all correct. Then suddenly, something that came from the inode cache is calling the tracefs inode cache to free it, and that's where the bug is happening.

This really looks like another bug that the recent changes have made more predominate.

-- Steve

Ciao, Thorsten (wearing his 'the Linux kernel's regression tracker' hat)

Everything you wanna know about Linux kernel regression tracking: https://linux-regtracking.leemhuis.info/about/#tldr If I did something stupid, please tell me, as explained on that page.

#regzbot poke

On 02.06.24 09:32, Ilkka Naulapää wrote:

...

sorry longer delay, been a bit busy but here is the result from that new patch. Only applied this patch so if the previous one is needed also, let me know and I'll rerun it.

--Ilkka

On Thu, May 30, 2024 at 5:00 PM Steven Rostedt rostedt@goodmis.org wrote:

...

On Thu, 30 May 2024 16:02:37 +0300 Ilkka Naulapää digirigawa@gmail.com wrote:

...

applied your patch and here's the output.

Unfortunately, it doesn't give me any new information. I added one more BUG on, want to try this? Otherwise, I'm pretty much at a lost. :-/

-- Steve

diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c index de5b72216b1a..a090495e78c9 100644 --- a/fs/tracefs/inode.c +++ b/fs/tracefs/inode.c @@ -39,13 +39,17 @@ static struct inode *tracefs_alloc_inode(struct super_block *sb) return NULL;

    ti->flags = 0;

•   ti->magic = 20240823;
  
    return &ti->vfs_inode;
  

}

static void tracefs_free_inode(struct inode *inode) {

•   kmem_cache_free(tracefs_inode_cachep, get_tracefs(inode));
  

•   struct tracefs_inode *ti = get_tracefs(inode);
  

•   BUG_ON(ti->magic != 20240823);
  

•   kmem_cache_free(tracefs_inode_cachep, ti);
  

}

static ssize_t default_read_file(struct file *file, char __user *buf, @@ -147,16 +151,6 @@ static const struct inode_operations tracefs_dir_inode_operations = { .rmdir = tracefs_syscall_rmdir, };

-struct inode *tracefs_get_inode(struct super_block *sb) -{

•   struct inode *inode = new_inode(sb);
  

•   if (inode) {
  

•           inode->i_ino = get_next_ino();
  

•           inode->i_atime = inode->i_mtime = inode_set_ctime_current(inode);
  

•   }
  

•   return inode;
  

-}

struct tracefs_mount_opts { kuid_t uid; kgid_t gid; @@ -384,6 +378,7 @@ static void tracefs_dentry_iput(struct dentry *dentry, struct inode *inode) return;

    ti = get_tracefs(inode);

•   BUG_ON(ti->magic != 20240823);
    if (ti && ti->flags & TRACEFS_EVENT_INODE)
            eventfs_set_ef_status_free(dentry);
    iput(inode);
  

@@ -568,6 +563,18 @@ struct dentry *eventfs_end_creating(struct dentry *dentry) return dentry; }

+struct inode *tracefs_get_inode(struct super_block *sb) +{

•   struct inode *inode = new_inode(sb);
  

•   BUG_ON(sb->s_op != &tracefs_super_operations);
  

•   if (inode) {
  

•           inode->i_ino = get_next_ino();
  

•           inode->i_atime = inode->i_mtime = inode_set_ctime_current(inode);
  

•   }
  

•   return inode;
  

+}

/**

• tracefs_create_file - create a file in the tracefs filesystem
• @name: a pointer to a string containing the name of the file to create.

diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h index 69c2b1d87c46..9059b8b11bb6 100644 --- a/fs/tracefs/internal.h +++ b/fs/tracefs/internal.h @@ -9,12 +9,15 @@ enum { struct tracefs_inode { unsigned long flags; void *private;

•   unsigned long           magic;
    struct inode            vfs_inode;
  

};

static inline struct tracefs_inode *get_tracefs(const struct inode *inode) {

•   return container_of(inode, struct tracefs_inode, vfs_inode);
  

•   struct tracefs_inode *ti = container_of(inode, struct tracefs_inode, vfs_inode);
  

•   BUG_ON(ti->magic != 20240823);
  

•   return ti;
  

}

struct dentry *tracefs_start_creating(const char *name, struct dentry *parent);

On 13.06.24 09:32, Ilkka Naulapää wrote:

On Wed, Jun 12, 2024 at 6:56 PM Steven Rostedt rostedt@goodmis.org wrote:

...

On Wed, 12 Jun 2024 15:36:22 +0200 "Linux regression tracking (Thorsten Leemhuis)" regressions@leemhuis.info wrote:

...

Ilkka or Steven, what happened to this? This thread looks stalled. I also was unsuccessful when looking for other threads related to this report or the culprit. Did it fall through the cracks or am I missing something here?

...

Honesty, I have no idea where the bug is. I can't reproduce it. [...]

Steven, thx for the update! And yeah, that's how it sometimes is. Given that we haven't seen similar reports (at least afaics) it's nothing I worry much about.

ok, so if you don't have any idea where this bug is after those debug patches, I'll try to find some time to bisect it as a last resort. Stay tuned.

Yeah, that would be great help. Thank you, too!

Ciao, Thorsten

...

...

On 02.06.24 09:32, Ilkka Naulapää wrote:

...

sorry longer delay, been a bit busy but here is the result from that new patch. Only applied this patch so if the previous one is needed also, let me know and I'll rerun it.

--Ilkka

On Thu, May 30, 2024 at 5:00 PM Steven Rostedt rostedt@goodmis.org wrote:

...

On Thu, 30 May 2024 16:02:37 +0300 Ilkka Naulapää digirigawa@gmail.com wrote:

...

applied your patch and here's the output.

Unfortunately, it doesn't give me any new information. I added one more BUG on, want to try this? Otherwise, I'm pretty much at a lost. :-/

-- Steve

diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c index de5b72216b1a..a090495e78c9 100644 --- a/fs/tracefs/inode.c +++ b/fs/tracefs/inode.c @@ -39,13 +39,17 @@ static struct inode *tracefs_alloc_inode(struct super_block *sb) return NULL;

    ti->flags = 0;

•   ti->magic = 20240823;
  
    return &ti->vfs_inode;
  

}

static void tracefs_free_inode(struct inode *inode) {

•   kmem_cache_free(tracefs_inode_cachep, get_tracefs(inode));
  

•   struct tracefs_inode *ti = get_tracefs(inode);
  

•   BUG_ON(ti->magic != 20240823);
  

•   kmem_cache_free(tracefs_inode_cachep, ti);
  

}

static ssize_t default_read_file(struct file *file, char __user *buf, @@ -147,16 +151,6 @@ static const struct inode_operations tracefs_dir_inode_operations = { .rmdir = tracefs_syscall_rmdir, };

-struct inode *tracefs_get_inode(struct super_block *sb) -{

•   struct inode *inode = new_inode(sb);
  

•   if (inode) {
  

•           inode->i_ino = get_next_ino();
  

•           inode->i_atime = inode->i_mtime = inode_set_ctime_current(inode);
  

•   }
  

•   return inode;
  

-}

struct tracefs_mount_opts { kuid_t uid; kgid_t gid; @@ -384,6 +378,7 @@ static void tracefs_dentry_iput(struct dentry *dentry, struct inode *inode) return;

    ti = get_tracefs(inode);

•   BUG_ON(ti->magic != 20240823);
    if (ti && ti->flags & TRACEFS_EVENT_INODE)
            eventfs_set_ef_status_free(dentry);
    iput(inode);
  

@@ -568,6 +563,18 @@ struct dentry *eventfs_end_creating(struct dentry *dentry) return dentry; }

+struct inode *tracefs_get_inode(struct super_block *sb) +{

•   struct inode *inode = new_inode(sb);
  

•   BUG_ON(sb->s_op != &tracefs_super_operations);
  

•   if (inode) {
  

•           inode->i_ino = get_next_ino();
  

•           inode->i_atime = inode->i_mtime = inode_set_ctime_current(inode);
  

•   }
  

•   return inode;
  

+}

/**

• tracefs_create_file - create a file in the tracefs filesystem
• @name: a pointer to a string containing the name of the file to create.

diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h index 69c2b1d87c46..9059b8b11bb6 100644 --- a/fs/tracefs/internal.h +++ b/fs/tracefs/internal.h @@ -9,12 +9,15 @@ enum { struct tracefs_inode { unsigned long flags; void *private;

•   unsigned long           magic;
    struct inode            vfs_inode;
  

};

static inline struct tracefs_inode *get_tracefs(const struct inode *inode) {

•   return container_of(inode, struct tracefs_inode, vfs_inode);
  

•   struct tracefs_inode *ti = container_of(inode, struct tracefs_inode, vfs_inode);
  

•   BUG_ON(ti->magic != 20240823);
  

•   return ti;
  

}

struct dentry *tracefs_start_creating(const char *name, struct dentry *parent);