当前位置:网站首页>tampercfg内核模块导致机器频繁crash

tampercfg内核模块导致机器频繁crash

2022-08-10 12:52:00 johnazhang

该问题,疑似rootkit或者哪个软件捆绑安装的模块(apache?),不过谷歌并搜不到相关信息。

【如何判断】

  1. 查看dmesg,堆栈有tampercore、tampercfg内核模块打印

2. 如果没有直接打印出tampercore、tampercfg,但是每次重启都是不同的位置踩内存(比如BUG、gpf等)

打开vmcore 后执行mod -t命令可以看到tampercfg、tampercore模块:

crash> mod -t
NAME        TAINTS
tampercfg   OE
tampercore  OE

【解决方案】

这个内核模块用rmmod是无法完全清理掉的,过一段时间又会自己装回来(目前暂无环境继续分析是谁装回来的)。

所以建议直接备份重装、并使用强密码

【其他一些细节】

这个模块会hook系统调用:

crash> p sys_call_table
sys_call_table = $2 = 
/*
* 提示:该行代码过长,系统自动注释不进行高亮。一键复制会移除系统注释 
* {0xffffffff95c4de60, 0xffffffffc0a007d0, 0xffffffffc0a00d30, 0xffffffff95c4c1f0, 0xffffffff95c534d0, 0xffffffff95c53500, 0xffffffff95c534e0, 0xffffffff95c64e40, 0xffffffff95c4cc80, 0xffffffff95a31ef0, 0xffffffff95bfea30, 0xffffffff95bfcce0, 0xffffffff95bfbf40, 0xffffffff95ab5430, 0xffffffff95ab42f0, 0xffffffff961934b0, 0xffffffff95c62ad0, 0xffffffff95c4e040, 0xffffffff95c4e100, 0xffffffff95c4e620, 0xffffffff95c4e730, 0xffffffff95c4b6f0, 0xffffffff95c57c30, 0xffffffff95c644a0, 0xffffffff95ada780, 0xffffffff95bffae0, 0xffffffff95c00200, 0xffffffff95bf74d0, 0xffffffff95becf50, 0xffffffff95cf50c0, 0xffffffff95cf58e0, 0xffffffff95cf5150, 0xffffffff95c6ec30, 0xffffffff95c6ebd0, 0xffffffff95ab57d0, 0xffffffff95acb600, 0xffffffff95aa3430, 0xffffffff95aafa30, 0xffffffff95aa3810, 0xffffffff95ab8bd0, 0xffffffff95c4ee80, 0xffffffff96035190, 0xffffffff960355b0, 0xffffffff96035590, 0xffffffff960355e0, 0xffffffff96035610, 0xffffffff96035960, 0xffffffff96035bd0, 0xffffffff96035830, 0xffffffff960354c0, 0xffffffff960354d0, 0xffffffff960355c0, 0xffffffff960355d0, 0xffffffff96035280, 0xffffffff96035640, 0xffffffff96035740, 0xffffffff96193270, 0xffffffff961932e0, 0xffffffff96193350, 0xffffffff96193430, 0xffffffff95aa2b70, 0xffffffff95aa2e10, 0xffffffff95ab4bb0, 0xffffffff95ab91e0, 0xffffffff95cf3720, 0xffffffff95cf3a60, 0xffffffff95cf37c0, 0xffffffff95cf5930, 0xffffffff95cf0320, 0xffffffff95cf07e0, 0xffffffff95cf0d70, 0xffffffff95cf03a0, 0xffffffff95c61200, 0xffffffff95cad390, 0xffffffff95c83230, 0xffffffff95c83250, 0xffffffff95c4b3c0, 0xffffffff95c4b3f0, 0xffffffff95c630a0, 0xffffffff95c699a0, 0xffffffff95c4b710, 0xffffffff95c4b7e0, 0xffffffffc0a00ad0, 0xffffffffc0a01140, 0xffffffffc0a00e80, 0xffffffffc0a00790, 0xffffffff95c60e70, 0xffffffffc0a009a0, 0xffffffff95c60b40, 0xffffffff95c53660, 0xffffffff95c4bb30, 0xffffffff95c4b9f0, 0xffffffff95c4bc70, 0xffffffff95c4bcc0, 0xffffffff95c4bc90, 0xffffffff95ab9e30, 0xffffffff95aa4090, 0xffffffff95ab9a30, 0xffffffff95ab9de0, 0xffffffff95aba3c0, 0xffffffff95ab8dd0, 0xffffffff95aab8d0, 0xffffffff95ab8c50, 0xffffffff95a9e0e0, 0xffffffff95ab8cb0, 0xffffffff95ab8420, 0xffffffff95ab8190, 0xffffffff95ab8c80, 0xffffffff95ab8ce0, 0xffffffff95ab8e60, 0xffffffff95ab8c20, 0xffffffff95ab9090, 0xffffffff95ab9120, 0xffffffff95ab8270, 0xffffffff95ab8010, 0xffffffff95acea40, 0xffffffff95aceb80, 0xffffffff95ab8540, 0xffffffff95ab8750, 0xffffffff95ab87f0, 0xffffffff95ab8980, 0xffffffff95ab9020, 0xffffffff95ab8a20, 0xffffffff95ab8b00, 0xffffffff95ab90b0, 0xffffffff95aaa070, 0xffffffff95aaa210, 0xffffffff95ab4410, 0xffffffff95ab4ba0, 0xffffffff95ab4c20, 0xffffffff95ab5890, 0xffffffff95ab4fd0, 0xffffffff95c83760, 0xffffffff95c60870, 0xffffffff95ac78b0, 0xffffffff95a9b7e0, 0xffffffff95c84840, 0xffffffff95c846c0, 0xffffffff95c846e0, 0xffffffff95c6f2b0, 0xffffffff95ab79d0, 0xffffffff95ab7750, 0xffffffff95ada070, 0xffffffff95ada2e0, 0xffffffff95ada040, 0xffffffff95ada260, 0xffffffff95ada850, 0xffffffff95ada890, 0xffffffff95ada8d0, 0xffffffff95bf8880, 0xffffffff95bf88e0, 0xffffffff95bf8960, 0xffffffff95bf8b00, 0xffffffff95c4c240, 0xffffffff95a30440, 0xffffffff95c73e70, 0xffffffff95aa9bf0, 0xffffffff95ab9e80, 0xffffffff95a2c1f0, 0xffffffff95aa4350, 0xffffffff95ab9cd0, 0xffffffff95c4b8b0, 0xffffffff95c83080, 0xffffffff95b21530, 0xffffffff95aa4260, 0xffffffff95c73ce0, 0xffffffff95c71cb0, 0xffffffff95c0e380, 0xffffffff95c0dca0, 0xffffffff95ab7fb0, 0xffffffff95ab9410, 0xffffffff95ab95e0, 0xffffffff961933c0, 0xffffffff95a2fc70, 0xffffffff95ac78b0, 0xffffffff95b1f2f0, 0xffffffff95b1c320, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95cc20c0, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95ab8c00, 0xffffffff95bcbd10, 0xffffffff95c76ee0, 0xffffffff95c76fe0, 0xffffffff95c770d0, 0xffffffff95c771e0, 0xffffffff95c772a0, 0xffffffff95c77350, 0xffffffff95c77420, 0xffffffff95c774d0, 0xffffffff95c77570, 0xffffffff95c77640, 0xffffffff95c77710, 0xffffffff95c777e0, 0xffffffff95ab4bf0, 0xffffffff95aa3fc0, 0xffffffff95b15bb0, 0xffffffff95adbf00, 0xffffffff95ada690, 0xffffffff95ac78b0, 0xffffffff95ca45c0, 0xffffffff95ca4950, 0xffffffff95ca5470, 0xffffffff95ca52f0, 0xffffffff95ca5310, 0xffffffff95ac78b0, 0xffffffff95ce0c50, 0xffffffff95c9ca00, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95bea0e0, 0xffffffff95c631d0, 0xffffffff95a9aba0, 0xffffffff95ab4060, 0xffffffff95cf3a50, 0xffffffff95bc30e0, 0xffffffff95ac5460, 0xffffffff95ac5ae0, 0xffffffff95ac5960, 0xffffffff95ac5a70, 0xffffffff95ac5cb0, 0xffffffff95ac5ec0, 0xffffffff95ac5f70, 0xffffffff95ac6030, 0xffffffff95ac60e0, 0xffffffff95aa2c30, 0xffffffff95c9d6b0, 0xffffffff95c9ca30, 0xffffffff95ab4bc0, 0xffffffff95c839d0, 0xffffffff95ac78b0, 0xffffffff95c1ce30, 0xffffffff95c1b950, 0xffffffff95c1b970, 0xffffffff95cf73f0, 0xffffffff95cf7790, 0xffffffff95cf7910, 0xffffffff95cf7c90, 0xffffffff95cf81d0, 0xffffffff95cf8620, 0xffffffff95b23560, 0xffffffff95aa2c80, 0xffffffff95cfc1c0, 0xffffffff95cfc3e0, 0xffffffff95cfdc40, 0xffffffff95c93dc0, 0xffffffff95c940f0, 0xffffffff95c993e0, 0xffffffff95c99400, 0xffffffff95c99750...}
*/
crash> sym 0xffffffff95c4de60
ffffffff95c4de60 (T) sys_read /usr/src/debug/kernel-3.10.0-1127.19.1.el7/linux-3.10.0-1127.19.1.el7.x86_64/fs/read_write.c: 569
crash> sym 0xffffffffc0a00d30
ffffffffc0a00d30 (t) tamperhack_open [tampercore] 

启动加载日志可以看到:

crash> log | grep tainting
[   15.908686] tampercfg: module verification failed: signature and/or required key missing - tainting kernel

一次踩内存的分析举例:

crash> bt
PID: 3141   TASK: ffff9413fba320e0  CPU: 3   COMMAND: "YDService"
 #0 [ffff9413fba7b960] machine_kexec at ffffffff95a66254 
 #1 [ffff9413fba7b9c0] __crash_kexec at ffffffff95b22ef2 
 #2 [ffff9413fba7ba90] crash_kexec at ffffffff95b22fe0 
 #3 [ffff9413fba7baa8] oops_end at ffffffff9618a798 
 #4 [ffff9413fba7bad0] no_context at ffffffff95a75d74 
 #5 [ffff9413fba7bb20] __bad_area_nosemaphore at ffffffff95a76042 
 #6 [ffff9413fba7bb70] bad_area_nosemaphore at ffffffff95a76164 
 #7 [ffff9413fba7bb80] __do_page_fault at ffffffff9618d750 
 #8 [ffff9413fba7bbf0] trace_do_page_fault at ffffffff9618da26 
 #9 [ffff9413fba7bc30] do_async_page_fault at ffffffff9618cfa2 
#10 [ffff9413fba7bc50] async_page_fault at ffffffff961897a8 
    [exception RIP: exit_creds+27]
    RIP: ffffffff95acd8ab  RSP: ffff9413fba7bd08  RFLAGS: 00010246
    RAX: 0000000000000000  RBX: ffff9414409f20e0  RCX: 0000000000000000
    RDX: ffffffff966c0ce0  RSI: ffff9414409f20e0  RDI: 0000000000000000
    RBP: ffff9413fba7bd10   R8: 0000000000000000   R9: ffff94146b229c00
    R10: 00000000000002b1  R11: 0000000000000000  R12: ffff9414409f20e0
    R13: ffff9414409f20e0  R14: 0000000000000001  R15: ffffa23101b3a170
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
#11 [ffff9413fba7bd18] __put_task_struct at ffffffff95a9839a 
#12 [ffff9413fba7bd38] wake_up_q at ffffffff95adb960 
#13 [ffff9413fba7bd60] futex_wake_op at ffffffff95b13193 
#14 [ffff9413fba7be48] do_futex at ffffffff95b1578a 
#15 [ffff9413fba7bed8] sys_futex at ffffffff95b15c30 
#16 [ffff9413fba7bf50] system_call_fastpath at ffffffff96192ed2 
    RIP: 00007f49c18200f6  RSP: 00007f4990060de8  RFLAGS: 00000202
    RAX: 00000000000000ca  RBX: 00000000025cdbe0  RCX: 0000000000000000
    RDX: 0000000000000001  RSI: 0000000000000085  RDI: 00000000025cdc5c
    RBP: 0000000000511800   R8: 00000000025cdc58   R9: 0000000004000001
    R10: 0000000000000001  R11: 0000000000000283  R12: 0000000000000400
    R13: 00007f4990060e50  R14: 00000000025b8620  R15: 0000000000000000
    ORIG_RAX: 00000000000000ca  CS: 0033  SS: 002b
	
crash> dis -r ffffffff95acd8ab
0xffffffff95acd890 <exit_creds>:        nopl   0x0(%rax,%rax,1) [FTRACE NOP]
0xffffffff95acd895 <exit_creds+5>:      push   %rbp
0xffffffff95acd896 <exit_creds+6>:      mov    %rsp,%rbp
0xffffffff95acd899 <exit_creds+9>:      push   %rbx
0xffffffff95acd89a <exit_creds+10>:     mov    0x670(%rdi),%rax
0xffffffff95acd8a1 <exit_creds+17>:     mov    %rdi,%rbx
0xffffffff95acd8a4 <exit_creds+20>:     mov    0x668(%rdi),%rdi
0xffffffff95acd8ab <exit_creds+27>:     mov    (%rax),%eax

rax、rdi是0,对应的task结构体被置空
	
crash> wake_q_head ffff9413fba7bdd0
struct wake_q_head {
  first = 0xffff9414409f30f8, 
  lastp = 0xffff9414409f30f8
}
crash> wake_q_node 0xffff9414409f30f8
struct wake_q_node {
  next = 0x0
}

这里为0不符合预期

对应代码:
void wake_up_q(struct wake_q_head *head)
{       
        struct wake_q_node *node = head->first;

        while (node != WAKE_Q_TAIL) {
                struct task_struct *task;

                task = container_of(node, struct task_struct, wake_q);
                BUG_ON(!task);	//注意,这里通过了BUG检查,说明这时候还不是0
                /* task can safely be re-inserted now */
                node = node->next;
                task->wake_q.next = NULL;
                
                /*
                 * wake_up_process() implies a wmb() to pair with the queueing
                 * in wake_q_add() so as not to miss wakeups.
                 */
                wake_up_process(task);
                put_task_struct(task);	//这里进入这个函数后,就再没有task为0的检查,后面被并发改成0了
        }
}

所以就是这个内核模块本身写的也有点问题,会去并发的改task结构体,导致其他的正常执行逻辑里,task为0跪了。

原网站

版权声明
本文为[johnazhang]所创,转载请带上原文链接,感谢
https://cloud.tencent.com/developer/article/2069524