RepoMirrors
/
kpatch
mirror of https://github.com/dynup/kpatch
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
kpatch/kpatch-build/create-diff-object.c

4192 lines
115 KiB
C
Raw Blame History

/*
* create-diff-object.c
*
* Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
* Copyright (C) 2013-2014 Josh Poimboeuf <jpoimboe@redhat.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA,
* 02110-1301, USA.
*/
/*
* This file contains the heart of the ELF object differencing engine.
*
* The tool takes two ELF objects from two versions of the same source
* file; a "orig" object and a "patched" object. These object need to have
* been compiled with the -ffunction-sections and -fdata-sections GCC options.
*
* The tool compares the objects at a section level to determine what
* sections have changed. Once a list of changed sections has been generated,
* various rules are applied to determine any object local sections that
* are dependencies of the changed section and also need to be included in
* the output object.
*/
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <err.h>
#include <gelf.h>
#include <argp.h>
#include <libgen.h>
#include <unistd.h>
#include "list.h"
#include "lookup.h"
#include "kpatch-patch.h"
#include "kpatch-elf.h"
#include "kpatch-intermediate.h"
#include "kpatch.h"
#define DIFF_FATAL(format, ...) \
({ \
fprintf(stderr, "ERROR: %s: " format "\n", childobj, ##__VA_ARGS__); \
errx(EXIT_STATUS_DIFF_FATAL, "unreconcilable difference"); \
})
char *childobj;
enum subsection {
SUBSECTION_NORMAL,
SUBSECTION_HOT,
SUBSECTION_UNLIKELY
};
enum loglevel loglevel = NORMAL;
bool KLP_ARCH;
int jump_label_errors, static_call_errors;
/*******************
* Data structures
* ****************/
struct special_section {
char *name;
enum architecture arch;
int (*group_size)(struct kpatch_elf *kelf, int offset);
bool (*group_filter)(struct lookup_table *lookup,
struct section *relasec, unsigned int offset,
unsigned int size);
};
/*************
* Functions
* **********/
static bool is_bundleable(struct symbol *sym)
{
if (sym->type == STT_FUNC &&
!strncmp(sym->sec->name, ".text.",6) &&
!strcmp(sym->sec->name + 6, sym->name))
return true;
if (sym->type == STT_FUNC &&
!strncmp(sym->sec->name, ".text.unlikely.",15) &&
(!strcmp(sym->sec->name + 15, sym->name) ||
(strstr(sym->name, ".cold") &&
!strncmp(sym->sec->name + 15, sym->name, strlen(sym->sec->name) - 15))))
return true;
if (sym->type == STT_FUNC &&
!strncmp(sym->sec->name, ".text.hot.",10) &&
!strcmp(sym->sec->name + 10, sym->name))
return true;
if (sym->type == STT_OBJECT &&
!strncmp(sym->sec->name, ".data.",6) &&
!strcmp(sym->sec->name + 6, sym->name))
return true;
if (sym->type == STT_OBJECT &&
!strncmp(sym->sec->name, ".data.rel.", 10) &&
!strcmp(sym->sec->name + 10, sym->name))
return true;
if (sym->type == STT_OBJECT &&
!strncmp(sym->sec->name, ".data.rel.ro.", 13) &&
!strcmp(sym->sec->name + 13, sym->name))
return true;
if (sym->type == STT_OBJECT &&
!strncmp(sym->sec->name, ".data.rel.ro.local.", 19) &&
!strcmp(sym->sec->name + 19, sym->name))
return 1;
if (sym->type == STT_OBJECT &&
!strncmp(sym->sec->name, ".data.rel.local.", 16) &&
!strcmp(sym->sec->name + 16, sym->name))
return 1;
if (sym->type == STT_OBJECT &&
!strncmp(sym->sec->name, ".rodata.",8) &&
!strcmp(sym->sec->name + 8, sym->name))
return true;
if (sym->type == STT_OBJECT &&
!strncmp(sym->sec->name, ".bss.",5) &&
!strcmp(sym->sec->name + 5, sym->name))
return true;
return false;
}
/* Symbol st_others value for powerpc */
#define STO_PPC64_LOCAL_BIT 5
#define STO_PPC64_LOCAL_MASK (7 << STO_PPC64_LOCAL_BIT)
#define PPC64_LOCAL_ENTRY_OFFSET(other) \
(((1 << (((other) & STO_PPC64_LOCAL_MASK) >> STO_PPC64_LOCAL_BIT)) >> 2) << 2)
/*
* On ppc64le, the function prologue generated by GCC 6+ has the sequence:
*
* .globl my_func
* .type my_func, @function
* .quad .TOC.-my_func
* my_func:
* .reloc ., R_PPC64_ENTRY ; optional
* ld r2,-8(r12)
* add r2,r2,r12
* .localentry my_func, .-my_func
*
* my_func is the global entry point, which, when called, sets up the TOC.
* .localentry is the local entry point, for calls to the function from within
* the object file. The local entry point is 8 bytes after the global entry
* point.
*/
static bool is_gcc6_localentry_bundled_sym(struct kpatch_elf *kelf,
struct symbol *sym)
{
switch(kelf->arch) {
case PPC64:
return ((PPC64_LOCAL_ENTRY_OFFSET(sym->sym.st_other) != 0) &&
sym->sym.st_value == 8);
case X86_64:
return false;
case S390:
return false;
default:
ERROR("unsupported arch");
}
return false;
}
/*
* On ppc64le, when a function references data, it does so indirectly, via the
* .toc section. So there are *two* levels of relas:
*
* 1) the original function rela, referring to the .toc section; and
*
* 2) the .toc section rela, referring to the data needed by the function.
*
* For example:
*
* Relocation section '.rela.text.netlink_release' at offset 0xcadf0 contains 44 entries:
* ...
* 0000000000000398 0000007300000032 R_PPC64_TOC16_HA 0000000000000000 .toc + 138
* 00000000000003a0 0000007300000040 R_PPC64_TOC16_LO_DS 0000000000000000 .toc + 138
*
* Relocation section '.rela.toc' at offset 0xcc6b0 contains 46 entries:
* ...
* 0000000000000138 0000002a00000026 R_PPC64_ADDR64 0000000000000000 .text.deferred_put_nlk_sk + 8
*
* The below function takes the "first level" rela as input, and, if it refers
* to .toc, returns the "second level" rela, which is the one that refers to
* the actual data symbol.
*
* In some rare cases, a .toc entry has constant data, and thus has no
* corresponding rela. In that case, NULL is returned.
*/
static struct rela *toc_rela(const struct rela *rela)
{
if (rela->type != R_PPC64_TOC16_HA &&
rela->type != R_PPC64_TOC16_LO_DS)
return (struct rela *)rela;
/* Only constants in toc */
if (!rela->sym->sec->rela)
return NULL;
/* Will return NULL for .toc constant entries */
return find_rela_by_offset(rela->sym->sec->rela,
(unsigned int)rela->addend);
}
/*
* When compiling with -ffunction-sections and -fdata-sections, almost every
* symbol gets its own dedicated section. We call such symbols "bundled"
* symbols. They're indicated by "sym->sec->sym == sym".
*/
static void kpatch_bundle_symbols(struct kpatch_elf *kelf)
{
struct symbol *sym;
unsigned int expected_offset;
list_for_each_entry(sym, &kelf->symbols, list) {
if (is_bundleable(sym)) {
if (sym->pfx)
expected_offset = 16;
else if (is_gcc6_localentry_bundled_sym(kelf, sym))
expected_offset = 8;
else
expected_offset = 0;
if (sym->sym.st_value != expected_offset) {
ERROR("symbol %s at offset %lu within section %s, expected %u",
sym->name, sym->sym.st_value,
sym->sec->name, expected_offset);
}
sym->sec->sym = sym;
}
}
}
static struct symbol *kpatch_lookup_parent(struct kpatch_elf *kelf,
const char *symname,
const char *child_suffix)
{
struct symbol *parent;
char *pname;
pname = strndup(symname, child_suffix - symname);
if (!pname)
ERROR("strndup");
parent = find_symbol_by_name(&kelf->symbols, pname);
free(pname);
return parent;
}
/*
* During optimization gcc may move unlikely execution branches into *.cold
* subfunctions. Some functions can also be split into multiple *.part
* functions.
* kpatch_detect_child_functions detects such subfunctions and
* crossreferences them with their parent functions through parent/child
* pointers.
*/
static void kpatch_detect_child_functions(struct kpatch_elf *kelf)
{
struct symbol *sym;
list_for_each_entry(sym, &kelf->symbols, list) {
char *childstr;
if (sym->type != STT_FUNC)
continue;
childstr = strstr(sym->name, ".cold");
if (childstr) {
sym->parent = kpatch_lookup_parent(kelf, sym->name,
childstr);
if (!sym->parent)
ERROR("failed to find parent function for %s",
sym->name);
} else {
childstr = strstr(sym->name, ".part.");
if (!childstr)
continue;
sym->parent = kpatch_lookup_parent(kelf, sym->name,
childstr);
}
if (sym->parent)
list_add_tail(&sym->subfunction_node, &sym->parent->children);
}
}
static bool is_dynamic_debug_symbol(struct symbol *sym)
{
if (sym->type == STT_OBJECT && !strcmp(sym->sec->name, "__verbose"))
return true;
if (sym->type == STT_OBJECT && !strcmp(sym->sec->name, "__dyndbg"))
return true;
if (sym->type == STT_SECTION && !strcmp(sym->name, "__verbose"))
return true;
if (sym->type == STT_SECTION && !strcmp(sym->name, "__dyndbg"))
return true;
return false;
}
static bool is_string_literal_section(struct section *sec)
{
return !strncmp(sec->name, ".rodata.", 8) && strstr(sec->name, ".str");
}
/*
* This function detects whether the given symbol is a "special" static local
* variable (for lack of a better term).
*
* Special static local variables should never be correlated and should always
* be included if they are referenced by an included function.
*/
static bool is_special_static(struct symbol *sym)
{
static char *var_names[] = {
"__key",
"__warned",
"__already_done.",
"__func__",
"__FUNCTION__",
"_rs",
"CSWTCH",
"_entry",
NULL,
};
char **var_name;
if (!sym)
return false;
/* pr_debug() uses static local variables in the __verbose or __dyndbg section */
if (is_dynamic_debug_symbol(sym))
return true;
if (sym->type == STT_SECTION) {
/* make sure section is bundled */
if (!sym->sec->sym)
return false;
/* use bundled object/function symbol for matching */
sym = sym->sec->sym;
}
if (sym->type != STT_OBJECT || sym->bind != STB_LOCAL)
return false;
if (!strcmp(sym->sec->name, ".data.once"))
return true;
for (var_name = var_names; *var_name; var_name++) {
size_t var_name_len = strlen(*var_name);
char buf[256];
snprintf(buf, 256, ".%s.", *var_name);
/* First look for gcc-style statics: '<var_name>.' */
if (!strncmp(sym->name, buf + 1, var_name_len + 1))
return true;
buf[var_name_len + 1] = '\0';
/* Next clang-style statics: '<function_name>.<var_name>' */
if (strstr(sym->name, buf))
return true;
}
return false;
}
static bool has_digit_tail(char *tail)
{
if (*tail != '.')
return false;
while (isdigit(*++tail))
;
if (!*tail)
return true;
return false;
}
/*
* Hack for __UNIQUE_ID(). The following should match:
*
* __UNIQUE_ID_ddebug1131.186
* __UNIQUE_ID_ddebug1132.187
*/
static int __kpatch_unique_id_strcmp(char *s1, char *s2)
{
/* match '__UNIQUE_ID_ddebug' */
while (*s1 == *s2) {
if (!*s1)
return 0;
s1++;
s2++;
}
/* skip digits before '.' or EOL */
while (isdigit(*s1))
s1++;
while (isdigit(*s2))
s2++;
if ((!*s1 || has_digit_tail(s1)) &&
(!*s2 || has_digit_tail(s2)))
return 0;
return 1;
}
/*
* This is like strcmp, but for gcc-mangled symbols. It skips the comparison
* of any substring which consists of '.' followed by any number of digits.
*/
static int kpatch_mangled_strcmp(char *s1, char *s2)
{
/*
* ELF string sections aren't mangled, though they look that way. Just
* compare them normally.
*/
if (strstr(s1, ".str1."))
return strcmp(s1, s2);
if (!strncmp(s1, "__UNIQUE_ID_", 12))
return __kpatch_unique_id_strcmp(s1, s2);
while (*s1 == *s2) {
if (!*s1)
return 0;
if (*s1 == '.' && isdigit(s1[1])) {
if (!isdigit(s2[1]))
return 1;
while (isdigit(*++s1))
;
while (isdigit(*++s2))
;
} else {
s1++;
s2++;
}
}
if ((!*s1 && has_digit_tail(s2)) ||
(!*s2 && has_digit_tail(s1)))
return 0;
return 1;
}
static bool rela_equal(struct rela *rela1, struct rela *rela2)
{
struct rela *rela_toc1, *rela_toc2;
unsigned long toc_data1 = 0, toc_data2 = 0; /* = 0 to prevent gcc warning */
if (rela1->type != rela2->type ||
rela1->offset != rela2->offset)
return false;
/*
* On x86, .altinstr_aux is used to store temporary code which allows
* static_cpu_has() to work before apply_alternatives() has run. This
* code is completely inert for modules, because apply_alternatives()
* runs during module init, before the module is fully formed. Any
* changed references to it (i.e. changed addend) can be ignored. As
* long as they're both references to .altinstr_aux, they can be
* considered equal, even if the addends differ.
*/
if (!strcmp(rela1->sym->name, ".altinstr_aux") &&
!strcmp(rela2->sym->name, ".altinstr_aux"))
return true;
/*
* With -mcmodel=large on ppc64le, GCC might generate entries in the .toc
* section for relocation symbol references. The .toc offsets may change
* between the original and patched .o, so comparing ".toc + offset" isn't
* right. Compare the .toc-based symbols by reading the corresponding relas
* from the .toc section.
*/
rela_toc1 = toc_rela(rela1);
if (!rela_toc1) {
/*
* .toc section entries are mostly place holder for relocation entries, specified
* in .rela.toc section. Sometimes, .toc section may have constants as entries.
* These constants are not reference to any symbols, but plain instructions mostly
* due to some arithmetics in the functions referring them.
*
* They are referred by the functions like normal .toc entries, these entries can
* not be resolved to any symbols.
*
* Disassembly of section .toc:
*
* 0000000000000000 <.toc>:
* ...
* 148: R_PPC64_ADDR64 .data.capacity_margin
* 150: 0b d7 a3 70 andi. r3,r5,55051
* 154: 3d 0a d7 a3 lhz r30,2621(r23)
* 158: R_PPC64_ADDR64 sched_max_numa_distance
*
* Relocation section '.rela.toc' at offset 0xadac0 contains 160 entries:
* Offset Info Type Symbol's Value Symbol's Name + Addend
* ...
* 0000000000000148 0000009100000026 R_PPC64_ADDR64 0000000000000000 .data.capacity_margin + 0
* 0000000000000158 000001a500000026 R_PPC64_ADDR64 0000000000000000 sched_max_numa_distance + 0
*
* Relocation section '.rela.text.select_task_rq_fair' at offset 0x90e98 contains 37 entries:
* Offset Info Type Symbol's Value Symbol's Name + Addend
* ...
* 00000000000004a0 0000008800000032 R_PPC64_TOC16_HA 0000000000000000 .toc + 148
* 00000000000004ac 0000008800000040 R_PPC64_TOC16_LO_DS 0000000000000000 .toc + 148
* 0000000000000514 0000008800000032 R_PPC64_TOC16_HA 0000000000000000 .toc + 150
* 000000000000051c 0000008800000040 R_PPC64_TOC16_LO_DS 0000000000000000 .toc + 150
*/
memcpy(&toc_data1, rela1->sym->sec->data->d_buf + rela1->addend, sizeof(toc_data1));
if (!toc_data1)
ERROR(".toc entry not found %s + %lx", rela1->sym->name, rela1->addend);
}
rela_toc2 = toc_rela(rela2);
if (!rela_toc2) {
memcpy(&toc_data2, rela2->sym->sec->data->d_buf + rela2->addend, sizeof(toc_data2));
if (!toc_data2)
ERROR(".toc entry not found %s + %lx", rela2->sym->name, rela2->addend);
}
if (!rela_toc1 && !rela_toc2)
return toc_data1 == toc_data2;
if (!rela_toc1 || !rela_toc2)
return false;
if (rela_toc1->string)
return rela_toc2->string && !strcmp(rela_toc1->string, rela_toc2->string);
if (rela_toc1->addend != rela_toc2->addend)
return false;
return !kpatch_mangled_strcmp(rela_toc1->sym->name, rela_toc2->sym->name);
}
static void kpatch_compare_correlated_rela_section(struct section *relasec)
{
struct rela *rela1, *rela2 = NULL;
/*
* On ppc64le, don't compare the .rela.toc section. The .toc and
* .rela.toc sections are included as standard elements.
*/
if (!strcmp(relasec->name, ".rela.toc")) {
relasec->status = SAME;
return;
}
rela2 = list_entry(relasec->twin->relas.next, struct rela, list);
list_for_each_entry(rela1, &relasec->relas, list) {
if (rela_equal(rela1, rela2)) {
rela2 = list_entry(rela2->list.next, struct rela, list);
continue;
}
relasec->status = CHANGED;
return;
}
relasec->status = SAME;
}
static void kpatch_compare_correlated_nonrela_section(struct section *sec)
{
struct section *sec1 = sec, *sec2 = sec->twin;
if (sec1->sh.sh_type != SHT_NOBITS &&
memcmp(sec1->data->d_buf, sec2->data->d_buf, sec1->data->d_size))
sec->status = CHANGED;
else
sec->status = SAME;
}
static void kpatch_compare_correlated_section(struct section *sec)
{
struct section *sec1 = sec, *sec2 = sec->twin;
/* Compare section headers (must match or fatal) */
if (sec1->sh.sh_type != sec2->sh.sh_type ||
sec1->sh.sh_flags != sec2->sh.sh_flags ||
sec1->sh.sh_entsize != sec2->sh.sh_entsize ||
(sec1->sh.sh_addralign != sec2->sh.sh_addralign &&
!is_text_section(sec1)))
DIFF_FATAL("%s section header details differ from %s", sec1->name, sec2->name);
/* Short circuit for mcount sections, we rebuild regardless */
if (!strcmp(sec->name, ".rela__mcount_loc") ||
!strcmp(sec->name, "__mcount_loc")) {
sec->status = SAME;
goto out;
}
if (sec1->sh.sh_size != sec2->sh.sh_size ||
sec1->data->d_size != sec2->data->d_size ||
(sec1->rela && !sec2->rela) ||
(sec2->rela && !sec1->rela)) {
sec->status = CHANGED;
goto out;
}
if (is_rela_section(sec))
kpatch_compare_correlated_rela_section(sec);
else
kpatch_compare_correlated_nonrela_section(sec);
out:
if (sec->status == CHANGED)
log_debug("section %s has changed\n", sec->name);
}
/*
* This function is not comprehensive, i.e. it doesn't detect immediate loads
* to *all* registers. It only detects those which have been found in the wild
* to be involved in the load of a __LINE__ immediate. If we miss some, that's
* ok, we'll find them later when someone notices a function falsely being
* reported as changed ;-)
*
* Right now we're only checking immediate loads to the registers corresponding
* to function arguments 2 and 3 for each respective arch's calling convention.
* (Argument 1 is typically the printf format string). Eventually we might
* want to consider just checking *all* registers which could conceivably be
* used as function arguments. But in practice, arg2 and arg3 seem to be the
* main ones, so for now, take a more conservative approach at the risk of
* failing to detect some of the more obscure __LINE__-only changed functions.
*/
static bool insn_is_load_immediate(struct kpatch_elf *kelf, void *addr)
{
unsigned char *insn = addr;
switch(kelf->arch) {
case X86_64:
/* arg2: mov $imm, %esi */
if (insn[0] == 0xbe)
return true;
/* arg3: mov $imm, %edx */
if (insn[0] == 0xba)
return true;
break;
case PPC64:
/*
* ppc64le insns are LE-encoded:
*
* 0a 00 80 38 li r4,10
* 47 14 a0 38 li r5,5191
*/
/* arg2: li r4, imm */
if (insn[3] == 0x38 && insn[2] == 0x80)
return true;
/* arg3: li r5, imm */
if (insn[3] == 0x38 && insn[2] == 0xa0)
return true;
break;
case S390:
/* arg2: lghi %r3, imm */
if (insn[0] == 0xa7 && insn[1] == 0x39)
return true;
/* arg3: lghi %r4, imm */
if (insn[0] == 0xa7 && insn[1] == 0x49)
return true;
break;
default:
ERROR("unsupported arch");
}
return false;
}
/*
* Determine if a section has changed only due to a __LINE__ number change,
* e.g. a WARN() or might_sleep() macro's embedding of the line number into an
* instruction operand.
*
* Warning: Hackery lies herein. It's hopefully justified by the fact that
* this issue is very common.
*
* Example WARN():
*
* 938: be 70 00 00 00 mov $0x70,%esi
* 93d: 48 c7 c7 00 00 00 00 mov $0x0,%rdi
* 940: R_X86_64_32S .rodata.tcp_conn_request.str1.8+0x88
* 944: c6 05 00 00 00 00 01 movb $0x1,0x0(%rip) # 94b <tcp_conn_request+0x94b>
* 946: R_X86_64_PC32 .data.unlikely-0x1
* 94b: e8 00 00 00 00 callq 950 <tcp_conn_request+0x950>
* 94c: R_X86_64_PC32 warn_slowpath_null-0x4
*
* Example might_sleep:
*
* 50f: be f7 01 00 00 mov $0x1f7,%esi
* 514: 48 c7 c7 00 00 00 00 mov $0x0,%rdi
* 517: R_X86_64_32S .rodata.do_select.str1.8+0x98
* 51b: e8 00 00 00 00 callq 520 <do_select+0x520>
* 51c: R_X86_64_PC32 ___might_sleep-0x4
*/
static bool kpatch_line_macro_change_only(struct kpatch_elf *kelf,
struct section *sec)
{
unsigned long offset, insn1_len, insn2_len;
void *data1, *data2, *insn1, *insn2;
struct rela *r, *rela;
bool found, found_any = false;
if (sec->status != CHANGED ||
is_rela_section(sec) ||
!is_text_section(sec) ||
sec->sh.sh_size != sec->twin->sh.sh_size ||
!sec->rela ||
sec->rela->status != SAME)
return false;
data1 = sec->twin->data->d_buf;
data2 = sec->data->d_buf;
for (offset = 0; offset < sec->sh.sh_size; offset += insn1_len) {
insn1 = data1 + offset;
insn2 = data2 + offset;
insn1_len = insn_length(kelf, insn1);
insn2_len = insn_length(kelf, insn2);
if (!insn1_len || !insn2_len)
ERROR("can't decode instruction in section %s at offset 0x%lx",
sec->name, offset);
if (insn1_len != insn2_len)
return false;
if (!memcmp(insn1, insn2, insn1_len))
continue;
/*
* Here we found a difference between two instructions of the
* same length. Only ignore the change if:
*
* a) the instructions match a known pattern of a '__LINE__'
* macro immediate value which was embedded in the
* instruction; and
*
* b) the instructions are followed by certain expected
* relocations.
*/
if (!insn_is_load_immediate(kelf, insn1) ||
!insn_is_load_immediate(kelf, insn2))
return false;
found = false;
list_for_each_entry(r, &sec->rela->relas, list) {
if (r->offset < offset + insn1_len)
continue;
rela = toc_rela(r);
if (!rela)
continue;
if (rela->string)
continue;
if (!strncmp(rela->sym->name, "__warned.", 9) ||
!strncmp(rela->sym->name, "__already_done.", 15) ||
!strncmp(rela->sym->name, "__func__.", 9))
continue;
if (!strncmp(rela->sym->name, "warn_slowpath_", 14) ||
!strcmp(rela->sym->name, "__warn_printk") ||
!strcmp(rela->sym->name, "__might_sleep") ||
!strcmp(rela->sym->name, "___might_sleep") ||
!strcmp(rela->sym->name, "__might_fault") ||
!strcmp(rela->sym->name, "printk") ||
!strcmp(rela->sym->name, "_printk") ||
!strcmp(rela->sym->name, "lockdep_rcu_suspicious") ||
!strcmp(rela->sym->name, "__btrfs_abort_transaction") ||
!strcmp(rela->sym->name, "__btrfs_handle_fs_error") ||
!strcmp(rela->sym->name, "__btrfs_panic")) {
found = true;
break;
}
return false;
}
if (!found)
return false;
found_any = true;
}
if (!found_any)
ERROR("no instruction changes detected for changed section %s",
sec->name);
return true;
}
/*
* Child functions with "*.cold" names don't have _fentry_ calls, but "*.part",
* often do. In the later case, it is not necessary to include the parent
* in the output object when the child function has changed.
*/
static bool kpatch_changed_child_needs_parent_profiling(struct symbol *sym)
{
struct symbol *child;
list_for_each_entry(child, &sym->children, subfunction_node) {
if (child->has_func_profiling)
continue;
if (child->sec->status == CHANGED ||
kpatch_changed_child_needs_parent_profiling(child))
return true;
}
return false;
}
static void kpatch_compare_sections(struct kpatch_elf *kelf)
{
struct section *sec;
struct list_head *seclist = &kelf->sections;
/* compare all sections */
list_for_each_entry(sec, seclist, list) {
if (sec->twin)
kpatch_compare_correlated_section(sec);
else
sec->status = NEW;
}
/* exclude WARN-only, might_sleep changes */
list_for_each_entry(sec, seclist, list) {
if (kpatch_line_macro_change_only(kelf, sec)) {
log_debug("reverting macro / line number section %s status to SAME\n",
sec->name);
sec->status = SAME;
}
}
/* sync symbol status */
list_for_each_entry(sec, seclist, list) {
if (is_rela_section(sec)) {
if (sec->base->sym && sec->base->sym->status != CHANGED)
sec->base->sym->status = sec->status;
} else {
struct symbol *sym = sec->sym;
if (sym && sym->status != CHANGED)
sym->status = sec->status;
if (sym && sym->status == SAME &&
kpatch_changed_child_needs_parent_profiling(sym))
sym->status = CHANGED;
}
}
}
static enum subsection kpatch_subsection_type(struct section *sec)
{
if (!strncmp(sec->name, ".text.unlikely.", 15))
return SUBSECTION_UNLIKELY;
if (!strncmp(sec->name, ".text.hot.", 10))
return SUBSECTION_HOT;
return SUBSECTION_NORMAL;
}
static bool kpatch_subsection_changed(struct section *sec1, struct section *sec2)
{
return kpatch_subsection_type(sec1) != kpatch_subsection_type(sec2);
}
static struct symbol *kpatch_get_correlated_parent(struct symbol *sym)
{
while (sym->parent && !sym->parent->twin)
sym = sym->parent;
return sym->parent;
}
static void kpatch_compare_correlated_symbol(struct symbol *sym)
{
struct symbol *sym1 = sym, *sym2 = sym->twin;
if (sym1->sym.st_info != sym2->sym.st_info ||
(sym1->sec && !sym2->sec) ||
(sym2->sec && !sym1->sec))
DIFF_FATAL("symbol info mismatch: %s", sym1->name);
/*
* If two symbols are correlated but their sections are not, then the
* symbol has changed sections. This is only allowed if the symbol is
* moving out of an ignored section, or moving between normal/hot/unlikely
* subsections.
*/
if (sym1->sec && sym2->sec && sym1->sec->twin != sym2->sec) {
if ((sym2->sec->twin && sym2->sec->twin->ignore) ||
kpatch_subsection_changed(sym1->sec, sym2->sec))
sym->status = CHANGED;
else
DIFF_FATAL("symbol changed sections: %s", sym1->name);
}
if (sym1->type == STT_OBJECT &&
sym1->sym.st_size != sym2->sym.st_size)
DIFF_FATAL("object size mismatch: %s", sym1->name);
if (sym1->sym.st_shndx == SHN_UNDEF ||
sym1->sym.st_shndx == SHN_ABS)
sym1->status = SAME;
/*
* The status of LOCAL symbols is dependent on the status of their
* matching section and is set during section comparison.
*/
}
static void kpatch_compare_symbols(struct list_head *symlist)
{
struct symbol *sym;
list_for_each_entry(sym, symlist, list) {
if (sym->twin)
kpatch_compare_correlated_symbol(sym);
else
sym->status = NEW;
log_debug("symbol %s is %s\n", sym->name, status_str(sym->status));
}
}
#define CORRELATE_ELEMENT(_e1_, _e2_, kindstr) \
do { \
typeof(_e1_) e1 = (_e1_); \
typeof(_e2_) e2 = (_e2_); \
e1->twin = e2; \
e2->twin = e1; \
/* set initial status, might change */ \
e1->status = e2->status = SAME; \
if (strcmp(e1->name, e2->name)) { \
/* Rename mangled element */ \
log_debug("renaming %s %s to %s\n", \
kindstr, e2->name, e1->name); \
e2->name = strdup(e1->name); \
if (!e2->name) \
ERROR("strdup"); \
} \
} while (0)
#define UNCORRELATE_ELEMENT(_elem_) \
do { \
typeof(_elem_) elem = (_elem_); \
elem->twin->twin = NULL; \
elem->twin = NULL; \
} while (0)
static void __kpatch_correlate_section(struct section *sec_orig,
struct section *sec_patched)
{
CORRELATE_ELEMENT(sec_orig, sec_patched, "section");
}
static void kpatch_correlate_symbol(struct symbol *sym_orig,
struct symbol *sym_patched)
{
CORRELATE_ELEMENT(sym_orig, sym_patched, "symbol");
if (sym_orig->lookup_table_file_sym && !sym_patched->lookup_table_file_sym)
sym_patched->lookup_table_file_sym = sym_orig->lookup_table_file_sym;
}
static void kpatch_correlate_static_local(struct symbol *sym_orig,
struct symbol *sym_patched)
{
CORRELATE_ELEMENT(sym_orig, sym_patched, "static local");
}
static void kpatch_correlate_section(struct section *sec_orig,
struct section *sec_patched)
{
__kpatch_correlate_section(sec_orig, sec_patched);
if (is_rela_section(sec_orig)) {
__kpatch_correlate_section(sec_orig->base, sec_patched->base);
sec_orig = sec_orig->base;
sec_patched = sec_patched->base;
} else if (sec_orig->rela && sec_patched->rela) {
__kpatch_correlate_section(sec_orig->rela, sec_patched->rela);
}
if (sec_orig->secsym)
kpatch_correlate_symbol(sec_orig->secsym, sec_patched->secsym);
if (sec_orig->sym)
kpatch_correlate_symbol(sec_orig->sym, sec_patched->sym);
}
static void kpatch_correlate_sections(struct list_head *seclist_orig,
struct list_head *seclist_patched)
{
struct section *sec_orig, *sec_patched;
list_for_each_entry(sec_orig, seclist_orig, list) {
if (sec_orig->twin)
continue;
list_for_each_entry(sec_patched, seclist_patched, list) {
if (kpatch_mangled_strcmp(sec_orig->name, sec_patched->name) ||
sec_patched->twin)
continue;
if (is_ubsan_sec(sec_orig->name))
continue;
if (is_special_static(is_rela_section(sec_orig) ?
sec_orig->base->secsym :
sec_orig->secsym))
continue;
/*
* Group sections must match exactly to be correlated.
* Changed group sections are currently not supported.
*/
if (sec_orig->sh.sh_type == SHT_GROUP) {
if (sec_orig->data->d_size != sec_patched->data->d_size)
continue;
if (memcmp(sec_orig->data->d_buf, sec_patched->data->d_buf,
sec_orig->data->d_size))
continue;
}
kpatch_correlate_section(sec_orig, sec_patched);
break;
}
}
}
static void kpatch_correlate_symbols(struct list_head *symlist_orig,
struct list_head *symlist_patched)
{
struct symbol *sym_orig, *sym_patched;
list_for_each_entry(sym_orig, symlist_orig, list) {
if (sym_orig->twin)
continue;
list_for_each_entry(sym_patched, symlist_patched, list) {
if (kpatch_mangled_strcmp(sym_orig->name, sym_patched->name) ||
sym_orig->type != sym_patched->type || sym_patched->twin)
continue;
if (is_ubsan_sec(sym_orig->name))
continue;
if (is_special_static(sym_orig))
continue;
/*
* The .LCx symbols point to string literals in
* '.rodata.<func>.str1.*' sections. They get included
* in kpatch_include_standard_elements().
*/
if (sym_orig->type == STT_NOTYPE &&
!strncmp(sym_orig->name, ".LC", 3))
continue;
/* group section symbols must have correlated sections */
if (sym_orig->sec &&
sym_orig->sec->sh.sh_type == SHT_GROUP &&
sym_orig->sec->twin != sym_patched->sec)
continue;
kpatch_correlate_symbol(sym_orig, sym_patched);
break;
}
}
}
static void kpatch_compare_elf_headers(Elf *elf_orig, Elf *elf_patched)
{
GElf_Ehdr ehdr_orig, ehdr_patched;
if (!gelf_getehdr(elf_orig, &ehdr_orig))
ERROR("gelf_getehdr");
if (!gelf_getehdr(elf_patched, &ehdr_patched))
ERROR("gelf_getehdr");
if (memcmp(ehdr_orig.e_ident, ehdr_patched.e_ident, EI_NIDENT) ||
ehdr_orig.e_type != ehdr_patched.e_type ||
ehdr_orig.e_machine != ehdr_patched.e_machine ||
ehdr_orig.e_version != ehdr_patched.e_version ||
ehdr_orig.e_entry != ehdr_patched.e_entry ||
ehdr_orig.e_phoff != ehdr_patched.e_phoff ||
ehdr_orig.e_flags != ehdr_patched.e_flags ||
ehdr_orig.e_ehsize != ehdr_patched.e_ehsize ||
ehdr_orig.e_phentsize != ehdr_patched.e_phentsize ||
ehdr_orig.e_shentsize != ehdr_patched.e_shentsize)
DIFF_FATAL("ELF headers differ");
}
static void kpatch_check_program_headers(Elf *elf)
{
size_t ph_nr;
if (elf_getphdrnum(elf, &ph_nr))
ERROR("elf_getphdrnum");
if (ph_nr != 0)
DIFF_FATAL("ELF contains program header");
}
static void kpatch_mark_grouped_sections(struct kpatch_elf *kelf)
{
struct section *groupsec, *sec;
unsigned int *data, *end;
list_for_each_entry(groupsec, &kelf->sections, list) {
if (groupsec->sh.sh_type != SHT_GROUP)
continue;
data = groupsec->data->d_buf;
end = groupsec->data->d_buf + groupsec->data->d_size;
data++; /* skip first flag word (e.g. GRP_COMDAT) */
while (data < end) {
sec = find_section_by_index(&kelf->sections, *data);
if (!sec)
ERROR("group section not found");
sec->grouped = 1;
log_debug("marking section %s (%d) as grouped\n",
sec->name, sec->index);
data++;
}
}
}
static char *kpatch_section_function_name(struct section *sec)
{
if (is_rela_section(sec))
sec = sec->base;
return sec->sym ? sec->sym->name : sec->name;
}
static struct symbol *kpatch_find_uncorrelated_rela(struct section *relasec,
struct symbol *sym)
{
struct rela *rela, *rela_toc;
/* find the patched object's corresponding variable */
list_for_each_entry(rela, &relasec->relas, list) {
struct symbol *patched_sym;
rela_toc = toc_rela(rela);
if (!rela_toc)
continue; /* skip toc constants */
patched_sym = rela_toc->sym;
if (patched_sym->twin)
continue;
if (sym->type != patched_sym->type ||
(sym->type == STT_OBJECT &&
sym->sym.st_size != patched_sym->sym.st_size))
continue;
if (kpatch_mangled_strcmp(patched_sym->name, sym->name))
continue;
return patched_sym;
}
return NULL;
}
static struct symbol *kpatch_find_static_twin_in_children(struct symbol *parent,
struct symbol *sym)
{
struct symbol *child;
list_for_each_entry(child, &parent->children, subfunction_node) {
struct symbol *res;
/* Only look in children whose rela section differ from the parent's */
if (child->sec->rela == parent->sec->rela || !child->sec->rela)
continue;
res = kpatch_find_uncorrelated_rela(child->sec->rela, sym);
/* Need to go deeper */
if (!res)
res = kpatch_find_static_twin_in_children(child, sym);
if (res != NULL)
return res;
}
return NULL;
}
/*
* Given a static local variable symbol and a rela section which references it
* in the base object, find a corresponding usage of a similarly named symbol
* in the patched object.
*/
static struct symbol *kpatch_find_static_twin(struct section *relasec,
struct symbol *sym)
{
struct symbol *res;
if (!relasec->twin && relasec->base->sym) {
struct symbol *parent = NULL;
/*
* The static twin might have been in a .part. symbol in the
* original object that got removed in the patched object.
*/
parent = kpatch_get_correlated_parent(relasec->base->sym);
if (parent)
relasec = parent->sec->rela;
}
if (!relasec->twin)
return NULL;
res = kpatch_find_uncorrelated_rela(relasec->twin, sym);
if (res != NULL)
return res;
/* Look if reference might have moved to child functions' sections */
if (relasec->twin->base->sym)
return kpatch_find_static_twin_in_children(relasec->twin->base->sym,
sym);
return NULL;
}
static bool kpatch_is_normal_static_local(struct symbol *sym)
{
if (sym->type != STT_OBJECT || sym->bind != STB_LOCAL)
return false;
if (!strncmp(sym->name, ".L", 2))
return false;
if (!strchr(sym->name, '.'))
return false;
if (is_special_static(sym))
return false;
return true;
}
static struct rela *kpatch_find_static_twin_ref(struct section *relasec,
struct symbol *sym)
{
struct rela *rela;
list_for_each_entry(rela, &relasec->relas, list) {
if (rela->sym == sym->twin)
return rela;
}
/* Reference to static variable might have moved to child function section */
if (relasec->base->sym) {
struct symbol *parent = relasec->base->sym;
struct symbol *child;
list_for_each_entry(child, &parent->children, subfunction_node) {
/* Only look in children whose rela section differ from the parent's */
if (child->sec->rela == parent->sec->rela ||
!child->sec->rela)
continue;
rela = kpatch_find_static_twin_ref(child->sec->rela, sym);
if (rela)
return rela;
}
}
return NULL;
}
/*
* gcc renames static local variables by appending a period and a number. For
* example, __foo could be renamed to __foo.31452. Unfortunately this number
* can arbitrarily change. Correlate them by comparing which functions
* reference them, and rename the patched symbols to match the base symbol
* names.
*
* Some surprising facts about static local variable symbols:
*
* - It's possible for multiple functions to use the same
* static local variable if the variable is defined in an
* inlined function.
*
* - It's also possible for multiple static local variables
* with the same name to be used in the same function if they
* have different scopes. (We have to assume that in such
* cases, the order in which they're referenced remains the
* same between the orig and patched objects, as there's no
* other way to distinguish them.)
*
* - Static locals are usually referenced by functions, but
* they can occasionally be referenced by data sections as
* well.
*/
static void kpatch_correlate_static_local_variables(struct kpatch_elf *orig,
struct kpatch_elf *patched)
{
struct symbol *sym, *patched_sym;
struct section *relasec;
struct rela *rela;
int bundled, patched_bundled;
/*
* First undo the correlations for all static locals. Two static
* locals can have the same numbered suffix in the orig and patched
* objects by coincidence.
*/
list_for_each_entry(sym, &orig->symbols, list) {
if (!kpatch_is_normal_static_local(sym))
continue;
if (sym->twin)
UNCORRELATE_ELEMENT(sym);
bundled = sym == sym->sec->sym;
if (bundled && sym->sec->twin) {
UNCORRELATE_ELEMENT(sym->sec);
if (sym->sec->secsym)
UNCORRELATE_ELEMENT(sym->sec->secsym);
if (sym->sec->rela)
UNCORRELATE_ELEMENT(sym->sec->rela);
}
}
/*
* Do the correlations: for each section reference to a static local,
* look for a corresponding reference in the section's twin.
*/
list_for_each_entry(relasec, &orig->sections, list) {
if (!is_rela_section(relasec) ||
is_debug_section(relasec) ||
!strcmp(relasec->name, ".rela.toc"))
continue;
list_for_each_entry(rela, &relasec->relas, list) {
if (!toc_rela(rela))
continue; /* skip toc constants */
sym = toc_rela(rela)->sym;
if (!kpatch_is_normal_static_local(sym))
continue;
if (sym->twin)
continue;
bundled = sym == sym->sec->sym;
if (bundled && sym->sec == relasec->base) {
/*
* A rare case where a static local data
* structure references itself. There's no
* reliable way to correlate this. Hopefully
* there's another reference to the symbol
* somewhere that can be used.
*/
log_debug("can't correlate static local %s's reference to itself\n",
sym->name);
continue;
}
patched_sym = kpatch_find_static_twin(relasec, sym);
if (!patched_sym)
DIFF_FATAL("reference to static local variable %s in %s was removed",
sym->name,
kpatch_section_function_name(relasec));
patched_bundled = patched_sym == patched_sym->sec->sym;
if (bundled != patched_bundled)
ERROR("bundle mismatch for symbol %s", sym->name);
if (!bundled && sym->sec->twin != patched_sym->sec)
ERROR("sections %s and %s aren't correlated for symbol %s",
sym->sec->name, patched_sym->sec->name, sym->name);
kpatch_correlate_static_local(sym, patched_sym);
if (bundled)
kpatch_correlate_section(sym->sec, patched_sym->sec);
}
}
/*
* Make sure that:
*
* 1. all the orig object's referenced static locals have been
* correlated; and
*
* 2. each reference to a static local in the orig object has a
* corresponding reference in the patched object (because a static
* local can be referenced by more than one section).
*/
list_for_each_entry(relasec, &orig->sections, list) {
if (!is_rela_section(relasec) ||
is_debug_section(relasec))
continue;
list_for_each_entry(rela, &relasec->relas, list) {
struct section *target_sec = relasec;
sym = rela->sym;
if (!kpatch_is_normal_static_local(sym))
continue;
if (!relasec->twin && relasec->base->sym) {
struct symbol *parent = NULL;
parent = kpatch_get_correlated_parent(relasec->base->sym);
if (parent)
target_sec = parent->sec->rela;
}
if (!sym->twin || !target_sec->twin)
DIFF_FATAL("reference to static local variable %s in %s was removed",
sym->name,
kpatch_section_function_name(target_sec));
if (!kpatch_find_static_twin_ref(target_sec->twin, sym))
DIFF_FATAL("static local %s has been correlated with %s, but patched %s is missing a reference to it",
sym->name, sym->twin->name,
kpatch_section_function_name(target_sec->twin));
}
}
/*
* Now go through the patched object and look for any uncorrelated
* static locals to see if we need to print any warnings about new
* variables.
*/
list_for_each_entry(relasec, &patched->sections, list) {
if (!is_rela_section(relasec) ||
is_debug_section(relasec))
continue;
list_for_each_entry(rela, &relasec->relas, list) {
sym = rela->sym;
if (!kpatch_is_normal_static_local(sym))
continue;
if (sym->twin)
continue;
log_normal("WARNING: unable to correlate static local variable %s used by %s, assuming variable is new\n",
sym->name,
kpatch_section_function_name(relasec));
return;
}
}
}
static void kpatch_correlate_elfs(struct kpatch_elf *kelf_orig,
struct kpatch_elf *kelf_patched)
{
kpatch_correlate_sections(&kelf_orig->sections, &kelf_patched->sections);
kpatch_correlate_symbols(&kelf_orig->symbols, &kelf_patched->symbols);
}
static void kpatch_compare_correlated_elements(struct kpatch_elf *kelf)
{
/* lists are already correlated at this point */
kpatch_compare_sections(kelf);
kpatch_compare_symbols(&kelf->symbols);
}
static bool is_callback_section(struct section *sec) {
static char *callback_sections[] = {
".kpatch.callbacks.pre_patch",
".kpatch.callbacks.post_patch",
".kpatch.callbacks.pre_unpatch",
".kpatch.callbacks.post_unpatch",
".rela.kpatch.callbacks.pre_patch",
".rela.kpatch.callbacks.post_patch",
".rela.kpatch.callbacks.pre_unpatch",
".rela.kpatch.callbacks.post_unpatch",
NULL,
};
char **callback_sec;
for (callback_sec = callback_sections; *callback_sec; callback_sec++)
if (!strcmp(sec->name, *callback_sec))
return true;
return false;
}
/*
* Mangle the relas a little. The compiler will sometimes use section symbols
* to reference local objects and functions rather than the object or function
* symbols themselves. We substitute the object/function symbols for the
* section symbol in this case so that the relas can be properly correlated and
* so that the existing object/function in vmlinux can be linked to.
*/
static void kpatch_replace_sections_syms(struct kpatch_elf *kelf)
{
struct section *relasec;
struct rela *rela;
struct symbol *sym;
long target_off;
bool found = false;
log_debug("\n");
list_for_each_entry(relasec, &kelf->sections, list) {
if (!is_rela_section(relasec) || is_debug_section(relasec))
continue;
list_for_each_entry(rela, &relasec->relas, list) {
if (rela->sym->type != STT_SECTION || !rela->sym->sec)
continue;
/*
* UBSAN data will be taken wholesale, no need to
* replace section symbols.
*/
if (is_ubsan_sec(rela->sym->name))
continue;
/*
* These sections don't have symbols associated with
* them:
*/
if (!strcmp(rela->sym->name, ".toc") ||
!strcmp(rela->sym->name, ".fixup") ||
!strcmp(rela->sym->name, ".altinstr_replacement") ||
!strcmp(rela->sym->name, ".altinstr_aux") ||
!strcmp(rela->sym->name, ".text..refcount") ||
!strncmp(rela->sym->name, "__ftr_alt_", 10))
continue;
/*
* Replace references to bundled sections with their
* symbols.
*/
if (rela->sym->sec->sym) {
rela->sym = rela->sym->sec->sym;
/*
* On ppc64le with GCC6+, even with
* -ffunction-sections, the function symbol
* starts 8 bytes past the beginning of the
* section, because the .TOC pointer is at the
* beginning, right before the code. So even
* though the symbol is bundled, we can't
* assume it's at offset 0 in the section.
*/
rela->addend -= rela->sym->sym.st_value;
continue;
}
target_off = rela_target_offset(kelf, relasec, rela);
/*
* Attempt to replace references to unbundled sections
* with their symbols.
*/
list_for_each_entry(sym, &kelf->symbols, list) {
long start, end;
if (sym->type == STT_SECTION ||
sym->sec != rela->sym->sec)
continue;
start = sym->sym.st_value;
end = sym->sym.st_value + sym->sym.st_size;
if (is_text_section(relasec->base) &&
!is_text_section(sym->sec) &&
rela->type == R_X86_64_32S &&
rela->addend == (long)sym->sec->sh.sh_size &&
end == (long)sym->sec->sh.sh_size) {
/*
* A special case where gcc needs a
* pointer to the address at the end of
* a data section.
*
* This is usually used with a compare
* instruction to determine when to end
* a loop. The code doesn't actually
* dereference the pointer so this is
* "normal" and we just replace the
* section reference with a reference
* to the last symbol in the section.
*
* Note that this only catches the
* issue when it happens at the end of
* a section. It can also happen in
* the middle of a section. In that
* case, the wrong symbol will be
* associated with the reference. But
* that's ok because:
*
* 1) This situation only occurs when
* gcc is trying to get the address
* of the symbol, not the contents
* of its data; and
*
* 2) Because kpatch doesn't allow data
* sections to change,
* &(var1+sizeof(var1)) will always
* be the same as &var2.
*/
} else if (target_off == start && target_off == end) {
/*
* Allow replacement for references to
* empty symbols.
*/
} else if (target_off < start || target_off >= end)
continue;
log_debug("%s: replacing %s+%ld reference with %s+%ld\n",
relasec->name,
rela->sym->name, rela->addend,
sym->name, rela->addend - start);
found = true;
rela->sym = sym;
rela->addend -= start;
break;
}
if (!found && !is_string_literal_section(rela->sym->sec) &&
strncmp(rela->sym->name, ".rodata", 7)) {
ERROR("%s+0x%x: can't find replacement symbol for %s+%ld reference",
relasec->base->name, rela->offset, rela->sym->name, rela->addend);
}
}
}
log_debug("\n");
}
static void kpatch_check_func_profiling_calls(struct kpatch_elf *kelf)
{
struct symbol *sym;
int errs = 0;
list_for_each_entry(sym, &kelf->symbols, list) {
if (sym->type != STT_FUNC || sym->status != CHANGED ||
(sym->parent && sym->parent->status == CHANGED))
continue;
if (!sym->twin->has_func_profiling) {
log_normal("function %s has no fentry/mcount call, unable to patch\n",
sym->name);
errs++;
}
}
if (errs)
DIFF_FATAL("%d function(s) can not be patched", errs);
}
static void kpatch_verify_patchability(struct kpatch_elf *kelf)
{
struct section *sec;
int errs = 0;
list_for_each_entry(sec, &kelf->sections, list) {
if (sec->status == CHANGED && !sec->include) {
log_normal("changed section %s not selected for inclusion\n",
sec->name);
errs++;
}
if (sec->status != SAME && sec->grouped) {
log_normal("changed section %s is part of a section group\n",
sec->name);
errs++;
}
if (sec->sh.sh_type == SHT_GROUP && sec->status == NEW) {
log_normal("new/changed group sections are not supported\n");
errs++;
}
/*
* ensure we aren't including .data.* or .bss.*
* (.data.unlikely and .data.once is ok b/c it only has __warned vars)
*/
if (sec->include && sec->status != NEW &&
(!strncmp(sec->name, ".data", 5) || !strncmp(sec->name, ".bss", 4)) &&
(strcmp(sec->name, ".data.unlikely") && strcmp(sec->name, ".data.once"))) {
log_normal("data section %s selected for inclusion\n",
sec->name);
errs++;
}
}
if (errs)
DIFF_FATAL("%d unsupported section change(s)", errs);
}
static void kpatch_include_symbol(struct symbol *sym);
static void kpatch_include_section(struct section *sec)
{
struct rela *rela;
/* Include the section and its section symbol */
if (sec->include)
return;
sec->include = 1;
if (sec->secsym)
sec->secsym->include = 1;
/*
* Include the section's rela section and then recursively include the
* symbols needed by its relas.
*/
if (!sec->rela)
return;
sec->rela->include = 1;
list_for_each_entry(rela, &sec->rela->relas, list)
kpatch_include_symbol(rela->sym);
}
static void kpatch_include_symbol(struct symbol *sym)
{
/*
* This function is called recursively from kpatch_include_section().
* Make sure we don't get into an endless loop.
*/
if (sym->include)
return;
/*
* The symbol gets included even if its section isn't needed, as it
* might be needed: either permanently for a rela, or temporarily for
* the later creation of a klp relocation.
*/
sym->include = 1;
/*
* For a function/object symbol, if it has a section, we only need to
* include the section if it has changed. Otherwise the symbol will be
* used by relas/klp_relocs to link to the real symbol externally.
*
* For section symbols, we always include the section because
* references to them can't otherwise be resolved externally.
*/
if (sym->sec && (sym->type == STT_SECTION || sym->status != SAME))
kpatch_include_section(sym->sec);
}
static void kpatch_include_standard_elements(struct kpatch_elf *kelf)
{
struct section *sec;
struct symbol *sym;
list_for_each_entry(sec, &kelf->sections, list) {
/*
* Include the following sections even if they haven't changed.
*
* Notes about some of the more interesting sections:
*
* - With -fdata-sections, .rodata is only used for:
*
* switch jump tables;
* KASAN data (with KASAN enabled, which is rare); and
* an ugly hack in vmx_vcpu_run().
*
* Those data are all local to the functions which use them.
* So it's safe to include .rodata.
*
* - On ppc64le, the .toc section is used for all data
* accesses.
*
* Note that if any of these sections have rela sections, they
* will also be included in their entirety. That may result in
* some extra (unused) klp relocations getting created, which
* should be harmless.
*/
if (!strcmp(sec->name, ".shstrtab") ||
!strcmp(sec->name, ".strtab") ||
!strcmp(sec->name, ".symtab") ||
!strcmp(sec->name, ".toc") ||
!strcmp(sec->name, ".rodata") ||
is_string_literal_section(sec)) {
kpatch_include_section(sec);
}
}
list_for_each_entry(sym, &kelf->symbols, list)
if (sym->sec && is_string_literal_section(sym->sec))
sym->include = 1;
/* include the NULL symbol */
list_entry(kelf->symbols.next, struct symbol, list)->include = 1;
}
static bool kpatch_include_callback_elements(struct kpatch_elf *kelf)
{
struct section *sec;
struct symbol *sym;
struct rela *rela;
bool found = false;
/* include load/unload sections */
list_for_each_entry(sec, &kelf->sections, list) {
if (!is_callback_section(sec))
continue;
sec->include = 1;
found = true;
if (is_rela_section(sec)) {
/* include callback dependencies */
rela = list_entry(sec->relas.next, struct rela, list);
sym = rela->sym;
log_normal("found callback: %s\n",sym->name);
kpatch_include_symbol(sym);
} else if (sec->secsym) {
sec->secsym->include = 1;
}
}
/* Strip temporary structure symbols used by the callback macros. */
list_for_each_entry(sym, &kelf->symbols, list) {
if (sym->type == STT_OBJECT && sym->sec &&
is_callback_section(sym->sec))
sym->include = 0;
}
return found;
}
static void kpatch_include_force_elements(struct kpatch_elf *kelf)
{
struct section *sec;
struct symbol *sym;
struct rela *rela;
/* include force sections */
list_for_each_entry(sec, &kelf->sections, list) {
if (!strcmp(sec->name, ".kpatch.force") ||
!strcmp(sec->name, ".rela.kpatch.force")) {
sec->include = 1;
if (!is_rela_section(sec)) {
/* .kpatch.force */
if (sec->secsym)
sec->secsym->include = 1;
continue;
}
/* .rela.kpatch.force */
list_for_each_entry(rela, &sec->relas, list)
log_normal("function '%s' marked with KPATCH_FORCE_UNSAFE!\n",
rela->sym->name);
}
}
/* strip temporary global kpatch_force_func_* symbols */
list_for_each_entry(sym, &kelf->symbols, list)
if (!strncmp(sym->name, "__kpatch_force_func_",
strlen("__kpatch_force_func_")))
sym->include = 0;
}
static int kpatch_include_new_globals(struct kpatch_elf *kelf)
{
struct symbol *sym;
int nr = 0;
list_for_each_entry(sym, &kelf->symbols, list) {
if (sym->bind == STB_GLOBAL && sym->sec &&
sym->status == NEW) {
kpatch_include_symbol(sym);
nr++;
}
}
return nr;
}
static int kpatch_include_changed_functions(struct kpatch_elf *kelf)
{
struct symbol *sym;
int changed_nr = 0;
list_for_each_entry(sym, &kelf->symbols, list) {
if (sym->status == CHANGED &&
sym->type == STT_FUNC) {
changed_nr++;
kpatch_include_symbol(sym);
if (sym->pfx)
kpatch_include_symbol(sym->pfx);
}
if (sym->type == STT_FILE)
sym->include = 1;
}
return changed_nr;
}
static void kpatch_print_changes(struct kpatch_elf *kelf)
{
struct symbol *sym;
list_for_each_entry(sym, &kelf->symbols, list) {
if (!sym->include || !sym->sec || sym->type != STT_FUNC ||
sym->parent || sym->is_pfx)
continue;
if (sym->status == NEW)
log_normal("new function: %s\n", sym->name);
else if (sym->status == CHANGED)
log_normal("changed function: %s\n", sym->name);
}
}
static void kpatch_migrate_symbols(struct list_head *src,
struct list_head *dst,
bool (*select)(struct symbol *))
{
struct symbol *sym, *safe;
list_for_each_entry_safe(sym, safe, src, list) {
if (select && !select(sym))
continue;
list_del(&sym->list);
list_add_tail(&sym->list, dst);
}
}
static void kpatch_migrate_included_elements(struct kpatch_elf *kelf, struct kpatch_elf **kelfout)
{
struct section *sec, *safesec;
struct symbol *sym, *safesym;
struct kpatch_elf *out;
/* allocate output kelf */
out = malloc(sizeof(*out));
if (!out)
ERROR("malloc");
memset(out, 0, sizeof(*out));
out->arch = kelf->arch;
INIT_LIST_HEAD(&out->sections);
INIT_LIST_HEAD(&out->symbols);
INIT_LIST_HEAD(&out->strings);
/* migrate included sections from kelf to out */
list_for_each_entry_safe(sec, safesec, &kelf->sections, list) {
if (!sec->include)
continue;
list_del(&sec->list);
list_add_tail(&sec->list, &out->sections);
sec->index = 0;
if (!is_rela_section(sec) && sec->secsym && !sec->secsym->include)
/* break link to non-included section symbol */
sec->secsym = NULL;
}
/* migrate included symbols from kelf to out */
list_for_each_entry_safe(sym, safesym, &kelf->symbols, list) {
if (!sym->include)
continue;
list_del(&sym->list);
list_add_tail(&sym->list, &out->symbols);
sym->index = 0;
sym->strip = SYMBOL_DEFAULT;
if (sym->sec && !sym->sec->include)
/* break link to non-included section */
sym->sec = NULL;
}
*kelfout = out;
}
static void kpatch_reorder_symbols(struct kpatch_elf *kelf)
{
LIST_HEAD(symbols);
/* migrate NULL sym */
kpatch_migrate_symbols(&kelf->symbols, &symbols, is_null_sym);
/* migrate LOCAL FILE sym */
kpatch_migrate_symbols(&kelf->symbols, &symbols, is_file_sym);
/* migrate LOCAL FUNC syms */
kpatch_migrate_symbols(&kelf->symbols, &symbols, is_local_func_sym);
/* migrate all other LOCAL syms */
kpatch_migrate_symbols(&kelf->symbols, &symbols, is_local_sym);
/* migrate all other (GLOBAL) syms */
kpatch_migrate_symbols(&kelf->symbols, &symbols, NULL);
list_replace(&symbols, &kelf->symbols);
}
static int bug_table_group_size(struct kpatch_elf *kelf, int offset)
{
static int size = 0;
char *str;
if (!size) {
str = getenv("BUG_STRUCT_SIZE");
if (!str)
ERROR("BUG_STRUCT_SIZE not set");
size = atoi(str);
}
return size;
}
static int ex_table_group_size(struct kpatch_elf *kelf, int offset)
{
static int size = 0;
char *str;
if (!size) {
str = getenv("EX_STRUCT_SIZE");
if (!str)
ERROR("EX_STRUCT_SIZE not set");
size = atoi(str);
}
return size;
}
static int jump_table_group_size(struct kpatch_elf *kelf, int offset)
{
static int size = 0;
char *str;
if (!size) {
str = getenv("JUMP_STRUCT_SIZE");
if (!str)
ERROR("JUMP_STRUCT_SIZE not set");
size = atoi(str);
}
return size;
}
static int printk_index_group_size(struct kpatch_elf *kelf, int offset)
{
static int size = 0;
char *str;
if (!size) {
str = getenv("PRINTK_INDEX_STRUCT_SIZE");
if (!str)
ERROR("PRINTK_INDEX_STRUCT_SIZE not set");
size = atoi(str);
}
return size;
}
static int parainstructions_group_size(struct kpatch_elf *kelf, int offset)
{
static int size = 0;
char *str;
if (!size) {
str = getenv("PARA_STRUCT_SIZE");
if (!str)
ERROR("PARA_STRUCT_SIZE not set");
size = atoi(str);
}
return size;
}
static int altinstructions_group_size(struct kpatch_elf *kelf, int offset)
{
static int size = 0;
char *str;
if (!size) {
str = getenv("ALT_STRUCT_SIZE");
if (!str)
ERROR("ALT_STRUCT_SIZE not set");
size = atoi(str);
}
return size;
}
static int smp_locks_group_size(struct kpatch_elf *kelf, int offset)
{
return 4;
}
static int static_call_sites_group_size(struct kpatch_elf *kelf, int offset)
{
static int size = 0;
char *str;
if (!size) {
str = getenv("STATIC_CALL_STRUCT_SIZE");
if (!str)
ERROR("STATIC_CALL_STRUCT_SIZE not set");
size = atoi(str);
}
return size;
}
static int call_sites_group_size(struct kpatch_elf *kelf, int offset)
{
return 4;
}
static int retpoline_sites_group_size(struct kpatch_elf *kelf, int offset)
{
return 4;
}
static int return_sites_group_size(struct kpatch_elf *kelf, int offset)
{
return 4;
}
static int fixup_entry_group_size(struct kpatch_elf *kelf, int offset)
{
static int size = 0;
char *str;
if (!size) {
str = getenv("FIXUP_STRUCT_SIZE");
if (!str)
ERROR("FIXUP_STRUCT_SIZE not set");
size = atoi(str);
}
return size;
}
static int fixup_lwsync_group_size(struct kpatch_elf *kelf, int offset)
{
return 8;
}
static int fixup_barrier_nospec_group_size(struct kpatch_elf *kelf, int offset)
{
return 8;
}
/*
* .s390_indirect_jump, .s390_indirect_call, .s390_indirect_branches,
* .s390_return_reg, .s390_return_mem contains indirect branch locations. This
* is an array of 32 bit elements. These sections could be used during runtime
* to replace the expolines with the normal indirect jump.
*/
static int s390_expolines_group_size(struct kpatch_elf *kelf, int offset)
{
return 4;
}
/*
* The rela groups in the .fixup section vary in size. The beginning of each
* .fixup rela group is referenced by the __ex_table section. To find the size
* of a .fixup rela group, we have to traverse the __ex_table relas.
*/
static int fixup_group_size(struct kpatch_elf *kelf, int offset)
{
struct section *relasec;
struct rela *rela;
int found;
relasec = find_section_by_name(&kelf->sections, ".rela__ex_table");
if (!relasec)
ERROR("missing .rela__ex_table section");
/* find beginning of this group */
found = 0;
list_for_each_entry(rela, &relasec->relas, list) {
if (!strcmp(rela->sym->name, ".fixup") &&
rela->addend == offset) {
found = 1;
break;
}
}
if (!found)
ERROR("can't find .fixup rela group at offset %d\n", offset);
/* find beginning of next group */
found = 0;
list_for_each_entry_continue(rela, &relasec->relas, list) {
if (!strcmp(rela->sym->name, ".fixup") &&
rela->addend > offset) {
found = 1;
break;
}
}
if (!found) {
/* last group */
struct section *fixupsec;
fixupsec = find_section_by_name(&kelf->sections, ".fixup");
if (!fixupsec)
ERROR("missing .fixup section");
return (int)(fixupsec->sh.sh_size - offset);
}
return (int)(rela->addend - offset);
}
static bool jump_table_group_filter(struct lookup_table *lookup,
struct section *relasec,
unsigned int group_offset,
unsigned int group_size)
{
struct rela *code = NULL, *key = NULL, *rela;
bool tracepoint = false, dynamic_debug = false;
struct lookup_result symbol;
int i = 0;
/*
* Here we hard-code knowledge about the contents of the jump_entry
* struct. It has three fields: code, target, and key. Each field has
* a relocation associated with it.
*/
list_for_each_entry(rela, &relasec->relas, list) {
if (rela->offset >= group_offset &&
rela->offset < group_offset + group_size) {
if (i == 0)
code = rela;
else if (i == 2)
key = rela;
i++;
}
}
if (i != 3 || !key || !code)
ERROR("BUG: __jump_table has an unexpected format");
if (!strncmp(key->sym->name, "__tracepoint_", 13))
tracepoint = true;
if (is_dynamic_debug_symbol(key->sym))
dynamic_debug = true;
if (KLP_ARCH) {
/*
* On older kernels (with .klp.arch support), jump labels
* aren't supported at all. Error out when they occur in a
* replacement function, with the exception of tracepoints and
* dynamic debug printks. An inert tracepoint or printk is
* harmless enough, but a broken jump label can cause
* unexpected behavior.
*/
if (tracepoint || dynamic_debug)
return false;
/*
* This will be upgraded to an error after all jump labels have
* been reported.
*/
log_normal("Found a jump label at %s()+0x%lx, using key %s. Jump labels aren't supported with this kernel. Use static_key_enabled() instead.\n",
code->sym->name, code->addend, key->sym->name);
jump_label_errors++;
return false;
}
/*
* On newer (5.8+) kernels, jump labels are supported in the case where
* the corresponding static key lives in vmlinux. That's because such
* kernels apply vmlinux-specific .klp.rela sections at the same time
* (in the klp module load) as normal relas, before jump label init.
* On the other hand, jump labels based on static keys which are
* defined in modules aren't supported, because late module patching
* can result in the klp relocations getting applied *after* the klp
* module's jump label init.
*/
if (lookup_symbol(lookup, key->sym, &symbol) &&
strcmp(symbol.objname, "vmlinux")) {
/* The static key lives in a module -- not supported */
/* Inert tracepoints and dynamic debug printks are harmless */
if (tracepoint || dynamic_debug)
return false;
/*
* This will be upgraded to an error after all jump label
* errors have been reported.
*/
log_normal("Found a jump label at %s()+0x%lx, using key %s, which is defined in a module. Use static_key_enabled() instead.\n",
code->sym->name, code->addend, key->sym->name);
jump_label_errors++;
return false;
}
/* The static key lives in vmlinux or the patch module itself */
/*
* If the jump label key lives in the '__dyndbg' section, make sure
* the section gets included, because we don't use klp relocs for
* dynamic debug symbols. For an example of such a key, see
* DYNAMIC_DEBUG_BRANCH().
*/
if (dynamic_debug)
kpatch_include_symbol(key->sym);
return true;
}
static bool static_call_sites_group_filter(struct lookup_table *lookup,
struct section *relasec,
unsigned int group_offset,
unsigned int group_size)
{
struct rela *code = NULL, *key = NULL, *rela;
bool tracepoint = false;
struct lookup_result symbol;
int i = 0;
/*
* Here we hard-code knowledge about the contents of the jump_entry
* struct. It has three fields: code, target, and key. Each field has
* a relocation associated with it.
*/
list_for_each_entry(rela, &relasec->relas, list) {
if (rela->offset >= group_offset &&
rela->offset < group_offset + group_size) {
if (i == 0)
code = rela;
else if (i == 1)
key = rela;
i++;
}
}
if (i != 2 || !key || !code)
ERROR("BUG: .static_call_sites has an unexpected format");
if (!strncmp(key->sym->name, "__SCK__tp_func_", 15))
tracepoint = true;
/*
* Static calls are only supported in the case where the corresponding
* static call key lives in vmlinux (see explanation in
* jump_table_group_filter).
*/
if (lookup_symbol(lookup, key->sym, &symbol) &&
strcmp(symbol.objname, "vmlinux")) {
/* The key lives in a module -- not supported */
/* Inert tracepoints are harmless */
if (tracepoint)
return false;
/*
* This will be upgraded to an error after all static call
* errors have been reported.
*/
log_normal("Found a static call at %s()+0x%lx, using key %s, which is defined in a module. Use KPATCH_STATIC_CALL() instead.\n",
code->sym->name, code->addend, key->sym->name);
static_call_errors++;
return false;
}
/* The key lives in vmlinux or the patch module itself */
return true;
}
static struct special_section special_sections[] = {
{
.name = "__bug_table",
.arch = X86_64 | PPC64 | S390,
.group_size = bug_table_group_size,
},
{
.name = ".fixup",
.arch = X86_64 | PPC64 | S390,
.group_size = fixup_group_size,
},
{
.name = "__ex_table", /* must come after .fixup */
.arch = X86_64 | PPC64 | S390,
.group_size = ex_table_group_size,
},
{
.name = "__jump_table",
.arch = X86_64 | PPC64 | S390,
.group_size = jump_table_group_size,
.group_filter = jump_table_group_filter,
},
{
.name = ".printk_index",
.arch = X86_64 | PPC64 | S390,
.group_size = printk_index_group_size,
},
{
.name = ".smp_locks",
.arch = X86_64,
.group_size = smp_locks_group_size,
},
{
.name = ".parainstructions",
.arch = X86_64,
.group_size = parainstructions_group_size,
},
{
.name = ".altinstructions",
.arch = X86_64 | S390,
.group_size = altinstructions_group_size,
},
{
.name = ".static_call_sites",
.arch = X86_64,
.group_size = static_call_sites_group_size,
.group_filter = static_call_sites_group_filter,
},
{
.name = ".call_sites",
.arch = X86_64,
.group_size = call_sites_group_size,
},
{
.name = ".retpoline_sites",
.arch = X86_64,
.group_size = retpoline_sites_group_size,
},
{
.name = ".return_sites",
.arch = X86_64,
.group_size = return_sites_group_size,
},
{
.name = "__ftr_fixup",
.arch = PPC64,
.group_size = fixup_entry_group_size,
},
{
.name = "__mmu_ftr_fixup",
.arch = PPC64,
.group_size = fixup_entry_group_size,
},
{
.name = "__fw_ftr_fixup",
.arch = PPC64,
.group_size = fixup_entry_group_size,
},
{
.name = "__lwsync_fixup",
.arch = PPC64,
.group_size = fixup_lwsync_group_size,
},
{
.name = "__barrier_nospec_fixup",
.arch = PPC64,
.group_size = fixup_barrier_nospec_group_size,
},
{
.name = ".s390_return_mem",
.arch = S390,
.group_size = s390_expolines_group_size,
},
{
.name = ".s390_return_reg",
.arch = S390,
.group_size = s390_expolines_group_size,
},
{
.name = ".s390_indirect_call",
.arch = S390,
.group_size = s390_expolines_group_size,
},
{
.name = ".s390_indirect_branches",
.arch = S390,
.group_size = s390_expolines_group_size,
},
{
.name = ".s390_indirect_jump",
.arch = S390,
.group_size = s390_expolines_group_size,
},
{},
};
static bool should_keep_rela_group(struct lookup_table *lookup,
struct section *relasec, unsigned int offset,
unsigned int size)
{
struct rela *rela;
bool found = false;
/* check if any relas in the group reference any changed functions */
list_for_each_entry(rela, &relasec->relas, list) {
if (rela->offset >= offset &&
rela->offset < offset + size &&
rela->sym->type == STT_FUNC &&
rela->sym->sec->include) {
found = true;
log_debug("new/changed symbol %s found in special section %s\n",
rela->sym->name, relasec->name);
}
}
if (!found)
return false;
return true;
}
/*
* When updating .fixup, the corresponding addends in .ex_table need to be
* updated too. Stash the result in rela.r_addend so that the calculation in
* fixup_group_size() is not affected.
*/
static void kpatch_update_ex_table_addend(struct kpatch_elf *kelf,
struct special_section *special,
int src_offset, int dest_offset,
int group_size)
{
struct rela *rela;
struct section *relasec;
relasec = find_section_by_name(&kelf->sections, ".rela__ex_table");
if (!relasec)
ERROR("missing .rela__ex_table section");
list_for_each_entry(rela, &relasec->relas, list) {
if (!strcmp(rela->sym->name, ".fixup") &&
rela->addend >= src_offset &&
rela->addend < src_offset + group_size)
rela->rela.r_addend = rela->addend - (src_offset - dest_offset);
}
}
static void kpatch_regenerate_special_section(struct kpatch_elf *kelf,
struct lookup_table *lookup,
struct special_section *special,
struct section *relasec)
{
struct rela *rela, *safe;
char *src, *dest;
unsigned int group_size, src_offset, dest_offset;
LIST_HEAD(newrelas);
src = relasec->base->data->d_buf;
/* alloc buffer for new base section */
dest = malloc(relasec->base->sh.sh_size);
if (!dest)
ERROR("malloc");
/* Restore the stashed r_addend from kpatch_update_ex_table_addend. */
if (!strcmp(special->name, "__ex_table")) {
list_for_each_entry(rela, &relasec->relas, list) {
if (!strcmp(rela->sym->name, ".fixup"))
rela->addend = rela->rela.r_addend;
}
}
src_offset = 0;
dest_offset = 0;
for ( ; src_offset < relasec->base->sh.sh_size; src_offset += group_size) {
group_size = special->group_size(kelf, src_offset);
/*
* In some cases the struct has padding at the end to ensure
* that all structs after it are properly aligned. But the
* last struct in the section may not be padded. In that case,
* shrink the group_size such that it still (hopefully)
* contains the data but doesn't go past the end of the
* section.
*/
if (src_offset + group_size > relasec->base->sh.sh_size)
group_size = (unsigned int)(relasec->base->sh.sh_size - src_offset);
if (!should_keep_rela_group(lookup, relasec, src_offset, group_size))
continue;
if (special->group_filter &&
!special->group_filter(lookup, relasec, src_offset, group_size))
continue;
/*
* Copy all relas in the group. It's possible that the relas
* aren't sorted (e.g. .rela.fixup), so go through the entire
* rela list each time.
*/
list_for_each_entry_safe(rela, safe, &relasec->relas, list) {
if (rela->offset >= src_offset &&
rela->offset < src_offset + group_size) {
/* copy rela entry */
list_del(&rela->list);
list_add_tail(&rela->list, &newrelas);
rela->offset -= src_offset - dest_offset;
rela->rela.r_offset = rela->offset;
rela->sym->include = 1;
if (!strcmp(special->name, ".fixup"))
kpatch_update_ex_table_addend(kelf, special,
src_offset,
dest_offset,
group_size);
}
}
/* copy base section group */
memcpy(dest + dest_offset, src + src_offset, group_size);
dest_offset += group_size;
}
if (jump_label_errors)
ERROR("Found %d unsupported jump label(s) in the patched code. Use static_key_enabled() instead.",
jump_label_errors);
if (static_call_errors)
ERROR("Found %d unsupported static call(s) in the patched code. Use KPATCH_STATIC_CALL() instead.",
static_call_errors);
if (!dest_offset) {
/* no changed or global functions referenced */
relasec->status = relasec->base->status = SAME;
relasec->include = relasec->base->include = 0;
free(dest);
return;
}
/* overwrite with new relas list */
list_replace(&newrelas, &relasec->relas);
/* include both rela and base sections */
relasec->include = 1;
relasec->base->include = 1;
/* include secsym so .kpatch.arch relas can point to section symbols */
if (relasec->base->secsym)
relasec->base->secsym->include = 1;
/*
* Update text section data buf and size.
*
* The rela section's data buf and size will be regenerated in
* kpatch_rebuild_rela_section_data().
*/
relasec->base->data->d_buf = dest;
relasec->base->data->d_size = dest_offset;
}
#define ORC_IP_PTR_SIZE 4
/*
* This function is similar to kpatch_regenerate_special_section(), but
* customized for the ORC-related sections. ORC is more special than the other
* special sections because each ORC entry is split into .orc_unwind (struct
* orc_entry) and .orc_unwind_ip.
*/
static void kpatch_regenerate_orc_sections(struct kpatch_elf *kelf)
{
struct rela *rela, *safe;
char *src, *dest, *str;
unsigned int src_idx = 0, dest_idx = 0, orc_entry_size;
struct section *orc_sec, *ip_sec;
str = getenv("ORC_STRUCT_SIZE");
if (!str)
return;
orc_entry_size = atoi(str);
if (!orc_entry_size)
ERROR("bad ORC_STRUCT_SIZE");
LIST_HEAD(newrelas);
orc_sec = find_section_by_name(&kelf->sections, ".orc_unwind");
ip_sec = find_section_by_name(&kelf->sections, ".orc_unwind_ip");
if (!orc_sec || !ip_sec)
return;
if (orc_sec->sh.sh_size % orc_entry_size != 0)
ERROR("bad .orc_unwind size");
if (ip_sec->sh.sh_size !=
(orc_sec->sh.sh_size / orc_entry_size) * ORC_IP_PTR_SIZE)
ERROR(".orc_unwind/.orc_unwind_ip size mismatch");
src = orc_sec->data->d_buf;
dest = malloc(orc_sec->sh.sh_size);
if (!dest)
ERROR("malloc");
list_for_each_entry_safe(rela, safe, &ip_sec->rela->relas, list) {
if (rela->sym->type != STT_FUNC || !rela->sym->sec->include)
goto next;
/* copy orc entry */
memcpy(dest + (dest_idx * orc_entry_size),
src + (src_idx * orc_entry_size),
orc_entry_size);
/* move ip rela */
list_del(&rela->list);
list_add_tail(&rela->list, &newrelas);
rela->offset = dest_idx * ORC_IP_PTR_SIZE;
rela->sym->include = 1;
dest_idx++;
next:
src_idx++;
}
if (!dest_idx) {
/* no changed or global functions referenced */
orc_sec->status = ip_sec->status = ip_sec->rela->status = SAME;
orc_sec->include = ip_sec->include = ip_sec->rela->include = 0;
free(dest);
return;
}
/* overwrite with new relas list */
list_replace(&newrelas, &ip_sec->rela->relas);
/* include the sections */
orc_sec->include = ip_sec->include = ip_sec->rela->include = 1;
/*
* Update data buf/size.
*
* The ip section can keep its old (zeroed data), though its size has
* possibly decreased. The ip rela section's data buf and size will be
* regenerated in kpatch_rebuild_rela_section_data().
*/
orc_sec->data->d_buf = dest;
orc_sec->data->d_size = dest_idx * orc_entry_size;
ip_sec->data->d_size = dest_idx * ORC_IP_PTR_SIZE;
}
static void kpatch_check_relocations(struct kpatch_elf *kelf)
{
struct rela *rela;
struct section *relasec;
long sec_size;
long sec_off;
list_for_each_entry(relasec, &kelf->sections, list) {
if (!is_rela_section(relasec))
continue;
list_for_each_entry(rela, &relasec->relas, list) {
if (!rela->sym->sec)
continue;
sec_size = rela->sym->sec->data->d_size;
sec_off = (long)rela->sym->sym.st_value +
rela_target_offset(kelf, relasec, rela);
/*
* This check isn't perfect: we still allow relocations
* to the end of a section. There are real instances
* of that, including ORC entries, LOCKDEP=n
* zero-length '__key' passing, and the loop edge case
* described in kpatch_replace_sections_syms(). For
* now, just allow all such cases.
*/
if (sec_off < 0 || sec_off > sec_size) {
ERROR("%s+0x%x: out-of-range relocation %s+%lx",
relasec->base->name, rela->offset,
rela->sym->name, rela->addend);
}
}
}
}
static void kpatch_include_debug_sections(struct kpatch_elf *kelf)
{
struct section *sec;
struct rela *rela, *saferela;
/* include all .debug_* sections */
list_for_each_entry(sec, &kelf->sections, list) {
if (is_debug_section(sec)) {
sec->include = 1;
if (!is_rela_section(sec) && sec->secsym)
sec->secsym->include = 1;
}
}
/*
* Go through the .rela.debug_ sections and strip entries
* referencing unchanged symbols
*/
list_for_each_entry(sec, &kelf->sections, list) {
if (!is_rela_section(sec) || !is_debug_section(sec))
continue;
list_for_each_entry_safe(rela, saferela, &sec->relas, list)
if (!rela->sym->sec->include)
list_del(&rela->list);
}
}
static void kpatch_mark_ignored_sections(struct kpatch_elf *kelf)
{
struct section *sec, *strsec, *ignoresec;
struct rela *rela;
char *name;
/* Ignore any discarded sections */
list_for_each_entry(sec, &kelf->sections, list) {
if (!strncmp(sec->name, ".discard", 8) ||
!strncmp(sec->name, ".rela.discard", 13) ||
!strncmp(sec->name, ".llvm_addrsig", 13) ||
!strncmp(sec->name, ".llvm.", 6))
sec->ignore = 1;
if (kelf->arch == X86_64) {
if (!strcmp(sec->name, ".rela__patchable_function_entries") ||
!strcmp(sec->name, "__patchable_function_entries"))
sec->ignore = 1;
}
}
sec = find_section_by_name(&kelf->sections, ".kpatch.ignore.sections");
if (!sec)
return;
list_for_each_entry(rela, &sec->rela->relas, list) {
strsec = rela->sym->sec;
strsec->status = CHANGED;
/*
* Include the string section here. This is because the
* KPATCH_IGNORE_SECTION() macro is passed a literal string
* by the patch author, resulting in a change to the string
* section. If we don't include it, then we will potentially
* get a "changed section not included" error in
* kpatch_verify_patchability() if no other function based change
* also changes the string section. We could try to exclude each
* literal string added to the section by KPATCH_IGNORE_SECTION()
* from the section data comparison, but this is a simpler way.
*/
strsec->include = 1;
if (strsec->secsym)
strsec->secsym->include = 1;
name = strsec->data->d_buf + rela->addend;
ignoresec = find_section_by_name(&kelf->sections, name);
if (!ignoresec)
ERROR("KPATCH_IGNORE_SECTION: can't find %s", name);
log_normal("ignoring section: %s\n", name);
if (is_rela_section(ignoresec))
ignoresec = ignoresec->base;
ignoresec->ignore = 1;
if (ignoresec->twin)
ignoresec->twin->ignore = 1;
}
}
static void kpatch_mark_ignored_sections_same(struct kpatch_elf *kelf)
{
struct section *sec;
struct symbol *sym;
list_for_each_entry(sec, &kelf->sections, list) {
if (!sec->ignore)
continue;
sec->status = SAME;
if (!is_rela_section(sec)) {
if (sec->secsym)
sec->secsym->status = SAME;
if (sec->rela)
sec->rela->status = SAME;
}
list_for_each_entry(sym, &kelf->symbols, list) {
if (sym->sec != sec)
continue;
sym->status = SAME;
}
}
/* strip temporary global __UNIQUE_ID_kpatch_ignore_section_* symbols */
list_for_each_entry(sym, &kelf->symbols, list)
if (!strncmp(sym->name, "__UNIQUE_ID_kpatch_ignore_section_",
strlen("__UNIQUE_ID_kpatch_ignore_section_")))
sym->status = SAME;
}
static void kpatch_mark_ignored_children_same(struct symbol *sym)
{
struct symbol *child;
list_for_each_entry(child, &sym->children, subfunction_node) {
child->status = SAME;
kpatch_mark_ignored_children_same(child);
}
}
static void kpatch_mark_ignored_functions_same(struct kpatch_elf *kelf)
{
struct section *sec;
struct symbol *sym;
struct rela *rela;
sec = find_section_by_name(&kelf->sections, ".kpatch.ignore.functions");
if (!sec)
return;
list_for_each_entry(rela, &sec->rela->relas, list) {
if (!rela->sym->sec)
ERROR("expected bundled symbol");
if (rela->sym->type != STT_FUNC)
ERROR("expected function symbol");
log_normal("ignoring function: %s\n", rela->sym->name);
if (rela->sym->status != CHANGED)
log_normal("NOTICE: no change detected in function %s, unnecessary KPATCH_IGNORE_FUNCTION()?\n", rela->sym->name);
rela->sym->status = SAME;
rela->sym->sec->status = SAME;
kpatch_mark_ignored_children_same(rela->sym);
if (rela->sym->sec->secsym)
rela->sym->sec->secsym->status = SAME;
if (rela->sym->sec->rela)
rela->sym->sec->rela->status = SAME;
}
/* strip temporary global kpatch_ignore_func_* symbols */
list_for_each_entry(sym, &kelf->symbols, list)
if (!strncmp(sym->name, "__kpatch_ignore_func_",
strlen("__kpatch_ignore_func_")))
sym->status = SAME;
}
static void kpatch_create_kpatch_arch_section(struct kpatch_elf *kelf, char *objname)
{
struct special_section *special;
struct symbol *strsym;
struct section *sec, *karch_sec;
struct rela *rela;
int nr, index = 0;
if (!KLP_ARCH)
return;
nr = sizeof(special_sections) / sizeof(special_sections[0]);
karch_sec = create_section_pair(kelf, ".kpatch.arch", sizeof(struct kpatch_arch), nr);
/* lookup strings symbol */
strsym = find_symbol_by_name(&kelf->symbols, ".kpatch.strings");
if (!strsym)
ERROR("can't find .kpatch.strings symbol");
for (special = special_sections; special->name; special++) {
if ((special->arch & kelf->arch) == 0)
continue;
if (strcmp(special->name, ".parainstructions") &&
strcmp(special->name, ".altinstructions"))
continue;
sec = find_section_by_name(&kelf->sections, special->name);
if (!sec)
continue;
/* entries[index].sec */
ALLOC_LINK(rela, &karch_sec->rela->relas);
rela->sym = sec->secsym;
rela->type = absolute_rela_type(kelf);
rela->addend = 0;
rela->offset = (unsigned int)(index * sizeof(struct kpatch_arch) + \
offsetof(struct kpatch_arch, sec));
/* entries[index].objname */
ALLOC_LINK(rela, &karch_sec->rela->relas);
rela->sym = strsym;
rela->type = absolute_rela_type(kelf);
rela->addend = offset_of_string(&kelf->strings, objname);
rela->offset = (unsigned int)(index * sizeof(struct kpatch_arch) + \
offsetof(struct kpatch_arch, objname));
index++;
}
karch_sec->data->d_size = index * sizeof(struct kpatch_arch);
karch_sec->sh.sh_size = karch_sec->data->d_size;
}
static void kpatch_process_special_sections(struct kpatch_elf *kelf,
struct lookup_table *lookup)
{
struct special_section *special;
struct section *sec;
struct symbol *sym;
struct rela *rela;
int altinstr = 0;
for (special = special_sections; special->name; special++) {
if ((special->arch & kelf->arch) == 0)
continue;
sec = find_section_by_name(&kelf->sections, special->name);
if (!sec || !sec->rela)
continue;
kpatch_regenerate_special_section(kelf, lookup, special, sec->rela);
if (!strcmp(special->name, ".altinstructions") && sec->include)
altinstr = 1;
}
/*
* The following special sections don't have relas which reference
* non-included symbols, so their entire rela section can be included.
*/
list_for_each_entry(sec, &kelf->sections, list) {
if (strcmp(sec->name, ".altinstr_replacement"))
continue;
/*
* Only include .altinstr_replacement if .altinstructions
* is also included.
*/
if (!altinstr)
break;
/* include base section */
sec->include = 1;
/* include all symbols in the section */
list_for_each_entry(sym, &kelf->symbols, list)
if (sym->sec == sec)
sym->include = 1;
/* include rela section */
if (sec->rela) {
sec->rela->include = 1;
/* include all symbols referenced by relas */
list_for_each_entry(rela, &sec->rela->relas, list)
kpatch_include_symbol(rela->sym);
}
}
if (KLP_ARCH) {
/*
* The following special sections aren't supported with older
* kernels, so make sure we don't ever try to include them.
* Otherwise the kernel will see the jump table during module
* loading and get confused. Generally it should be safe to
* exclude them, it just means that you can't modify jump
* labels and enable tracepoints in a patched function.
*/
list_for_each_entry(sec, &kelf->sections, list) {
if (strcmp(sec->name, "__jump_table") &&
strcmp(sec->name, "__tracepoints") &&
strcmp(sec->name, "__tracepoints_ptrs") &&
strcmp(sec->name, "__tracepoints_strings"))
continue;
sec->status = SAME;
sec->include = 0;
if (sec->rela) {
sec->rela->status = SAME;
sec->rela->include = 0;
}
}
}
kpatch_regenerate_orc_sections(kelf);
}
static void kpatch_create_patches_sections(struct kpatch_elf *kelf,
struct lookup_table *table,
char *objname)
{
int nr, index, objname_offset;
struct section *sec, *relasec;
struct symbol *sym, *strsym;
struct rela *rela;
struct lookup_result symbol;
struct kpatch_patch_func *funcs;
/* count patched functions */
nr = 0;
list_for_each_entry(sym, &kelf->symbols, list) {
if (sym->type != STT_FUNC || sym->status != CHANGED ||
sym->parent)
continue;
nr++;
}
/* create text/rela section pair */
sec = create_section_pair(kelf, ".kpatch.funcs", sizeof(*funcs), nr);
relasec = sec->rela;
funcs = sec->data->d_buf;
/* lookup strings symbol */
strsym = find_symbol_by_name(&kelf->symbols, ".kpatch.strings");
if (!strsym)
ERROR("can't find .kpatch.strings symbol");
/* add objname to strings */
objname_offset = offset_of_string(&kelf->strings, objname);
/* populate sections */
index = 0;
list_for_each_entry(sym, &kelf->symbols, list) {
if (sym->type != STT_FUNC || sym->status != CHANGED ||
sym->parent)
continue;
if (!lookup_symbol(table, sym, &symbol))
ERROR("can't find symbol '%s' in symbol table", sym->name);
if (sym->bind == STB_LOCAL && symbol.global)
ERROR("can't find local symbol '%s' in symbol table", sym->name);
log_debug("lookup for %s: obj=%s sympos=%lu size=%lu",
sym->name, symbol.objname, symbol.sympos,
symbol.size);
/*
* Convert global symbols to local so other objects in the
* patch module (like the patch callback object's init code)
* won't link to this function and call it before its
* relocations have been applied.
*/
sym->bind = STB_LOCAL;
sym->sym.st_info = (unsigned char)
GELF_ST_INFO(sym->bind, sym->type);
/* add entry in text section */
funcs[index].old_addr = symbol.addr;
funcs[index].old_size = symbol.size;
funcs[index].new_size = sym->sym.st_size;
funcs[index].sympos = symbol.sympos;
/*
* Add a relocation that will populate the
* funcs[index].new_addr field at module load time.
*/
ALLOC_LINK(rela, &relasec->relas);
rela->sym = sym;
rela->type = absolute_rela_type(kelf);
rela->addend = 0;
rela->offset = (unsigned int)(index * sizeof(*funcs));
/*
* Add a relocation that will populate the funcs[index].name
* field.
*/
ALLOC_LINK(rela, &relasec->relas);
rela->sym = strsym;
rela->type = absolute_rela_type(kelf);
rela->addend = offset_of_string(&kelf->strings, sym->name);
rela->offset = (unsigned int)(index * sizeof(*funcs) +
offsetof(struct kpatch_patch_func, name));
/*
* Add a relocation that will populate the funcs[index].objname
* field.
*/
ALLOC_LINK(rela, &relasec->relas);
rela->sym = strsym;
rela->type = absolute_rela_type(kelf);
rela->addend = objname_offset;
rela->offset = (unsigned int)(index * sizeof(*funcs) +
offsetof(struct kpatch_patch_func,objname));
index++;
}
/* sanity check, index should equal nr */
if (index != nr)
ERROR("size mismatch in funcs sections");
}
static bool kpatch_is_core_module_symbol(char *name)
{
return (!strcmp(name, "kpatch_shadow_alloc") ||
!strcmp(name, "kpatch_shadow_free") ||
!strcmp(name, "kpatch_shadow_get"));
}
static bool is_expoline(struct kpatch_elf *kelf, char *name)
{
return kelf->arch == S390 && !strncmp(name, "__s390_indirect_jump_r", 22);
}
static int function_ptr_rela(const struct rela *rela)
{
const struct rela *rela_toc = toc_rela(rela);
return (rela_toc && rela_toc->sym->type == STT_FUNC &&
!rela_toc->sym->parent &&
rela_toc->addend == (int)rela_toc->sym->sym.st_value &&
(rela->type == R_X86_64_32S ||
rela->type == R_PPC64_TOC16_HA ||
rela->type == R_PPC64_TOC16_LO_DS));
}
static bool need_klp_reloc(struct kpatch_elf *kelf, struct lookup_table *table,
struct section *relasec, const struct rela *rela)
{
struct lookup_result symbol;
if (is_debug_section(relasec))
return false;
/*
* These references are treated specially by the module loader and
* should never be converted to klp relocations.
*/
if (rela->type == R_PPC64_REL16_HA || rela->type == R_PPC64_REL16_LO ||
rela->type == R_PPC64_ENTRY)
return false;
/* v5.13+ kernels use relative jump labels */
if (rela->type == R_PPC64_REL64 && strcmp(relasec->name, ".rela__jump_table"))
return false;
/*
* On powerpc, the function prologue generated by GCC 6 has the
* sequence:
*
* .globl my_func
* .type my_func, @function
* .quad .TOC.-my_func
* my_func:
* .reloc ., R_PPC64_ENTRY ; optional
* ld r2,-8(r12)
* add r2,r2,r12
* .localentry my_func, .-my_func
*
* The R_PPC64_ENTRY is optional and its symbol might have an empty
* name. Leave it as a normal rela.
*/
if (rela->type == R_PPC64_ENTRY)
return false;
/*
* Allow references to core module symbols to remain as normal
* relas. They should be exported.
*/
if (kpatch_is_core_module_symbol(rela->sym->name))
return false;
/*
* Allow references to s390 expolines to remain as normal relas. They
* will be generated in the module by the kernel module link.
*/
if (is_expoline(kelf, rela->sym->name))
return false;
if (rela->sym->sec) {
/*
* Internal symbols usually don't need klp relocations, because
* they live in the patch module and can be relocated normally.
*
* There's one exception: function pointers.
*
* If the rela references a function pointer, we convert it to
* a klp relocation, so that the function pointer will refer to
* the original function rather than the patched function.
* This can prevent crashes in cases where the function pointer
* is called asynchronously after the patch module has been
* unloaded.
*/
if (!function_ptr_rela(rela))
return false;
/*
* Function pointers which refer to _nested_ functions are a
* special case. They are not supposed to be visible outside
* of the function that defines them. Their names may differ
* in the original and the patched kernels which makes it
* difficult to use klp relocations. Fortunately, nested
* functions are rare and are unlikely to be used as
* asynchronous callbacks, so the patched code can refer to
* them directly. It seems, one can only distinguish such
* functions by their names containing a dot. Other kinds of
* functions with such names (e.g. optimized copies of
* functions) are unlikely to be used as callbacks.
*
* Function pointers to *new* functions don't have this issue,
* just use a normal rela for them.
*/
return toc_rela(rela)->sym->status != NEW &&
!strchr(toc_rela(rela)->sym->name, '.');
}
if (!lookup_symbol(table, rela->sym, &symbol)) {
/*
* Assume the symbol lives in another .o in the patch module.
* A normal rela should work.
*/
return false;
}
if (rela->sym->bind == STB_LOCAL) {
if (symbol.global)
ERROR("can't find local symbol '%s' in symbol table",
rela->sym->name);
/*
* The symbol is (formerly) local. Use a klp relocation to
* access the original version of the symbol in the patched
* object.
*/
return true;
}
if (symbol.exported) {
if (is_gcc6_localentry_bundled_sym(kelf, rela->sym)) {
/*
* On powerpc, the symbol is global and exported, but
* it was also in the changed object file. In this
* case the rela refers to the 'localentry' point, so a
* normal rela wouldn't work. Force a klp relocation
* so it can be handled correctly by the livepatch
* relocation code.
*/
return true;
}
if (!strcmp(symbol.objname, "vmlinux")) {
/*
* The symbol is exported by vmlinux. Use a normal
* rela.
*/
return false;
}
/*
* The symbol is exported by the to-be-patched module, or by
* another module which the patched module depends on. Use a
* klp relocation because of late module loading: the patch
* module may be loaded before the to-be-patched (or other)
* module.
*/
return true;
}
if (symbol.global) {
/*
* The symbol is global in the to-be-patched object, but not
* exported. Use a klp relocation to work around the fact that
* it's an unexported sybmbol.
*/
return true;
}
/*
* The symbol is global and not exported, but it's not in the parent
* object. The only explanation is that it's defined in another object
* in the patch module. A normal rela should resolve it.
*/
return false;
}
/*
* kpatch_create_intermediate_sections()
*
* The primary purpose of this function is to convert some relocations to klp
* relocations.
*
* If the patched code refers to a symbol, for example, if it calls a function
* or stores a pointer to a function somewhere or accesses some global data,
* the address of that symbol must be resolved somehow before the patch is
* applied.
*
* If the symbol lives outside the patch module, and if it's not exported by
* vmlinux (e.g., with EXPORT_SYMBOL) then the rela needs to be converted to a
* klp relocation so the livepatch code can resolve it at runtime.
*/
static void kpatch_create_intermediate_sections(struct kpatch_elf *kelf,
struct lookup_table *table,
char *objname,
char *pmod_name)
{
int nr, index;
struct section *relasec, *ksym_sec, *krela_sec;
struct rela *rela, *rela2, *safe;
struct symbol *strsym, *ksym_sec_sym;
struct kpatch_symbol *ksyms;
struct kpatch_relocation *krelas;
struct lookup_result symbol;
bool special;
bool vmlinux = !strcmp(objname, "vmlinux");
struct special_section *s;
/* count rela entries that need to be dynamic */
nr = 0;
list_for_each_entry(relasec, &kelf->sections, list) {
if (!is_rela_section(relasec))
continue;
if (!strcmp(relasec->name, ".rela.kpatch.funcs"))
continue;
list_for_each_entry(rela, &relasec->relas, list) {
/* upper bound on number of kpatch relas and symbols */
nr++;
/*
* We set 'need_klp_reloc' here in the first pass
* because the .toc section's 'need_klp_reloc' values
* are dependent on all the other sections. Otherwise,
* if we did this analysis in the second pass, we'd
* have to convert .toc klp relocations at the very end.
*
* Specifically, this is needed for the powerpc
* internal symbol function pointer check which is done
* via .toc indirection in need_klp_reloc().
*/
if (need_klp_reloc(kelf, table, relasec, rela))
toc_rela(rela)->need_klp_reloc = true;
}
}
/* create .kpatch.relocations text/rela section pair */
krela_sec = create_section_pair(kelf, ".kpatch.relocations", sizeof(*krelas), nr);
krelas = krela_sec->data->d_buf;
/* create .kpatch.symbols text/rela section pair */
ksym_sec = create_section_pair(kelf, ".kpatch.symbols", sizeof(*ksyms), nr);
ksyms = ksym_sec->data->d_buf;
/* create .kpatch.symbols section symbol (to set rela->sym later) */
ALLOC_LINK(ksym_sec_sym, &kelf->symbols);
ksym_sec_sym->sec = ksym_sec;
ksym_sec_sym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_SECTION);
ksym_sec_sym->type = STT_SECTION;
ksym_sec_sym->bind = STB_LOCAL;
ksym_sec_sym->name = ".kpatch.symbols";
/* lookup strings symbol */
strsym = find_symbol_by_name(&kelf->symbols, ".kpatch.strings");
if (!strsym)
ERROR("can't find .kpatch.strings symbol");
/* populate sections */
index = 0;
list_for_each_entry(relasec, &kelf->sections, list) {
if (!is_rela_section(relasec))
continue;
if (!strcmp(relasec->name, ".rela.kpatch.funcs") ||
!strcmp(relasec->name, ".rela.kpatch.relocations") ||
!strcmp(relasec->name, ".rela.kpatch.symbols"))
continue;
special = false;
for (s = special_sections; s->name; s++) {
if ((s->arch & kelf->arch) == 0)
continue;
if (!strcmp(relasec->base->name, s->name))
special = true;
}
list_for_each_entry_safe(rela, safe, &relasec->relas, list) {
if (!rela->need_klp_reloc) {
rela->sym->strip = SYMBOL_USED;
continue;
}
/*
* Starting with Linux 5.8, .klp.arch sections are no
* longer supported: now that vmlinux relocations are
* written early, before paravirt and alternative
* module init, .klp.arch is technically not needed.
*
* For sanity we just need to make sure that there are
* no .klp.rela.{module}.{section} sections for special
* sections. Otherwise there might be ordering issues,
* if the .klp.relas are applied after the module
* special section init code (e.g., apply_paravirt)
* runs due to late module patching.
*/
if (!KLP_ARCH && !vmlinux && special)
ERROR("unsupported klp relocation reference to symbol '%s' in module-specific special section '%s'",
rela->sym->name, relasec->base->name);
if (!lookup_symbol(table, rela->sym, &symbol))
ERROR("can't find symbol '%s' in symbol table",
rela->sym->name);
log_debug("lookup for %s: obj=%s sympos=%lu",
rela->sym->name, symbol.objname,
symbol.sympos);
/* Fill in ksyms[index] */
if (vmlinux)
ksyms[index].src = symbol.addr;
else
/* for modules, src is discovered at runtime */
ksyms[index].src = 0;
ksyms[index].sympos = symbol.sympos;
ksyms[index].type = rela->sym->type;
ksyms[index].bind = rela->sym->bind;
/* add rela to fill in ksyms[index].name field */
ALLOC_LINK(rela2, &ksym_sec->rela->relas);
rela2->sym = strsym;
rela2->type = absolute_rela_type(kelf);
rela2->addend = offset_of_string(&kelf->strings, rela->sym->name);
rela2->offset = (unsigned int)(index * sizeof(*ksyms) + \
offsetof(struct kpatch_symbol, name));
/* add rela to fill in ksyms[index].objname field */
ALLOC_LINK(rela2, &ksym_sec->rela->relas);
rela2->sym = strsym;
rela2->type = absolute_rela_type(kelf);
rela2->addend = offset_of_string(&kelf->strings, symbol.objname);
rela2->offset = (unsigned int)(index * sizeof(*ksyms) + \
offsetof(struct kpatch_symbol, objname));
/* Fill in krelas[index] */
if (is_gcc6_localentry_bundled_sym(kelf, rela->sym) &&
rela->addend == (int)rela->sym->sym.st_value)
rela->addend -= rela->sym->sym.st_value;
krelas[index].addend = rela->addend;
krelas[index].type = rela->type;
krelas[index].external = !vmlinux && symbol.exported;
/* add rela to fill in krelas[index].dest field */
ALLOC_LINK(rela2, &krela_sec->rela->relas);
if (!relasec->base->secsym) {
struct symbol *sym;
/*
* Newer toolchains are stingy with their
* section symbols, create one if it doesn't
* exist already.
*/
ALLOC_LINK(sym, &kelf->symbols);
sym->sec = relasec->base;
sym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_SECTION);
sym->type = STT_SECTION;
sym->bind = STB_LOCAL;
sym->name = relasec->base->name;
relasec->base->secsym = sym;
}
rela2->sym = relasec->base->secsym;
rela2->type = absolute_rela_type(kelf);
rela2->addend = rela->offset;
rela2->offset = (unsigned int)(index * sizeof(*krelas) + \
offsetof(struct kpatch_relocation, dest));
/* add rela to fill in krelas[index].objname field */
ALLOC_LINK(rela2, &krela_sec->rela->relas);
rela2->sym = strsym;
rela2->type = absolute_rela_type(kelf);
rela2->addend = offset_of_string(&kelf->strings, objname);
rela2->offset = (unsigned int)(index * sizeof(*krelas) + \
offsetof(struct kpatch_relocation, objname));
/* add rela to fill in krelas[index].ksym field */
ALLOC_LINK(rela2, &krela_sec->rela->relas);
rela2->sym = ksym_sec_sym;
rela2->type = absolute_rela_type(kelf);
rela2->addend = (unsigned int)(index * sizeof(*ksyms));
rela2->offset = (unsigned int)(index * sizeof(*krelas) + \
offsetof(struct kpatch_relocation, ksym));
/*
* Mark the referred to symbol for removal but
* only if it is not from this object file.
* The symbols from this object file may be needed
* later (for example, they may have relocations
* of their own which should be processed).
*/
if (!rela->sym->sec && rela->sym->strip != SYMBOL_USED)
rela->sym->strip = SYMBOL_STRIP;
list_del(&rela->list);
free(rela);
index++;
}
}
/* set size to actual number of ksyms/krelas */
ksym_sec->data->d_size = index * sizeof(struct kpatch_symbol);
ksym_sec->sh.sh_size = ksym_sec->data->d_size;
krela_sec->data->d_size = index * sizeof(struct kpatch_relocation);
krela_sec->sh.sh_size = krela_sec->data->d_size;
}
static void kpatch_create_callbacks_objname_rela(struct kpatch_elf *kelf, char *objname)
{
struct section *sec;
struct rela *rela;
struct symbol *strsym;
int objname_offset;
struct callback { char *name; int offset; };
static struct callback callbacks[] = {
{ .name = ".rela.kpatch.callbacks.pre_patch",
.offset = offsetof(struct kpatch_pre_patch_callback, objname) },
{ .name = ".rela.kpatch.callbacks.post_patch",
.offset = offsetof(struct kpatch_post_patch_callback, objname) },
{ .name = ".rela.kpatch.callbacks.pre_unpatch",
.offset = offsetof(struct kpatch_pre_unpatch_callback, objname) },
{ .name = ".rela.kpatch.callbacks.post_unpatch",
.offset = offsetof(struct kpatch_post_patch_callback, objname) },
{ .name = NULL, .offset = 0 },
};
struct callback *callbackp;
/* lookup strings symbol */
strsym = find_symbol_by_name(&kelf->symbols, ".kpatch.strings");
if (!strsym)
ERROR("can't find .kpatch.strings symbol");
/* add objname to strings */
objname_offset = offset_of_string(&kelf->strings, objname);
list_for_each_entry(sec, &kelf->sections, list) {
for (callbackp = callbacks; callbackp->name; callbackp++) {
if (!strcmp(callbackp->name, sec->name)) {
ALLOC_LINK(rela, &sec->relas);
rela->sym = strsym;
rela->type = absolute_rela_type(kelf);
rela->addend = objname_offset;
rela->offset = callbackp->offset;
break;
}
}
}
}
/*
* This function basically reimplements the functionality of the Linux
* recordmcount script, so that patched functions can be recognized by ftrace.
*
* TODO: Eventually we can modify recordmount so that it recognizes our bundled
* sections as valid and does this work for us.
*/
static void kpatch_create_mcount_sections(struct kpatch_elf *kelf)
{
int nr, index;
struct section *sec, *relasec;
struct symbol *sym;
struct rela *rela, *mcount_rela;
void **funcs;
unsigned long insn_offset = 0;
nr = 0;
list_for_each_entry(sym, &kelf->symbols, list)
if (sym->type == STT_FUNC && sym->status != SAME &&
sym->has_func_profiling)
nr++;
/* create text/rela section pair */
sec = create_section_pair(kelf, "__mcount_loc", sizeof(void*), nr);
relasec = sec->rela;
/* populate sections */
index = 0;
list_for_each_entry(sym, &kelf->symbols, list) {
if (sym->type != STT_FUNC || sym->status == SAME)
continue;
if (!sym->has_func_profiling) {
log_debug("function %s has no fentry/mcount call, no mcount record is needed\n",
sym->name);
continue;
}
switch(kelf->arch) {
case PPC64: {
bool found = false;
list_for_each_entry(rela, &sym->sec->rela->relas, list)
if (!strcmp(rela->sym->name, "_mcount")) {
found = true;
break;
}
if (!found)
ERROR("%s: unexpected missing call to _mcount()", __func__);
insn_offset = rela->offset;
break;
}
case X86_64: {
unsigned char *insn;
void *newdata;
rela = list_first_entry(&sym->sec->rela->relas, struct rela, list);
/*
* For "call fentry", the relocation points to 1 byte past the
* beginning of the instruction.
*/
insn_offset = rela->offset - 1;
/*
* R_X86_64_NONE is only generated by older versions of
* kernel/gcc which use the mcount script. There's a
* NOP instead of a call to fentry.
*/
if (rela->type != R_X86_64_NONE)
break;
/* Make a writable copy of the text section data */
newdata = malloc(sym->sec->data->d_size);
if (!newdata)
ERROR("malloc");
memcpy(newdata, sym->sec->data->d_buf, sym->sec->data->d_size);
sym->sec->data->d_buf = newdata;
insn = newdata;
/*
* Replace the NOP with a call to fentry. The fentry
* rela symbol is already there, just need to change
* the relocation type accordingly.
*/
insn = sym->sec->data->d_buf;
if (insn[0] != 0xf)
ERROR("%s: unexpected instruction at the start of the function", sym->name);
insn[0] = 0xe8;
insn[1] = 0;
insn[2] = 0;
insn[3] = 0;
insn[4] = 0;
rela->type = R_X86_64_PC32;
break;
}
case S390: {
insn_offset = sym->sym.st_value;
break;
}
default:
ERROR("unsupported arch");
}
/*
* 'rela' points to the mcount/fentry call.
*
* Create a .rela__mcount_loc entry which also points to it.
*/
ALLOC_LINK(mcount_rela, &relasec->relas);
mcount_rela->sym = sym;
mcount_rela->type = absolute_rela_type(kelf);
mcount_rela->addend = insn_offset - sym->sym.st_value;
mcount_rela->offset = (unsigned int) (index * sizeof(*funcs));
index++;
}
/* sanity check, index should equal nr */
if (index != nr)
ERROR("size mismatch in funcs sections");
}
/*
* This function strips out symbols that were referenced by changed rela
* sections, but the rela entries that referenced them were converted to
* klp relocations and are no longer needed.
*/
static void kpatch_strip_unneeded_syms(struct kpatch_elf *kelf,
struct lookup_table *table)
{
struct symbol *sym, *safe;
list_for_each_entry_safe(sym, safe, &kelf->symbols, list) {
if (sym->strip == SYMBOL_STRIP) {
list_del(&sym->list);
free(sym);
}
}
}
static void kpatch_create_strings_elements(struct kpatch_elf *kelf)
{
struct section *sec;
struct symbol *sym;
/* create .kpatch.strings */
/* allocate section resources */
ALLOC_LINK(sec, &kelf->sections);
sec->name = ".kpatch.strings";
/* set data */
sec->data = malloc(sizeof(*sec->data));
if (!sec->data)
ERROR("malloc");
sec->data->d_type = ELF_T_BYTE;
/* set section header */
sec->sh.sh_type = SHT_PROGBITS;
sec->sh.sh_entsize = 1;
sec->sh.sh_addralign = 1;
sec->sh.sh_flags = SHF_ALLOC;
/* create .kpatch.strings section symbol (reuse sym variable) */
ALLOC_LINK(sym, &kelf->symbols);
sym->sec = sec;
sym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_SECTION);
sym->type = STT_SECTION;
sym->bind = STB_LOCAL;
sym->name = ".kpatch.strings";
}
static void kpatch_build_strings_section_data(struct kpatch_elf *kelf)
{
struct string *string;
struct section *sec;
size_t size;
char *strtab;
sec = find_section_by_name(&kelf->sections, ".kpatch.strings");
if (!sec)
ERROR("can't find .kpatch.strings");
/* determine size */
size = 0;
list_for_each_entry(string, &kelf->strings, list)
size += strlen(string->name) + 1;
/* allocate section resources */
strtab = malloc(size);
if (!strtab)
ERROR("malloc");
sec->data->d_buf = strtab;
sec->data->d_size = size;
/* populate strings section data */
list_for_each_entry(string, &kelf->strings, list) {
strcpy(strtab, string->name);
strtab += strlen(string->name) + 1;
}
}
/*
* Don't allow sibling calls from patched functions on ppc64le. Before doing a
* sibling call, the patched function restores the stack to its caller's stack.
* The kernel-generated stub then writes the patch module's r2 (toc) value to
* the caller's stack, corrupting it, eventually causing a panic after it
* returns to the caller and the caller tries to use the livepatch module's toc
* value.
*
* In theory we could instead a) generate a custom stub, or b) modify the
* kernel livepatch_handler code to save/restore the stack r2 value, but this
* is easier for now.
*/
static void kpatch_no_sibling_calls_ppc64le(struct kpatch_elf *kelf)
{
struct symbol *sym;
unsigned char *insn;
unsigned int offset;
int sibling_call_errors = 0;
if (kelf->arch != PPC64)
return;
list_for_each_entry(sym, &kelf->symbols, list) {
if (sym->type != STT_FUNC || sym->status != CHANGED)
continue;
for (offset = 0; offset < sym->sec->data->d_size; offset += 4) {
insn = sym->sec->data->d_buf + offset;
/*
* The instruction 0x48000000 can be assumed to be a
* sibling call:
*
* Bits 0-5 (opcode) == 0x9: unconditional branch
* Bit 30 (absolute) == 0: relative address
* Bit 31 (link) == 0: doesn't set LR (not a call)
*
* Bits 6-29 (branch address) == zero, which means
* it's either a branch to self (infinite loop), or
* there's a REL24 relocation for the address which
* will be written by the linker or the kernel.
*/
if (insn[3] != 0x48 || insn[2] != 0x00 ||
insn[1] != 0x00 || insn[0] != 0x00)
continue;
/* Make sure it's not a branch-to-self: */
if (!find_rela_by_offset(sym->sec->rela, offset))
continue;
log_normal("Found an unsupported sibling call at %s()+0x%lx. Add __attribute__((optimize(\"-fno-optimize-sibling-calls\"))) to %s() definition.\n",
sym->name, sym->sym.st_value + offset, sym->name);
sibling_call_errors++;
}
}
if (sibling_call_errors)
ERROR("Found %d unsupported sibling call(s) in the patched code.",
sibling_call_errors);
}
/* Check which functions have fentry/mcount calls; save this info for later use. */
static void kpatch_find_func_profiling_calls(struct kpatch_elf *kelf)
{
struct symbol *sym;
struct rela *rela;
unsigned char *insn;
list_for_each_entry(sym, &kelf->symbols, list) {
if (sym->type != STT_FUNC || sym->is_pfx ||
!sym->sec || !sym->sec->rela)
continue;
switch(kelf->arch) {
case PPC64:
list_for_each_entry(rela, &sym->sec->rela->relas, list) {
if (!strcmp(rela->sym->name, "_mcount")) {
sym->has_func_profiling = 1;
break;
}
}
break;
case X86_64:
rela = list_first_entry(&sym->sec->rela->relas, struct rela,
list);
if ((rela->type != R_X86_64_NONE &&
rela->type != R_X86_64_PC32 &&
rela->type != R_X86_64_PLT32) ||
strcmp(rela->sym->name, "__fentry__"))
continue;
sym->has_func_profiling = 1;
break;
case S390:
/* Check for compiler generated fentry nop - jgnop 0 */
insn = sym->sec->data->d_buf;
if (insn[0] == 0xc0 && insn[1] == 0x04 &&
insn[2] == 0x00 && insn[3] == 0x00 &&
insn[4] == 0x00 && insn[5] == 0x00)
sym->has_func_profiling = 1;
break;
default:
ERROR("unsupported arch");
}
}
}
struct arguments {
char *args[7];
bool debug, klp_arch;
};
static char args_doc[] = "original.o patched.o parent-name parent-symtab Module.symvers patch-module-name output.o";
static struct argp_option options[] = {
{"debug", 'd', NULL, 0, "Show debug output" },
{"klp-arch", 'a', NULL, 0, "Kernel supports .klp.arch section" },
{ NULL }
};
static error_t parse_opt (int key, char *arg, struct argp_state *state)
{
/* Get the input argument from argp_parse, which we
know is a pointer to our arguments structure. */
struct arguments *arguments = state->input;
switch (key)
{
case 'd':
arguments->debug = 1;
break;
case 'a':
arguments->klp_arch = 1;
break;
case ARGP_KEY_ARG:
if (state->arg_num >= 7)
/* Too many arguments. */
argp_usage (state);
arguments->args[state->arg_num] = arg;
break;
case ARGP_KEY_END:
if (state->arg_num < 7)
/* Not enough arguments. */
argp_usage (state);
break;
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
static struct argp argp = { options, parse_opt, args_doc, NULL };
int main(int argc, char *argv[])
{
struct kpatch_elf *kelf_orig, *kelf_patched, *kelf_out;
struct arguments arguments;
int num_changed, callbacks_exist, new_globals_exist;
struct lookup_table *lookup;
struct section *relasec, *symtab;
char *orig_obj, *patched_obj, *parent_name;
char *parent_symtab, *mod_symvers, *patch_name, *output_obj;
memset(&arguments, 0, sizeof(arguments));
argp_parse (&argp, argc, argv, 0, NULL, &arguments);
if (arguments.debug)
loglevel = DEBUG;
if (arguments.klp_arch)
KLP_ARCH = true;
elf_version(EV_CURRENT);
orig_obj = arguments.args[0];
patched_obj = arguments.args[1];
parent_name = arguments.args[2];
parent_symtab = arguments.args[3];
mod_symvers = arguments.args[4];
patch_name = arguments.args[5];
output_obj = arguments.args[6];
childobj = basename(orig_obj);
kelf_orig = kpatch_elf_open(orig_obj);
kelf_patched = kpatch_elf_open(patched_obj);
kpatch_find_func_profiling_calls(kelf_orig);
kpatch_find_func_profiling_calls(kelf_patched);
kpatch_compare_elf_headers(kelf_orig->elf, kelf_patched->elf);
kpatch_check_program_headers(kelf_orig->elf);
kpatch_check_program_headers(kelf_patched->elf);
kpatch_bundle_symbols(kelf_orig);
kpatch_bundle_symbols(kelf_patched);
kpatch_detect_child_functions(kelf_orig);
kpatch_detect_child_functions(kelf_patched);
lookup = lookup_open(parent_symtab, parent_name, mod_symvers, kelf_orig);
kpatch_mark_grouped_sections(kelf_patched);
kpatch_replace_sections_syms(kelf_orig);
kpatch_replace_sections_syms(kelf_patched);
kpatch_correlate_elfs(kelf_orig, kelf_patched);
kpatch_correlate_static_local_variables(kelf_orig, kelf_patched);
/*
* After this point, we don't care about kelf_orig anymore.
* We access its sections via the twin pointers in the
* section, symbol, and rela lists of kelf_patched.
*/
kpatch_mark_ignored_sections(kelf_patched);
kpatch_compare_correlated_elements(kelf_patched);
kpatch_mark_ignored_functions_same(kelf_patched);
kpatch_mark_ignored_sections_same(kelf_patched);
kpatch_check_func_profiling_calls(kelf_patched);
kpatch_elf_teardown(kelf_orig);
kpatch_elf_free(kelf_orig);
kpatch_include_standard_elements(kelf_patched);
num_changed = kpatch_include_changed_functions(kelf_patched);
callbacks_exist = kpatch_include_callback_elements(kelf_patched);
kpatch_include_force_elements(kelf_patched);
new_globals_exist = kpatch_include_new_globals(kelf_patched);
kpatch_include_debug_sections(kelf_patched);
kpatch_process_special_sections(kelf_patched, lookup);
kpatch_print_changes(kelf_patched);
kpatch_dump_kelf(kelf_patched);
kpatch_verify_patchability(kelf_patched);
if (!num_changed && !new_globals_exist) {
if (callbacks_exist)
log_debug("no changed functions were found, but callbacks exist\n");
else {
log_debug("no changed functions were found\n");
return EXIT_STATUS_NO_CHANGE;
}
}
/* this is destructive to kelf_patched */
kpatch_migrate_included_elements(kelf_patched, &kelf_out);
/*
* Teardown kelf_patched since we shouldn't access sections or symbols
* through it anymore. Don't free however, since our section and symbol
* name fields still point to strings in the Elf object owned by
* kpatch_patched.
*/
kpatch_elf_teardown(kelf_patched);
kpatch_no_sibling_calls_ppc64le(kelf_out);
/* create strings, patches, and klp relocation sections */
kpatch_create_strings_elements(kelf_out);
kpatch_create_patches_sections(kelf_out, lookup, parent_name);
kpatch_create_intermediate_sections(kelf_out, lookup, parent_name, patch_name);
kpatch_create_kpatch_arch_section(kelf_out, parent_name);
kpatch_create_callbacks_objname_rela(kelf_out, parent_name);
kpatch_build_strings_section_data(kelf_out);
kpatch_create_mcount_sections(kelf_out);
/*
* At this point, the set of output sections and symbols is
* finalized. Reorder the symbols into linker-compliant
* order and index all the symbols and sections. After the
* indexes have been established, update index data
* throughout the structure.
*/
kpatch_reorder_symbols(kelf_out);
kpatch_strip_unneeded_syms(kelf_out, lookup);
kpatch_reindex_elements(kelf_out);
/*
* Update rela section headers and rebuild the rela section data
* buffers from the relas lists.
*/
symtab = find_section_by_name(&kelf_out->sections, ".symtab");
if (!symtab)
ERROR("missing .symtab section");
list_for_each_entry(relasec, &kelf_out->sections, list) {
if (!is_rela_section(relasec))
continue;
relasec->sh.sh_link = symtab->index;
relasec->sh.sh_info = relasec->base->index;
kpatch_rebuild_rela_section_data(relasec);
}
kpatch_check_relocations(kelf_out);
kpatch_create_shstrtab(kelf_out);
kpatch_create_strtab(kelf_out);
kpatch_create_symtab(kelf_out);
kpatch_dump_kelf(kelf_out);
kpatch_write_output_elf(kelf_out, kelf_patched->elf, output_obj, 0664);
lookup_close(lookup);
kpatch_elf_free(kelf_patched);
kpatch_elf_teardown(kelf_out);
kpatch_elf_free(kelf_out);
return EXIT_STATUS_SUCCESS;
}
Reference in New Issue View Git Blame Copy Permalink