RepoMirrors/mpv
1
0
Fork 0
mirror of https://github.com/mpv-player/mpv synced 2025-01-24 16:43:28 +00:00
Code Issues Releases Wiki Activity

bstr: add UTF-8 validation and sanitation functions

Browse Source
This commit is contained in:
wm4 2013-08-15 18:48:05 +02:00
parent 04bc16f6ea
commit 380fa71fc7
2 changed files with 81 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
mpvcore/bstr.c
View File

@ -273,7 +273,7 @@ int bstr_decode_utf8(struct bstr s, struct bstr *out_next)
return -1;
codepoint &= 127 >> bytes;
for (int n = 1; n < bytes; n++) {
int tmp = s.start[0];
int tmp = (unsigned char)s.start[0];
if ((tmp & 0xC0) != 0x80)
return -1;
codepoint = (codepoint << 6) | (tmp & ~0xC0);
@ -285,6 +285,69 @@ int bstr_decode_utf8(struct bstr s, struct bstr *out_next)
return codepoint;
}
int bstr_validate_utf8(struct bstr s)
{
while (s.len) {
if (bstr_decode_utf8(s, &s) < 0) {
// Try to guess whether the sequence was just cut-off.
unsigned int codepoint = (unsigned char)s.start[0];
int bytes = bstr_parse_utf8_code_length(codepoint);
if (bytes > 1 && s.len < 6) {
// Manually check validity of left bytes
for (int n = 1; n < bytes; n++) {
if (n >= s.len) {
// Everything valid until now - just cut off.
return -(bytes - s.len);
}
int tmp = (unsigned char)s.start[n];
if ((tmp & 0xC0) != 0x80)
break;
}
}
return -8;
}
}
return 0;
}
static void append_bstr(bstr *buf, bstr s)
{
buf->start = talloc_realloc(NULL, buf->start, unsigned char, buf->len + s.len);
memcpy(buf->start + buf->len, s.start, s.len);
buf->len += s.len;
}
struct bstr bstr_sanitize_utf8_latin1(void *talloc_ctx, struct bstr s)
{
bstr new = {0};
bstr left = s;
unsigned char *first_ok = s.start;
while (left.len) {
int r = bstr_decode_utf8(left, &left);
if (r < 0) {
append_bstr(&new, (bstr){first_ok, left.start - first_ok});
uint32_t codepoint = (unsigned char)left.start[0];
char data[8];
uint8_t tmp;
char *output = data;
PUT_UTF8(codepoint, tmp, *output++ = tmp;);
append_bstr(&new, (bstr){data, output - data});
left.start += 1;
left.len -= 1;
first_ok = left.start;
}
}
if (!new.start)
return s;
if (first_ok != left.start)
append_bstr(&new, (bstr){first_ok, left.start - first_ok});
// For convenience
append_bstr(&new, (bstr){"\0", 1});
new.len -= 1;
talloc_steal(talloc_ctx, new.start);
return new;
}
bool bstr_case_startswith(struct bstr s, struct bstr prefix)
{
struct bstr start = bstr_splice(s, 0, prefix.len);

17
mpvcore/bstr.h
View File

@ -92,6 +92,23 @@ int bstr_decode_utf8(struct bstr str, struct bstr *out_next);
// On error, -1 is returned. On success, it returns a value in the range [1, 4].
int bstr_parse_utf8_code_length(unsigned char b);
// Return >= 0 if the string is valid UTF-8, otherwise negative error code.
// Embedded \0 bytes are considered valid.
// This returns -N if the UTF-8 string was likely just cut-off in the middle of
// an UTF-8 sequence: -1 means 1 byte was missing, -5 5 bytes missing.
// If the string was likely not cut off, -8 is returned.
// Use (return_value > -8) to check whether the string is valid UTF-8 or valid
// but cut-off UTF-8.
int bstr_validate_utf8(struct bstr s);
// Force the input string to valid UTF-8. If invalid UTF-8 encoding is
// encountered, the invalid bytes are interpreted as Latin-1.
// Embedded \0 bytes are considered valid.
// If replacement happens, a newly allocated string is returned (with a \0
// byte added past its end for convenience). The string is allocated via
// talloc, with talloc_ctx as parent.
struct bstr bstr_sanitize_utf8_latin1(void *talloc_ctx, struct bstr s);
// Return the text before the next line break, and return it. Change *rest to
// point to the text following this line break. (rest can be NULL.)
// Line break characters are not stripped.