Merge pull request #1132 from prometheus/fix-quoting-and-escaping

Support escape sequences in strings and add raw strings
This commit is contained in:
Julius Volz 2015-10-08 20:51:18 +02:00
commit 0088aa4d45
6 changed files with 502 additions and 25 deletions

View File

@ -16,6 +16,7 @@ package promql
import (
"fmt"
"strings"
"unicode"
"unicode/utf8"
)
@ -465,6 +466,9 @@ func lexStatements(l *lexer) stateFn {
case r == '"' || r == '\'':
l.stringOpen = r
return lexString
case r == '`':
l.stringOpen = r
return lexRawString
case isAlpha(r) || r == ':':
l.backup()
return lexKeywordOrIdentifier
@ -523,6 +527,9 @@ func lexInsideBraces(l *lexer) stateFn {
case r == '"' || r == '\'':
l.stringOpen = r
return lexString
case r == '`':
l.stringOpen = r
return lexRawString
case r == '=':
if l.next() == '~' {
l.emit(itemEQLRegex)
@ -583,16 +590,79 @@ func lexValueSequence(l *lexer) stateFn {
return lexValueSequence
}
// lexEscape scans a string escape sequence. The initial escaping character (\)
// has already been seen.
//
// NOTE: This function as well as the helper function digitVal() and associated
// tests have been adapted from the corresponding functions in the "go/scanner"
// package of the Go standard library to work for Prometheus-style strings.
// None of the actual escaping/quoting logic was changed in this function - it
// was only modified to integrate with our lexer.
func lexEscape(l *lexer) {
var n int
var base, max uint32
ch := l.next()
switch ch {
case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', l.stringOpen:
return
case '0', '1', '2', '3', '4', '5', '6', '7':
n, base, max = 3, 8, 255
case 'x':
ch = l.next()
n, base, max = 2, 16, 255
case 'u':
ch = l.next()
n, base, max = 4, 16, unicode.MaxRune
case 'U':
ch = l.next()
n, base, max = 8, 16, unicode.MaxRune
case eof:
l.errorf("escape sequence not terminated")
default:
l.errorf("unknown escape sequence %#U", ch)
}
var x uint32
for n > 0 {
d := uint32(digitVal(ch))
if d >= base {
if ch == eof {
l.errorf("escape sequence not terminated")
}
l.errorf("illegal character %#U in escape sequence", ch)
}
x = x*base + d
ch = l.next()
n--
}
if x > max || 0xD800 <= x && x < 0xE000 {
l.errorf("escape sequence is an invalid Unicode code point")
}
}
// digitVal returns the digit value of a rune or 16 in case the rune does not
// represent a valid digit.
func digitVal(ch rune) int {
switch {
case '0' <= ch && ch <= '9':
return int(ch - '0')
case 'a' <= ch && ch <= 'f':
return int(ch - 'a' + 10)
case 'A' <= ch && ch <= 'F':
return int(ch - 'A' + 10)
}
return 16 // Larger than any legal digit val.
}
// lexString scans a quoted string. The initial quote has already been seen.
func lexString(l *lexer) stateFn {
Loop:
for {
switch l.next() {
case '\\':
if r := l.next(); r != eof && r != '\n' {
break
}
fallthrough
lexEscape(l)
case eof, '\n':
return l.errorf("unterminated quoted string")
case l.stringOpen:
@ -603,6 +673,21 @@ Loop:
return lexStatements
}
// lexRawString scans a raw quoted string. The initial quote has already been seen.
func lexRawString(l *lexer) stateFn {
Loop:
for {
switch l.next() {
case eof:
return l.errorf("unterminated raw string")
case l.stringOpen:
break Loop
}
}
l.emit(itemString)
return lexStatements
}
// lexSpace scans a run of space characters. One space has already been seen.
func lexSpace(l *lexer) stateFn {
for isSpace(l.peek()) {

View File

@ -43,9 +43,9 @@ type ParseErr struct {
func (e *ParseErr) Error() string {
if e.Line == 0 {
return fmt.Sprintf("Parse error at char %d: %s", e.Pos, e.Err)
return fmt.Sprintf("parse error at char %d: %s", e.Pos, e.Err)
}
return fmt.Sprintf("Parse error at line %d, char %d: %s", e.Line, e.Pos, e.Err)
return fmt.Sprintf("parse error at line %d, char %d: %s", e.Line, e.Pos, e.Err)
}
// ParseStmts parses the input and returns the resulting statements or any ocurring error.
@ -401,21 +401,21 @@ Loop:
p.errorf("summary must not be defined twice")
}
hasSum = true
sum = trimOne(p.expect(itemString, ctx).val)
sum = p.unquoteString(p.expect(itemString, ctx).val)
case itemDescription:
if hasDesc {
p.errorf("description must not be defined twice")
}
hasDesc = true
desc = trimOne(p.expect(itemString, ctx).val)
desc = p.unquoteString(p.expect(itemString, ctx).val)
case itemRunbook:
if hasRunbook {
p.errorf("runbook must not be defined twice")
}
hasRunbook = true
runbook = trimOne(p.expect(itemString, ctx).val)
runbook = p.unquoteString(p.expect(itemString, ctx).val)
default:
p.backup()
@ -654,8 +654,7 @@ func (p *parser) primaryExpr() Expr {
return &NumberLiteral{model.SampleValue(f)}
case t.typ == itemString:
s := t.val[1 : len(t.val)-1]
return &StringLiteral{s}
return &StringLiteral{p.unquoteString(t.val)}
case t.typ == itemLeftBrace:
// Metric selector without metric name.
@ -843,7 +842,7 @@ func (p *parser) labelMatchers(operators ...itemType) metric.LabelMatchers {
p.errorf("operator must be one of %q, is %q", operators, op)
}
val := trimOne(p.expect(itemString, ctx).val)
val := p.unquoteString(p.expect(itemString, ctx).val)
// Map the item to the respective match type.
var matchType metric.MatchType
@ -1104,6 +1103,14 @@ func (p *parser) checkType(node Node) (typ model.ValueType) {
return
}
func (p *parser) unquoteString(s string) string {
unquoted, err := strutil.Unquote(s)
if err != nil {
p.errorf("error unquoting string %q: %s", s, err)
}
return unquoted
}
func parseDuration(ds string) (time.Duration, error) {
dur, err := strutil.StringToDuration(ds)
if err != nil {
@ -1114,14 +1121,3 @@ func parseDuration(ds string) (time.Duration, error) {
}
return dur, nil
}
// trimOne removes the first and last character from a string.
func trimOne(s string) string {
if len(s) > 0 {
s = s[1:]
}
if len(s) > 0 {
s = s[:len(s)-1]
}
return s
}

View File

@ -1016,6 +1016,54 @@ var testExpr = []struct {
fail: true,
errMsg: `no valid expression found`,
},
// String quoting and escape sequence interpretation tests.
{
input: `"double-quoted string \" with escaped quote"`,
expected: &StringLiteral{
Val: "double-quoted string \" with escaped quote",
},
}, {
input: `'single-quoted string \' with escaped quote'`,
expected: &StringLiteral{
Val: "single-quoted string ' with escaped quote",
},
}, {
input: "`backtick-quoted string`",
expected: &StringLiteral{
Val: "backtick-quoted string",
},
}, {
input: `"\a\b\f\n\r\t\v\\\" - \xFF\377\u1234\U00010111\U0001011111☺"`,
expected: &StringLiteral{
Val: "\a\b\f\n\r\t\v\\\" - \xFF\377\u1234\U00010111\U0001011111☺",
},
}, {
input: `'\a\b\f\n\r\t\v\\\' - \xFF\377\u1234\U00010111\U0001011111☺'`,
expected: &StringLiteral{
Val: "\a\b\f\n\r\t\v\\' - \xFF\377\u1234\U00010111\U0001011111☺",
},
}, {
input: "`" + `\a\b\f\n\r\t\v\\\"\' - \xFF\377\u1234\U00010111\U0001011111☺` + "`",
expected: &StringLiteral{
Val: `\a\b\f\n\r\t\v\\\"\' - \xFF\377\u1234\U00010111\U0001011111☺`,
},
}, {
input: "`\\``",
fail: true,
errMsg: "could not parse remaining input",
}, {
input: `"\`,
fail: true,
errMsg: "escape sequence not terminated",
}, {
input: `"\c"`,
fail: true,
errMsg: "unknown escape sequence U+0063 'c'",
}, {
input: `"\x."`,
fail: true,
errMsg: "illegal character U+002E '.' in escape sequence",
},
}
func TestParseExpressions(t *testing.T) {

223
util/strutil/quote.go Normal file
View File

@ -0,0 +1,223 @@
// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package strutil
import (
"errors"
"unicode/utf8"
)
// ErrSyntax indicates that a value does not have the right syntax for the target type.
var ErrSyntax = errors.New("invalid syntax")
// Unquote interprets s as a single-quoted, double-quoted, or backquoted
// Prometheus query language string literal, returning the string value that s
// quotes.
//
// NOTE: This function as well as the necessary helper functions below
// (unquoteChar, contains, unhex) and associated tests have been adapted from
// the corresponding functions in the "strconv" package of the Go standard
// library to work for Prometheus-style strings. Go's special-casing for single
// quotes was removed and single quoted strings are now treated the same as
// double quoted ones.
func Unquote(s string) (t string, err error) {
n := len(s)
if n < 2 {
return "", ErrSyntax
}
quote := s[0]
if quote != s[n-1] {
return "", ErrSyntax
}
s = s[1 : n-1]
if quote == '`' {
if contains(s, '`') {
return "", ErrSyntax
}
return s, nil
}
if quote != '"' && quote != '\'' {
return "", ErrSyntax
}
if contains(s, '\n') {
return "", ErrSyntax
}
// Is it trivial? Avoid allocation.
if !contains(s, '\\') && !contains(s, quote) {
return s, nil
}
var runeTmp [utf8.UTFMax]byte
buf := make([]byte, 0, 3*len(s)/2) // Try to avoid more allocations.
for len(s) > 0 {
c, multibyte, ss, err := unquoteChar(s, quote)
if err != nil {
return "", err
}
s = ss
if c < utf8.RuneSelf || !multibyte {
buf = append(buf, byte(c))
} else {
n := utf8.EncodeRune(runeTmp[:], c)
buf = append(buf, runeTmp[:n]...)
}
}
return string(buf), nil
}
// unquoteChar decodes the first character or byte in the escaped string
// or character literal represented by the string s.
// It returns four values:
//
// 1) value, the decoded Unicode code point or byte value;
// 2) multibyte, a boolean indicating whether the decoded character requires a multibyte UTF-8 representation;
// 3) tail, the remainder of the string after the character; and
// 4) an error that will be nil if the character is syntactically valid.
//
// The second argument, quote, specifies the type of literal being parsed
// and therefore which escaped quote character is permitted.
// If set to a single quote, it permits the sequence \' and disallows unescaped '.
// If set to a double quote, it permits \" and disallows unescaped ".
// If set to zero, it does not permit either escape and allows both quote characters to appear unescaped.
func unquoteChar(s string, quote byte) (value rune, multibyte bool, tail string, err error) {
// easy cases
switch c := s[0]; {
case c == quote && (quote == '\'' || quote == '"'):
err = ErrSyntax
return
case c >= utf8.RuneSelf:
r, size := utf8.DecodeRuneInString(s)
return r, true, s[size:], nil
case c != '\\':
return rune(s[0]), false, s[1:], nil
}
// Hard case: c is backslash.
if len(s) <= 1 {
err = ErrSyntax
return
}
c := s[1]
s = s[2:]
switch c {
case 'a':
value = '\a'
case 'b':
value = '\b'
case 'f':
value = '\f'
case 'n':
value = '\n'
case 'r':
value = '\r'
case 't':
value = '\t'
case 'v':
value = '\v'
case 'x', 'u', 'U':
n := 0
switch c {
case 'x':
n = 2
case 'u':
n = 4
case 'U':
n = 8
}
var v rune
if len(s) < n {
err = ErrSyntax
return
}
for j := 0; j < n; j++ {
x, ok := unhex(s[j])
if !ok {
err = ErrSyntax
return
}
v = v<<4 | x
}
s = s[n:]
if c == 'x' {
// Single-byte string, possibly not UTF-8.
value = v
break
}
if v > utf8.MaxRune {
err = ErrSyntax
return
}
value = v
multibyte = true
case '0', '1', '2', '3', '4', '5', '6', '7':
v := rune(c) - '0'
if len(s) < 2 {
err = ErrSyntax
return
}
for j := 0; j < 2; j++ { // One digit already; two more.
x := rune(s[j]) - '0'
if x < 0 || x > 7 {
err = ErrSyntax
return
}
v = (v << 3) | x
}
s = s[2:]
if v > 255 {
err = ErrSyntax
return
}
value = v
case '\\':
value = '\\'
case '\'', '"':
if c != quote {
err = ErrSyntax
return
}
value = rune(c)
default:
err = ErrSyntax
return
}
tail = s
return
}
// contains reports whether the string contains the byte c.
func contains(s string, c byte) bool {
for i := 0; i < len(s); i++ {
if s[i] == c {
return true
}
}
return false
}
func unhex(b byte) (v rune, ok bool) {
c := rune(b)
switch {
case '0' <= c && c <= '9':
return c - '0', true
case 'a' <= c && c <= 'f':
return c - 'a' + 10, true
case 'A' <= c && c <= 'F':
return c - 'A' + 10, true
}
return
}

125
util/strutil/quote_test.go Normal file
View File

@ -0,0 +1,125 @@
// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package strutil
import (
"testing"
)
type quoteTest struct {
in string
out string
ascii string
}
var quotetests = []quoteTest{
{"\a\b\f\r\n\t\v", `"\a\b\f\r\n\t\v"`, `"\a\b\f\r\n\t\v"`},
{"\\", `"\\"`, `"\\"`},
{"abc\xffdef", `"abc\xffdef"`, `"abc\xffdef"`},
{"\u263a", `"☺"`, `"\u263a"`},
{"\U0010ffff", `"\U0010ffff"`, `"\U0010ffff"`},
{"\x04", `"\x04"`, `"\x04"`},
}
type unQuoteTest struct {
in string
out string
}
var unquotetests = []unQuoteTest{
{`""`, ""},
{`"a"`, "a"},
{`"abc"`, "abc"},
{`"☺"`, "☺"},
{`"hello world"`, "hello world"},
{`"\xFF"`, "\xFF"},
{`"\377"`, "\377"},
{`"\u1234"`, "\u1234"},
{`"\U00010111"`, "\U00010111"},
{`"\U0001011111"`, "\U0001011111"},
{`"\a\b\f\n\r\t\v\\\""`, "\a\b\f\n\r\t\v\\\""},
{`"'"`, "'"},
{`''`, ""},
{`'a'`, "a"},
{`'abc'`, "abc"},
{`'☺'`, "☺"},
{`'hello world'`, "hello world"},
{`'\xFF'`, "\xFF"},
{`'\377'`, "\377"},
{`'\u1234'`, "\u1234"},
{`'\U00010111'`, "\U00010111"},
{`'\U0001011111'`, "\U0001011111"},
{`'\a\b\f\n\r\t\v\\\''`, "\a\b\f\n\r\t\v\\'"},
{`'"'`, "\""},
{"``", ``},
{"`a`", `a`},
{"`abc`", `abc`},
{"`☺`", ``},
{"`hello world`", `hello world`},
{"`\\xFF`", `\xFF`},
{"`\\377`", `\377`},
{"`\\`", `\`},
{"`\n`", "\n"},
{"` `", ` `},
}
var misquoted = []string{
``,
`"`,
`"a`,
`"'`,
`b"`,
`"\"`,
`"\9"`,
`"\19"`,
`"\129"`,
`'\'`,
`'\9'`,
`'\19'`,
`'\129'`,
`"\x1!"`,
`"\U12345678"`,
`"\z"`,
"`",
"`xxx",
"`\"",
`"\'"`,
`'\"'`,
"\"\n\"",
"\"\\n\n\"",
"'\n'",
}
func TestUnquote(t *testing.T) {
for _, tt := range unquotetests {
if out, err := Unquote(tt.in); err != nil && out != tt.out {
t.Errorf("Unquote(%#q) = %q, %v want %q, nil", tt.in, out, err, tt.out)
}
}
// Run the quote tests too, backward.
for _, tt := range quotetests {
if in, err := Unquote(tt.out); in != tt.in {
t.Errorf("Unquote(%#q) = %q, %v, want %q, nil", tt.out, in, err, tt.in)
}
}
for _, s := range misquoted {
if out, err := Unquote(s); out != "" || err != ErrSyntax {
t.Errorf("Unquote(%#q) = %q, %v want %q, %v", s, out, err, "", ErrSyntax)
}
}
}

View File

@ -53,7 +53,7 @@ func TestQuery(t *testing.T) {
{
queryStr: "",
status: http.StatusOK,
bodyRe: `{"type":"error","value":"Parse error at char 1: no expression found in input","version":1}`,
bodyRe: `{"type":"error","value":"parse error at char 1: no expression found in input","version":1}`,
},
{
queryStr: "expr=1.4",
@ -83,7 +83,7 @@ func TestQuery(t *testing.T) {
{
queryStr: "expr=(badexpression",
status: http.StatusOK,
bodyRe: `{"type":"error","value":"Parse error at char 15: unclosed left parenthesis","version":1}`,
bodyRe: `{"type":"error","value":"parse error at char 15: unclosed left parenthesis","version":1}`,
},
}