tokenizer.go

package goxpath

import (
	"fmt"
	"io"
	"strconv"
	"strings"
	"unicode"
)

type tokenType int

const (
	// tokAny contains any token.
	tokAny tokenType = iota
	// tokString contains any characters including whitespace.
	tokString
	// tokVarname represents a variable name.
	tokVarname
	// tokNumber represents a float64.
	tokNumber
	// tokOperator contains a single or double letter operator or path
	// separator.
	tokOperator
	// tokOpenParen is an opening parenthesis (.
	tokOpenParen
	// tokCloseParen is a closing parenthesis ).
	tokCloseParen
	// tokOpenBracket is an opening bracket [.
	tokOpenBracket
	// tokCloseBracket is a closing bracket ].
	tokCloseBracket
	// tokQName is a QName (which might contain one colon).
	tokQName
	// tokComma represents a comma
	tokComma
	// tokDoubleColon represents a word with two colons a the end (axis for
	// example)
	tokDoubleColon
	// tokOpenBrace is an opening curly brace {.
	tokOpenBrace
	// tokCloseBrace is a closing curly brace }.
	tokCloseBrace
	// tokEQName is a URIQualifiedName: Q{namespace}localname.
	tokEQName
)

func (tt tokenType) String() string {
	switch tt {
	case tokAny:
		return "Any token"
	case tokString:
		return "string"
	case tokVarname:
		return "variable name"
	case tokNumber:
		return "number"
	case tokOperator:
		return "operator"
	case tokOpenParen:
		return "open paren"
	case tokCloseParen:
		return "close paren"
	case tokOpenBracket:
		return "open bracket"
	case tokCloseBracket:
		return "close bracket"
	case tokQName:
		return "QName"
	case tokComma:
		return "comma"
	case tokOpenBrace:
		return "open brace"
	case tokCloseBrace:
		return "close brace"
	case tokEQName:
		return "EQName"
	}
	return "--"
}

type token struct {
	Value any
	Typ   tokenType
}

func (tok *token) isNCName() bool {
	if tok.Typ != tokQName {
		return false
	}
	tokAsString := tok.Value.(string)
	return !strings.ContainsRune(tokAsString, ':')
}

type tokens []token

func (toks tokens) String() string {
	var str []string
	for _, t := range toks {
		str = append(str, t.String())
	}
	return strings.Join(str, " ")
}

func (tok token) String() string {
	switch tok.Typ {
	case tokVarname:
		return "$" + tok.Value.(string)
	case tokOpenParen:
		return "("
	case tokCloseParen:
		return ")"
	case tokOpenBracket:
		return "["
	case tokCloseBracket:
		return "]"
	case tokOpenBrace:
		return "{"
	case tokCloseBrace:
		return "}"
	}

	switch v := tok.Value.(type) {
	case string:
		return v
	case float64:
		return fmt.Sprintf("%f", v)
	default:
		fmt.Printf("v %#v\n", v)
		return "***"
	}
}

// Tokenlist represents units of XPath language elements.
type Tokenlist struct {
	pos           int
	toks          tokens
	attributeMode bool // for Name Test
}

func (tl *Tokenlist) nexttokIsTyp(typ tokenType) bool {
	tok, err := tl.peek()
	if err != nil {
		return false
	}
	return tok.Typ == typ
}

func (tl *Tokenlist) readIfTokenFollow(toks []token) bool {
	pos := tl.pos
	savepos := tl.pos
	for _, tok := range toks {
		peekTok, err := tl.peek()
		if err != nil {
			tl.pos = savepos
			return false
		}
		if peekTok.Value == tok.Value && peekTok.Typ == tok.Typ {
			pos++
		} else {
			tl.pos = savepos
			return false
		}
		tl.read()
	}
	return true
}

// nexttokIsValue looks at the next token and returns true if the value matches.
// Does not move the pointer forward.
func (tl *Tokenlist) nexttokIsValue(val string) bool {
	tok, err := tl.peek()
	if err != nil {
		return false
	}
	if str, ok := tok.Value.(string); ok {
		return str == val
	}
	return false
}

func (tl *Tokenlist) readNexttokIfIsOneOfValue(val []string) (string, bool) {
	tok, err := tl.peek()
	if err != nil {
		return "", false
	}
	if str, ok := tok.Value.(string); ok {
		for _, v := range val {
			if str == v {
				tl.read()
				return v, true
			}
		}
	}
	return "", false
}

func (tl *Tokenlist) readNexttokIfIsOneOfValueAndType(val []string, typ tokenType) (string, bool) {
	tok, err := tl.peek()
	if err != nil {
		return "", false
	}
	if tok.Typ != typ {
		return fmt.Sprint(tok.Value), false
	}
	if str, ok := tok.Value.(string); ok {
		for _, v := range val {
			if str == v {
				tl.read()
				return v, true
			}
		}
	}
	return "", false
}

func (tl *Tokenlist) peek() (*token, error) {
	if len(tl.toks) == tl.pos {
		return nil, io.EOF
	}
	return &tl.toks[tl.pos], nil
}

func (tl *Tokenlist) read() (*token, error) {
	if len(tl.toks) == tl.pos {
		return nil, io.EOF
	}
	tl.pos++
	return &tl.toks[tl.pos-1], nil
}

func (tl *Tokenlist) unread() error {
	tl.pos--
	return nil
}

// skipType reads the next token and returns an error if the token type does not
// match.
func (tl *Tokenlist) skipType(typ tokenType) error {
	var val *token
	var err error
	if val, err = tl.read(); err != nil {
		return err
	}
	if val.Typ != typ {
		return fmt.Errorf("expect %s, got %s", typ, val.Typ)
	}
	return nil
}

func (tl *Tokenlist) skipNCName(name string) error {
	var val *token
	var err error
	if val, err = tl.read(); err != nil {
		return err
	}
	if !val.isNCName() {
		return fmt.Errorf("expect %s, got %s", name, val.Typ)
	}
	if str, ok := val.Value.(string); ok && str != name {
		return fmt.Errorf("expect %s, got %s", name, str)
	}
	return nil
}

// getNum reads a numeric literal and returns the appropriately typed value:
// - IntegerLiteral (digits only) → int
// - DecimalLiteral (digits with '.') → XSDecimal
// - DoubleLiteral (has 'e'/'E') → XSDouble
func getNum(sr *strings.Reader) any {
	var buf []byte
	hasExp := false
	hasDot := false
	for {
		r, _, err := sr.ReadRune()
		if err != nil {
			break
		}
		if r >= '0' && r <= '9' {
			buf = append(buf, byte(r))
		} else if r == '.' {
			hasDot = true
			buf = append(buf, byte(r))
		} else if (r == 'e' || r == 'E') && !hasExp {
			buf = append(buf, byte(r))
			hasExp = true
		} else if (r == '+' || r == '-') && hasExp && len(buf) > 0 && (buf[len(buf)-1] == 'e' || buf[len(buf)-1] == 'E') {
			buf = append(buf, byte(r))
		} else {
			sr.UnreadRune()
			break
		}
	}
	s := string(buf)
	f, _ := strconv.ParseFloat(s, 64)

	if hasExp {
		return XSDouble(f)
	}
	if hasDot {
		return XSDecimal(f)
	}
	// Integer literal — parse as int64 to preserve precision
	if i, err := strconv.ParseInt(s, 10, 64); err == nil {
		return int(i)
	}
	// Overflow — store as XSDecimal to preserve the value as best as possible
	return XSDecimal(f)
}

// getNumWithPrefix parses a number literal starting with a given prefix (e.g., ".").
func getNumWithPrefix(prefix string, sr *strings.Reader) any {
	var buf []byte
	buf = append(buf, []byte(prefix)...)
	hasExp := false
	hasDot := strings.Contains(prefix, ".")
	for {
		r, _, err := sr.ReadRune()
		if err != nil {
			break
		}
		if r >= '0' && r <= '9' {
			buf = append(buf, byte(r))
		} else if r == '.' && !hasDot {
			hasDot = true
			buf = append(buf, byte(r))
		} else if (r == 'e' || r == 'E') && !hasExp {
			buf = append(buf, byte(r))
			hasExp = true
		} else if (r == '+' || r == '-') && hasExp && len(buf) > 0 && (buf[len(buf)-1] == 'e' || buf[len(buf)-1] == 'E') {
			buf = append(buf, byte(r))
		} else {
			sr.UnreadRune()
			break
		}
	}
	s := string(buf)
	f, _ := strconv.ParseFloat(s, 64)
	if hasExp {
		return XSDouble(f)
	}
	if hasDot {
		return XSDecimal(f)
	}
	if i, err := strconv.Atoi(s); err == nil {
		return i
	}
	return int(f)
}

// getQName reads until any non QName rune is found.
func getQName(sr *strings.Reader) (string, error) {
	var word []rune
	var hasColon bool
	for {
		r, _, err := sr.ReadRune()
		if err == io.EOF {
			break
		}
		if err != nil {
			return "", err
		}
		if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' || r == '-' || r == '.' || r == '·' || r == '‿' || r == '⁀' {
			word = append(word, r)
		} else if r == ':' {
			if hasColon {
				sr.UnreadRune()
				break
			}
			word = append(word, r)
			hasColon = true
		} else {
			sr.UnreadRune()
			break
		}
	}
	return string(word), nil
}

// getEQName reads the rest of an EQName after "Q{" has been consumed.
// It reads the namespace URI up to "}", then the local name.
// Returns "namespace}localname" as the token value.
func getEQName(sr *strings.Reader) (string, error) {
	var ns []rune
	for {
		r, _, err := sr.ReadRune()
		if err != nil {
			return "", fmt.Errorf("unterminated Q{} EQName")
		}
		if r == '}' {
			break
		}
		ns = append(ns, r)
	}
	localName, err := getQName(sr)
	if err != nil {
		return "", err
	}
	if localName == "" {
		return "", fmt.Errorf("missing local name in Q{%s}", string(ns))
	}
	return string(ns) + "}" + localName, nil
}

func getDelimitedString(sr *strings.Reader) (string, error) {
	var str []rune
	delim, _, err := sr.ReadRune()
	if err != nil {
		return "", err
	}
	for {
		r, _, err := sr.ReadRune()
		if err == io.EOF {
			break
		}
		if err != nil {
			return "", err
		}
		if r == delim {
			// XPath escaped delimiter: "" → " or '' → '
			next, _, err := sr.ReadRune()
			if err == nil && next == delim {
				str = append(str, delim)
				continue
			}
			if err == nil {
				sr.UnreadRune()
			}
			break
		} else {
			str = append(str, r)
		}

	}
	return string(str), err
}

func getComment(sr *strings.Reader) (string, error) {
	lvl := 1
	var this, next rune
	var err error
	var comment []rune
	for {
		if this, _, err = sr.ReadRune(); err != nil {
			if err == io.EOF {
				// todo error handling
			}
			return "", err
		}
		if next, _, err = sr.ReadRune(); err != nil {
			if err == io.EOF {
				// todo error handling
			}
			return "", err
		}

		if this == ':' && next == ')' {
			lvl--
			if lvl == 0 {
				break
			}
		}

		if this == '(' && next == ':' {
			lvl++
		}

		comment = append(comment, this)
		if next == ':' || next == '(' {
			sr.UnreadRune()
		} else {
			comment = append(comment, next)
		}

	}

	return string(comment), nil
}

func stringToTokenlist(str string) (*Tokenlist, error) {
	var tokens []token
	sr := strings.NewReader(str)
	for {
		r, _, err := sr.ReadRune()
		if err == io.EOF {
			break
		}
		if err != nil {
			break
		}
		if '0' <= r && r <= '9' {
			sr.UnreadRune()
			tokens = append(tokens, token{getNum(sr), tokNumber})
		} else if r == '.' {
			nextRune, _, err := sr.ReadRune()
			if err == io.EOF {
				tokens = append(tokens, token{".", tokOperator})
				break
			}
			if err != nil {
				return nil, err
			}
			if '0' <= nextRune && nextRune <= '9' {
				// Put back the digit, then parse number starting with "."
				sr.UnreadRune()
				tokens = append(tokens, token{getNumWithPrefix(".", sr), tokNumber})
			} else if nextRune == '.' {
				tokens = append(tokens, token{"..", tokOperator})
			} else {
				sr.UnreadRune()
				tokens = append(tokens, token{".", tokOperator})
			}
		} else if r == '+' || r == '-' || r == '*' || r == '?' || r == '@' || r == '#' {
			tokens = append(tokens, token{string(r), tokOperator})
		} else if r == '=' {
			nextRune, _, err := sr.ReadRune()
			if err == io.EOF {
				tokens = append(tokens, token{"=", tokOperator})
				break
			}
			if err != nil {
				return nil, err
			}
			if nextRune == '>' {
				tokens = append(tokens, token{"=>", tokOperator})
			} else {
				tokens = append(tokens, token{"=", tokOperator})
				sr.UnreadRune()
			}
		} else if r == '|' {
			nextRune, _, err := sr.ReadRune()
			if err == io.EOF {
				tokens = append(tokens, token{"|", tokOperator})
				break
			}
			if err != nil {
				return nil, err
			}
			if nextRune == '|' {
				tokens = append(tokens, token{"||", tokOperator})
			} else {
				tokens = append(tokens, token{"|", tokOperator})
				sr.UnreadRune()
			}
		} else if r == ',' {
			tokens = append(tokens, token{string(r), tokComma})
		} else if r == '>' || r == '<' {
			nextRune, _, err := sr.ReadRune()
			if err != nil {
				return nil, err
			}
			if nextRune == '=' || nextRune == r {
				tokens = append(tokens, token{string(r) + string(nextRune), tokOperator})
			} else {
				tokens = append(tokens, token{string(r), tokOperator})
				sr.UnreadRune()
			}
		} else if r == '!' {
			nextRune, _, err := sr.ReadRune()
			if err != nil {
				// standalone ! at end of input
				tokens = append(tokens, token{"!", tokOperator})
				break
			}
			if nextRune == '=' {
				tokens = append(tokens, token{"!=", tokOperator})
			} else {
				tokens = append(tokens, token{"!", tokOperator})
				sr.UnreadRune()
			}
		} else if r == '/' || r == ':' {
			nextRune, _, err := sr.ReadRune()
			if err == io.EOF {
				tokens = append(tokens, token{string(r), tokOperator})
				break
			}
			if err != nil {
				return nil, err
			}
			if nextRune == r {
				tokens = append(tokens, token{string(r) + string(r), tokOperator})
			} else {
				tokens = append(tokens, token{string(r), tokOperator})
				sr.UnreadRune()
			}
		} else if r == '[' {
			tokens = append(tokens, token{r, tokOpenBracket})
		} else if r == ']' {
			tokens = append(tokens, token{r, tokCloseBracket})
		} else if r == '$' {
			qname, err := getQName(sr)
			if err != nil {
				return nil, err
			}
			tokens = append(tokens, token{qname, tokVarname})
		} else if unicode.IsSpace(r) {
			// ignore whitespace
		} else if unicode.IsLetter(r) {
			sr.UnreadRune()
			word, err := getQName(sr)
			if err != nil {
				return nil, err
			}
			// Check for EQName: Q{namespace}localname
			if word == "Q" {
				nextRune, _, err := sr.ReadRune()
				if err == nil && nextRune == '{' {
					eqname, err := getEQName(sr)
					if err != nil {
						return nil, err
					}
					tokens = append(tokens, token{eqname, tokEQName})
					continue
				}
				if err == nil {
					sr.UnreadRune()
				}
			}
			nextRune, _, err := sr.ReadRune()
			if err == io.EOF {
				tokens = append(tokens, token{word, tokQName})
				break
			}
			if nextRune == ':' {
				tokens = append(tokens, token{strings.TrimSuffix(word, ":"), tokDoubleColon})
			} else {
				sr.UnreadRune()
				tokens = append(tokens, token{word, tokQName})
			}

		} else if r == '\'' || r == '"' {
			sr.UnreadRune()
			str, err := getDelimitedString(sr)
			if err != nil {
				return nil, err
			}
			tokens = append(tokens, token{str, tokString})
		} else if r == '(' {
			nextRune, _, err := sr.ReadRune()
			if err == io.EOF {
				// what should we do?
				return nil, fmt.Errorf("parse error, unbalanced ( at end")
			}
			if err != nil {
				return nil, err
			}
			if nextRune == ':' {
				_, err := getComment(sr)
				if err != nil {
					return nil, err
				}
				// comments are ignored
				// tokens = append(tokens, token{cmt, TokAny})
			} else {
				sr.UnreadRune()
				tokens = append(tokens, token{r, tokOpenParen})
			}
		} else if r == ')' {
			tokens = append(tokens, token{r, tokCloseParen})
		} else if r == '{' {
			tokens = append(tokens, token{r, tokOpenBrace})
		} else if r == '}' {
			tokens = append(tokens, token{r, tokCloseBrace})
		} else {
			return nil, fmt.Errorf("Invalid char for xpath expression %q", string(r))
		}
	}
	tl := Tokenlist{pos: 0, toks: tokens}
	return &tl, nil
}