// Tokenize transforms the provided input into a slice of tokens or returns a ParseError
439651
func Tokenize(usage string) ([]*Token, error) {
	pos := 0
	var res []*Token
	var (
		tk = func(t TokenType, v string) {
			res = append(res, &Token{t, v, pos})
		}

		tkp = func(t TokenType, v string, p int) {
			res = append(res, &Token{t, v, p})
		}

		err = func(msg string) *ParseError {
			return &ParseError{usage, msg, pos}
		}
	)
	eof := len(usage)
	for pos < eof {
		switch c := usage[pos]; c {
		case ' ':
			pos++
		case '\t':
			pos++
		case '[':
			tk(TTOpenSq, "[")
			pos++
		case ']':
			tk(TTCloseSq, "]")
			pos++
		case '(':
			tk(TTOpenPar, "(")
			pos++
		case ')':
			tk(TTClosePar, ")")
			pos++
		case '|':
			tk(TTChoice, "|")
			pos++
		case '.':
			start := pos
			pos++
			if pos >= eof || usage[pos] != '.' {
				return nil, err("Unexpected end of usage, was expecting '..'")
			}
			pos++
			if pos >= eof || usage[pos] != '.' {
				return nil, err("Unexpected end of usage, was expecting '.'")
			}
			tkp(TTRep, "...", start)
			pos++
		case '-':
			start := pos
			pos++
			if pos >= eof {
				return nil, err("Unexpected end of usage, was expecting an option name")
			}

			switch o := usage[pos]; {
			case isLetter(o):
				pos++
				for ; pos < eof; pos++ {
					ok := isLetter(usage[pos])
					if !ok {
						break
					}
				}
				typ := TTShortOpt
				opt := usage[start:pos]
				if pos-start > 2 {
					typ = TTOptSeq
					opt = opt[1:]
				}
				tkp(typ, opt, start)
				if pos < eof && usage[pos] == '-' {
					return nil, err("Invalid syntax")
				}
			case o == '-':
				pos++
				if pos == eof || usage[pos] == ' ' {
					tkp(TTDoubleDash, "--", start)
					continue
				}
				for pos0 := pos; pos < eof; pos++ {
					ok := isOkLongOpt(usage[pos], pos == pos0)
					if !ok {
						break
					}
				}
				opt := usage[start:pos]
				if len(opt) == 2 {
					return nil, err("Was expecting a long option name")
				}
				tkp(TTLongOpt, opt, start)
			}

		case '=':
			start := pos
			pos++
			if pos >= eof || usage[pos] != '<' {
				return nil, err("Unexpected end of usage, was expecting '=<'")
			}
			closed := false
			for ; pos < eof; pos++ {
				closed = usage[pos] == '>'
				if closed {
					break
				}
			}
			if !closed {
				return nil, err("Unclosed option value")
			}
			if pos-start == 2 {
				return nil, err("Was expecting an option value")
			}
			pos++
			value := usage[start:pos]

			tkp(TTOptValue, value, start)

		default:
			switch {
			case isUppercase(c):
				start := pos
				for pos = pos + 1; pos < eof; pos++ {
					if !isOkInArg(usage[pos]) {
						break
					}
				}
				s := usage[start:pos]
				typ := TTArg
				if s == "OPTIONS" {
					typ = TTOptions
				}
				tkp(typ, s, start)
			default:
				return nil, err("Unexpected input")
			}

		}
	}

	return res, nil
}
222915
func Register(name string, t Tokenizer) {
	if tokenizers == nil {
		tokenizers = map[string]Tokenizer{}
	}
	tokenizers[name] = t
}
852203
func (i Iterator) Tokens() []Token {
	var out []Token
	for t := i(); t != EOF; t = i() {
		out = append(out, t)
	}
	return out
}
432131
func (i tokenType) String() string {
	s := tokenName[i]
	if s == "" {
		return fmt.Sprintf("t_unknown_%d", int(i))
	}
	return s
}
608871
func (l *lexer) token() token {
	for {
		select {
		case token := <-l.tokens:
			return token
		default:
			l.state = l.state(l)
		}
	}
}
700925
func Decode(token string) int {
  number := 0
  idx    := 0.0
  chars  := []byte(alphabet)

  charsLength := float64(len(chars))
  tokenLength := float64(len(token))

  for _, c := range []byte(token) {
    power := tokenLength - (idx + 1)
    index := bytes.IndexByte(chars, c)
    number += index * int(math.Pow(charsLength, power))
    idx++
  }

  return number
}
166102
func Collect(input string) []Token {
	var result []Token

	l := Scan(input)
	for {
		token := l.NextToken()
		result = append(result, token)

		if token.Kind == TokenEOF || token.Kind == TokenError {
			break
		}
	}

	return result
}
65140
func NewTokenizer(r io.Reader) *Tokenizer {
	return &Tokenizer{
		InStream: r,
		buf:      make([]byte, defaultBufSize),
	}
}
1043125
func NewTokenizer(input io.RuneReader) *Tokenizer {
	assert2(input != nil, "input must not be nil")
	return &Tokenizer{
		TokenStreamImpl: NewTokenStream(),
		inputPending:    input,
		Input:           ILLEGAL_STATE_READER,
	}
}
497657
func Parse(tokenString string, keyFunc Keyfunc) (*Token, error) {
	return new(Parser).Parse(tokenString, keyFunc)
}
593627
func parseSize(sizeStr string) (int64, error) {
	matches := sizeRegex.FindStringSubmatch(sizeStr)
	if len(matches) != 4 {
		return -1, fmt.Errorf("invalid size: '%s'", sizeStr)
	}

	size, err := strconv.ParseFloat(matches[1], 64)
	if err != nil {
		return -1, err
	}

	uMap := map[string]int64{"k": KB, "m": MB, "g": GB, "t": TB, "p": PB}
	unitPrefix := strings.ToLower(matches[3])
	if mul, ok := uMap[unitPrefix]; ok {
		size *= float64(mul)
	}

	return int64(size), nil
}
304861
func tokenize(s string) []string {
	/*
		TODO: Properly detect if the string starts with a ( and ends with a )
		and that those two form a matching pair.

		Eg: (foo) -> true; (foo)(bar) -> false;
	*/
	if s[0] == '(' && s[len(s)-1] == ')' {
		s = s[1 : len(s)-1]
	}
	ret := []string{}
	for chunk := range split(s) {
		if len(chunk) != 0 {
			if chunk[0] == '(' && chunk[len(chunk)-1] == ')' {
				ret = append(ret, chunk)
			} else {
				for _, token := range pat.FindAllStringSubmatch(chunk, -1) {
					ret = append(ret, token[0])
				}
			}
		} else {
			fmt.Printf("Empty chunk in string '%s'\n", s)
		}
	}
	return ret
}
354715
func NewTokenLimiter(count uint) *TokenLimiter {
	tl := &TokenLimiter{count: count, ch: make(chan *Token, count)}
	for i := uint(0); i < count; i++ {
		tl.ch <- &Token{}
	}

	return tl
}
412882
func (p *PragmaticSegmenter) Tokenize(text string) []string {
	return p.processor.process(text)
}
356254
func (s *IndicesAnalyzeService) Tokenizer(tokenizer string) *IndicesAnalyzeService {
	s.request.Tokenizer = tokenizer
	return s
}
275874
func (bc Bytecode) Tokens() ([]string, error) {
	// Overestimate of capacity in the presence of pushes
	tokens := make([]string, 0, len(bc))

	for i := 0; i < len(bc); i++ {
		op, ok := asm.GetOpCode(bc[i])
		if !ok {
			return tokens, fmt.Errorf("did not recognise byte %#x at position %v as an OpCode:\n %s",
				bc[i], i, lexingPositionString(bc, i, tokens))
		}
		pushes := op.Pushes()
		tokens = append(tokens, op.Name())
		if pushes > 0 {
			// This is a PUSH<N> OpCode so consume N bytes from the input, render them as hex, and skip to next OpCode
			if i+pushes >= len(bc) {
				return tokens, fmt.Errorf("token %v of input is %s but not enough input remains to push %v: %s",
					i, op.Name(), pushes, lexingPositionString(bc, i, tokens))
			}
			pushedBytes := bc[i+1 : i+pushes+1]
			tokens = append(tokens, fmt.Sprintf("0x%s", pushedBytes))
			i += pushes
		}
	}
	return tokens, nil
}
728451
func (z *Tokenizer) Token() Token {
	t := Token{Type: z.tt}
	switch z.tt {
	case TextToken, CommentToken, DoctypeToken:
		t.Data = string(z.Text())
	case StartTagToken, SelfClosingTagToken, EndTagToken:
		name, moreAttr := z.TagName()
		for moreAttr {
			var key, val []byte
			key, val, moreAttr = z.TagAttr()
			t.Attr = append(t.Attr, Attribute{"", atom.String(key), string(val)})
		}
		if a := atom.Lookup(name); a != 0 {
			t.DataAtom, t.Data = a, a.String()
		} else {
			t.DataAtom, t.Data = 0, string(name)
		}
	}
	return t
}