Dictionaries container for Tokenizer
@param {DoubleArray} trie
@param {TokenInfoDictionary} token_info_dictionary
@param {ConnectionCosts} connection_costs
@param {UnknownDictionary} unknown_dictionary
@constructor
197075
function DynamicDictionaries(trie, token_info_dictionary, connection_costs, unknown_dictionary) {
    if (trie != null) {
        this.trie = trie;
    } else {
        this.trie = doublearray.builder(0).build([
            {k: "", v: 1}
        ]);
    }
    if (token_info_dictionary != null) {
        this.token_info_dictionary = token_info_dictionary;
    } else {
        this.token_info_dictionary = new TokenInfoDictionary();
    }
    if (connection_costs != null) {
        this.connection_costs = connection_costs;
    } else {
        // backward_size * backward_size
        this.connection_costs = new ConnectionCosts(0, 0);
    }
    if (unknown_dictionary != null) {
        this.unknown_dictionary = unknown_dictionary;
    } else {
        this.unknown_dictionary = new UnknownDictionary();
    }
}
923600
function(dictionary) {
            var o = [];
            var val;
            for (var prop in dictionary) {
                val = '' + dictionary[prop];
                o.push(encodeURIComponent(prop) + '=' + encodeURIComponent(dictionary[prop]));
            }
            return o.join('&');
        }
679203
def adrest_include(parser, token):
    
    bits = token.split_contents()[1:]
    args, kwargs = parse_bits(
        parser, bits, ['content'], 'args', 'kwargs', tuple(),
        False, 'adrest_include')
    return AdrestInclusionNode(False, args, kwargs)
952102
def tokens(cls, tokens):
        
        return cls(Lnk.TOKENS, tuple(map(int, tokens)))
655784
def set_dictionary(dictionary)
      dict = StringValue dictionary

      err = Zlib.deflateSetDictionary pointer, dict, dict.length

      Zlib.handle_error err, message

      dictionary
    end
854859
function Tokenizer(text) {
        this._state = TEXT;
        this._buffer = text;
        this._sectionStart = 0;
        this._sectionStartPos = {line: 0, ch: 0};
        this._index = 0;
        this._indexPos = {line: 0, ch: 0};
        this._special = 0; // 1 for script, 2 for style
        this._token = null;
        this._nextToken = null;
    }
723345
def GetTokenInformation(token, information_class):
    
    data_size = ctypes.wintypes.DWORD()
    ctypes.windll.advapi32.GetTokenInformation(
        token, information_class.num, 0, 0, ctypes.byref(data_size)
    )
    data = ctypes.create_string_buffer(data_size.value)
    handle_nonzero_success(
        ctypes.windll.advapi32.GetTokenInformation(
            token,
            information_class.num,
            ctypes.byref(data),
            ctypes.sizeof(data),
            ctypes.byref(data_size),
        )
    )
    return ctypes.cast(data, ctypes.POINTER(TOKEN_USER)).contents
929751
def parse_connection_option(
    header: str, pos: int, header_name: str
) -> Tuple[ConnectionOption, int]:
    
    item, pos = parse_token(header, pos, header_name)
    return cast(ConnectionOption, item), pos
882134
function getPrefixTokens (node) {
    const prefixes = []

    let token = tokenStore.getFirstToken(node)
    while (token != null && token.range[1] <= node.key.range[0]) {
      prefixes.push(token)
      token = tokenStore.getTokenAfter(token)
    }
    while (isLeftParen(last(prefixes)) || isLeftBracket(last(prefixes))) {
      prefixes.pop()
    }

    return prefixes
  }
961611
def persist_trie_data_dict(self, trie_data_dict: Dict[Hash32, bytes]) -> None:
        
        with self.db.atomic_batch() as db:
            for key, value in trie_data_dict.items():
                db[key] = value
35698
def _construct_linebreak_token(self, d: Dict) -> List[Dict]:
        

        result = []
        num_break = int(d["length"][0]) if d["length"] else 1
        if num_break:
            s = ''
            for i in range(num_break):
                s += '\n'
            this_token = {attrs.LOWER: s}
            result.append(this_token)
            s += ' '
            this_token = {attrs.LOWER: s}
            result.append(this_token)
        result = self._add_common_constrain(result, d)

        return result
1235151
def parse(self, dictionary=None, end_token=None):
        
        self.level += 1
        name = []
        if dictionary is None:
            dictionary = self.config
        while True:
            token = self.next()
            if token is None:
                # we have got to the end of the config
                break
            t_type = token["type"]
            t_value = token["value"]
            if t_type == "newline":
                continue
            elif t_value == end_token:
                self.level -= 1
                return
            elif t_type == "literal":
                value = self.remove_quotes(t_value)
                if not name and not re.match("[a-zA-Z_]", value):
                    self.error("Invalid name")
                name.append(value)
            elif t_type == "function":
                self.error("Name expected")
            elif t_type == "operator":
                name = " ".join(name)
                if not name:
                    self.error("Name expected")
                elif t_value == "+=" and name not in dictionary:
                    # deal with encoded names
                    if name.split(":")[0] not in dictionary:
                        # order is treated specially
                        if not (self.level == 1 and name == "order"):
                            self.error("{} does not exist".format(name))
                if t_value in ["{"]:
                    if self.current_module:
                        self.check_child_friendly(self.current_module[-1])
                    self.check_module_name(name)
                    self.current_module.append(name)
                value = self.assignment(token)
                # order is treated specially to create a list
                if self.level == 1 and name == "order":
                    if not value:
                        self.error("Invalid module")
                    self.check_module_name(value, offset=1)
                    dictionary.setdefault(name, []).append(value)
                # assignment of  module definition
                elif t_value == "{":
                    # If this is an py3status module and in a container and has
                    # no instance name then give it an anon one.  This allows
                    # us to have multiple non-instance named modules defined
                    # without them clashing.
                    if (
                        self.level > 1
                        and " " not in name
                        and name not in I3S_MODULE_NAMES
                    ):
                        name = "{} _anon_module_{}".format(name, self.anon_count)
                        self.anon_count += 1
                    dictionary[name] = value
                # assignment of value
                elif t_value == "=":
                    try:
                        name, value = self.process_value(
                            name, value, self.current_module[-1]
                        )
                    except IndexError:
                        self.error("Missing {", previous=True)
                    dictionary[name] = value
                # appending to existing values
                elif t_value == "+=":
                    dictionary[name] += value
                else:
                    self.error("Unexpected character")
                name = []
609553
function parseItemOrGroup(tokenizer) {
  var tkn = tokenizer.next();
  if (!tkn) return null;
  if (tkn.typ == '{') {
    return parseGroup(tokenizer);
  }
  return {tag: tkn.typ, children: tkn.txt};
}
21798
def add_token_to_namespace(self, token: str, namespace: str = 'tokens') -> int:
        
        if not isinstance(token, str):
            raise ValueError("Vocabulary tokens must be strings, or saving and loading will break."
                             "  Got %s (with type %s)" % (repr(token), type(token)))
        if token not in self._token_to_index[namespace]:
            index = len(self._token_to_index[namespace])
            self._token_to_index[namespace][token] = index
            self._index_to_token[namespace][index] = token
            return index
        else:
            return self._token_to_index[namespace][token]
66752
function appInfoFromDict (dict) {
  const id = dict.WIRApplicationIdentifierKey;
  const isProxy = _.isString(dict.WIRIsApplicationProxyKey)
    ? dict.WIRIsApplicationProxyKey.toLowerCase() === 'true'
    : dict.WIRIsApplicationProxyKey;
  const entry = {
    id,
    isProxy,
    name: dict.WIRApplicationNameKey,
    bundleId: dict.WIRApplicationBundleIdentifierKey,
    hostId: dict.WIRHostApplicationIdentifierKey,
    isActive: dict.WIRIsApplicationActiveKey,
    isAutomationEnabled: !!dict.WIRRemoteAutomationEnabledKey,
  };

  return [id, entry];
}
345497
function writeTxInfo(tokenInfo) {
  fs.writeFile("token-tx.json", JSON.stringify(tokenInfo, null, 2), function(
    err
  ) {
    if (err) return console.error(err)
    console.log(`token-tx.json written successfully.`)
  })
}
422842
function SemanticChecker(tokens) {
    this._tokens = tokens;
    this._index = -1;
    this._tree = null;
    this._stack = [];
    if (!this.checkExp()) {
        throw [0, "This is not a understandable expression!"];
    }
    if (this.tkn() !== null) {
        throw [this._index, "Unexpected end of expression! Maybe you forgot a \"(\" somewhere."];
    }
}