Convert a PyParsing data dictionary to a PyBEL fusion data dictionary.

    :param tokens: A PyParsing data dictionary representing a fusion
    :type tokens: ParseResult
548283
def _fusion_to_dsl(tokens) -> FusionBase:
    
    func = tokens[FUNCTION]
    fusion_dsl = FUNC_TO_FUSION_DSL[func]
    member_dsl = FUNC_TO_DSL[func]

    partner_5p = member_dsl(
        namespace=tokens[FUSION][PARTNER_5P][NAMESPACE],
        name=tokens[FUSION][PARTNER_5P][NAME]
    )

    partner_3p = member_dsl(
        namespace=tokens[FUSION][PARTNER_3P][NAMESPACE],
        name=tokens[FUSION][PARTNER_3P][NAME]
    )

    range_5p = _fusion_range_to_dsl(tokens[FUSION][RANGE_5P])
    range_3p = _fusion_range_to_dsl(tokens[FUSION][RANGE_3P])

    return fusion_dsl(
        partner_5p=partner_5p,
        partner_3p=partner_3p,
        range_5p=range_5p,
        range_3p=range_3p,
    )
1343561
def save_tokens(tokens)
      f = File.open(TOKEN_PATH, 'w')
      YAML::dump(tokens, f)
      f.close
    end
1364673
def parse_tokens(self, tokens):
        
        if len(tokens) == 1:
            return list(), tokens, {"kubectl": self.ast.help}
        else:
            tokens.reverse()
        parsed, unparsed, suggestions = self.treewalk(self.ast, parsed=list(), unparsed=tokens)
        if not suggestions and unparsed:
            # TODO: @vogxn: This is hack until we include expected value types for each option and argument.
            # Whenver we recieve no suggestions but are left with unparsed tokens we pop the value and walk the
            # tree again without values
            logger.debug("unparsed tokens remain, possible value encountered")
            unparsed.pop()
            parsed.reverse()
            unparsed.extend(parsed)
            logger.debug("resuming treewalk with tokens: %s", unparsed)
            return self.treewalk(self.ast, parsed=list(), unparsed=unparsed)
        else:
            return parsed, unparsed, suggestions
651242
function parse(str, options = {}) {
    let tokens = tokenize(str)
    let data = []

    tokens.forEach((item, idx) => {
        let [ label, token ] = item
        if (renderRules[label]) {
            data.push(renderRules[label](token))
        }else {
            data.push([ RENDER_TYPE_PLAIN, token ])
        }
    })
    return data
}
238620
def tokens2ids(tokens: Iterable[str], vocab: Dict[str, int]) -> List[int]:
    
    return [vocab.get(w, vocab[C.UNK_SYMBOL]) for w in tokens]
366331
public Set<AuthorisationData> convertIdentifyingTokenList(Set<IdentifyingToken> tokens) {
        Set<AuthorisationData> resultList = new HashSet<>();

        for (IdentifyingToken token : tokens) {
            resultList.add(convertIdentifyingToken(token));
        }

        return resultList;
    }
496079
def tokenize(self, token_list):
    """"""
    return [
        self._vocab_dict.get(token, self._vocab_dict[self.UNK])
        for token in token_list
    ]
879318
def convert_examples_to_features(examples, seq_length, tokenizer):
    """"""

    features = []
    for (ex_index, example) in enumerate(examples):
        tokens_a = tokenizer.tokenize(example.text_a)

        tokens_b = None
        if example.text_b:
            tokens_b = tokenizer.tokenize(example.text_b)

        if tokens_b:
            # Modifies `tokens_a` and `tokens_b` in place so that the total
            # length is less than the specified length.
            # Account for [CLS], [SEP], [SEP] with "- 3"
            _truncate_seq_pair(tokens_a, tokens_b, seq_length - 3)
        else:
            # Account for [CLS] and [SEP] with "- 2"
            if len(tokens_a) > seq_length - 2:
                tokens_a = tokens_a[0:(seq_length - 2)]

        # The convention in BERT is:
        # (a) For sequence pairs:
        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
        #  type_ids:   0   0  0    0    0     0      0   0    1  1  1   1  1   1
        # (b) For single sequences:
        #  tokens:   [CLS] the dog is hairy . [SEP]
        #  type_ids:   0   0   0   0  0     0   0
        #
        # Where "type_ids" are used to indicate whether this is the first
        # sequence or the second sequence. The embedding vectors for `type=0` and
        # `type=1` were learned during pre-training and are added to the wordpiece
        # embedding vector (and position vector). This is not *strictly* necessary
        # since the [SEP] token unambigiously separates the sequences, but it makes
        # it easier for the model to learn the concept of sequences.
        #
        # For classification tasks, the first vector (corresponding to [CLS]) is
        # used as as the "sentence vector". Note that this only makes sense because
        # the entire model is fine-tuned.
        tokens = []
        input_type_ids = []
        tokens.append("[CLS]")
        input_type_ids.append(0)
        for token in tokens_a:
            tokens.append(token)
            input_type_ids.append(0)
        tokens.append("[SEP]")
        input_type_ids.append(0)

        if tokens_b:
            for token in tokens_b:
                tokens.append(token)
                input_type_ids.append(1)
            tokens.append("[SEP]")
            input_type_ids.append(1)

        input_ids = tokenizer.convert_tokens_to_ids(tokens)

        # The mask has 1 for real tokens and 0 for padding tokens. Only real
        # tokens are attended to.
        input_mask = [1] * len(input_ids)

        # Zero-pad up to the sequence length.
        while len(input_ids) < seq_length:
            input_ids.append(0)
            input_mask.append(0)
            input_type_ids.append(0)

        assert len(input_ids) == seq_length
        assert len(input_mask) == seq_length
        assert len(input_type_ids) == seq_length

        if ex_index < 5:
            logger.info("*** Example ***")
            logger.info("unique_id: %s" % (example.unique_id))
            logger.info("tokens: %s" % " ".join([str(x) for x in tokens]))
            logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
            logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask]))
            logger.info(
                "input_type_ids: %s" % " ".join([str(x) for x in input_type_ids]))

        features.append(
            InputFeatures(
                unique_id=example.unique_id,
                tokens=tokens,
                input_ids=input_ids,
                input_mask=input_mask,
                input_type_ids=input_type_ids))
    return features
607764
public CommonTree parse(final CommonTokenStream tokens)
            throws RecognizerException {
        if (_log.isDebugEnabled()) {
            _log.debug("3. Parsing tokens: {}", tokens.toString());
        }
        try {
            CobolStructureParser parser = new CobolStructureParserImpl(tokens,
                    getErrorHandler());
            cobdata_return parserResult = parser.cobdata();
            if (parser.getNumberOfSyntaxErrors() != 0 || parserResult == null) {
                throw (new RecognizerException("Parsing failed. "
                        + parser.getNumberOfSyntaxErrors() + " syntax errors."
                        + " Last error was "
                        + getErrorHistory().get(getErrorHistory().size() - 1)));
            }
            return (CommonTree) parserResult.getTree();
        } catch (RecognitionException e) {
            throw (new RecognizerException(e));
        }
    }
89372
def parse_nested(
    bels: list, char_locs: CharLocs, parsed: Parsed, errors: Errors
) -> Tuple[Parsed, Errors]:
    

    for sp in char_locs[
        "nested_parens"
    ]:  # sp = start parenthesis, ep = end parenthesis
        ep, level = char_locs["nested_parens"][sp]
        if ep == -1:
            ep = len(bels) + 1
        parsed[(sp, ep)] = {"type": "Nested", "span": (sp, ep)}

    return parsed, errors
337211
def _parse_complement(self, tokens):
        
        tokens.pop(0)  # Pop 'complement'
        tokens.pop(0)  # Pop '('
        res = self._parse_nested_interval(tokens)
        tokens.pop(0)  # Pop ')'
        res.switch_strand()
        return res
544560
private void convertTokenDefinitions() throws GrammarException,
	    TreeException {
	tokenVisibility.clear();
	Map<String, ParseTreeNode> helpers = getHelperTokens();
	Map<String, ParseTreeNode> tokens = getTokens();
	convertTokenDefinitions(helpers, tokens);
    }
1320730
def make_tokens(parse_buffer):
    
    tokens = []
    for token_type, result in parse_buffer:
        token = token_type(result)
        if token is not None:
            tokens.append(token)
    return tokens
231713
def get_dict(parser, token):
    
    bits = token.contents.split(' ')
    return GetDict(bits[1], bits[2], ((len(bits) > 3) and bits[3]) or '', ((len(bits) > 4) and bits[4]) or '', ((len(bits) > 5) and bits[5]) or '')
997188
function parser(tokens) {
  // add Close token before each Else
  const toks = tokens.reduce((prev, tok) => {
    if (tok.tokenType === 'Else') {
      return [...prev, new Close(), tok];
    }

    return [...prev, tok];
  }, []);

  const [body] = parse(toks, '', 1);
  return body;
}
1364543
def datatype_from_token(self, token):
        

        if token.type == SystemRDLParser.ID:
            # Is an identifier for either an enum or struct type

            typ = self.compiler.namespace.lookup_type(get_ID_text(token))
            if typ is None:
                self.msg.fatal(
                    "Type '%s' is not defined" % get_ID_text(token),
                    SourceRef.from_antlr(token)
                )

            if rdltypes.is_user_enum(typ) or rdltypes.is_user_struct(typ):
                return typ
            else:
                self.msg.fatal(
                    "Type '%s' is not a struct or enum" % get_ID_text(token),
                    SourceRef.from_antlr(token)
                )

        else:
            return self._DataType_Map[token.type]
1118818
def tokenize(self, tw):
        
        toks = defaultdict(lambda: [])
        for field in self.fields:
            if '.' in field:
                parts = field.split('.')
                value = tw.js
                for p in parts:
                    value = value[p]
            else:
                value = tw.js[field]
            toks[field].extend(self.do_tokenize(value))
        return dict([(f, v) for f, v in toks.items()])