Decomposes a MARC "code" into tag, ind1, ind2, subcode
486376
def decompose_code(code):
    
    code = "%-6s" % code
    ind1 = code[3:4]
    if ind1 == " ": ind1 = "_"
    ind2 = code[4:5]
    if ind2 == " ": ind2 = "_"
    subcode = code[5:6]
    if subcode == " ": subcode = None
    return (code[0:3], ind1, ind2, subcode)
411850
def coarse_tag_str(pos_seq):
	""""""
	global tag2coarse
	tags = [tag2coarse.get(tag, 'O') for tag in pos_seq]
	return ''.join(tags)
4591
void decompileArrayLiteral(ArrayLiteral node) {
        decompiler.addToken(Token.LB);
        List<AstNode> elems = node.getElements();
        int size = elems.size();
        for (int i = 0; i < size; i++) {
            AstNode elem = elems.get(i);
            decompile(elem);
            if (i < size - 1) {
                decompiler.addToken(Token.COMMA);
            }
        }
        decompiler.addToken(Token.RB);
    }
1011872
def parse_subtags(subtags, expect=EXTLANG):
    
    # We parse the parts of a language code recursively: each step of
    # language code parsing handles one component of the code, recurses
    # to handle the rest of the code, and adds what it found onto the
    # list of things that were in the rest of the code.
    #
    # This could just as well have been iterative, but the loops would have
    # been convoluted.
    #
    # So here's the base case.
    if not subtags:
        return []

    # There's a subtag that comes next. We need to find out what it is.
    #
    # The primary thing that distinguishes different types of subtags is
    # length, but the subtags also come in a specified order. The 'expect'
    # parameter keeps track of where we are in that order. expect=REGION,
    # for example, means we're expecting a region code, or anything later
    # (because everything but the language is optional).
    subtag = subtags[0]
    tag_length = len(subtag)

    # In the usual case, our goal is to recognize what kind of tag this is,
    # and set it in 'tagtype' -- as an integer, so we can compare where it
    # should go in order. You can see the enumerated list of tagtypes above,
    # where the SUBTAG_TYPES global is defined.
    tagtype = None

    if tag_length == 0 or tag_length > 8:
        # Unless you're inside a private use tag or something -- in which case,
        # you're not in this function at the moment -- every component needs to
        # be between 1 and 8 characters.
        subtag_error(subtag, '1-8 characters')

    elif tag_length == 1:
        # A one-character subtag introduces an extension, which can itself have
        # sub-subtags, so we dispatch to a different function at this point.
        #
        # We don't need to check anything about the order, because extensions
        # necessarily come last.
        return parse_extension(subtags)

    elif tag_length == 2:
        if subtag.isalpha():
            # Two-letter alphabetic subtags are regions. These are the only
            # two-character subtags after the language.
            tagtype = REGION

    elif tag_length == 3:
        if subtag.isalpha():
            # Three-letter alphabetic subtags are 'extended languages'.
            # It's allowed for there to be up to three of them in a row, so we
            # need another function to enforce that. Before we dispatch to that
            # function, though, we need to check whether we're in the right
            # place in order.
            if expect <= EXTLANG:
                return parse_extlang(subtags)
            else:
                order_error(subtag, EXTLANG, expect)
        elif subtag.isdigit():
            # Three-digit subtags are broad regions, such as Latin America
            # (419).
            tagtype = REGION

    elif tag_length == 4:
        if subtag.isalpha():
            # Four-letter alphabetic subtags are scripts.
            tagtype = SCRIPT
        elif subtag[0].isdigit():
            # Four-character subtags that start with a digit are variants.
            tagtype = VARIANT

    else:
        # Tags of length 5-8 are variants.
        tagtype = VARIANT

    # That's the end of the big elif block for figuring out what kind of
    # subtag we have based on its length. Now we should do something with that
    # kind of subtag.

    if tagtype is None:
        # We haven't recognized a type of tag. This subtag just doesn't fit the
        # standard.
        subtag_error(subtag)

    elif tagtype < expect:
        # We got a tag type that was supposed to appear earlier in the order.
        order_error(subtag, tagtype, expect)

    else:
        # We've recognized a subtag of a particular type. If it's a region or
        # script, we expect the next subtag to be a strictly later type, because
        # there can be at most one region and one script. Otherwise, we expect
        # the next subtag to be the type we got or later.

        if tagtype in (SCRIPT, REGION):
            expect = tagtype + 1
        else:
            expect = tagtype

        # Get the name of this subtag type instead of its integer value.
        typename = SUBTAG_TYPES[tagtype]

        # Some subtags are conventionally written with capitalization. Apply
        # those conventions.
        if tagtype == SCRIPT:
            subtag = subtag.title()
        elif tagtype == REGION:
            subtag = subtag.upper()

        # Recurse on the remaining subtags.
        return [(typename, subtag)] + parse_subtags(subtags[1:], expect)
1079464
def _process_tags(self):
        """"""
        tags = []

        while True:
            tag_bf = unpack_ui16(self._src)
            tag_type = tag_bf >> 6   # upper 10 bits
            if tag_type == 0:
                # the end
                break
            tag_len = tag_bf & 0x3f  # last 6 bits
            if tag_len == 0x3f:
                # the length is the next four bytes!
                tag_len = unpack_ui32(self._src)

            try:
                tag_name = TAG_NAMES[tag_type]
            except KeyError:
                # malformed SWF, create and unknown object with malformed tag
                tag_payload = self._src.read(tag_len)
                _dict = {
                    '__str__': _repr,
                    '__repr__': _repr,
                    'name': 'UnspecifiedObject(tag={!r})'.format(tag_type),
                }
                tag = type("UnknownObject", (SWFObject,), _dict)()
                tag.raw_payload = tag_payload
                tags.append(tag)
                continue

            try:
                tag_meth = getattr(self, "_handle_tag_" + tag_name.lower())
            except AttributeError:
                if self.unknown_alert:
                    raise ValueError("Unknown tag: " + repr(tag_name))

                tag_payload = self._src.read(tag_len)
                _dict = {'__str__': _repr, '__repr__': _repr, 'name': tag_name}
                tag = type("UnknownObject", (SWFObject,), _dict)()
                tag.raw_payload = tag_payload
                tags.append(tag)
                continue

            # we know the tag type, and have the handler, let's process it
            prev_pos = self._src.tell()
            self._src.guard = tag_len
            try:
                with ReadQuantityController(self._src, tag_len):
                    tag = tag_meth()
                assert tag is not None, tag_name
            except ValueError:
                # an attempt to read too much happened; create a failing
                # object with the raw payload
                self._src.guard = None
                self._src.seek(prev_pos)
                tag_payload = self._src.read(tag_len)
                _dict = {'__str__': _repr, '__repr__': _repr, 'name': tag_name}
                tag = type("FailingObject", (SWFObject,), _dict)()
                tag.raw_payload = tag_payload
            tags.append(tag)
        return tags
169750
function(code) {
    var lines = [];
    code.split(/\n/).forEach(line => {
      var segs = line.split(/(\s*#)/);
      var ln = {code: segs[0]};
      if (segs.length > 1)
        ln.comment = segs.reduce((c, seg, i) => i > 1 ? c + seg : seg, '');
      lines.push(ln);
    });
    return lines;
  }
798054
def extract_separator
      if meta_tags[:separator] == false
        # Special case: if separator is hidden, do not display suffix/prefix
        prefix = separator = suffix = ''
      else
        prefix    = extract_separator_section(:prefix, ' ')
        separator = extract_separator_section(:separator, '|')
        suffix    = extract_separator_section(:suffix, ' ')
      end
      delete(:separator, :prefix, :suffix)

      TextNormalizer.safe_join([prefix, separator, suffix], '')
    end
957984
def _stripped_codes(codes):
    """"""
    return tuple([
        code.strip() for code in codes.split(',')
        if code.strip()
    ])
1103788
def delimiters_to_re(delimiters):
    """"""

    # caching
    delimiters = tuple(delimiters)
    if delimiters in re_delimiters:
        re_tag = re_delimiters[delimiters]
    else:
        open_tag, close_tag = delimiters

        # escape
        open_tag = ''.join([c if c.isalnum() else '\\' + c for c in open_tag])
        close_tag = ''.join([c if c.isalnum() else '\\' + c for c in close_tag])

        re_tag = re.compile(open_tag + r'([#^>&{/!=]?)\s*(.*?)\s*([}=]?)' + close_tag, re.DOTALL)
        re_delimiters[delimiters] = re_tag

    return re_tag
1064483
function parseCodeChunkToAST(src, options) {
    src = src
    .replace(/@/g, '\uFFDA')
    .replace(/#/g, '\uFFDB')
    ;
    var ast = recast.parse(src);
    return ast;
}
383680
def _slice_replace(code, index, old, new):
        """"""
        nodes = [str(node) for node in code.get(index)]
        substring = "".join(nodes).replace(old, new)
        code.nodes[index] = parse_anything(substring).nodes
749920
def decompile(f):
    
    co = f.__code__
    args, kwonly, varargs, varkwargs = paramnames(co)
    annotations = f.__annotations__ or {}
    defaults = list(f.__defaults__ or ())
    kw_defaults = f.__kwdefaults__ or {}

    if f.__name__ == '<lambda>':
        node = ast.Lambda
        body = pycode_to_body(co, DecompilationContext(in_lambda=True))[0]
        extra_kwargs = {}
    else:
        node = ast.FunctionDef
        body = pycode_to_body(co, DecompilationContext(in_function_block=True))
        extra_kwargs = {
            'decorator_list': [],
            'returns': annotations.get('return')
        }

    return node(
        name=f.__name__,
        args=make_function_arguments(
            args=args,
            kwonly=kwonly,
            varargs=varargs,
            varkwargs=varkwargs,
            defaults=defaults,
            kw_defaults=kw_defaults,
            annotations=annotations,
        ),
        body=body,
        **extra_kwargs
    )
1108908
def tokenize_tags(tags_string):
    

    # text is parsed in two steps:
    # the first step extract every single world that is 3 > chars long
    # and that contains only alphanumeric characters, underscores and dashes
    tags_string = tags_string.lower().strip(",")
    single_words = set([w[:100] for w in re.split(';|,|\*|\n| ', tags_string)
                          if len(w) >= 3 and re.match("^[A-Za-z0-9_-]*$", w)])
    # the second step divide the original string using comma as separator
    comma_separated = set([t[:100] for t in tags_string.split(",") if t])
    # resulting set are merged using union
    return list(single_words | comma_separated)
533436
function tokenize(source, options){
  var result = [];
  var tagStack = [];
  var lastTag = { children: result };
  var parseTag = false;
  var token;
  var state = TEXT;
  var pos = 0;
  var textStateEndPos = 0;
  var textEndPos;
  var bufferPos;
  var startPos;
  var m;

  /** @cut */ var attrMap;  // map to detect attribute duplicates
  /** @cut */ result.source_ = source;
  /** @cut */ result.warns = [];

  // skip starting whitespaces
  if (!options || options.trim !== false)
  {
    pos = textStateEndPos = source.match(/^\s*/)[0].length;
    source = source.trimRight();  // TODO: replace for basis.string.trimRight
  }

  while (pos < source.length || state != TEXT)
  {
    state.lastIndex = pos;
    startPos = pos;

    m = state.exec(source);

    if (!m || m.index !== pos)
    {
      // treat broken comment reference as comment content
      if (state == REFERENCE && token && token.type == TYPE_COMMENT)
      {
        state = COMMENT;
        continue;
      }

      if (parseTag)
        lastTag = tagStack.pop();

      if (token)
        lastTag.children.pop();

      if (token = lastTag.children.pop())
      {
        if (token.type == TYPE_TEXT && !token.refs)
          textStateEndPos -= token.value.length;
        else
          lastTag.children.push(token);
      }

      parseTag = false;
      state = TEXT;
      continue;
    }

    pos = state.lastIndex;

    //stat[state] = (stat[state] || 0) + 1;
    switch (state)
    {
      case TEXT:

        textEndPos = startPos + m[1].length;

        if (textStateEndPos != textEndPos)
        {
          var sourceText = textStateEndPos == startPos
            ? m[1]
            : source.substring(textStateEndPos, textEndPos);

          sourceText = sourceText.replace(/\s*(\r\n?|\n\r?)\s*/g, '');

          if (sourceText)
            lastTag.children.push({
              type: TYPE_TEXT,
              value: sourceText,
              range: {
                start_: textStateEndPos,
                end_: textEndPos
              }
            });
        }

        textStateEndPos = textEndPos;

        if (m[3])
        {
          lastTag.children.push({
            type: TYPE_TEXT,
            refs: ['l10n:' + m[3]],
            value: '{l10n:' + m[3] + '}',
            range: {
              start_: textEndPos,
              end_: pos
            }
          });
        }
        else if (m[2] == '{')
        {
          bufferPos = pos - 1;
          lastTag.children.push(token = {
            type: TYPE_TEXT,
            range: {
              start_: textEndPos,
              end_: textEndPos
            }
          });
          state = REFERENCE;
        }
        else if (m[4])
        {
          if (m[4] == '/')
          {
            token = null;
            state = CLOSE_TAG;
          }
          else //if (m[3] == '!--')
          {
            lastTag.children.push(token = {
              type: TYPE_COMMENT,
              range: {
                start_: textEndPos,
                end_: textEndPos
              }
            });

            if (m[5])
            {
              bufferPos = pos - m[5].length;
              state = REFERENCE;
            }
            else
            {
              bufferPos = pos;
              state = COMMENT;
            }
          }
        }
        else if (m[2]) // m[2] == '<' open tag
        {
          parseTag = true;
          tagStack.push(lastTag);

          lastTag.children.push(token = {
            type: TYPE_ELEMENT,
            attrs: [],
            children: [],
            range: {
              start_: textEndPos,
              end_: textEndPos
            }
          });
          lastTag = token;

          state = TAG_NAME;

          // create attribute map
          /** @cut */ attrMap = {};
        }

        break;

      case CLOSE_TAG:
        if (m[1] !== (lastTag.prefix ? lastTag.prefix + ':' : '') + lastTag.name)
        {
          lastTag.children.push({
            type: TYPE_TEXT,
            value: '</' + m[0],
            range: {
              start_: startPos - 2,
              end_: startPos + m[0].length
            }
          });
          /** @cut */ result.warns.push(['Wrong close tag: ' + source.substr(startPos - 2, m[0].length + 2), lastTag.children[lastTag.children.length - 1]]);
        }
        else
        {
          lastTag.range.end_ = startPos + m[0].length;
          lastTag = tagStack.pop();
        }

        state = TEXT;
        break;

      case TAG_NAME:
      case ATTRIBUTE_NAME_OR_END:
        if (m[2] == ':')
        {
          if (token.prefix)      // prefix was before, break tag parse
            state = BREAK_TAG_PARSE;
          else
            token.prefix = m[1];

          break;
        }

        if (m[1])
        {
          // store name (it may be null when check for attribute and end)
          token.name = m[1];
          token.range.end_ = startPos + m[1].length;

          // store attribute
          if (token.type == TYPE_ATTRIBUTE)
          {
            // detect attribute duplicates
            /** @cut */ var fullName = (token.prefix ? token.prefix + ':' : '') + token.name;
            /** @cut */ if (Object.prototype.hasOwnProperty.call(attrMap, fullName))
            /** @cut */   result.warns.push(['Duplicate attribute: ' + fullName, token]);
            /** @cut */ attrMap[fullName] = true;

            lastTag.attrs.push(token);
          }
        }

        if (m[2] == '{')
        {
          if (token.type == TYPE_ELEMENT)
            state = REFERENCE;
          else
            state = BREAK_TAG_PARSE;

          break;
        }

        if (m[3]) // end tag declaration
        {
          parseTag = false;
          lastTag.range.end_ = pos;

          if (m[3] == '/>' ||
              (!lastTag.prefix && SINGLETON_TAG.test(lastTag.name)))
          {
            /** @cut */ if (m[3] != '/>')
            /** @cut */   result.warns.push(['Tag <' + lastTag.name + '> doesn\'t closed explicit (use `/>` as tag ending)', lastTag]);

            lastTag = tagStack.pop();
          }
          else
          {
            // otherwise m[3] == '>'
            if (lastTag.prefix == 'b' && lastTag.name in TAG_IGNORE_CONTENT)
            {
              state = TAG_IGNORE_CONTENT[lastTag.name];
              break;
            }
          }

          state = TEXT;
          break;
        }

        if (m[2] == '=') // ATTRIBUTE_NAME_OR_END only
        {
          state = ATTRIBUTE_VALUE;
          break;
        }

        // m[2] == '\s+' next attr, state doesn't change
        token = {
          type: TYPE_ATTRIBUTE,
          range: {
            start_: pos,
            end_: pos
          }
        };
        state = ATTRIBUTE_NAME_OR_END;
        break;

      case COMMENT:
        token.value = source.substring(bufferPos, pos - 3);
        token.range.end_ = pos;
        state = TEXT;
        break;

      case REFERENCE:
        // add reference to token list name
        if (token.refs)
          token.refs.push(m[1]);
        else
          token.refs = [m[1]];

        // go next
        if (m[2] != '|') // m[2] == '}\s*'
        {
          if (token.type == TYPE_TEXT)
          {
            pos -= m[2].length - 1;
            token.value = source.substring(bufferPos, pos);
            token.range.end_ = pos;
            state = TEXT;
          }
          else if (token.type == TYPE_COMMENT)
          {
            state = COMMENT;
          }
          else if (token.type == TYPE_ATTRIBUTE && source[pos] == '=')
          {
            pos++;
            state = ATTRIBUTE_VALUE;
          }
          else // ATTRIBUTE || ELEMENT
          {
            token = {
              type: TYPE_ATTRIBUTE,
              range: {
                start_: pos,
                end_: pos
              }
            };
            state = ATTRIBUTE_NAME_OR_END;
          }
        }

        // continue to collect references
        break;

      case ATTRIBUTE_VALUE:
        token.value = m[1].replace(QUOTE_UNESCAPE, '"');
        token.range.end_ = pos;
        token.valueRange = {
          start_: startPos + 1,
          end_: startPos + 1 + m[1].length
        };

        token = {
          type: TYPE_ATTRIBUTE,
          range: {
            start_: pos,
            end_: pos
          }
        };
        state = ATTRIBUTE_NAME_OR_END;

        break;

      case TAG_IGNORE_CONTENT.text:
      case TAG_IGNORE_CONTENT.style:
        lastTag.children.push({
          type: TYPE_TEXT,
          value: m[1],
          range: {
            start_: startPos,
            end_: startPos + m[1].length
          }
        });

        lastTag = tagStack.pop();

        state = TEXT;
        break;

      default:
        throw SYNTAX_ERROR; // Must never to be here; bug in parser otherwise
    }

    if (state == TEXT)
      textStateEndPos = pos;
  }

  if (textStateEndPos != pos)
    lastTag.children.push({
      type: TYPE_TEXT,
      value: source.substring(textStateEndPos, pos),
      range: {
        start_: textStateEndPos,
        end_: pos
      }
    });

  // process attributes binding, join text tokens, normalize ranges and add location info
  postProcessing(result, options || {}, source);

  /** @cut */ if (lastTag.name)
  /** @cut */   result.warns.push(['No close tag for <' + (lastTag.prefix ? lastTag.prefix + ':' : '') + lastTag.name + '>', lastTag]);
  /** @cut */
  /** @cut */ if (!result.warns.length)
  /** @cut */   result.warns = false;

  result.templateTokens = true;

  return result;
}
1157465
def _clean_code(self, code):
        
        lines = code.split("\n")
        min_indent = 100
        for line in lines:
            if line.strip() != "":
                indent = len(line) - len(line.lstrip())
                min_indent = min(indent, min_indent)
        if min_indent > 0:
            lines = [line[min_indent:] for line in lines]
        code = "\n".join(lines)
        return code
625519
def tag(self, tokens):
        """"""
        if not self._loaded_model:
            self.load(self.model)
        tags = [None] * len(tokens)
        norm = self._normalize(tokens)
        length = len(norm)
        # A set of allowed indexes for matches to start or end at
        delims = [0] + [i for span in [m.span() for m in self.delimiters.finditer(norm)] for i in span] + [length]
        # Token indices
        token_at_index = []
        for i, t in enumerate(tokens):
            token_at_index.extend([i] * (len(self.lexicon[t].normalized) + 1))
        start_i = 0
        end_i = 1
        matches = {}
        next_start = end_i
        # TODO: This could be a little more efficient by skipping indexes forward to next delim points.
        while True:
            current = norm[start_i:end_i]
            if self._dawg.has_keys_with_prefix(current):
                # print('%s:%s:%s' % (start_i, end_i, current))
                # If the current span is in the dawg, and isn't followed by an alphanumeric character
                if current in self._dawg and start_i in delims and end_i in delims:
                    # print(current)
                    # Subsequent longer matches with same start_i will overwrite values in matches dict
                    matches[start_i] = (start_i, end_i, current)
                    # We can skip forward to after this match next time we increment start_i
                    next_start = end_i
                # Increment end_i provided we aren't already at the end of the input
                if end_i < length:
                    end_i += 1
                    continue
            # Increment start_i provided we aren't already at the end of the input
            start_i = next_start
            if start_i >= length - 1:
                break
            end_i = start_i + 1
            next_start = end_i
        # Apply matches as tags to the relevant tokens
        for start_i, end_i, current in matches.values():
            start_token = token_at_index[start_i]
            end_token = token_at_index[end_i]
            # Possible for match to start in 'I' token from prev match. Merge matches by not overwriting to 'B'.
            if not tags[start_token] == 'I-%s' % self.entity:
                tags[start_token] = 'B-%s' % self.entity
            tags[start_token+1:end_token+1] = ['I-%s' % self.entity] * (end_token - start_token)
        tokentags = list(zip(tokens, tags))
        return tokentags
541376
def mrc_to_marc(mrc):
    
    # ignore blank lines
    lines = [
        line
        for line in mrc.splitlines()
        if line.strip()
    ]

    def split_to_parts(lines):
        for line in lines:
            first_part, second_part = line.split(" L ", 1)

            yield line, first_part, second_part.lstrip()

    control_lines = []
    data_lines = []
    for line, first_part, second_part in split_to_parts(lines):
        if second_part.startswith("$"):
            data_lines.append(line)
        else:
            control_lines.append(line)

    # convert controlfield lines
    record = MARCXMLRecord()
    record.oai_marc = True
    for line, descr, content in split_to_parts(control_lines):
        record.controlfields[descr.strip()[:3]] = content

    def get_subfield_dict(line):
        fields = (
            (field[0], field[1:])
            for field in line.split("$$")[1:]
        )

        fields_dict = defaultdict(list)
        for key, val in fields:
            fields_dict[key].append(val)

        return fields_dict

    # convert datafield lines
    for line, descr, content_line in split_to_parts(data_lines):
        name = descr[:3]
        i1 = descr[3]
        i2 = descr[4]

        record.add_data_field(
            name,
            i1,
            i2,
            get_subfield_dict(content_line)
        )

    return record.to_XML()