Convert emoji lexicon file to a dictionary
482548
def make_emoji_dict(self):
        
        emoji_dict = {}
        for line in self.emoji_full_filepath.split('\n'):
            (emoji, description) = line.strip().split('\t')[0:2]
            emoji_dict[emoji] = description
        return emoji_dict
714993
def to_dict(self):
        '''
        '''
        try:
            name = self.file.name
        except AttributeError:
            name = None

        try:
            size = self.size()
        except OSError:
            size = None

        return {
            'filename': name,
            'length': size,
            'content_size': size,
        }
1273076
def to_dict(self, info_filename, tree_filename, dash_f_e=False):
        
        logger.debug('info_filename: {} {}'
                     .format(info_filename, '(FOUND)' if os.path.exists(info_filename) else '(NOT FOUND)'))
        logger.debug('tree_filename: {} {}'
                     .format(tree_filename, '(FOUND)' if os.path.exists(tree_filename) else '(NOT FOUND)'))
        if dash_f_e:
            return self._dash_f_e_to_dict(info_filename, tree_filename)
        else:
            return self._to_dict(info_filename, tree_filename)
225327
def _to_dict(self):
        """"""
        _dict = {}
        if hasattr(self, 'concepts') and self.concepts is not None:
            _dict['concepts'] = self.concepts._to_dict()
        if hasattr(self, 'emotion') and self.emotion is not None:
            _dict['emotion'] = self.emotion._to_dict()
        if hasattr(self, 'entities') and self.entities is not None:
            _dict['entities'] = self.entities._to_dict()
        if hasattr(self, 'keywords') and self.keywords is not None:
            _dict['keywords'] = self.keywords._to_dict()
        if hasattr(self, 'metadata') and self.metadata is not None:
            _dict['metadata'] = self.metadata._to_dict()
        if hasattr(self, 'relations') and self.relations is not None:
            _dict['relations'] = self.relations._to_dict()
        if hasattr(self, 'semantic_roles') and self.semantic_roles is not None:
            _dict['semantic_roles'] = self.semantic_roles._to_dict()
        if hasattr(self, 'sentiment') and self.sentiment is not None:
            _dict['sentiment'] = self.sentiment._to_dict()
        if hasattr(self, 'categories') and self.categories is not None:
            _dict['categories'] = self.categories._to_dict()
        if hasattr(self, 'syntax') and self.syntax is not None:
            _dict['syntax'] = self.syntax._to_dict()
        return _dict
1294154
def etree_to_dict(t, trim=True, **kw):
    
    """
    d = {t.tag: {} if t.attrib else None}
    children = list(t)
    etree_to_dict_w_args = partial(etree_to_dict, trim=trim, **kw)

    if children:
        dd = defaultdict(list)
        d = {t.tag: {}}

        for dc in map(etree_to_dict_w_args, children):
            for k, v in dc.iteritems():
                # do not add Comment instance to the key
                if k is not etree.Comment:
                    dd[k].append(v)

        d[t.tag] = {k: v[0] if len(v) == 1 else v for k, v in dd.iteritems()}

    if t.attrib:
        d[t.tag].update(('@' + k, v) for k, v in t.attrib.iteritems())
    if trim and t.text:
        t.text = t.text.strip()
    if t.text:
        if t.tag is etree.Comment and not kw.get('without_comments'):
            # adds a comments node
            d['#comments'] = t.text
        elif children or t.attrib:
            d[t.tag]['#text'] = t.text
        else:
            d[t.tag] = t.text
    return d
1334718
def _build_unicode_character_database(self):
        
        filename = "UnicodeData.txt"
        current_dir = os.path.abspath(os.path.dirname(__file__))
        tag = re.compile(r"<\w+?>")
        with codecs.open(os.path.join(current_dir, filename), mode="r", encoding="utf-8") as fp:
            for line in fp:
                if not line.strip():
                    continue
                data = line.strip().split(";")
                # Replace the start/end range markers with their proper derived names.
                if data[1].endswith((u"First>", u"Last>")) and _is_derived(int(data[0], 16)):
                    data[1] = _get_nr_prefix(int(data[0], 16))
                    if data[1].startswith("HANGUL SYLLABLE"):  # For Hangul syllables, use naming rule NR1
                        data[1] += _get_hangul_syllable_name(int(data[0], 16))
                    else:  # Others should use naming rule NR2
                        data[1] += data[0]
                data[3] = int(data[3])  # Convert the Canonical Combining Class value into an int.
                if data[5]:  # Convert the contents of the decomposition into characters, preserving tag info.
                    data[5] = u" ".join([_hexstr_to_unichr(s) if not tag.match(s) else s for s in data[5].split()])
                for i in [6, 7, 8]:  # Convert the decimal, digit and numeric fields to either ints or fractions.
                    if data[i]:
                        if "/" in data[i]:
                            data[i] = Fraction(data[i])
                        else:
                            data[i] = int(data[i])
                for i in [12, 13, 14]:  # Convert the uppercase, lowercase and titlecase fields to characters.
                    if data[i]:
                        data[i] = _hexstr_to_unichr(data[i])
                lookup_name = _uax44lm2transform(data[1])
                uc_data = UnicodeCharacter(u"U+" + data[0], *data[1:])
                self._unicode_character_database[int(data[0], 16)] = uc_data
                self._name_database[lookup_name] = uc_data
        # Fill out the "compressed" ranges of UnicodeData.txt i.e. fill out the remaining characters per the Name
        # Derivation Rules.  See the Unicode Standard, ch. 4, section 4.8, Unicode Name Property
        for lookup_range, prefix_string in _nr_prefix_strings.items():
            exemplar = self._unicode_character_database.__getitem__(lookup_range[0])
            for item in lookup_range:
                hex_code = _padded_hex(item)
                new_name = prefix_string
                if prefix_string.startswith("HANGUL SYLLABLE"):  # For Hangul, use naming rule NR1
                    new_name += _get_hangul_syllable_name(item)
                else:  # Everything else uses naming rule NR2
                    new_name += hex_code
                uc_data = exemplar._replace(code=u"U+" + hex_code, name=new_name)
                self._unicode_character_database[item] = uc_data
                self._name_database[_uax44lm2transform(new_name)] = uc_data
59553
def utf8_normalize(input_filename, comment_char='#', to_upper=False):
    
    # Prepare the input dictionary file in UTF-8 and NFC
    temp_dict_fd, output_filename = tempfile.mkstemp()

    logging.debug('to_nfc from file {} to file {}'.format(input_filename, output_filename))

    with codecs.open(output_filename, 'w', 'utf-8') as f:
        with codecs.open(input_filename, 'r', 'utf-8') as input_f:
            lines = sorted([unicodedata.normalize(UTF8_NORMALIZATION, l)
                            for l in input_f.readlines()
                            if not l.strip().startswith(comment_char)])
            if to_upper:
                lines = [l.upper() for l in lines]

            f.writelines(lines)

    return output_filename
1360995
function getPandocId (text) {
  text = text
    .replace(emojiRegex(), '') // Strip emojis
    .toLowerCase()
    .trim()
    .replace(/%25|%/ig, '')   // remove single % signs
    .replace(RE_ENTITIES, '') // remove xml/html entities
    .replace(RE_SPECIALS, '') // single chars that are removed but not [.-]
    .replace(/\s+/g, '-')     // whitespaces
    .replace(/^-+|-+$/g, '')  // heading/ tailing hyphen
    .replace(RE_CJK, '')      // CJK punctuations that are removed

  if (/^[0-9-]+$/.test(text)) {
    text = 'section'
  }
  return text
}
762992
def to_array(self):
        
        array = super(Sticker, self).to_array()
        array['file_id'] = u(self.file_id)  # py2: type unicode, py3: type str

        array['width'] = int(self.width)  # type int
        array['height'] = int(self.height)  # type int
        if self.thumb is not None:
            array['thumb'] = self.thumb.to_array()  # type PhotoSize

        if self.emoji is not None:
            array['emoji'] = u(self.emoji)  # py2: type unicode, py3: type str

        if self.set_name is not None:
            array['set_name'] = u(self.set_name)  # py2: type unicode, py3: type str

        if self.mask_position is not None:
            array['mask_position'] = self.mask_position.to_array()  # type MaskPosition

        if self.file_size is not None:
            array['file_size'] = int(self.file_size)  # type int
        return array
765173
def replace_emoji(text)
      text.gsub(/:([^\s:])+:/) do |emoji|

        emoji_code = emoji #.gsub("|", "_")
        emoji      = emoji_code.gsub(":", "")

        if MdEmoji::EMOJI.include?(emoji)
          file_name    = "#{emoji.gsub('+', 'plus')}.png"
          default_size = %{height="20" width="20"}

          %{<img src="/assets/emojis/#{file_name}" class="emoji" } +
            %{title="#{emoji_code}" alt="#{emoji_code}" #{default_size}>}
        else
          emoji_code
        end
      end
    end
1140266
def _decode_embedded_dict(src):
    '''
    
    '''
    output = {}
    for key, val in six.iteritems(src):
        if isinstance(val, dict):
            val = _decode_embedded_dict(val)
        elif isinstance(val, list):
            val = _decode_embedded_list(val)  # pylint: disable=redefined-variable-type
        elif isinstance(val, bytes):
            try:
                val = val.decode()
            except UnicodeError:
                pass
        if isinstance(key, bytes):
            try:
                key = key.decode()
            except UnicodeError:
                pass
        output[key] = val
    return output
199386
def dict(self):
    ''''''
    return {
        'raw': self.raw,
        'start': self.start,
        'end': self.end,
        'entities': [(e.tag, e) for e in self.entities],
        'tokens': self.tokens,
        'words': self.words,
        'pos_tags': self.pos_tags,
        'language': self.language.code,
        # TODO: 'polarity': self.polarity,
    }
322008
def emoji_list(server, n=1):
    """"""
    global EMOJI
    if EMOJI is None:
        EMOJI = EmojiCache(server)
    return EMOJI.get(n)
527609
def _to_dict(self):
        """"""
        _dict = {}
        if hasattr(self, 'limit') and self.limit is not None:
            _dict['limit'] = self.limit
        if hasattr(self, 'mentions') and self.mentions is not None:
            _dict['mentions'] = self.mentions
        if hasattr(self, 'model') and self.model is not None:
            _dict['model'] = self.model
        if hasattr(self, 'sentiment') and self.sentiment is not None:
            _dict['sentiment'] = self.sentiment
        if hasattr(self, 'emotion') and self.emotion is not None:
            _dict['emotion'] = self.emotion
        return _dict
417448
def _to_dict(self):
        """"""
        _dict = {}
        if hasattr(self, 'text') and self.text is not None:
            _dict['text'] = self.text
        if hasattr(self, 'type') and self.type is not None:
            _dict['type'] = self.type
        if hasattr(self, 'evidence') and self.evidence is not None:
            _dict['evidence'] = [x._to_dict() for x in self.evidence]
        return _dict
316873
def _convert_to_lexicon(self, record):
        

        name = record['Name']
        if self.domain not in name:
            name = "{}.{}".format(name, self.domain)

        processed_record = {
            'type': record['Type'],
            'name': '{0}.{1}'.format(record['Name'], self.domain),
            'ttl': record['TTL'],
            'content': record['Address'],
            'id': record['HostId']
        }

        return processed_record
523971
def _to_dict(self):
        """"""
        _dict = {}
        if hasattr(self, 'word') and self.word is not None:
            _dict['word'] = self.word
        if hasattr(self, 'sounds_like') and self.sounds_like is not None:
            _dict['sounds_like'] = self.sounds_like
        if hasattr(self, 'display_as') and self.display_as is not None:
            _dict['display_as'] = self.display_as
        if hasattr(self, 'count') and self.count is not None:
            _dict['count'] = self.count
        if hasattr(self, 'source') and self.source is not None:
            _dict['source'] = self.source
        if hasattr(self, 'error') and self.error is not None:
            _dict['error'] = [x._to_dict() for x in self.error]
        return _dict