Latent Sentiment Analyis on random sampling of twitter search results for words listed in cased_tokens
962515
def lsa_twitter(cased_tokens):
    

    # Only 5 of these tokens are saved for a no_below=2 filter:
    #   PyCons NLPS #PyCon2016 #NaturalLanguageProcessing #naturallanguageprocessing
    if cased_tokens is None:
        cased_tokens = ('PyConOpenSpaces PyCon PyCon2017 PyCon2018 PyCon2016 PyCon2015 OpenSpace PyconTutorial ' +
                        'NLP NaturalLanguageProcessing NLPInAction NaturalLanguageProcessingInAction NLPIA Twote Twip'
                        ).split()
        cased_tokens += [s + 's' for s in cased_tokens]

        cased_tokens += 'TotalGood TotalGoods HobsonLane Hob Hobs TotalGood.com ' \
                        'www.TotalGood.com http://www.TotalGood.com https://www.TotalGood.com'.split()

    allcase_tokens = cased_tokens + [s.lower() for s in cased_tokens]
    allcase_tokens += [s.title() for s in cased_tokens]
    allcase_tokens += [s.upper() for s in cased_tokens]
    KEEP_TOKENS = allcase_tokens + ['#' + s for s in allcase_tokens]

    # takes 15 minutes and 10GB of RAM for 500k tweets if you keep all 20M unique tokens/names URLs
    vocab_path = os.path.join(BIGDATA_PATH, 'vocab939370.pkl')
    if os.path.isfile(vocab_path):
        print('Loading vocab: {} ...'.format(vocab_path))
        vocab = Dictionary.load(vocab_path)
        print(' len(vocab) loaded: {}'.format(len(vocab.dfs)))
    else:
        tweets_path = os.path.join(BIGDATA_PATH, 'tweets.csv.gz')
        print('Loading tweets: {} ...'.format(tweets_path))
        tweets = read_csv(tweets_path)
        tweets = pd.np.array(tweets.text.str.split())
        with gzip.open(os.path.join(BIGDATA_PATH, 'tweets.txt.gz'), 'w') as f:
            for tokens in tweets:
                f.write((' '.join(tokens) + '\n').encode('utf-8'))
        # tweets['text'] = tweets.text.apply(lambda s: eval(s).decode('utf-8'))
        # tweets['user'] = tweets.user.apply(lambda s: eval(s).decode('utf-8'))
        # tweets.to_csv('tweets.csv.gz', compression='gzip')
        print('Computing vocab from {} tweets...'.format(len(tweets)))
        vocab = Dictionary(tweets, no_below=NO_BELOW, no_above=NO_ABOVE, keep_tokens=set(KEEP_TOKENS))

    vocab.filter_extremes(no_below=NO_BELOW, no_above=NO_ABOVE, keep_n=KEEP_N, keep_tokens=set(KEEP_TOKENS))
    print(' len(vocab) after filtering: {}'.format(len(vocab.dfs)))

    # no time at all, just a bookeeping step, doesn't actually compute anything
    tfidf = TfidfModel(id2word=vocab, dictionary=vocab)
    tfidf.save(os.path.join(BIGDATA_PATH, 'tfidf{}.pkl'.format(len(vocab.dfs))))

    tweets = [vocab.doc2bow(tw) for tw in tweets]
    json.dump(tweets, gzip.open(os.path.join(BIGDATA_PATH, 'tweet_bows.json.gz'), 'w'))

    gc.collect()

    # LSA is more useful name than LSA
    lsa = LsiModel(tfidf[tweets], num_topics=200, id2word=vocab, extra_samples=100, power_iters=2)

    return lsa
533689
def _filter(self, text, context, encoding):
        """"""

        docstrings = []
        strings = []
        comments = []
        prev_token_type = tokenize.NEWLINE
        indent = ''
        name = None
        stack = [(context, 0, self.MODULE)]
        last_comment = False
        possible_fmt_str = None

        src = io.StringIO(text)

        for token in tokenizer(src.readline):
            token_type = token[0]
            value = token[1]
            line = str(token[2][0])
            line_num = token[2][0]

            value = token[1]
            line = str(token[2][0])

            # Track function and class ancestry
            if token_type == tokenize.NAME:
                possible_fmt_str = None
                if value in ('def', 'class'):
                    name = value
                elif name:
                    parent = stack[-1][2]
                    prefix = ''
                    if parent != self.MODULE:
                        prefix = '.' if parent == self.CLASS else ', '
                    if name == 'class':
                        stack.append(('%s%s' % (prefix, value), len(indent), self.CLASS))
                    elif name == 'def':
                        stack.append(('%s%s()' % (prefix, value), len(indent), self.FUNCTION))
                    name = None
                elif not F_SUPPORT and value in FMT_STR:
                    possible_fmt_str = (prev_token_type, value)
            elif token_type != tokenize.STRING:
                possible_fmt_str = None

            if token_type == tokenize.COMMENT and self.comments:
                # Capture comments
                if (
                    self.group_comments and
                    last_comment and
                    comments and (comments[-1][2] + 1) == line_num
                ):
                    # Group multiple consecutive comments
                    comments[-1][0] += '\n' + value[1:]
                    comments[-1][2] = line_num
                else:
                    if len(stack) > 1:
                        loc = "%s(%s): %s" % (stack[0][0], line, ''.join([crumb[0] for crumb in stack[1:]]))
                    else:
                        loc = "%s(%s)" % (stack[0][0], line)
                    comments.append([value[1:], loc, line_num])
            if token_type == tokenize.STRING:
                # Capture docstrings.
                # If we captured an `INDENT` or `NEWLINE` previously we probably have a docstring.
                # `NL` means end of line, but not the end of the Python code line (line continuation).
                if (
                    (prev_token_type in PREV_DOC_TOKENS) or
                    (possible_fmt_str and possible_fmt_str[0] in PREV_DOC_TOKENS)
                ):
                    if self.docstrings:
                        value = value.strip()
                        if possible_fmt_str and value.startswith(("'", "\"")):
                            value = possible_fmt_str[1] + value
                        string, is_bytes = self.process_strings(value, docstrings=True)
                        if string:
                            loc = "%s(%s): %s" % (stack[0][0], line, ''.join([crumb[0] for crumb in stack[1:]]))
                            docstrings.append(
                                filters.SourceText(string, loc, 'utf-8', 'py-docstring')
                            )
                elif self.strings:
                    value = value.strip()
                    if possible_fmt_str and value.startswith(("'", "\"")):
                        value = possible_fmt_str[1] + value
                    string, is_bytes = self.process_strings(value)
                    if string:
                        loc = "%s(%s): %s" % (stack[0][0], line, ''.join([crumb[0] for crumb in stack[1:]]))
                        strings.append(filters.SourceText(string, loc, 'utf-8', 'py-string'))

            if token_type == tokenize.INDENT:
                indent = value
            elif token_type == tokenize.DEDENT:
                indent = indent[:-4]
                if len(stack) > 1 and len(indent) <= stack[-1][1]:
                    stack.pop()

            # We purposefully avoid storing comments as comments can come before docstrings,
            # and that can mess up our logic. So if the token is a comment we won't track it,
            # and comments are always followed with `NL` so we ignore that as well.
            # We only care that docstrings are preceded by `NEWLINE`.
            if token_type != tokenize.COMMENT and not (last_comment and token_type == tokenize.NL):
                prev_token_type = token_type
                last_comment = False
            else:
                last_comment = True

        final_comments = []
        for comment in comments:
            final_comments.append(filters.SourceText(textwrap.dedent(comment[0]), comment[1], encoding, 'py-comment'))

        return docstrings + final_comments + strings
576645
def search(self, q, max_id=None, since_id=None, lang=None,
               result_type='recent', geocode=None, max_pages=None):
        
        url = "https://api.twitter.com/1.1/search/tweets.json"
        params = {
            "count": 100,
            "q": q,
            "include_ext_alt_text": 'true'
        }
        if lang is not None:
            params['lang'] = lang
        if result_type in ['mixed', 'recent', 'popular']:
            params['result_type'] = result_type
        else:
            params['result_type'] = 'recent'
        if geocode is not None:
            params['geocode'] = geocode

        retrieved_pages = 0
        reached_end = False

        while True:
            if since_id:
                # Make the since_id inclusive, so we can avoid retrieving
                # an empty page of results in some cases
                params['since_id'] = str(int(since_id) - 1)
            if max_id:
                params['max_id'] = max_id

            resp = self.get(url, params=params)
            retrieved_pages += 1

            statuses = resp.json()["statuses"]

            if len(statuses) == 0:
                log.info("no new tweets matching %s", params)
                break

            for status in statuses:
                # We've certainly reached the end of new results
                if since_id is not None and status['id_str'] == str(since_id):
                    reached_end = True
                    break

                yield status

            if reached_end:
                log.info("no new tweets matching %s", params)
                break

            if max_pages is not None and retrieved_pages == max_pages:
                log.info("reached max page limit for %s", params)
                break

            max_id = str(int(status["id_str"]) - 1)
70198
function getKeyphrases(results, maximum) {
  var stemmedPhrases = {}
  var initialWords = []
  var stemmedPhrase
  var index
  var length
  var otherIndex
  var keyword
  var matches
  var phrase
  var stems
  var score
  var first
  var match

  // Iterate over all grouped important words...
  for (keyword in results) {
    matches = results[keyword].matches
    length = matches.length
    index = -1

    // Iterate over every occurence of a certain keyword...
    while (++index < length) {
      phrase = findPhrase(matches[index])
      stemmedPhrase = stemmedPhrases[phrase.value]
      first = phrase.nodes[0]

      match = {
        nodes: phrase.nodes,
        parent: matches[index].parent
      }

      // If we've detected the same stemmed phrase somewhere.
      if (own.call(stemmedPhrases, phrase.value)) {
        // Add weight per phrase to the score of the phrase.
        stemmedPhrase.score += stemmedPhrase.weight

        // If this is the first time we walk over the phrase (exact match but
        // containing another important word), add it to the list of matching
        // phrases.
        if (initialWords.indexOf(first) === -1) {
          initialWords.push(first)
          stemmedPhrase.matches.push(match)
        }
      } else {
        otherIndex = -1
        score = -1
        stems = phrase.stems

        initialWords.push(first)

        // For every stem in phrase, add its score to score.
        while (stems[++otherIndex]) {
          score += results[stems[otherIndex]].score
        }

        stemmedPhrases[phrase.value] = {
          score: score,
          weight: score,
          stems: stems,
          value: phrase.value,
          matches: [match]
        }
      }
    }
  }

  for (stemmedPhrase in stemmedPhrases) {
    phrase = stemmedPhrases[stemmedPhrase]

    // Modify its score to be the rounded result of multiplying it with the
    // number of occurances, and dividing it by the ammount of words in the
    // phrase.
    phrase.score = Math.round(
      (phrase.score * phrase.matches.length) / phrase.stems.length
    )
  }

  return filterResults(stemmedPhrases, maximum)
}
835510
function Brain(dataset) {
  // The root of the tree represents the start of the query.
  this.root = new Node(this)

  // List of token matchers: functions from tokens
  // (see src/tokenizers/tokenize.js)
  // to either nothing (if no match) or {tag, length, data}.
  this.tokenMatchers = tokenize.defaultTokenMatchers()

  // Number of examples per label.
  this.numExamples = new Map()
  // Are weights built?
  this.weightsNeedBuilding = true
  this.links = []

  this.learn = this.learn.bind(this)
  if (dataset != null) {
    dataset.forEach(this.learn)
  }
}
41595
def cat_tweets(filename='all_tweets.json.gz', path=DATA_PATH, ext='.json', save_tmp=False, verbosity=1, numtweets=10000000, ignore_suspicious=True):
    """"""
    log.info('Finding {} files in {}...'.format(ext, path))
    meta_files = find_files(path=path, ext=ext)
    meta_files = [meta for meta in meta_files
                  if re.match(r'^[-#@a-z ]*201[5-6]-[0-9]{2}-[0-9]{2}.*', meta['name'])]
    #  '\s[0-9]{2}[:][0-9]{2}[:][0-9]{2}[.][0-9]+[.]json(.gz)?$', meta['name'])]
    log.info('Found {} files that look like tweetget dumps.'.format(len(meta_files)))
    print([mf['name'] for mf in meta_files])
    total_size = sum([meta['size'] for meta in meta_files])
    if verbosity > 0:
        pbar = progressbar.ProgressBar(maxval=(total_size + 1.) / 1e6)
        pbar.start()
    else:
        pbar = None

    loaded_size = 0
    df_all = pd.DataFrame()
    for meta in meta_files:
        with (gzip.open(meta['path']) if ext.endswith('.gz') else open(meta['path'])) as fin:
            js = pd.json.load(fin)
        if not len(js):
            continue
        df = pd.io.json.json_normalize(js)
        # json entries were dumped in reverse time order (most recent first)
        df.drop_duplicates(['id'], keep='first', inplace=True)
        df.set_index('id', drop=True, inplace=True)
        if 'geo.coordinates' in df.columns:
            latlon = np.array([(ll[0], ll[1]) if isinstance(ll, list) else (np.nan, np.nan) for ll in df['geo.coordinates']])
            for i, ll in enumerate(latlon):
                try:
                    latlon[i] = float(ll[0]), float(ll[1])
                except ValueError:
                    latlon[i] = np.nan, np.nan
            ll = list(zip(*latlon))
            df['lat'] = ll[0]
            df['lon'] = ll[1]
            df_all = df_all.append(df)
        else:
            df['lat'] = np.nan * np.ones(len(df))
            df['lon'] = np.nan * np.ones(len(df))
            if ignore_suspicious:
                log.info('\nOddly the DataFrame in {} didnt have a geo.coordinates column.'.format(meta['path']))
                log.warn('\nSkipping {} suspicious tweets.'.format(len(df)))
            else:
                df_all = df_all.append(df)
        # this would be a good time to incrementally save these rows to disc
        del df
        loaded_size += meta['size']
        if len(df_all) >= numtweets:
            # FIXME use average rate of 400 tweets/MB to calculate better progressbar size at start
            break
        if save_tmp:
            save_tweets(df_all, path=path, filename='tmp.csv')
        if pbar:
            pbar.update(loaded_size / 1e6)
    print(len(df_all))
    bigger_better_cols = [c for c in df_all.columns if c.endswith('_at') or '_count' in c]
    print(bigger_better_cols)
    print(all(c in df_all.columns for c in bigger_better_cols))
    df_all = df_all.sort_values(by=bigger_better_cols, inplace=False)
    hashable_cols = [c for c in df_all.columns if df_all[c].dtype not in (list,)]
    df_all = df_all.drop_duplicates(subset=hashable_cols, inplace=False)
    if pbar:
        pbar.finish()
    log.info('Loaded {} unique tweets.'.format(len(df_all)))
    return df_all
185592
function createWordIndex(exampleData) {
  const index = {};
  const keys = ['shortdesc', 'title', 'tags'];
  exampleData.forEach((data, i) => {
    keys.forEach(key => {
      let text = data[key];
      if (Array.isArray(text)) {
        text = text.join(' ');
      }
      const words = text ? text.split(/\W+/) : [];
      words.forEach(word => {
        if (word) {
          word = word.toLowerCase();
          let counts = index[word];
          if (counts) {
            if (index in counts) {
              counts[i] += 1;
            } else {
              counts[i] = 1;
            }
          } else {
            counts = {};
            counts[i] = 1;
            index[word] = counts;
          }
        }
      });
    });
  });
  return index;
}
612431
def analysis(self):
        """"""
        if not self.is_tagged(ANALYSIS):
            self.tag_analysis()
        return [word[ANALYSIS] for word in self.words]
967257
def feature_detector(self, tokens, index, history):
        

        word = tokens[index]
        if index == 0: # At the beginning of the sentence
            prevword = prevprevword = None
            prevtag = prevprevtag = None
            #word = word.lower() # Lowercase at the beginning of sentence
        elif index == 1:
            prevword = tokens[index-1] # Note: no lowercase
            prevprevword = None
            prevtag = history[index-1]
            prevprevtag = None
        else:
            prevword = tokens[index-1]
            prevprevword = tokens[index-2]
            prevtag = history[index-1]
            prevprevtag = history[index-2]

        if re.match('[0-9]+([\.,][0-9]*)?|[0-9]*[\.,][0-9]+$', word):
            # Included "," as decimal point
            shape = 'number'
        elif re.compile('\W+$', re.UNICODE).match(word):
            # Included unicode flag
            shape = 'punct'
        elif re.match('([A-ZÄÖÜ]+[a-zäöüß]*-?)+$', word):
            # Included dash for dashed words and umlauts
            shape = 'upcase'
        elif re.match('[a-zäöüß]+', word):
            # Included umlauts
            shape = 'downcase'
        elif re.compile("\w+", re.UNICODE).match(word):
            # Included unicode flag
            shape = 'mixedcase'
        else:
            shape = 'other'

        features = {
            'prevtag': prevtag,
            'prevprevtag': prevprevtag,
            'word': word,
            'word.lower': word.lower(),
            'suffix3': word.lower()[-3:],
            #'suffix2': word.lower()[-2:],
            #'suffix1': word.lower()[-1:],
            'preffix1': word[:1], # included
            'prevprevword': prevprevword,
            'prevword': prevword,
            'prevtag+word': '%s+%s' % (prevtag, word),
            'prevprevtag+word': '%s+%s' % (prevprevtag, word),
            'prevword+word': '%s+%s' % (prevword, word),
            'shape': shape
            }
        return features
262561
def get_analogy_task_tokens(args):
    """"""
    tokens = set()
    for _, _, dataset in iterate_analogy_datasets(args):
        tokens.update(
            itertools.chain.from_iterable(
                (d[0], d[1], d[2], d[3]) for d in dataset))
    return tokens
724623
function Tokenizer(str, tags) {
            this.str = str;
            this.index = 0;
            this.len = str.length;
            this.lineno = 0;
            this.colno = 0;

            this.in_code = false;

            tags = tags || {};
            this.tags = {
                BLOCK_START: tags.blockStart || BLOCK_START,
                BLOCK_END: tags.blockEnd || BLOCK_END,
                VARIABLE_START: tags.variableStart || VARIABLE_START,
                VARIABLE_END: tags.variableEnd || VARIABLE_END,
                COMMENT_START: tags.commentStart || COMMENT_START,
                COMMENT_END: tags.commentEnd || COMMENT_END
            };
        }
1316904
def tweets_for(query_type, args, per_user=None):
    
    lookup = {"query_type": query_type, "value": args[0]}
    try:
        tweets = Tweet.objects.get_for(**lookup)
    except TwitterQueryException:
        return []
    if per_user is not None:
        _tweets = defaultdict(list)
        for tweet in tweets:
            if len(_tweets[tweet.user_name]) < per_user:
                _tweets[tweet.user_name].append(tweet)
        tweets = sum(_tweets.values(), [])
        tweets.sort(key=lambda t: t.created_at, reverse=True)
    if len(args) > 1 and str(args[-1]).isdigit():
        tweets = tweets[:int(args[-1])]
    return tweets
1068344
def vectorize(self, token_list):
        '''
        
        '''
        sentence_list = [token_list]
        test_observed_arr = self.__setup_dataset(sentence_list, self.__token_master_list)
        pred_arr = self.__controller.inference(test_observed_arr)
        return self.__controller.get_feature_points()
158809
def _tokenize_wkt(tokens):
    
    negative = False
    for t in tokens:
        if t == '-':
            negative = True
            continue
        else:
            if negative:
                yield '-%s' % t
            else:
                yield t
            negative = False
707322
def find_tracked_words(self, tokens):
        
        tracked_words = []

        for i in range(len(tokens)):
            for j in range(i + 1, len(tokens) + 1):
                phrase = tokens[i:j]
                status = self.has_tokens(phrase)

                if status is not None:
                    if status is True:
                        tracked_words.append(TokenTrie.Token(phrase, i, j - 1))
                    elif status is False:
                        break

        return tracked_words
1122112
def detect_intent_with_sentiment_analysis(project_id, session_id, texts,
                                          language_code):
    """"""
    import dialogflow_v2beta1 as dialogflow
    session_client = dialogflow.SessionsClient()

    session_path = session_client.session_path(project_id, session_id)
    print('Session path: {}\n'.format(session_path))

    for text in texts:
        text_input = dialogflow.types.TextInput(
            text=text, language_code=language_code)

        query_input = dialogflow.types.QueryInput(text=text_input)

        # Enable sentiment analysis
        sentiment_config = dialogflow.types.SentimentAnalysisRequestConfig(
            analyze_query_text_sentiment=True)

        # Set the query parameters with sentiment analysis
        query_params = dialogflow.types.QueryParameters(
            sentiment_analysis_request_config=sentiment_config)

        response = session_client.detect_intent(
            session=session_path, query_input=query_input,
            query_params=query_params)

        print('=' * 20)
        print('Query text: {}'.format(response.query_result.query_text))
        print('Detected intent: {} (confidence: {})\n'.format(
            response.query_result.intent.display_name,
            response.query_result.intent_detection_confidence))
        print('Fulfillment text: {}\n'.format(
            response.query_result.fulfillment_text))
        # Score between -1.0 (negative sentiment) and 1.0 (positive sentiment).
        print('Query Text Sentiment Score: {}\n'.format(
            response.query_result.sentiment_analysis_result
            .query_text_sentiment.score))
        print('Query Text Sentiment Magnitude: {}\n'.format(
            response.query_result.sentiment_analysis_result
            .query_text_sentiment.magnitude))
625682
def get_strings_from_utterance(tokenized_utterance: List[Token]) -> Dict[str, List[int]]:
    
    string_linking_scores: Dict[str, List[int]] = defaultdict(list)

    for index, token in enumerate(tokenized_utterance):
        for string in ATIS_TRIGGER_DICT.get(token.text.lower(), []):
            string_linking_scores[string].append(index)

    token_bigrams = bigrams([token.text for token in tokenized_utterance])
    for index, token_bigram in enumerate(token_bigrams):
        for string in ATIS_TRIGGER_DICT.get(' '.join(token_bigram).lower(), []):
            string_linking_scores[string].extend([index,
                                                  index + 1])

    trigrams = ngrams([token.text for token in tokenized_utterance], 3)
    for index, trigram in enumerate(trigrams):
        if trigram[0] == 'st':
            natural_language_key = f'st. {trigram[2]}'.lower()
        else:
            natural_language_key = ' '.join(trigram).lower()
        for string in ATIS_TRIGGER_DICT.get(natural_language_key, []):
            string_linking_scores[string].extend([index,
                                                  index + 1,
                                                  index + 2])
    return string_linking_scores