Tokenizes lookup fields and returns all matching buckets in the
index.
547513
private List<Bucket> lookup(Record record) {
    List<Bucket> buckets = new ArrayList();
    for (Property p : config.getLookupProperties()) {
      String propname = p.getName();
      Collection<String> values = record.getValues(propname);
      if (values == null)
        continue;

      for (String value : values) {
        String[] tokens = StringUtils.split(value);
        for (int ix = 0; ix < tokens.length; ix++) {
          Bucket b = store.lookupToken(propname, tokens[ix]);
          if (b == null || b.records == null)
            continue;
          long[] ids = b.records;
          if (DEBUG)
            System.out.println(propname + ", " + tokens[ix] + ": " + b.nextfree + " (" + b.getScore() + ")");
          buckets.add(b);
        }
      }
    }

    return buckets;
  }
1244596
private int getFieldIndexes(ISyncHandler<?, ? extends ISyncableData> handler, String... syncNames)
	{
		int indexes = 0;
		for (String str : syncNames)
		{
			ObjectData od = handler.getObjectData(str);
			if (od != null)
				indexes |= 1 << od.getIndex();
		}
		return indexes;
	}
922144
function(fieldOrSpec) {
  var fieldHash = {};
  var indexes = [];
  var keys;

  // Get all the fields accordingly
  if ('string' === typeof fieldOrSpec) {
    // 'type'
    indexes.push(fieldOrSpec + '_' + 1);
    fieldHash[fieldOrSpec] = 1;
  } else if (Array.isArray(fieldOrSpec)) {
    fieldOrSpec.forEach(function(f) {
      if ('string' === typeof f) {
        // [{location:'2d'}, 'type']
        indexes.push(f + '_' + 1);
        fieldHash[f] = 1;
      } else if (Array.isArray(f)) {
        // [['location', '2d'],['type', 1]]
        indexes.push(f[0] + '_' + (f[1] || 1));
        fieldHash[f[0]] = f[1] || 1;
      } else if (isObject(f)) {
        // [{location:'2d'}, {type:1}]
        keys = Object.keys(f);
        keys.forEach(function(k) {
          indexes.push(k + '_' + f[k]);
          fieldHash[k] = f[k];
        });
      } else {
        // undefined (ignore)
      }
    });
  } else if (isObject(fieldOrSpec)) {
    // {location:'2d', type:1}
    keys = Object.keys(fieldOrSpec);
    keys.forEach(function(key) {
      indexes.push(key + '_' + fieldOrSpec[key]);
      fieldHash[key] = fieldOrSpec[key];
    });
  }

  return {
    name: indexes.join('_'),
    keys: keys,
    fieldHash: fieldHash
  };
}
1348670
def [](lookup)
      if lookup.is_a?(Integer)
        children.fetch(lookup)
      else
        found = find_all(lookup)
        case found.size
        when 1
          found.first
        when 0
          raise "No children of type #{lookup}"
        else
          raise "Ambiguous lookup #{lookup}, found #{found}."
        end
      end
    end
522008
def regex_matches_to_indexed_words(matches):
    
    words = [dict(v, **dict([('sentence', i)]))
             for i, s in enumerate(matches['sentences'])
             for k, v in s.items() if k != 'length']
    return words
66047
def list_objects(Bucket, Delimiter=None, EncodingType=None, Prefix=None,
                 FetchOwner=False, StartAfter=None, region=None, key=None,
                 keyid=None, profile=None):
    '''
    

    '''

    try:
        Contents = []
        args = {'Bucket': Bucket, 'FetchOwner': FetchOwner}
        args.update({'Delimiter': Delimiter}) if Delimiter else None
        args.update({'EncodingType': EncodingType}) if Delimiter else None
        args.update({'Prefix': Prefix}) if Prefix else None
        args.update({'StartAfter': StartAfter}) if StartAfter else None
        conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)
        IsTruncated = True
        while IsTruncated:
            ret = conn.list_objects_v2(**args)
            IsTruncated = ret.get('IsTruncated', False)
            if IsTruncated in ('True', 'true', True):
                args['ContinuationToken'] = ret['NextContinuationToken']
            Contents += ret.get('Contents', [])
        return {'Contents': Contents}
    except ClientError as e:
        return {'error': __utils__['boto3.get_error'](e)}
610520
Map<String, Integer> getPrefixesIds(String field, List<String> prefixes) {
    LinkedHashMap<String, Long> refs = getPrefixRefs(field);
    if (refs != null) {
      List<String> list = new ArrayList<>(refs.keySet());
      Map<String, Integer> result = new HashMap<>();
      for (String prefix : prefixes) {
        int id = list.indexOf(prefix);
        if (id >= 0) {
          result.put(prefix, id + 1);
        }
      }
      return result;
    } else {
      return null;
    }
  }
151516
private static Map<GroupHit, Spans> collectSpansForOccurences(
      Set<GroupHit> occurences, Set<String> prefixes, String field,
      IndexSearcher searcher, LeafReaderContext lrc) throws IOException {
    Map<GroupHit, Spans> list = new HashMap<>();
    IndexReader reader = searcher.getIndexReader();
    final float boost = 0;
    for (GroupHit hit : occurences) {
      MtasSpanQuery queryHit = createQueryFromGroupHit(prefixes, field, hit);
      if (queryHit != null) {
        MtasSpanQuery queryHitRewritten = queryHit.rewrite(reader);
        SpanWeight weight = queryHitRewritten.createWeight(searcher, false,
            boost);
        Spans spans = weight.getSpans(lrc, SpanWeight.Postings.POSITIONS);
        if (spans != null) {
          list.put(hit, spans);
        }
      }
    }
    return list;
  }
401599
Map<String, Double> findMatchQueries(
      String matchedWordsString, Map<String, String> collectExpandedQueryMap) {
    Map<String, Double> qualifiedQueries = new HashMap<>();
    Set<String> matchedWords = splitIntoTerms(matchedWordsString);
    for (Entry<String, String> entry : collectExpandedQueryMap.entrySet()) {
      Set<String> wordsInQuery = splitIntoTerms(entry.getKey());
      if (wordsInQuery.containsAll(matchedWords)) {
        qualifiedQueries.put(
            entry.getKey(),
            NGramDistanceAlgorithm.stringMatching(matchedWordsString, entry.getKey()));
      }
    }
    return qualifiedQueries;
  }
1365554
def _get_subset_matches(self, query):
        
        for field_name, db_index in self._get_db_fields(query):
            attr = getattr(query, field_name)

            if attr is None:  # don't use unset attributes
                continue

            results = db_index.get(attr, OOTreeSet())

            if results:
                yield results
1368635
def versioned_bucket_lister(bucket, prefix='', delimiter='',
                            key_marker='', version_id_marker='', headers=None):
    
    more_results = True
    k = None
    while more_results:
        rs = bucket.get_all_versions(prefix=prefix, key_marker=key_marker,
                                     version_id_marker=version_id_marker,
                                     delimiter=delimiter, headers=headers,
                                     max_keys=999)
        for k in rs:
            yield k
        key_marker = rs.next_key_marker
        version_id_marker = rs.next_version_id_marker
        more_results= rs.is_truncated
416290
function getFields() {
                return storage.headObject({
                  Bucket: bucket,
                  Key: storageObject.Key
                })
                  .then(function(storageObjectHead) {

                    return {
                      name: path.parse(storageObject.Key).name,
                      location: storageObject.Key,
                      size: storageObjectHead.ContentLength,
                      contentType: storageObjectHead.ContentType,
                      type: services.helpers.mediaType(storageObjectHead.ContentType)
                    };
                  });
              }
823424
private Terms filterBuckets(Terms buckets, KunderaQuery query)
    {
        Expression havingClause = query.getSelectStatement().getHavingClause();

        if (!(havingClause instanceof NullExpression) && havingClause != null)
        {
            Expression conditionalExpression = ((HavingClause) havingClause).getConditionalExpression();

            for (Iterator<Bucket> bucketIterator = buckets.getBuckets().iterator(); bucketIterator.hasNext();)
            {
                InternalAggregations internalAgg = (InternalAggregations) bucketIterator.next().getAggregations();
                if (!isValidBucket(internalAgg, query, conditionalExpression))
                {
                    bucketIterator.remove();
                }
            }
        }
        return buckets;
    }
27333
def get_all_accounts(bucket, region='us-west-2', json_path='accounts.json', **filters):
    """"""
    swag_opts = {
        'swag.type': 's3',
        'swag.bucket_name': bucket,
        'swag.bucket_region': region,
        'swag.data_file': json_path,
        'swag.schema_version': 1
    }

    swag = SWAGManager(**parse_swag_config_options(swag_opts))
    accounts = swag.get_all()
    accounts = [account for account in accounts['accounts'] if is_sub_dict(filters, account)]
    return {'accounts': accounts}
1227801
public List<String> getBioDictionaryMatch(final String[] tokens) {

    final List<String> entitiesList = new ArrayList<String>();

    String prefix = "-" + BioCodec.START;
    String gazEntry = null;
    String searchSpan = null;
    // iterative over tokens from the beginning
    for (int i = 0; i < tokens.length; i++) {
      gazEntry = null;
      int j;
      // iterate over tokens from the end
      for (j = tokens.length - 1; j >= i; j--) {
        // create span for search in dictionary Map; the first search takes as
        // span
        // the whole sentence
        searchSpan = createSpan(tokens, i, j);
        gazEntry = lookup(searchSpan.toLowerCase());
        if (gazEntry != null) {
          break;
        }
      }
      prefix = "-" + BioCodec.START;
      // multi-token case
      if (gazEntry != null) {
        while (i < j) {
          entitiesList.add((gazEntry + prefix).intern());
          prefix = "-" + BioCodec.CONTINUE;
          i++;
        }
      }
      // one word case or last member of span
      if (gazEntry != null) {
        entitiesList.add((gazEntry + prefix).intern());
      } else {
        entitiesList.add(BioCodec.OTHER);
      }
    }
    return entitiesList;
  }
1305674
function matcherFromTokens( tokens, context, xml ) {
	var token, matcher,
		i = 0;

	for ( ; (token = tokens[i]); i++ ) {
		if ( Expr.relative[ token.part ] ) {
			matcher = addCombinator( matcher, Expr.relative[ token.part ], context );
		} else {
			token.captures.push( context, xml );
			matcher = addMatcher( matcher, Expr.filter[ token.part ].apply( null, token.captures ) );
		}
	}

	return matcher;
}
437487
def start_indexing(self):
        
        for filepath in self.filepaths:
            with open(filepath) as fp:
                blob = fp.read()
                self.words.extend(self.tokenize(blob))