/*
Each record contains a field with a geohash value
485678
function (record, index) {
		var geohash = this.options.geohashField ? L.Util.getFieldValue(record, this.options.geohashField) : index;
		var locationInfo = GeoHash.decodeGeoHash(geohash);
		var bounds;

		if (locationInfo.latitude[2] && locationInfo.longitude[2]) {
			bounds = new L.LatLngBounds(new L.LatLng(locationInfo.latitude[0], locationInfo.longitude[0]), new L.LatLng(locationInfo.latitude[1], locationInfo.longitude[1]));
		}

		return {
			location: bounds,
			text: geohash,
			center: bounds.getCenter()
		};
	}
245808
public Collection<Record> findCandidateMatches(Record record) {
    if (directory == null)
      init();

    // if we have a geoprop it means that's the only way to search
    if (geoprop != null) {
      String value = record.getValue(geoprop.getName());
      if (value != null) {
        Filter filter = geoprop.geoSearch(value);
        return maintracker.doQuery(new MatchAllDocsQuery(), filter);
      }
    }

    // ok, we didn't do a geosearch, so proceed as normal.
    // first we build the combined query for all lookup properties
    BooleanQuery query = new BooleanQuery();
    for (Property prop : config.getLookupProperties()) {
      Collection<String> values = record.getValues(prop.getName());
      if (values == null)
        continue;
      for (String value : values)
        parseTokens(query, prop.getName(), value,
                    prop.getLookupBehaviour() == Property.Lookup.REQUIRED,
                    prop.getHighProbability());
    }

    // do the query
    return maintracker.doQuery(query);
  }

  /**
   * Stores state to disk and closes all open resources.
   */
  public void close() {
    if (directory == null)
      return;

    try {
      iwriter.close();
      directory.close();
      if (reader != null)
        reader.close();
    } catch (IOException e) {
      throw new DukeException(e);
    }
  }

  public String toString() {
    return "LuceneDatabase, max-search-hits: " + max_search_hits +
      ", min-relevance: " + min_relevance + ", fuzzy: " + fuzzy_search +
      ", boost-mode: " + boost_mode + ", path: " + path + "\n  " + directory;
  }

  // ----- INTERNALS

  private void init() {
    try {
      openIndexes(overwrite);
      openSearchers();
      initSpatial();
    } catch (Exception e) {
      // initialization failed, so clean up to prevent leaving us in an
      // inconsistent state https://github.com/larsga/Duke/issues/226
      directory = null;
      throw new DukeException(e);
    }
  }

  private void openIndexes(boolean overwrite) throws IOException {
    if (directory == null) {
      try {
        if (path == null)
          directory = new RAMDirectory();
        else {
          //directory = new MMapDirectory(new File(config.getPath()));
          // as per http://wiki.apache.org/lucene-java/ImproveSearchingSpeed
          // we use NIOFSDirectory, provided we're not on Windows
          if (Utils.isWindowsOS())
            directory = FSDirectory.open(new File(path));
          else
            directory = NIOFSDirectory.open(new File(path));
        }

        IndexWriterConfig cfg =
          new IndexWriterConfig(Version.LUCENE_CURRENT, analyzer);
        cfg.setOpenMode(overwrite ? IndexWriterConfig.OpenMode.CREATE :
                                    IndexWriterConfig.OpenMode.APPEND);
        iwriter = new IndexWriter(directory, cfg);
        iwriter.commit(); // so that the searcher doesn't fail
      } catch (IndexNotFoundException e) {
        if (!overwrite) {
          // the index was not there, so make a new one
          directory = null; // ensure we really do try again
          openIndexes(true);
        } else
          throw new DukeException(e);
      }
    }
  }

  public void openSearchers() throws IOException {
    reader = DirectoryReader.open(directory);
    searcher = new IndexSearcher(reader);
  }

  /**
   * Parses the query. Using this instead of a QueryParser in order
   * to avoid thread-safety issues with Lucene's query parser.
   *
   * @param fieldName the name of the field
   * @param value the value of the field
   * @return the parsed query
   */
  private Query parseTokens(String fieldName, String value) {
    BooleanQuery searchQuery = new BooleanQuery();
    if (value != null) {
      Analyzer analyzer = new KeywordAnalyzer();

      try {
        TokenStream tokenStream =
          analyzer.tokenStream(fieldName, new StringReader(value));
        tokenStream.reset();
        CharTermAttribute attr =
          tokenStream.getAttribute(CharTermAttribute.class);

        while (tokenStream.incrementToken()) {
          String term = attr.toString();
          Query termQuery = new TermQuery(new Term(fieldName, term));
          searchQuery.add(termQuery, Occur.SHOULD);
        }
      } catch (IOException e) {
        throw new DukeException("Error parsing input string '" + value + "' " +
                                "in field " + fieldName);
      }
    }

    return searchQuery;
  }

  /**
   * Parses Lucene query.
   * @param required Iff true, return only records matching this value.
   */
  private void parseTokens(BooleanQuery parent, String fieldName,
                             String value, boolean required, double probability) {
    value = escapeLucene(value);
    if (value.length() == 0)
      return;

    try {
      TokenStream tokenStream =
        analyzer.tokenStream(fieldName, new StringReader(value));
      tokenStream.reset();
      CharTermAttribute attr =
        tokenStream.getAttribute(CharTermAttribute.class);

      Float boost = getBoostFactor(probability, BoostMode.QUERY);

      while (tokenStream.incrementToken()) {
        String term = attr.toString();
        Query termQuery;
        if (fuzzy_search && isFuzzy(fieldName))
          termQuery = new FuzzyQuery(new Term(fieldName, term));
        else
          termQuery = new TermQuery(new Term(fieldName, term));

        if (boost != null)
          termQuery.setBoost(boost);
        parent.add(termQuery, required ? Occur.MUST : Occur.SHOULD);
      }
    } catch (IOException e) {
      throw new DukeException("Error parsing input string '"+value+"' "+
                              "in field " + fieldName);
    }
  }

  private boolean isFuzzy(String fieldName) {
    Comparator c = config.getPropertyByName(fieldName).getComparator();
    return c != null && c.isTokenized();
  }

  private String escapeLucene(String query) {
    char[] tmp = new char[query.length() * 2];
    int count = 0;
    for (int ix = 0; ix < query.length(); ix++) {
      char ch = query.charAt(ix);
      if (ch == '*' || ch == '?' || ch == '!' || ch == '&' || ch == '(' ||
          ch == ')' || ch == '-' || ch == '+' || ch == ':' || ch == '"' ||
          ch == '[' || ch == ']' || ch == '~' || ch == '{' || ch == '}' ||
          ch == '^' || ch == '|')
        tmp[count++] = '\\'; // these characters must be escaped
      tmp[count++] = ch;
    }

    return new String(tmp, 0, count).trim();
  }

  public Collection<Record> lookup(Property property, String value) {
    Query query = parseTokens(property.getName(), value);
    return maintracker.doQuery(query);
  }

  /**
   * The tracker is used to estimate the size of the query result
   * we should ask Lucene for. This parameter is the single biggest
   * influence on search performance, but setting it too low causes
   * matches to be missed. We therefore try hard to estimate it as
   * correctly as possible.
   *
   * The tracker uses a ring buffer of recent result sizes to
   * estimate the result size.
   */
  class EstimateResultTracker {
    private int limit;
    /**
     * Ring buffer containing n last search result sizes, except for
     * searches which found nothing.
     */
    private int[] prevsizes;
    private int sizeix; // position in prevsizes

    public EstimateResultTracker() {
      this.limit = 100;
      this.prevsizes = new int[10];
    }

    public Collection<Record> doQuery(Query query) {
      return doQuery(query, null);
    }

    public Collection<Record> doQuery(Query query, Filter filter) {
      List<Record> matches;
      try {
        ScoreDoc[] hits;

        int thislimit = Math.min(limit, max_search_hits);
        while (true) {
          hits = searcher.search(query, filter, thislimit).scoreDocs;
          if (hits.length < thislimit || thislimit == max_search_hits)
            break;
          thislimit = thislimit * 5;
        }

        matches = new ArrayList(Math.min(hits.length, max_search_hits));
        for (int ix = 0; ix < hits.length &&
                         hits[ix].score >= min_relevance; ix++)

          matches.add(new DocumentRecord(hits[ix].doc,
                                         searcher.doc(hits[ix].doc)));

        if (hits.length > 0) {
          synchronized(this) {
            prevsizes[sizeix++] = matches.size();
            if (sizeix == prevsizes.length) {
              sizeix = 0;
              limit = Math.max((int) (average() * SEARCH_EXPANSION_FACTOR), limit);
            }
          }
        }
      } catch (IOException e) {
        throw new DukeException(e);
      }
      return matches;
    }

    private double average() {
      int sum = 0;
      int ix = 0;
      for (; ix < prevsizes.length && prevsizes[ix] != 0; ix++)
        sum += prevsizes[ix];
      return sum / (double) ix;
    }
  }

  /**
   * Checks to see if we need the spatial support, and if so creates
   * the necessary context objects.
   */
  private void initSpatial() {
    // FIXME: for now, we only use geosearch if that's the only way to
    // find suitable records, since we don't know how to combine
    // geosearch ranking with normal search ranking.
    if (config.getLookupProperties().size() != 1)
      return;

    Property prop = config.getLookupProperties().iterator().next();
    if (!(prop.getComparator() instanceof GeopositionComparator))
      return;

    geoprop = new GeoProperty(prop);
  }

  public enum BoostMode {
    /**
     * Boost fields at query time.
     */
    QUERY,
    /**
     * Boost fields at index time. This means records must be
     * reindexed to change the boosting.
     */
    INDEX,
    /**
     * Don't boost fields.
     */
    NONE;
  }

  private Float getBoostFactor(double probability, BoostMode phase) {
    Float boost = null;
    if (phase == boost_mode) {
      double p = Math.min(0.99, probability); // don't divide by zero
      boost = (float) Math.sqrt(1.0 / ((1.0 - p) * 2.0));
    }
    return boost;
  }
}
1017990
@Override
    public IDocumentQuery<T> containsAll(String fieldName, Collection<Object> values) {
        _containsAll(fieldName, values);
        return this;
    }
616736
function cacheHas(cache, value) {
        var map = cache.__data__;
        if (isKeyable(value)) {
          var data = map.__data__, hash = typeof value == 'string' ? data.string : data.hash;
          return hash[value] === HASH_UNDEFINED;
        }
        return map.has(value);
      }
910719
function(record, extraNames) {
      var fieldNames = record.getFields().map(function (field) {
        return field.getName();
      });

      if (extraNames) {
        fieldNames = fieldNames.concat(extraNames);
      }
      return fieldNames;
    }
346621
@SuppressWarnings("unchecked")
  protected int hashFields(int hash, Map<FieldDescriptor, Object> map) {
    for (Map.Entry<FieldDescriptor, Object> entry : map.entrySet()) {
      FieldDescriptor field = entry.getKey();
      Object value = entry.getValue();
      hash = (37 * hash) + field.getNumber();
      if (field.getType() != FieldDescriptor.Type.ENUM){
        hash = (53 * hash) + value.hashCode();
      } else if (field.isRepeated()) {
        List<? extends EnumLite> list = (List<? extends EnumLite>) value;
        hash = (53 * hash) + hashEnumList(list);
      } else {
        hash = (53 * hash) + hashEnum((EnumLite) value);
      }
    }
    return hash;
  }
216947
function(seriesKeys) {
        var fieldValues = _.values(seriesKeys),
            matchingSeriesDef = _.find(this._attributes.series, function(seriesDef) {
                return _.contains(fieldValues, seriesDef.name);
            });
        if (!matchingSeriesDef) {
            return _.find(this._attributes.series, function(seriesDef) {
                return !seriesDef.name;
            });
        }
        return matchingSeriesDef;
    }
123550
function wlCheck(wl, record, value) {
  if (!value) value = 'allow';

  let wlItem = _.get(wl, record);

  if (_.isArray(wlItem)) return wlItem.indexOf(value) !== -1;

  return wlItem === value;
}
128095
private void set(String fieldName, Object value) {
    try {
      String subField;
      Iterator<String> levels = Splitter.on(".").split(fieldName).iterator();
      GenericRecord toInsert = record;
      subField = levels.next();
      Object subRecord = toInsert;

      while (levels.hasNext()) {
        if (subRecord instanceof GenericRecord) {
          subRecord = ((GenericRecord)subRecord).get(subField);
        } else if (subRecord instanceof List) {
          subRecord = ((List)subRecord).get(Integer.parseInt(subField));
        } else if (subRecord instanceof Map) {
          subRecord = ((Map)subRecord).get(subField);
        }

        if (subRecord == null) {
          throw new FieldDoesNotExistException("Field " + subField + " not found when trying to set " + fieldName);
        }
        subField = levels.next();
      }

      if (!(subRecord instanceof GenericRecord)) {
        throw new IllegalArgumentException("Field " + fieldName + " does not refer to a record type.");
      }

      toInsert = (GenericRecord)subRecord;
      Object oldValue = toInsert.get(subField);

      toInsert.put(subField, value);

      Schema.Field changedField = toInsert.getSchema().getField(subField);
      GenericData genericData = GenericData.get();

      boolean valid = genericData
          .validate(changedField.schema(), genericData.getField(toInsert, changedField.name(), changedField.pos()));
      if (!valid) {
        toInsert.put(subField, oldValue);
        throw new IncorrectTypeException(
            "Incorrect type - can't insert a " + value.getClass().getCanonicalName() + " into an Avro record of type "
                + changedField.schema().getType().toString());
      }
    } catch (AvroRuntimeException e) {
      throw new FieldDoesNotExistException("Field not found setting name " + fieldName, e);
    }
  }
932208
function (record) {
      var recordToSend = { object: record };

      recordToSend.columnValues = this.getRecordColumnValues(record);
      recordToSend.searchKeywords = this.getRecordKeywords(record);
      recordToSend.filterValues = this.getRecordFilterValues(record);
      recordToSend.color = this.getRecordColor(record);

      return recordToSend;
    }
1084238
def getAllValuesForFieldsAsArray( dbid, fieldNames = nil, query = nil, qid = nil, qname = nil, clist = nil, slist = nil, fmt = "structured", options = nil )
      ret = []
      valuesForFields = getAllValuesForFields(dbid, fieldNames, query, qid, qname, clist, slist,fmt,options)
      if valuesForFields
         fieldNames ||= getFieldNames(@dbid) 
         if fieldNames and fieldNames[0]
            ret = Array.new(valuesForFields[fieldNames[0]].length) 
            fieldType = {}
            fieldNames.each{|field|fieldType[field]=lookupFieldTypeByName(field)}
            valuesForFields.each{ |field,values|
               values.each_index { |i| 
                  ret[i] ||= {} 
                  ret[i][field]=formatFieldValue(values[i],fieldType[field])
               }
            }
         end
      end
      ret
   end
70634
def eachRecord( records = @records )
      if records and block_given?
         records.each { |record|
            if record.is_a?( REXML::Element) and record.name == "record"
               @record = record
               yield record
            end
         }
      end
      nil
   end
1080469
function getMatchingRecord(relationship, included, parents) {
        var circular, match;

        circular = findWhere(
            parents,
            {
                id: relationship.id,
                type: relationship.type
            }
        );

        if (circular) {
            return relationship;
        }

        var key = relationship.type + '-' + relationship.id;
        match = included[key];

        // IF: No match or match is the same as parent, return the relationship information
        if (!match) {
            return relationship;
        }

        populateRelatedFields(match, included, parents);

        // IF: relationship defined meta, merge with record provided meta
        var contextSpecificMeta = {};
        if (relationship && isObject(relationship.meta)) {
            contextSpecificMeta = extend({}, match.meta, relationship.meta);
        }

        return flatten(match, contextSpecificMeta);
    }
802624
def min( dbid, fieldNames, query = nil, qid = nil, qname = nil, clist = nil, slist = nil, fmt = "structured", options = nil )
      min = {}
      hasValues = false
      iterateRecords(dbid,fieldNames,query,qid,qname,clist,slist,fmt,options){|record|
         fieldNames.each{|field|
            value = record[field]
            if value
               baseFieldType = lookupBaseFieldTypeByName(field)
               case baseFieldType
                  when "int32","int64","bool" 
                     value = record[field].to_i
                  when "float"   
                     value = record[field].to_f
               end
            end
            if min[field].nil? or  (value and value < min[field])
               min[field] = value
               hasValues = true
            end
         }
      }
      min = nil if not hasValues
      min
   end
891900
def count( dbid, fieldNames, query = nil, qid = nil, qname = nil, clist = nil, slist = nil, fmt = "structured", options = nil )
      count = {}
      fieldNames.each{|field| count[field]=0 }
      hasValues = false
      iterateRecords(dbid,fieldNames,query,qid,qname,clist,slist,fmt,options){|record|
         fieldNames.each{|field|
            if record[field] and record[field].length > 0
               count[field] += 1
               hasValues = true
            end
         }
      }
      count = nil if not hasValues
      count
   end
271770
function (ndefRecords) {

        var encoded = [],
            tnf_byte,
            record_type,
            payload_length,
            id_length,
            i,
            mb, me, // messageBegin, messageEnd
            cf = false, // chunkFlag TODO implement
            sr, // boolean shortRecord
            il; // boolean idLengthFieldIsPresent

        for(i = 0; i < ndefRecords.length; i++) {

            mb = (i === 0);
            me = (i === (ndefRecords.length - 1));
            sr = (ndefRecords[i].payload.length < 0xFF);
            il = (ndefRecords[i].id.length > 0);
            tnf_byte = ndef.encodeTnf(mb, me, cf, sr, il, ndefRecords[i].tnf);
            encoded.push(tnf_byte);

            // type is stored as String, converting to bytes for storage
            record_type = util.stringToBytes(ndefRecords[i].type);
            encoded.push(record_type.length);

            if (sr) {
                payload_length = ndefRecords[i].payload.length;
                encoded.push(payload_length);
            } else {
                payload_length = ndefRecords[i].payload.length;
                // 4 bytes
                encoded.push((payload_length >> 24));
                encoded.push((payload_length >> 16));
                encoded.push((payload_length >> 8));
                encoded.push((payload_length & 0xFF));
            }

            if (il) {
                id_length = ndefRecords[i].id.length;
                encoded.push(id_length);
            }

            encoded = encoded.concat(record_type);

            if (il) {
                encoded = encoded.concat(ndefRecords[i].id);
            }

            encoded = encoded.concat(ndefRecords[i].payload);
        }

        return encoded;
    }
54271
def eachField( record = @record )
      if record and block_given?
         record.each{ |field|
             if field.is_a?( REXML::Element) and field.name == "f" and field.attributes["id"]
                @field = field
                yield field
             end
         }
      end
      nil      
   end