Migrate old Elasticsearch config settings to new ClusterConfig based ones.
186895
@Override
    public void upgrade() {
        // All default rotation strategy settings.
        final MessageCountRotationStrategyConfig messageCountRotationStrategyConfig = clusterConfigService.get(MessageCountRotationStrategyConfig.class);
        final SizeBasedRotationStrategyConfig sizeBasedRotationStrategyConfig = clusterConfigService.get(SizeBasedRotationStrategyConfig.class);
        final TimeBasedRotationStrategyConfig timeBasedRotationStrategyConfig = clusterConfigService.get(TimeBasedRotationStrategyConfig.class);

        if (messageCountRotationStrategyConfig == null) {
            final MessageCountRotationStrategyConfig countConfig = MessageCountRotationStrategyConfig.create(elasticsearchConfiguration.getMaxDocsPerIndex());
            clusterConfigService.write(countConfig);
            LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_docs_per_index", countConfig);
        }
        if (sizeBasedRotationStrategyConfig == null) {
            final SizeBasedRotationStrategyConfig sizeConfig = SizeBasedRotationStrategyConfig.create(elasticsearchConfiguration.getMaxSizePerIndex());
            clusterConfigService.write(sizeConfig);
            LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_size_per_index", sizeConfig);
        }
        if (timeBasedRotationStrategyConfig == null) {
            final TimeBasedRotationStrategyConfig timeConfig = TimeBasedRotationStrategyConfig.create(elasticsearchConfiguration.getMaxTimePerIndex());
            clusterConfigService.write(timeConfig);
            LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_time_per_index", timeConfig);
        }

        // All default retention strategy settings
        final ClosingRetentionStrategyConfig closingRetentionStrategyConfig = clusterConfigService.get(ClosingRetentionStrategyConfig.class);
        final DeletionRetentionStrategyConfig deletionRetentionStrategyConfig = clusterConfigService.get(DeletionRetentionStrategyConfig.class);

        if (closingRetentionStrategyConfig == null) {
            final ClosingRetentionStrategyConfig closingConfig = ClosingRetentionStrategyConfig.create(elasticsearchConfiguration.getMaxNumberOfIndices());
            clusterConfigService.write(closingConfig);
            LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_number_of_indices", closingConfig);
        }

        if (deletionRetentionStrategyConfig == null) {
            final DeletionRetentionStrategyConfig deletionConfig = DeletionRetentionStrategyConfig.create(elasticsearchConfiguration.getMaxNumberOfIndices());
            clusterConfigService.write(deletionConfig);
            LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_number_of_indices", deletionConfig);
        }

        // Selected rotation and retention strategies.
        final IndexManagementConfig indexManagementConfig = clusterConfigService.get(IndexManagementConfig.class);
        if (indexManagementConfig == null) {
            final Class<? extends RotationStrategy> rotationStrategyClass;
            switch (elasticsearchConfiguration.getRotationStrategy()) {
                case "size":
                    rotationStrategyClass = SizeBasedRotationStrategy.class;
                    break;
                case "time":
                    rotationStrategyClass = TimeBasedRotationStrategy.class;
                    break;
                case "count":
                    rotationStrategyClass = MessageCountRotationStrategy.class;
                    break;
                default:
                    LOG.warn("Unknown retention strategy \"{}\"", elasticsearchConfiguration.getRotationStrategy());
                    rotationStrategyClass = MessageCountRotationStrategy.class;
            }

            final Class<? extends RetentionStrategy> retentionStrategyClass;
            switch (elasticsearchConfiguration.getRetentionStrategy()) {
                case "close":
                    retentionStrategyClass = ClosingRetentionStrategy.class;
                    break;
                case "delete":
                    retentionStrategyClass = DeletionRetentionStrategy.class;
                    break;
                default:
                    LOG.warn("Unknown retention strategy \"{}\"", elasticsearchConfiguration.getRetentionStrategy());
                    retentionStrategyClass = DeletionRetentionStrategy.class;
            }

            final IndexManagementConfig config = IndexManagementConfig.create(
                    rotationStrategyClass.getCanonicalName(),
                    retentionStrategyClass.getCanonicalName());
            clusterConfigService.write(config);
            LOG.info("Migrated \"{}\" and \"{}\" setting: {}", "rotation_strategy", "retention_strategy", config);
        }

        final SearchesClusterConfig searchesClusterConfig = clusterConfigService.get(SearchesClusterConfig.class);
        if (searchesClusterConfig == null) {
            final SearchesClusterConfig config = SearchesClusterConfig.createDefault();
            LOG.info("Creating searches cluster config: {}", config);
            clusterConfigService.write(config);
        }
    }
614503
def update_config(self, config):
    
    path = self._path() + '/config'
    resp = self._get_resource_root().put(path, data = config_to_json(config))
    return json_to_config(resp)
1208641
def _convert_to_clusters(c):
    """"""
    new_dict = {}
    n_cluster = 0
    logger.debug("_convert_to_cluster: loci %s" % c.loci2seq.keys())
    for idl in c.loci2seq:
        n_cluster += 1
        new_c = cluster(n_cluster)
        #new_c.id_prev = c.id
        new_c.loci2seq[idl] = c.loci2seq[idl]
        new_dict[n_cluster] = new_c
    logger.debug("_convert_to_cluster: new ids %s" % new_dict.keys())
    return new_dict
946885
def reset_elasticsearch(self):
        

        print("""
Cleanup Elasticsearch database:
    This will truncate all tables and reset the whole Elasticsearch database.
              """)

        confirm = self.no_confirm

        if not confirm:
            confirm = 'yes' in builtins.input(
                """
Do you really want to do this? Write 'yes' to confirm: {yes}"""
                    .format(yes='yes' if confirm else ''))

        if not confirm:
            print("Did not type yes. Thus aborting.")
            return

        try:
            # initialize DB connection
            es = Elasticsearch([self.elasticsearch["host"]],
                               http_auth=(self.elasticsearch["username"], self.elasticsearch["secret"]),
                               port=self.elasticsearch["port"],
                               use_ssl=self.elasticsearch["use_ca_certificates"],
                               verify_certs=self.elasticsearch["use_ca_certificates"],
                               ca_certs=self.elasticsearch["ca_cert_path"],
                               client_cert=self.elasticsearch["client_cert_path"],
                               client_key=self.elasticsearch["client_key_path"])

            print("Resetting Elasticsearch database...")
            es.indices.delete(index=self.elasticsearch["index_current"], ignore=[400, 404])
            es.indices.delete(index=self.elasticsearch["index_archive"], ignore=[400, 404])
        except ConnectionError as error:
            self.log.error("Failed to connect to Elasticsearch. "
                           "Please check if the database is running and the config is correct: %s" % error)
178356
def with_updated_configuration(self, options=None,
                                   attribute_options=None):
        
        return self._mapping.with_updated_configuration(options=options,
                                                        attribute_options=
                                                            attribute_options)
1033325
def put_mapping(self, app, index=None):
        
        for resource, resource_config in self._get_elastic_resources().items():
            datasource = resource_config.get('datasource', {})

            if not is_elastic(datasource):
                continue

            if datasource.get('source', resource) != resource:  # only put mapping for core types
                continue

            properties = self._get_mapping_properties(resource_config)

            kwargs = {
                'index': index or self._resource_index(resource),
                'doc_type': resource,
                'body': properties,
            }

            try:
                self.elastic(resource).indices.put_mapping(**kwargs)
            except elasticsearch.exceptions.RequestError:
                logger.exception('mapping error, updating settings resource=%s' % resource)
671249
function buildIndicesRollover (opts) {
  // eslint-disable-next-line no-unused-vars
  const { makeRequest, ConfigurationError, handleError, snakeCaseKeys } = opts
  /**
   * Perform a [indices.rollover](http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-rollover-index.html) request
   *
   * @param {string} alias - The name of the alias to rollover
   * @param {string} new_index - The name of the rollover index
   * @param {boolean} include_type_name - Whether a type should be included in the body of the mappings.
   * @param {time} timeout - Explicit operation timeout
   * @param {boolean} dry_run - If set to true the rollover action will only be validated but not actually performed even if a condition matches. The default is false
   * @param {time} master_timeout - Specify timeout for connection to master
   * @param {string} wait_for_active_shards - Set the number of active shards to wait for on the newly created rollover index before the operation returns.
   * @param {object} body - The conditions that needs to be met for executing rollover
   */

  const acceptedQuerystring = [
    'include_type_name',
    'timeout',
    'dry_run',
    'master_timeout',
    'wait_for_active_shards',
    'pretty',
    'human',
    'error_trace',
    'source',
    'filter_path'
  ]

  const snakeCase = {
    includeTypeName: 'include_type_name',
    dryRun: 'dry_run',
    masterTimeout: 'master_timeout',
    waitForActiveShards: 'wait_for_active_shards',
    errorTrace: 'error_trace',
    filterPath: 'filter_path'
  }

  return function indicesRollover (params, options, callback) {
    options = options || {}
    if (typeof options === 'function') {
      callback = options
      options = {}
    }
    if (typeof params === 'function' || params == null) {
      callback = params
      params = {}
      options = {}
    }

    // check required parameters
    if (params['alias'] == null) {
      const err = new ConfigurationError('Missing required parameter: alias')
      return handleError(err, callback)
    }

    // check required url components
    if ((params['new_index'] != null || params['newIndex'] != null) && (params['alias'] == null)) {
      const err = new ConfigurationError('Missing required parameter of the url: alias')
      return handleError(err, callback)
    }

    // validate headers object
    if (options.headers != null && typeof options.headers !== 'object') {
      const err = new ConfigurationError(`Headers should be an object, instead got: ${typeof options.headers}`)
      return handleError(err, callback)
    }

    var warnings = []
    var { method, body, alias, newIndex, new_index, ...querystring } = params
    querystring = snakeCaseKeys(acceptedQuerystring, snakeCase, querystring, warnings)

    if (method == null) {
      method = 'POST'
    }

    var ignore = options.ignore
    if (typeof ignore === 'number') {
      options.ignore = [ignore]
    }

    var path = ''

    if ((alias) != null && (new_index || newIndex) != null) {
      path = '/' + encodeURIComponent(alias) + '/' + '_rollover' + '/' + encodeURIComponent(new_index || newIndex)
    } else {
      path = '/' + encodeURIComponent(alias) + '/' + '_rollover'
    }

    // build request object
    const request = {
      method,
      path,
      body: body || '',
      querystring
    }

    options.warnings = warnings.length === 0 ? null : warnings
    return makeRequest(request, options, callback)
  }
}
1254820
def migrate(ctx, start_version, end_version):
    """"""
    if ctx.type == 'file':
        if ctx.data_file:
            file_path = ctx.data_file
        else:
            file_path = os.path.join(ctx.data_file, ctx.namespace + '.json')

        # todo make this more like alemebic and determine/load versions automatically
        with open(file_path, 'r') as f:
            data = json.loads(f.read())

        data = run_migration(data, start_version, end_version)
        with open(file_path, 'w') as f:
            f.write(json.dumps(data))
891518
def migrate(schema_file, config_file):
    ''''''

    schema = load_schema_file(open(schema_file, 'r'))
    config = load_config_file(config_file, open(config_file, 'r').read())

    config = append_existing_values(schema, config)

    migrated_config = generate_config_parser(config)
    migrated_config.write(sys.stdout)
1106995
def _merge_cluster(old, new):
    """"""
    logger.debug("_merge_cluster: %s to %s" % (old.id, new.id))
    logger.debug("_merge_cluster: add idls %s" % old.loci2seq.keys())
    for idl in old.loci2seq:
        # if idl in new.loci2seq:
        #    new.loci2seq[idl] = list(set(new.loci2seq[idl] + old.loci2seq[idl]))
        # new.loci2seq[idl] = old.loci2seq[idl]
        new.add_id_member(old.loci2seq[idl], idl)
    return new
11788
def convert_aws_args(aws_args):
    """"""
    if not isinstance(aws_args, dict):
        raise errors.InvalidConfiguration(
            'Elastic DocManager config option "aws" must be a dict'
        )
    old_session_kwargs = dict(
        region="region_name",
        access_id="aws_access_key_id",
        secret_key="aws_secret_access_key",
    )
    new_kwargs = {}
    for arg in aws_args:
        if arg in old_session_kwargs:
            new_kwargs[old_session_kwargs[arg]] = aws_args[arg]
        else:
            new_kwargs[arg] = aws_args[arg]
    return new_kwargs
319895
def upgrade(check_only, yes):
    '''
    
    '''
    from utils.es import Elasticsearch
    from libreantdb import DB, migration
    from libreantdb.exceptions import MappingsException

    try:
        db = DB(Elasticsearch(hosts=conf['ES_HOSTS']),
                              index_name=conf['ES_INDEXNAME'])
        if not db.es.indices.exists(db.index_name):
            die("The specified index does not exists: {}".format(db.index_name))

        # Migrate old special `_timestamp` field into the new `_insertion_date`
        num_to_update = migration.elements_without_insertion_date(db.es, db.index_name)
        if num_to_update > 0:
            if check_only:
                exit(123)

            if yes or click.confirm("{} entries miss the '_insertion_date' field. Do you want to proceed and update those entries?".format(num_to_update),
                             prompt_suffix='',
                             default=False):
                migration.migrate_timestamp(db.es, db.index_name)
            else:
                exit(0)

        # Upgrade the index mappings and reindex if necessary
        try:
            db.update_mappings()
        except MappingsException:
            if check_only:
                exit(123)
            count = db.es.count(index=db.index_name)['count']
            if yes or click.confirm("Some old or wrong mappings has been found for the index '"+ db.index_name +"'.\n"\
                                    "In order to upgrade them it is necessary to reindex '"+ str(count) +"' entries.\n"\
                                    "Are you sure you want to proceed?",
                                     prompt_suffix='',
                                     default=False):
                    db.reindex()

    except Exception as e:
        if conf.get('DEBUG', False):
            raise
        else:
            die(str(e))
264310
def get_default_config(self):
        
        config = super(ElasticSearchCollector, self).get_default_config()
        config.update({
            'host':           '127.0.0.1',
            'port':           9200,
            'user':           '',
            'password':       '',
            'instances':      [],
            'scheme':         'http',
            'path':           'elasticsearch',
            'stats':          ['jvm', 'thread_pool', 'indices'],
            'logstash_mode': False,
            'cluster':       False,
        })
        return config
217237
def create_mapping(self):
        """"""
        try:
            self.document_class.init()
            self._mapping_created = True
        except IllegalOperation as error:
            if error.args[0].startswith('You cannot update analysis configuration'):
                # Ignore mapping update errors, which are thrown even when the analysis
                # configuration stays the same.
                # TODO: Remove this when https://github.com/elastic/elasticsearch-dsl-py/pull/272 is merged.
                return

            raise
932006
def init(self):
        """"""
        if not self.export_enable:
            return None

        self.index='{}-{}'.format(self.index, datetime.utcnow().strftime("%Y.%m.%d"))
        template_body =  {
          "mappings": {
            "glances": {
              "dynamic_templates": [
                {
                  "integers": {
                    "match_mapping_type": "long",
                    "mapping": {
                      "type": "integer"
                    }
                  }
                },
                {
                  "strings": {
                    "match_mapping_type": "string",
                    "mapping": {
                      "type": "text",
                      "fields": {
                        "raw": {
                          "type":  "keyword",
                          "ignore_above": 256
                        }
                      }
                    }
                  }
                }
              ]
            }
          }
        }

        try:
            es = Elasticsearch(hosts=['{}:{}'.format(self.host, self.port)])
        except Exception as e:
            logger.critical("Cannot connect to ElasticSearch server %s:%s (%s)" % (self.host, self.port, e))
            sys.exit(2)
        else:
            logger.info("Connected to the ElasticSearch server %s:%s" % (self.host, self.port))

        try:
            index_count = es.count(index=self.index)['count']
        except Exception as e:
            # Index did not exist, it will be created at the first write
            # Create it...
            es.indices.create(index=self.index,body=template_body)
        else:
            logger.info("The index %s exists and holds %s entries." % (self.index, index_count))

        return es
801955
def _construct_config(self, basedir, port, name=None, isreplset=False):
        """"""
        if isreplset:
            return self._construct_replset(basedir=basedir, portstart=port,
                                           name=name,
                                           num_nodes=list(range(
                                               self.args['config'])),
                                           arbiter=False, extra='--configsvr')
        else:
            datapath = self._create_paths(basedir, name)
            self._construct_mongod(os.path.join(datapath, 'db'),
                                   os.path.join(datapath, 'mongod.log'),
                                   port, replset=None, extra='--configsvr')
685390
def algolia_reindex(batch_size = AlgoliaSearch::IndexSettings::DEFAULT_BATCH_SIZE, synchronous = false)
      return if algolia_without_auto_index_scope
      algolia_configurations.each do |options, settings|
        next if algolia_indexing_disabled?(options)
        next if options[:slave] || options[:replica]

        # fetch the master settings
        master_index = algolia_ensure_init(options, settings)
        master_settings = master_index.get_settings rescue {} # if master doesn't exist yet
        master_settings.merge!(JSON.parse(settings.to_settings.to_json)) # convert symbols to strings

        # remove the replicas of the temporary index
        master_settings.delete :slaves
        master_settings.delete 'slaves'
        master_settings.delete :replicas
        master_settings.delete 'replicas'

        # init temporary index
        index_name = algolia_index_name(options)
        tmp_options = options.merge({ :index_name => "#{index_name}.tmp" })
        tmp_options.delete(:per_environment) # already included in the temporary index_name
        tmp_settings = settings.dup
        tmp_index = algolia_ensure_init(tmp_options, tmp_settings, master_settings)

        algolia_find_in_batches(batch_size) do |group|
          if algolia_conditional_index?(tmp_options)
            # select only indexable objects
            group = group.select { |o| algolia_indexable?(o, tmp_options) }
          end
          objects = group.map { |o| tmp_settings.get_attributes(o).merge 'objectID' => algolia_object_id_of(o, tmp_options) }
          tmp_index.save_objects(objects)
        end

        move_task = SafeIndex.move_index(tmp_index.name, index_name)
        master_index.wait_task(move_task["taskID"]) if synchronous || options[:synchronous]
      end
      nil
    end