Imports Elasticsearch index data.

@param Manager         $manager
@param string          $filename
@param OutputInterface $output
@param array           $options
182961
public function importIndex(
        Manager $manager,
        $filename,
        OutputInterface $output,
        $options
    ) {
        $reader = $this->getReader($manager, $this->getFilePath($filename), $options);

        $progress = new ProgressBar($output, $reader->count());
        $progress->setRedrawFrequency(100);
        $progress->start();

        $bulkSize = $options['bulk-size'];
        foreach ($reader as $key => $document) {
            $data = $document['_source'];
            $data['_id'] = $document['_id'];

            if (array_key_exists('fields', $document)) {
                $data = array_merge($document['fields'], $data);
            }

            $manager->bulk('index', $document['_type'], $data);

            if (($key + 1) % $bulkSize == 0) {
                $manager->commit();
            }

            $progress->advance();
        }

        $manager->commit();

        $progress->finish();
        $output->writeln('');
    }
353850
public static function gather($options = array()) {

		// set default vars
		$exportClean        = (isset($options["exportClean"])) ? $options["exportClean"] : false;
		$exportFiles        = (isset($options["exportClean"])) ? $options["exportFiles"] : false;
		$dispatcherInstance = Dispatcher::getInstance();

		// cleaning the var for use below, i know this is stupid
		$options = array();

		// dispatch that the data gather has started
		$event = new PatternDataEvent($options);
		$dispatcherInstance->dispatch("patternData.gatherStart",$event);

		// load up the rules for parsing patterns and the directories
		self::loadRules($options);

		// dispatch that the rules are loaded
		$event = new PatternDataEvent($options);
		$dispatcherInstance->dispatch("patternData.rulesLoaded",$event);

		// iterate over the patterns & related data and regenerate the entire site if they've changed
		// seems a little silly to use symfony finder here. not really giving me any power
		if (!is_dir(Config::getOption("patternSourceDir"))) {
			Console::writeError("having patterns is important. please make sure you've installed a starterkit and/or that ".Console::getHumanReadablePath(Config::getOption("patternSourceDir"))." exists...");
		}

		$patternSourceDir = Config::getOption("patternSourceDir");
		$patternObjects = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($patternSourceDir, \RecursiveDirectoryIterator::FOLLOW_SYMLINKS | \FilesystemIterator::SKIP_DOTS), \RecursiveIteratorIterator::SELF_FIRST);

		// sort the returned objects
		$patternObjects = iterator_to_array($patternObjects);
		ksort($patternObjects);

		/**
     * @var string $name
     * @var \SplFileInfo $object
     */
    foreach ($patternObjects as $name => $object) {

			$ext      = $object->getExtension();
			$isDir    = $object->isDir();
			$isFile   = $object->isFile();

			$path     = str_replace($patternSourceDir.DIRECTORY_SEPARATOR,"",$object->getPath());
			$pathName = str_replace($patternSourceDir.DIRECTORY_SEPARATOR,"",$object->getPathname());
			$name     = $object->getFilename();
			$depth    = substr_count($pathName,DIRECTORY_SEPARATOR);

			// iterate over the rules and see if the current file matches one, if so run the rule
			foreach (self::$rules as $rule) {
				if ($rule->test($depth, $ext, $isDir, $isFile, $name)) {
					$rule->run($depth, $ext, $path, $pathName, $name);
				}
			}

		}

		// dispatch that the data is loaded
		$event = new PatternDataEvent($options);
		$dispatcherInstance->dispatch("patternData.dataLoaded",$event);

		// make sure all of the appropriate pattern data is pumped into $this->d for rendering patterns
		$dataLinkExporter       = new DataLinkExporter();
		$dataLinkExporter->run();

		// make sure all of the appropriate pattern data is pumped into $this->d for rendering patterns
		$dataMergeExporter       = new DataMergeExporter();
		$dataMergeExporter->run();

		// dispatch that the raw pattern helper is about to start
		$event = new PatternDataEvent($options);
		$dispatcherInstance->dispatch("patternData.rawPatternHelperStart",$event);

		// add the lineage info to PatternData::$store
		$rawPatternHelper        = new RawPatternHelper();
		$rawPatternHelper->run();

		// dispatch that the raw pattern helper is ended
		$event = new PatternDataEvent($options);
		$dispatcherInstance->dispatch("patternData.rawPatternHelperEnd",$event);

		// dispatch that the lineage helper is about to start
		$event = new PatternDataEvent($options);
		$dispatcherInstance->dispatch("patternData.lineageHelperStart",$event);

		// add the lineage info to PatternData::$store
		$lineageHelper           = new LineageHelper();
		$lineageHelper->run();

		// dispatch that the lineage helper is ended
		$event = new PatternDataEvent($options);
		$dispatcherInstance->dispatch("patternData.lineageHelperEnd",$event);


		// dispatch that the pattern state helper is about to start
		$event = new PatternDataEvent($options);
		$dispatcherInstance->dispatch("patternData.patternStateHelperStart",$event);

		// using the lineage info update the pattern states on PatternData::$store
		$patternStateHelper      = new PatternStateHelper();
		$patternStateHelper->run();

		// dispatch that the pattern state helper is ended
		$event = new PatternDataEvent($options);
		$dispatcherInstance->dispatch("patternData.patternStateHelperEnd",$event);

		// set-up code pattern paths
		$ppdExporter             = new PatternPathSrcExporter();
		$patternPathSrc          = $ppdExporter->run();
		$options                 = array();
		$options["patternPaths"] = $patternPathSrc;

		// dispatch that the code helper is about to start
		$event = new PatternDataEvent($options);
		$dispatcherInstance->dispatch("patternData.codeHelperStart",$event);

		// render out all of the patterns and store the generated info in PatternData::$store
		$options["exportFiles"]  = $exportFiles;
		$options["exportClean"]  = $exportClean;
		$patternCodeHelper       = new PatternCodeHelper($options);
		$patternCodeHelper->run();

		// dispatch that the pattern code helper is ended
		$event = new PatternDataEvent($options);
		$dispatcherInstance->dispatch("patternData.patternCodeHelperEnd",$event);

		// dispatch that the gather has ended
		$event = new PatternDataEvent($options);
		$dispatcherInstance->dispatch("patternData.gatherEnd",$event);

	}
163169
protected function execute(InputInterface $input, OutputInterface $output)
    {
        if (null === $this->em) {
            $this->em = $this->getContainer()->get('doctrine.orm.entity_manager');
        }

        $configRepository = $this->em->getRepository('KunstmaanDashboardBundle:AnalyticsConfig');
        $configId = $input->getOption('config');
        $configs = [];

        try {
            if ($configId) {
                $configs[] = $configRepository->find($configId);
            } else {
                $configs = $configRepository->findAll();
            }

            foreach ($configs as $config) {
                $this->em->remove($config);
            }
            $this->em->flush();
            $output->writeln('<fg=green>Config flushed</fg=green>');
        } catch (\Exception $e) {
            $output->writeln('<fg=red>'.$e->getMessage().'</fg=red>');
        }
    }
332317
public function getOutput($input, array $options = array())
    {
        $this->log(sprintf('Getting output from file (%s)', $input));

        return $this->getGenerator()->getOutput($input, $options);
    }
912280
public function importIndex(
        Repository $repository,
        string $data,
        string $format
    ) {
        $items = $this
            ->exporterCollection
            ->getExporterByName($format)
            ->formatToItems($data);

        $repository->addItems($items);
        $repository->flush();
    }
958998
public function createBucket($bucketName, array $options = [])
    {
        $this->openManager();

        $this->manager->createBucket($bucketName, $options);

        // ToDo: https://github.com/matrozov/yii2-couchbase/issues/1
        sleep(5);
    }
1200227
private void importDataArchive(
            final Resource resource,
            final ArchiveInputStream resourceStream,
            BatchImportOptions options) {

        final File tempDir = Files.createTempDir();
        try {
            ArchiveEntry archiveEntry;
            while ((archiveEntry = resourceStream.getNextEntry()) != null) {
                final File entryFile = new File(tempDir, archiveEntry.getName());
                if (!archiveEntry.isDirectory()) {
                    entryFile.getParentFile().mkdirs();
                    IOUtils.copy(
                            new CloseShieldInputStream(resourceStream),
                            new FileOutputStream(entryFile));
                }
            }

            importDataDirectory(tempDir, null, options);
        } catch (IOException e) {
            throw new RuntimeException(
                    "Failed to extract data from '"
                            + resource
                            + "' to '"
                            + tempDir
                            + "' for batch import.",
                    e);
        } finally {
            FileUtils.deleteQuietly(tempDir);
        }
    }
858140
public function importFromArray(array $options)
    {
        foreach ($options as $key => $value) {
            switch (true) {
                case $key === 'args':
                    $this->args = $value;
                    break;
                case is_integer($key):
                    $this->args[] = $value;
                    break;
                default:
                    $this->options[$key] = $value;
                    break;
            }
        }
    }
333053
function index(options) {
  const {
    file,
    config,
    logStart,
    logNotFound,
    logEnd,
    defaultContent = defaultFileContent,
  } = options;

  logger.log(logStart);

  return readConfig(config)
    .then(input => validateExtensions(input))
    .then(input => createStrings(input))
    .then(input => writeExtensionFile({
      input,
      file,
      defaultContent,
      logNotFound,
      logEnd,
    }));
}
1291479
private void esIndex(String index, String id, String json, String pipeline) {
        logger.debug("Indexing {}/{}?pipeline={}", index, id, pipeline);
        logger.trace("JSon indexed : {}", json);

        if (!closed) {
            esClient.index(index, id, json, pipeline);
        } else {
            logger.warn("trying to add new file while closing crawler. Document [{}]/[{}] has been ignored", index, id);
        }
    }
360939
protected function execute(InputInterface $input, OutputInterface $output)
    {
        $io = new SymfonyStyle($input, $output);
        $io->title('Start ke_search indexer process');
        $indexerRunner = GeneralUtility::makeInstance(IndexerRunner::class);
        $indexerRunner->logger->log('notice', 'Indexer process started by command controller.');
        $indexerResponse = $indexerRunner->startIndexing(
            true, [], 'CLI'
        );

        if (strstr($indexerResponse, 'You can\'t start the indexer twice')) {
            $io->warning(
                'Indexing lock is set. You can\'t start the indexer process twice.' . chr(10) . chr(10)
                . 'If lock was not reset because of indexer errors you can use ke_search:removelock command.'
            );
        } else {

            // set custom style
            $titleStyle = new OutputFormatterStyle('blue', 'black', array('bold'));
            $output->getFormatter()->setStyle('title', $titleStyle);
            $warning = false;

            foreach (explode(chr(10), $indexerResponse) as $line) {

                // skip empty lines
                if (empty(strip_tags($line))) {
                    continue;
                }

                // format lines and catch warnings
                if (strstr($line, 'There were errors')) {
                    $warning = strip_tags(str_replace('[DANGER] ', '', $line));
                } else if (strstr($line, 'Index contains')) {
                    $io->writeln('<info>' . strip_tags($line) . '</info>');
                } else if (strstr($line, '<p><b>')) {
                    $io->writeln(chr(10) . '<title>' . strip_tags($line) . '</>');
                } else {
                    $io->writeln(strip_tags($line));
                }
            }

            if ($warning !== false) {
                $io->warning($warning);
            }

            $io->success('Indexer process completed.');
        }
    }
799471
public function connectOutputToLogging(OutputInterface $output, $command)
    {
        static $alreadyConnected = false;
        $helper = $this;

        // ignore any second or later invocations of this method
        if ($alreadyConnected) {
            return;
        }

        /** @var Dispatcher $eventDispatcher  */
        $eventDispatcher = $command->getService('event_dispatcher');

        $eventDispatcher->addListener(
            'parser.file.pre',
            function (PreFileEvent $event) use ($output) {
                $output->writeln('Parsing <info>'.$event->getFile().'</info>');
            }
        );

        $eventDispatcher->addListener(
            'system.log',
            function (LogEvent $event) use ($command, $helper, $output) {
                $helper->logEvent($output, $event, $command);
            }
        );

        $alreadyConnected = true;
    }
847420
public function createIndexFile($sources, $outputDir)
    {
      // Make the configuration available to the index template
      $configuration = $this->configuration;

      ob_start();
      include($this->configuration->get('index_template'));

      $indexContents = ob_get_clean();

      $outputFile = $outputDir . '/index.html';

      file_put_contents($outputFile, $indexContents);
    }
1041989
protected function getIndexCreate($output, $schema, $table, $indexName)
    {
        if ($indexName == 'PRIMARY') {
            return $output;
        }
        $indexes = $schema['tables'][$table]['indexes'];
        $indexSequences = $indexes[$indexName];

        $indexFields = $this->getIndexFields($indexSequences);
        $indexOptions = $this->getIndexOptions(array_values($indexSequences)[0]);

        $output[] = sprintf("%sif(\$this->table('%s')->hasIndex('%s')) {", $this->ind2, $table, $indexName);
        $output[] = sprintf("%s%s", $this->ind, $this->getIndexRemove($table, $indexName));
        $output[] = sprintf("%s}", $this->ind2);
        $output[] = sprintf("%s\$this->table(\"%s\")->addIndex(%s, %s)->save();", $this->ind2, $table, $indexFields,
            $indexOptions);

        return $output;
    }
230377
public function buildIndex($basedir, OutputInterface $output, array $options = array())
    {
        $target = $this->ensureBuildDir($basedir, $output);

        $built_iterator = new DirectoryIterator($target);
        $index = $target . DIRECTORY_SEPARATOR . 'index.rst';

        $title = str_replace('\\', '\\\\', $this->reflection->getName());
        if (isset($options['title'])) {
            $title = $options['title'];
        }

        $depth = substr_count($this->reflection->getName(), '\\');

        $template = str_repeat($this->titles[$depth], strlen($title)) . "\n";
        $template .= $title . "\n";
        $template .= str_repeat($this->titles[$depth], strlen($title)) . "\n\n";
        $template .= $this->getNamespaceElement();

        $template .= ".. toctree::\n\n";

        foreach ($built_iterator as $file) {
            if ($file->isDot()) continue;
            if ($file->isFile() && !$file->getExtension() == 'rst') continue;
            if ($file->isFile() && substr($file->getBaseName(), 0, 1) == '.') continue;
            if ($file->getBaseName() == 'index.rst') continue;

            $template .= '   ' . pathinfo($file->getPathName(), PATHINFO_FILENAME);

            if ($file->isDir()) {
                $template .= '/index';
            }

            $template .= "\n";
        }

        file_put_contents($index, $template);
    }
534714
protected function execute(InputInterface $input, OutputInterface $output)
    {
        $this->getHelper('phpdocumentor_logger')->connectOutputToLogging($output, $this);

        $converter = $this->getConverter($input);
        $converter->setOption('title', $input->getOption('title'));

        $output->writeln('Collect all documents');
        $files = $this->buildCollection(
            $input->getArgument('source'),
            $converter->getDefinition()->getInputFormat()->getExtensions()
        );

        $output->writeln('Converting documents');
        $files = $converter->convert($files, $this->getTemplate($input));

        $output->writeln('Writing converted documents to disk');
        $this->writeToDisk($files, $input->getOption('target'));

        $output->writeln('Writing assets to disk');
        $converter->getAssets()->copyTo($input->getOption('target'));
    }
970284
protected function createFields(Manager $manager)
    {
        $fieldMap = [];
        $instance = $this->getDataObjectInstance();
        $extraDataObjects = $this->nestedDataObjectClasses();
        $this->fields->removeDuplicates('Name');

        if (!$this->fields->exists()) {
            $this->addFields(
                Config::inst()->get(self::class, 'default_fields')
            );
        }

        $resolver = function ($obj, $args, $context, $info) {
            /**
             * @var DataObject $obj
             */
            $field = $obj->obj($info->fieldName);
            // return the raw field value, or checks like `is_numeric()` fail
            if ($field instanceof DBField && $field->isInternalGraphQLType()) {
                return $field->getValue();
            }
            return $field;
        };

        foreach ($this->fields as $fieldData) {
            $fieldName = $fieldData->Name;
            if (!StaticSchema::inst()->isValidFieldName($instance, $fieldName)) {
                throw new InvalidArgumentException(
                    sprintf(
                        'Invalid field "%s" on %s',
                        $fieldName,
                        $this->getDataObjectClass()
                    )
                );
            }

            $result = $instance->obj($fieldName);

            if ($result instanceof SS_List) {
                throw new InvalidArgumentException(
                    sprintf(
                        'Fieldname %s added to %s returns a list. This should be defined as a nested query using addNestedQuery(%s)',
                        $fieldName,
                        $this->getDataObjectClass(),
                        $fieldName
                    )
                );
            }

            if ($result instanceof DBField) {
                /** @var DBField|TypeCreatorExtension $result */
                $fieldMap[$fieldName] = [];
                $fieldMap[$fieldName]['type'] = $result->getGraphQLType($manager);
                $fieldMap[$fieldName]['resolve'] = $resolver;
                $fieldMap[$fieldName]['description'] = $fieldData->Description;
            }
        }

        foreach ($extraDataObjects as $fieldName => $className) {
            $description = $this->getFieldDescription($fieldName);
            $fieldMap[$fieldName] = [
                'type' => StaticSchema::inst()->fetchFromManager($className, $manager),
                'description' => $description,
                'resolve' => $resolver,
            ];
        }

        foreach ($this->nestedQueries as $name => $scaffolder) {
            $scaffold = $scaffolder->scaffold($manager);
            $scaffold['name'] = $name;
            $fieldMap[$name] = $scaffold;
        }

        return $fieldMap;
    }