Index build method.

Its responsibility is to call the parseFile
and generate methods from generator.
328633
public function build()
    {
        $files = $this->iterator->getFiles();
        $generators = $this->generatorRegistry->generators();

        foreach ($files as $file) {
            foreach ($generators as $generator) {
                $generator->parseFile($file);
            }
        }

        foreach ($generators as $generator) {
            $generator->generate();
        }
    }
899363
def _build_indices(self):
        """"""
        result = {key: OrderedDict() for key in LINES_WITH_ID}
        for line in self.lines:
            if line.key in LINES_WITH_ID:
                result.setdefault(line.key, OrderedDict())
                if line.mapping["ID"] in result[line.key]:
                    warnings.warn(
                        ("Seen {} header more than once: {}, using first" "occurence").format(
                            line.key, line.mapping["ID"]
                        ),
                        DuplicateHeaderLineWarning,
                    )
                else:
                    result[line.key][line.mapping["ID"]] = line
            else:
                result.setdefault(line.key, [])
                result[line.key].append(line)
        return result
561217
protected synchronized void makeIndex() throws Exception {
        File indexFile = getIndexFile(filename);
        if (indexFile.exists()) try {
            loadIndex(indexFile);
            setIndexCreated(true);
            return;
        } catch (Exception x) {
            logger.warn(x.getMessage());
        }
        indexCreated = false;
        long now = System.currentTimeMillis();
        int recordLength = 1000;
        int maxRecords = 1;
        int maxRecordLength = 0;
        maxRecords = (int) raFile.length() / recordLength;
        if (maxRecords == 0) maxRecords = 1;
        index = new long[maxRecords][3];

        String s = null;
        long start = 0;
        long end = 0;
        raFile.seek(0);
        records = 0;
        recordLength = 0;
        while ((s = raFile.readLine()) != null) {
            if (start == -1) start = raFile.getFilePointer();
            if (isRecordEnd(s)) {
                //fireFrameRead();
                if (records >= maxRecords) {
                    index = resize(index,
                            records
                                    + (int) (records + (raFile.length() - records * raFile.getFilePointer())
                                            / recordLength));
                }
                end += 4;
                index[records][0] = start;
                index[records][1] = end - start;
                index[records][2] = -1;
                if (maxRecordLength < index[records][1]) maxRecordLength = (int) index[records][1];
                records++;
                recordLength += end - start;

                start = raFile.getFilePointer();
            } else {
                end = raFile.getFilePointer();
            }

        }
        b = new byte[maxRecordLength];
        //fireFrameRead();
        logger.info("Index created in " + (System.currentTimeMillis() - now) + " ms.");
        try {
            saveIndex(indexFile);
        } catch (Exception x) {
            logger.error(x);
        }
    }
698144
def _generate_index(root, folder, paths,
                    bots_index=False, bots_index_paths=()):
    """"""
    # Determine the namespaces listed here (as sub folders)
    # and the files (.html files) that we should link to
    namespaces = []
    files = []
    INDEX = 'index.html'
    BOT_INDEX = 'botindex.html'

    for item in (bots_index_paths or folder.iterdir()):
        if item.is_dir():
            namespaces.append(item)
        elif item.name not in (INDEX, BOT_INDEX):
            files.append(item)

    # Now that everything is setup, write the index.html file
    filename = folder / (BOT_INDEX if bots_index else INDEX)
    with DocsWriter(root, filename, _get_path_for_type) as docs:
        # Title should be the current folder name
        docs.write_head(str(folder).replace(os.path.sep, '/').title(),
                        css_path=paths['css'],
                        default_css=paths['default_css'])

        docs.set_menu_separator(paths['arrow'])
        _build_menu(docs)
        docs.write_title(str(filename.parent.relative_to(root))
                         .replace(os.path.sep, '/').title())

        if bots_index:
            docs.write_text('These are the methods that you may be able to '
                            'use as a bot. Click <a href="{}">here</a> to '
                            'view them all.'.format(INDEX))
        else:
            docs.write_text('Click <a href="{}">here</a> to view the methods '
                            'that you can use as a bot.'.format(BOT_INDEX))
        if namespaces:
            docs.write_title('Namespaces', level=3)
            docs.begin_table(4)
            namespaces.sort()
            for namespace in namespaces:
                # For every namespace, also write the index of it
                namespace_paths = []
                if bots_index:
                    for item in bots_index_paths:
                        if item.parent == namespace:
                            namespace_paths.append(item)

                _generate_index(root, namespace, paths,
                                bots_index, namespace_paths)

                docs.add_row(
                    namespace.stem.title(),
                    link=namespace / (BOT_INDEX if bots_index else INDEX))

            docs.end_table()

        docs.write_title('Available items')
        docs.begin_table(2)

        files = [(f, _find_title(f)) for f in files]
        files.sort(key=lambda t: t[1])

        for file, title in files:
            docs.add_row(title, link=file)

        docs.end_table()
        docs.end_body()
508847
def build(self, sources=None, tables=None, stage=None, force=False):
        

        from operator import attrgetter
        from itertools import groupby
        from .concurrent import build_mp, unify_mp
        from ambry.bundle.events import TAG

        self.log('==== Building ====')
        self.state = self.STATES.BUILDING
        self.commit()

        class SourceSet(object):
            """Container for sources that can reload them after they get expired from the session"""
            def __init__(self, bundle, v):
                self.bundle = bundle
                self.sources = v
                self._s_vids = [s.vid for s in self.sources]

            def reload(self):
                self.sources = [self.bundle.source(vid) for vid in self._s_vids]

            def __iter__(self):
                for s in self.sources:
                    yield s

            def __len__(self):
                return len(self._s_vids)

        self._run_events(TAG.BEFORE_BUILD, 0)

        resolved_sources = SourceSet(self, self._resolve_sources(sources, tables, stage=stage,
                                                                 predicate=lambda s: s.is_processable))
        with self.progress.start('build', stage, item_total=len(resolved_sources)) as ps:

            if len(resolved_sources) == 0:
                ps.update(message='No sources', state='skipped')
                self.log('No processable sources, skipping build stage {}'.format(stage))
                return True

            if not self.pre_build(force):
                ps.update(message='Pre-build failed', state='skipped')
                return False

            if force:
                self._reset_build(resolved_sources)

            resolved_sources.reload()

            e = [
                (stage, SourceSet(self, list(stage_sources)))
                for stage, stage_sources in groupby(sorted(resolved_sources, key=attrgetter('stage')),
                                                    attrgetter('stage'))

                ]

            for stage, stage_sources in e:

                stage_sources.reload()

                for s in stage_sources:
                    s.state = self.STATES.WAITING
                self.commit()

                stage_sources.reload()

                self.log('Processing {} sources, stage {} ; first 10: {}'
                         .format(len(stage_sources), stage, [x.name for x in stage_sources.sources[:10]]))
                self._run_events(TAG.BEFORE_BUILD, stage)

                if self.multi:

                    try:
                        # The '1' for chunksize ensures that the subprocess only gets one
                        # source to build. Combined with maxchildspertask = 1 in the pool,
                        # each process will only handle one source before exiting.

                        args = [(self.identity.vid, stage, source.vid, force) for source in stage_sources]
                        pool = self.library.process_pool(limited_run=self.limited_run)
                        r = pool.map_async(build_mp, args, 1)
                        completed_sources = r.get()

                        ps.add('Finished MP building {} sources. Starting MP coalescing'
                               .format(len(completed_sources)))

                        partition_names = [(self.identity.vid, k) for k, v
                                           in self.collect_segment_partitions().items()]

                        r = pool.map_async(unify_mp, partition_names, 1)

                        completed_partitions = r.get()

                        ps.add('Finished MP coalescing {} partitions'.format(len(completed_partitions)))

                        pool.close()
                        pool.join()

                    except KeyboardInterrupt:
                        self.log('Got keyboard interrrupt; terminating workers')
                        pool.terminate()

                else:

                    for i, source in enumerate(stage_sources):
                        id_ = ps.add(message='Running source {}'.format(source.name),
                                     source=source, item_count=i, state='running')

                        self.build_source(stage, source, ps, force=force)

                        ps.update(message='Finished processing source', state='done')

                        # This bit seems to solve a problem where the records from the ps.add above
                        # never gets closed out.
                        ps.get(id_).state = 'done'
                        self.progress.commit()

                    self.unify_partitions()

                self._run_events(TAG.AFTER_BUILD, stage)

        self.state = self.STATES.BUILT
        self.commit()

        self._run_events(TAG.AFTER_BUILD, 0)

        self.close_session()

        self.log('==== Done Building ====')
        self.buildstate.commit()
        self.state = self.STATES.BUILT
        self.commit()
        return True
665561
def _build_indexes(self):
        
        if isinstance(self._data, list):
            for d in self._data:
                if not isinstance(d, dict):
                    err = u'Cannot build index for non Dict type.'
                    self._tcex.log.error(err)
                    raise RuntimeError(err)

                data_obj = DataObj(d)
                self._master_index.setdefault(id(data_obj), data_obj)

                for key, value in d.items():
                    # bcs - update this
                    # if not isinstance(value, (types.StringType, float, int)):
                    # TODO: This is not Python 3 ready
                    if not isinstance(value, (float, int, str)):
                        # For comparison operators the value needs to be a StringType
                        self._tcex.log.debug(u'Can only build index String Types.')
                        continue

                    self._indexes.setdefault(key, {}).setdefault(value, []).append(data_obj)
        else:
            err = u'Only *List* data type is currently supported'
            self._tcex.log.error(err)
            raise RuntimeError(err)
924404
def create_contents_and_index
      contents = []
      index    = []

      (@files+@classes).sort.each do |entry|
	content_entry = { "c_name" => entry.name, "ref" => entry.path }
	index << { "name" => entry.name, "aref" => entry.path }

	internals = []

	methods = entry.build_method_summary_list(entry.path)

	content_entry["methods"] = methods unless methods.empty?
        contents << content_entry
	index.concat methods
      end

      values = { "contents" => contents }
      template = TemplatePage.new(RDoc::Page::CONTENTS)
      File.open("contents.hhc", "w") do |f|
	template.write_html_on(f, values)
      end

      values = { "index" => index }
      template = TemplatePage.new(RDoc::Page::CHM_INDEX)
      File.open("index.hhk", "w") do |f|
	template.write_html_on(f, values)
      end      
    end
1233608
@Override
	public void generateAPIFiles(List<ApiImplementor> implementors) throws IOException {
		// Generate index first
		this.generateWikiIndex();
		for (ApiImplementor imp : implementors) {
			this.generateAPIFiles(imp);
		}
		this.methods = 0;
		this.generateWikiFull();
		System.out.println("Generated a total of " + methods + " methods");
	}
791797
public final void execute(@NotNull final Set<File> files) throws ParseException, GenerateException {

        Contract.requireArgNotNull("files", files);

        LOG.info("Executing incremental build ({} files)", files.size());
        if (LOG.isDebugEnabled()) {
            for (final File file : files) {
                LOG.debug(file.toString());
            }
        }

        if (files.size() == 0) {
            // Nothing to do...
            return;
        }

        // Parse models & generate
        final Parsers parsers = config.getParsers();
        if (parsers == null) {
            LOG.warn("No parsers element");
        } else {
            final List<ParserConfig> parserConfigs = parsers.getList();
            if (parserConfigs == null) {
                LOG.warn("No parsers configured");
            } else {
                for (final ParserConfig pc : parserConfigs) {
                    final Parser<Object> pars = pc.getParser();
                    if (pars instanceof IncrementalParser) {
                        final IncrementalParser<?> parser = (IncrementalParser<?>) pars;
                        final Object model = parser.parse(files);
                        final List<GeneratorConfig> generatorConfigs = config.findGeneratorsForParser(pc.getName());
                        for (final GeneratorConfig gc : generatorConfigs) {
                            final Generator<Object> generator = gc.getGenerator();
                            generator.generate(model, true);
                        }
                    } else {
                        LOG.debug("No incremental parser: {}", pars.getClass().getName());
                    }
                }
            }
        }

    }
1367416
function(obj, index) {
    var funcKey = [];

    for(var key in obj) {
      if(obj.hasOwnProperty(key)) {

        // If this is a function, generate it later.
        if(typeof obj[key] === 'function') {
          funcKey.push(key);
          continue;
        }
        obj[key] = generate.call(obj, obj[key], index);
      }
    }

    // parse function
    funcKey.forEach(function(key) {
      obj[key] = generate.call(obj, obj[key], index);
    });

    return obj;
  }
998354
def generate_index(credentials, instance_config, instance_name,
                   script_dir, genome_file, output_dir, annotation_file=None,
                   splice_overhang=100,
                   num_threads=8, chromosome_bin_bits=18,
                   genome_memory_limit=31000000000,
                   self_destruct=True,
                   **kwargs):
    """"""

    template = _TEMPLATE_ENV.get_template(
        os.path.join('generate_index.sh'))
    startup_script = template.render(
        script_dir=script_dir,
        genome_file=genome_file,
        annotation_file=annotation_file,
        splice_overhang=splice_overhang,
        output_dir=output_dir,
        num_threads=num_threads,
        chromosome_bin_bits=chromosome_bin_bits,
        genome_memory_limit=genome_memory_limit,
        self_destruct=self_destruct)

    op_name = instance_config.create_instance(
        credentials, instance_name, startup_script=startup_script, **kwargs)

    return op_name
101562
public static void generate(ConfigurationImpl configuration,
                                IndexBuilder indexbuilder) {
        SplitIndexWriter indexgen;
        DocPath filename = DocPath.empty;
        DocPath path = DocPaths.INDEX_FILES;
        try {
            Set<Object> keys = new TreeSet<>(Arrays.asList(indexbuilder.elements()));
            keys.addAll(configuration.tagSearchIndexKeys);
            List<Object> elements = new ArrayList<>(keys);
            ListIterator<Object> li = elements.listIterator();
            while (li.hasNext()) {
                Object ch = li.next();
                filename = DocPaths.indexN(li.nextIndex());
                indexgen = new SplitIndexWriter(configuration,
                        path.resolve(filename),
                        indexbuilder, elements, li.previousIndex(), li.nextIndex());
                indexgen.generateIndexFile((Character) ch);
                if (!li.hasNext()) {
                    indexgen.createSearchIndexFiles();
                }
                indexgen.close();
            }
        } catch (IOException exc) {
            configuration.standardmessage.error(
                        "doclet.exception_encountered",
                        exc.toString(), filename.getPath());
            throw new DocletAbortException(exc);
        }
    }
164258
function generateUtilityFiles(next) {
    fs.copy(path.join(getConfig('template.folder'), 'index.js'), path.join(getConfig('output.folder'), 'index.js'), function (err) {
        if (err) { next(err); return; }
        if (getConfig('output.log')) { logger.info('(Created) Index file: ' + path.resolve(path.join(getConfig('output.folder'), 'index.js'))); }
        fs.copy(path.join(getConfig('template.folder'), '..', 'utils.js'), path.join(getConfig('output.folder'), 'utils.js'), function (err) {
            if (err) { next(err); return; }
            next(null);
        });
    });
    try {
        overriddenAliases = require(path.join(process.cwd(), getConfig('output.folder'), 'alias.json'));
    } catch (ignore) {}
}
877398
def _slice_generator(self, index):
        """"""
        start, stop, step = index.indices(len(self))
        for i in range(start, stop, step):
            yield self.get_blob(i)
234556
def build(self):
        '''
        '''

        X_factory, mX_factory, category_idx_store, term_idx_store, metadata_idx_store, y \
            = self._init_term_doc_matrix_variables()
        parse_pipeline = ParsePipelineFactory(self.get_nlp(),
                                              X_factory,
                                              mX_factory,
                                              category_idx_store,
                                              term_idx_store,
                                              metadata_idx_store,
                                              y,
                                              self)
        df = self._clean_and_filter_nulls_and_empties_from_dataframe()
        tdm = self._apply_pipeline_and_get_build_instance(X_factory,
                                                          mX_factory,
                                                          category_idx_store,
                                                          df,
                                                          parse_pipeline,
                                                          term_idx_store,
                                                          metadata_idx_store,
                                                          y)
        return tdm
861904
private void buildIndex(String startfolder) throws Exception {

        File folder = new File(startfolder);
        // get all subresources

        File[] subresources = folder.listFiles();
        // now loop through all subresources and add them to the index list
        for (int i = 0; i < subresources.length; i++) {
            try {

                String relativeFSName = subresources[i].getAbsolutePath().substring(m_inputDir.length() + 1);
                String absoluteVFSName = getVfsName(
                    relativeFSName,
                    subresources[i].getName(),
                    subresources[i].isFile());
                m_report.print(Messages.get().container(Messages.RPT_CREATE_INDEX_0), I_CmsReport.FORMAT_NOTE);
                m_report.print(
                    org.opencms.report.Messages.get().container(
                        org.opencms.report.Messages.RPT_ARGUMENT_1,
                        relativeFSName.replace('\\', '/')));
                m_report.print(org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_DOTS_0));
                m_report.print(Messages.get().container(Messages.RPT_ARROW_RIGHT_0), I_CmsReport.FORMAT_NOTE);
                m_report.print(
                    org.opencms.report.Messages.get().container(
                        org.opencms.report.Messages.RPT_ARGUMENT_1,
                        absoluteVFSName));
                m_report.print(org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_DOTS_0));
                m_fileIndex.put(subresources[i].getAbsolutePath().replace('\\', '/'), absoluteVFSName);
                // if the subresource is a folder, get all subresources of it as well
                if (subresources[i].isDirectory()) {
                    buildIndex(subresources[i].getAbsolutePath());
                }
                m_report.println(
                    org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_OK_0),
                    I_CmsReport.FORMAT_OK);
            } catch (Exception e) {
                LOG.error(e.getLocalizedMessage(), e);
                m_report.println(e);
            }
        }
    }
302672
def generateIndex(self, refresh=0, refresh_index=0):
        """"""
        open(self.index_file, "wt").write(self.renderIndex(refresh=refresh, refresh_index=refresh_index))