Import activities for a single document into timeline.
325704
def _activity_import_doc(self, time_doc, activities):
        '''
        
        '''
        batch_updates = [time_doc]
        # We want to consider only activities that happend before time_doc
        # do not move this, because time_doc._start changes
        # time_doc['_start'] is a timestamp, whereas act[0] is a datetime
        # we need to be sure to convert act[0] (when) to timestamp!
        td_start = time_doc['_start']
        activities = filter(lambda act: (act[0] < td_start and
                                         act[1] in time_doc), activities)
        creation_field = self.lconfig.get('cfield')
        # make sure that activities are sorted by when descending
        activities.sort(reverse=True, key=lambda o: o[0])
        new_doc = {}
        for when, field, removed, added in activities:
            last_doc = batch_updates.pop()
            # check if this activity happened at the same time as the last one,
            # if it did then we need to group them together
            if last_doc['_end'] == when:
                new_doc = deepcopy(last_doc)
                last_doc = batch_updates.pop()
            else:
                new_doc = deepcopy(last_doc)
                new_doc['_start'] = when
                new_doc['_end'] = when
                last_doc['_start'] = when
            last_val = last_doc[field]

            new_val, inconsistent = self._activity_backwards(new_doc[field],
                                                             removed, added)
            new_doc[field] = new_val
            # Check if the object has the correct field value.
            if inconsistent:
                self._log_inconsistency(last_doc, last_val, field,
                                        removed, added, when)
                new_doc['_e'] = {} if not new_doc.get('_e') else new_doc['_e']
                # set curreupted field value to the the value that was added
                # and continue processing as if that issue didn't exist
                new_doc['_e'][field] = added
            # Add the objects to the batch
            batch_updates.extend([last_doc, new_doc])
        # try to set the _start of the first version to the creation time
        try:
            # set start to creation time if available
            last_doc = batch_updates[-1]
            if creation_field:
                # again, we expect _start to be epoch float...
                creation_ts = dt2ts(last_doc[creation_field])
                if creation_ts < last_doc['_start']:
                    last_doc['_start'] = creation_ts
                elif len(batch_updates) == 1:
                    # we have only one version, that we did not change
                    return []
                else:
                    pass  # leave as-is
        except Exception as e:
            logger.error('Error updating creation time; %s' % e)
        return batch_updates
1137476
def bulk_upsert(self, docs, namespace, timestamp):
        
        for doc in docs:
            self.upsert(doc, namespace, timestamp)
471217
def document_single_builder(builder):
  """"""
  mod_name = builder.__class__.__module__
  cls_name = builder.__class__.__name__
  mod_file = sys.modules[mod_name].__file__
  if mod_file.endswith("pyc"):
    mod_file = mod_file[:-1]

  description_prefix = ""


  if builder.builder_configs:
    # Dataset with configs; document each one
    config_docs = []
    for config in builder.BUILDER_CONFIGS:
      builder = tfds.builder(builder.name, config=config)
      info = builder.info
      # TODO(rsepassi): document the actual config object
      config_doc = SINGLE_CONFIG_ENTRY.format(
          builder_name=builder.name,
          config_name=config.name,
          description=config.description,
          version=config.version,
          feature_information=make_feature_information(info),
          size=tfds.units.size_str(info.size_in_bytes),
      )
      config_docs.append(config_doc)
    out_str = DATASET_WITH_CONFIGS_ENTRY.format(
        snakecase_name=builder.name,
        module_and_class="%s.%s" % (tfds_mod_name(mod_name), cls_name),
        cls_url=cls_url(mod_name),
        config_names="\n".join([
            CONFIG_BULLET.format(name=config.name,
                                 description=config.description,
                                 version=config.version,
                                 size=tfds.units.size_str(tfds.builder(
                                     builder.name, config=config)
                                                          .info.size_in_bytes))
            for config in builder.BUILDER_CONFIGS]),
        config_cls="%s.%s" % (tfds_mod_name(mod_name),
                              type(builder.builder_config).__name__),
        configs="\n".join(config_docs),
        urls=format_urls(info.urls),
        url=url_from_info(info),
        supervised_keys=str(info.supervised_keys),
        citation=make_citation(info.citation),
        statistics_information=make_statistics_information(info),
        description=builder.info.description,
        description_prefix=description_prefix,
    )
  else:
    info = builder.info
    out_str = DATASET_ENTRY.format(
        snakecase_name=builder.name,
        module_and_class="%s.%s" % (tfds_mod_name(mod_name), cls_name),
        cls_url=cls_url(mod_name),
        description=info.description,
        description_prefix=description_prefix,
        version=info.version,
        feature_information=make_feature_information(info),
        statistics_information=make_statistics_information(info),
        urls=format_urls(info.urls),
        url=url_from_info(info),
        supervised_keys=str(info.supervised_keys),
        citation=make_citation(info.citation),
        size=tfds.units.size_str(info.size_in_bytes),
    )

  out_str = schema_org(builder) + "\n" + out_str
  return out_str
1055812
def import_doc(self, file_uris, docsearch, current_doc=None):
        

        doc = None
        docs = []
        pages = []

        file_uris = [self.fs.safe(uri) for uri in file_uris]
        imported = []
        for file_uri in file_uris:
            logger.info("Importing PDF from '%s'" % (file_uri))
            idx = 0

            for child in self.fs.recurse(file_uri):
                gc.collect()
                if not self.check_file_type(child):
                    continue
                h = PdfDoc.hash_file(self.fs, child)
                if docsearch.is_hash_in_index(h):
                    logger.info(
                        "Document %s already found in the index. Skipped",
                        child
                    )
                    continue
                imported.append(child)
                doc = PdfDoc(self.fs, docsearch.rootdir)
                error = doc.import_pdf(child)
                if error:
                    continue
                docs.append(doc)
                pages += [p for p in doc.pages]
                idx += 1
        return ImportResult(
            imported_file_uris=imported,
            select_doc=doc, new_docs=docs,
            new_docs_pages=pages,
            stats={
                _("PDF"): len(docs),
                _("Document(s)"): len(docs),
                _("Page(s)"): sum([d.nb_pages for d in docs]),
            },
        )
1371728
def parse_schedules(username, password, importer, progress, utc_start=None, 
                    utc_stop=None):
    """"""

    file_obj = get_file_object(username, password, utc_start, utc_stop)
    process_file_object(file_obj, importer, progress)
938797
def do_links(doc1, doc2, from_page = -1, to_page = -1, start_at = -1):
    ''''''
    #--------------------------------------------------------------------------
    # define skeletons for /Annots object texts
    #--------------------------------------------------------------------------
    annot_goto ='''<</Dest[%i 0 R /XYZ %g %g 0]/Rect[%s]/Subtype/Link>>'''

    annot_gotor = '''<</A<</D[%i /XYZ %g %g 0]/F<</F(%s)/UF(%s)/Type/Filespec
    >>/S/GoToR>>/Rect[%s]/Subtype/Link>>'''

    annot_gotor_n = "<</A<</D(%s)/F(%s)/S/GoToR>>/Rect[%s]/Subtype/Link>>"

    annot_launch = '''<</A<</F<</F(%s)/UF(%s)/Type/Filespec>>/S/Launch
    >>/Rect[%s]/Subtype/Link>>'''

    annot_uri = '''<</A<</S/URI/URI(%s)/Type/Action>>/Rect[%s]/Subtype/Link>>'''

    #--------------------------------------------------------------------------
    # internal function to create the actual "/Annots" object string
    #--------------------------------------------------------------------------
    def cre_annot(lnk, xref_dst, list_src, height):
        '''Create annotation object string for a passed-in link.'''

        # "from" rectangle is always there. Note: y-coords are from bottom!

        r = lnk["from"]
        rect = "%g %g %g %g" % (r.x0, height - r.y0, r.x1, height - r.y1)
        if lnk["kind"] == LINK_GOTO:
            txt = annot_goto
            idx = list_src.index(lnk["page"])
            annot = txt % (xref_dst[idx], lnk["to"].x,
                           lnk["to"].y, rect)
        elif lnk["kind"] == LINK_GOTOR:
            if lnk["page"] >= 0:
                txt = annot_gotor
                pnt = lnk.get("to", Point(0, 0))          # destination point
                if type(pnt) is not Point:
                    pnt = Point(0, 0)
                annot = txt % (lnk["page"], pnt.x, pnt.y,
                           lnk["file"], lnk["file"], rect)
            else:
                txt = annot_gotor_n
                to = getPDFstr(lnk["to"])
                to = to[1:-1]
                f = lnk["file"]
                annot = txt % (to, f, rect)

        elif lnk["kind"] == LINK_LAUNCH:
            txt = annot_launch
            annot = txt % (lnk["file"], lnk["file"], rect)
        elif lnk["kind"] == LINK_URI:
            txt = annot_uri
            annot = txt % (lnk["uri"], rect)
        else:
            annot = ""

        return annot
    #--------------------------------------------------------------------------

    # validate & normalize parameters
    if from_page < 0:
        fp = 0
    elif from_page >= doc2.pageCount:
        from_page = doc2.pageCount - 1
    else:
        fp = from_page

    if to_page < 0 or to_page >= doc2.pageCount:
        tp = doc2.pageCount - 1
    else:
        tp = to_page

    if start_at < 0:
        raise ValueError("do_links: 'start_at' arg must be >= 0")
    sa = start_at

    incr = 1 if fp <= tp else -1            # page range could be reversed
    # lists of source / destination page numbers
    list_src = list(range(fp, tp + incr, incr))
    list_dst = [sa + i for i in range(len(list_src))]
    # lists of source / destination page xref numbers
    xref_src = []
    xref_dst = []
    for i in range(len(list_src)):
        p_src = list_src[i]
        p_dst = list_dst[i]
        old_xref = doc2._getPageObjNumber(p_src)[0]
        new_xref = doc1._getPageObjNumber(p_dst)[0]
        xref_src.append(old_xref)
        xref_dst.append(new_xref)

    # create /Annots per copied page in destination PDF
    for i in range(len(xref_src)):
        page_src = doc2[list_src[i]]
        links = page_src.getLinks()
        if len(links) == 0:
            page_src = None
            continue
        height = page_src.bound().y1
        p_annots = ""
        page_dst = doc1[list_dst[i]]
        link_tab = []
        for l in links:
            if l["kind"] == LINK_GOTO and (l["page"] not in list_src):
                continue          # target not in copied pages
            annot_text = cre_annot(l, xref_dst, list_src, height)
            if not annot_text:
                print("cannot create /Annot for kind: " + str(l["kind"]))
            else:
                link_tab.append(annot_text)
        if len(link_tab) > 0:
            page_dst._addAnnot_FromString(link_tab)
        page_dst = None
        page_src = None
    return
342326
def bulk_upsert(self, docs, namespace, timestamp):
        """"""
        def docs_to_upsert():
            doc = None
            for doc in docs:
                # Remove metadata and redundant _id
                index, doc_type = self._index_and_mapping(namespace)
                doc_id = u(doc.pop("_id"))
                document_action = {
                    '_index': index,
                    '_type': doc_type,
                    '_id': doc_id,
                    '_source': self._formatter.format_document(doc)
                }
                document_meta = {
                    '_index': self.meta_index_name,
                    '_type': self.meta_type,
                    '_id': doc_id,
                    '_source': {
                        'ns': namespace,
                        '_ts': timestamp
                    }
                }
                yield document_action
                yield document_meta
            if doc is None:
                raise errors.EmptyDocsError(
                    "Cannot upsert an empty sequence of "
                    "documents into Elastic Search")
        try:
            kw = {}
            if self.chunk_size > 0:
                kw['chunk_size'] = self.chunk_size

            responses = streaming_bulk(client=self.elastic,
                                       actions=docs_to_upsert(),
                                       **kw)

            for ok, resp in responses:
                if not ok:
                    LOG.error(
                        "Could not bulk-upsert document "
                        "into ElasticSearch: %r" % resp)
            if self.auto_commit_interval == 0:
                self.commit()
        except errors.EmptyDocsError:
            # This can happen when mongo-connector starts up, there is no
            # config file, but nothing to dump
            pass
1272395
def import_data(self, fname):
        """"""
        if self.count():
            nsb = self.current_widget()
            nsb.refresh_table()
            nsb.import_data(filenames=fname)
            if self.dockwidget and not self.ismaximized:
                self.dockwidget.setVisible(True)
                self.dockwidget.raise_()
615629
def compose_from_transcript(files, search, searchtype):
    """"""

    final_segments = []

    if searchtype in ['re', 'word', 'franken', 'fragment']:
        if searchtype == 're':
            searchtype = 'sentence'

        segments = audiogrep.search(search, files, mode=searchtype, regex=True)
        for seg in segments:
            seg['file'] = seg['file'].replace('.transcription.txt', '')
            seg['line'] = seg['words']
            final_segments.append(seg)

    elif searchtype in ['hyper', 'pos']:
        for s in audiogrep.convert_timestamps(files):
            for w in s['words']:
                if search_line(w[0], search, searchtype):
                    seg = {
                        'file': s['file'].replace('.transcription.txt',''),
                        'line': w[0],
                        'start': float(w[1]),
                        'end': float(w[2])
                    }
                    final_segments.append(seg)

    return final_segments
1019815
def _load_timestamps(self):
        """"""
        timestamp_file = os.path.join(self.sequence_path, 'times.txt')

        # Read and parse the timestamps
        self.timestamps = []
        with open(timestamp_file, 'r') as f:
            for line in f.readlines():
                t = dt.timedelta(seconds=float(line))
                self.timestamps.append(t)

        # Subselect the chosen range of frames, if any
        if self.frames is not None:
            self.timestamps = [self.timestamps[i] for i in self.frames]
798400
def timeline_list(self, id, max_id=None, min_id=None, since_id=None, limit=None):
        
        id = self.__unpack_id(id)
        return self.timeline('list/{0}'.format(id), max_id=max_id,
                             min_id=min_id, since_id=since_id, limit=limit)
463957
def import_dashboards(session, data_stream, import_time=None):
    """"""
    current_tt = int(time.time())
    import_time = current_tt if import_time is None else import_time
    data = json.loads(data_stream.read(), object_hook=decode_dashboards)
    # TODO: import DRUID datasources
    for table in data['datasources']:
        type(table).import_obj(table, import_time=import_time)
    session.commit()
    for dashboard in data['dashboards']:
        Dashboard.import_obj(
            dashboard, import_time=import_time)
    session.commit()
223017
def process_document(self, doc):
        

        descriptions = doc.select_segments("projects[*].description")
        projects = doc.select_segments("projects[*]")

        for d, p in zip(descriptions, projects):

            # First phase of extraction
            names = doc.extract(self.name_extractor, d)
            p.store(names, "members")

            # Second phase of extraction
            students = []
            for name_extraction in names:
                students += doc.extract(self.student_extractor, name_extraction)
            p.store(students, "students")
        doc.kg.add_value("developer", json_path="projects[*].members[*]")
        doc.kg.add_value("owner", json_path="projects[*].students[*]")
        return list()
943891
def insert_doc(self, p_elem, docstring):
        """"""
        dtag = self.namespaces[self.a_uri] + ":documentation"
        elem = SchemaNode(dtag, text=docstring)
        p_elem.annots.append(elem)
347170
def recipients_for_timeline(timeline_class, activity)
      result = { }

      routes = timeline_class.routes_for_activity(activity.class)
      routes.each do |route|
        route.resolve(activity).each do |recipient|
          recipient_id = timeline_class.recipient_id(recipient)

          # keep only one route per recipient
          if result[recipient_id].nil?
            result[recipient_id] = { :rcpt => recipient, :route => route }
          end
        end
      end

      result.inject({ }) do |memo, (recipient_id, infos)|
        memo[infos[:rcpt]] = infos[:route]
        memo
      end
    end
857852
def on_import1(self, event):
        
        pmag_menu_dialogs.MoveFileIntoWD(self.parent, self.parent.WD)
330419
def write(self, items):
        
        if self._read_only:
            raise IOError("Cannot write, Connector created as Read Only")

        if len(items) == 0:
            logger.info(self.__log_prefix + " Nothing to write")
            return

        # Uploading info to the new ES
        rows = items.to_dict("index")
        docs = []
        for row_index in rows.keys():
            row = rows[row_index]
            item_id = row[self.AUTHOR_ORG] + '_' + row[self.PROJECT] + '_' \
                + row[self.TIMEFRAME] + '_' + row[self.AUTHOR_UUID]
            item_id = item_id.replace(' ', '').lower()

            doc = {
                "_index": self._es_index,
                "_type": "item",
                "_id": item_id,
                "_source": row
            }
            docs.append(doc)

        # TODO uncomment following lines for incremental version
        # # Delete old data if exists to ensure refreshing in case of deleted commits
        # timeframe = docs[0]['_source']['timeframe']
        # org = docs[0]['_source']['author_org_name']
        # project = docs[0]['_source']['project']
        # s = Search(using=self._es_conn, index=self._es_index)
        # s = s.filter('term', project=project)
        # s = s.filter('term', author_org_name=org)
        # s = s.filter('term', timeframe=timeframe)
        # response = s.execute()
        #
        # if response.hits.total > 0:
        #     response = s.delete()
        #     logger.info("[Onion] Deleted " + str(response.deleted) + " items for refreshing: " + timeframe + " "
        #                 + org + " " + project)

        # TODO exception and error handling
        helpers.bulk(self._es_conn, docs)
        logger.info(self.__log_prefix + " Written: " + str(len(docs)))