Enqueuing an ingest command from local files.
        :param file_descriptor: a FileDescriptor to be ingested.
        :param azure.kusto.ingest.IngestionProperties ingestion_properties: Ingestion properties.
190609
def ingest_from_file(self, file_descriptor, ingestion_properties):
        
        file_descriptors = list()
        containers = self._resource_manager.get_containers()

        if isinstance(file_descriptor, FileDescriptor):
            descriptor = file_descriptor
        else:
            descriptor = FileDescriptor(file_descriptor)

        file_descriptors.append(descriptor)
        blob_name = "{db}__{table}__{guid}__{file}".format(
            db=ingestion_properties.database,
            table=ingestion_properties.table,
            guid=uuid.uuid4(),
            file=descriptor.stream_name,
        )

        container_details = random.choice(containers)
        storage_client = CloudStorageAccount(container_details.storage_account_name, sas_token=container_details.sas)
        blob_service = storage_client.create_block_blob_service()

        blob_service.create_blob_from_stream(
            container_name=container_details.object_name, blob_name=blob_name, stream=descriptor.zipped_stream
        )
        url = blob_service.make_blob_url(container_details.object_name, blob_name, sas_token=container_details.sas)

        self.ingest_from_blob(
            BlobDescriptor(url, descriptor.size, descriptor.source_id), ingestion_properties=ingestion_properties
        )
769420
def get_descriptor(self, number):
        """"""
        # Create stdout file and get file descriptor
        stdout_path = os.path.join(self.config_dir,
                                   'pueue_process_{}.stdout'.format(number))
        if os.path.exists(stdout_path):
            os.remove(stdout_path)
        out_descriptor = open(stdout_path, 'w+')

        # Create stderr file and get file descriptor
        stderr_path = os.path.join(self.config_dir,
                                   'pueue_process_{}.stderr'.format(number))
        if os.path.exists(stderr_path):
            os.remove(stderr_path)
        err_descriptor = open(stderr_path, 'w+')

        self.descriptors[number] = {}
        self.descriptors[number]['stdout'] = out_descriptor
        self.descriptors[number]['stdout_path'] = stdout_path
        self.descriptors[number]['stderr'] = err_descriptor
        self.descriptors[number]['stderr_path'] = stderr_path
        return out_descriptor, err_descriptor
641894
def ingest_video(self, bcVideoId, videoUrl):
        ''''''

        ingestData = {
            "master": {"url": videoUrl},
            "profile": "balanced-nextgen-player"  # "balanced-high-definition"
        }
        url = '/videos/' + bcVideoId + '/ingest-requests'
        result = self._make_request(self.Ingest_Server, 'POST', url, data=ingestData)
        return result
872135
@Override
    public ManifestDescriptor fromFile(final String file) throws IllegalArgumentException, DescriptorImportException {
        return this.fromFile(new File(file));
    }
181959
def bundles():
    
    for bundle in sorted(bundles_module.bundles.keys()):
        if bundle.startswith('.'):
            # hide the test data
            continue
        try:
            ingestions = list(
                map(text_type, bundles_module.ingestions_for_bundle(bundle))
            )
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise
            ingestions = []

        # If we got no ingestions, either because the directory didn't exist or
        # because there were no entries, print a single message indicating that
        # no ingestions have yet been made.
        for timestamp in ingestions or ["<no ingestions>"]:
            click.echo("%s %s" % (bundle, timestamp))
995131
public static ApplicationDescriptor load( File f ) throws IOException {

		Properties properties = Utils.readPropertiesFile( f );
		return load( properties );
	}
847573
def _write_module_descriptor_file(handle, module_dir):
  
  readme = _module_descriptor_file(module_dir)
  readme_content = (
      "Module: %s\nDownload Time: %s\nDownloader Hostname: %s (PID:%d)" %
      (handle, str(datetime.datetime.today()), socket.gethostname(),
       os.getpid()))
  # The descriptor file has no semantic meaning so we allow 'overwrite' since
  # there is a chance that another process might have written the file (and
  # crashed), we just overwrite it.
  tf_utils.atomic_write_string_to_file(readme, readme_content, overwrite=True)
574706
def _invokeRead(self, fileIO, directory, filename, session, spatial=False,
                    spatialReferenceID=4236, replaceParamFile=None, **kwargs):
        
        path = os.path.join(directory, filename)

        if os.path.isfile(path):
            instance = fileIO()
            instance.projectFile = self
            instance.read(directory, filename, session, spatial=spatial,
                          spatialReferenceID=spatialReferenceID,
                          replaceParamFile=replaceParamFile, **kwargs)
            return instance
        else:
            self._readBatchOutputForFile(directory, fileIO, filename, session,
                                         spatial, spatialReferenceID, replaceParamFile)
1374880
public static Properties readProperties(File _file) {
        if (_file.exists()) {
            try {
                return readProperties(new FileInputStream(_file));
            } catch (FileNotFoundException _ex) {
                LOGGER.info("Could not load properties file: " + _file, _ex);
            }
        }
        return null;
    }
800309
def ingest(topic, text, **kwargs):
    
    if not text:
        raise ValueError('No text given to ingest for topic: ' + topic)
    data = {'topic': topic, 'text': text.strip()}
    data.update(kwargs)
    db.markovify.insert(data)
168884
def set_file_properties(ase, md5, timeout=None):
    # type: (blobxfer.models.azure.StorageEntity, str, int) -> None
    
    dir, fpath, _ = parse_file_path(ase.name)
    ase.client.set_file_properties(
        share_name=ase.container,
        directory_name=dir,
        file_name=fpath,
        content_settings=azure.storage.file.models.ContentSettings(
            content_type=blobxfer.util.get_mime_type(fpath),
            content_md5=md5,
            cache_control=ase.cache_control,
        ),
        timeout=timeout)
684302
def ingest(self, co, classname=None, code_objects={}, show_asm=None):
        
        tokens, customize = scan.Scanner21.ingest(self, co, classname, code_objects, show_asm)
        for t in tokens:
            if t.op == self.opc.UNPACK_LIST:
                t.kind = 'UNPACK_LIST_%d' % t.attr
            pass
        return tokens, customize
1165452
int ingestFSEdits() throws IOException {
    FSDirectory fsDir = fsNamesys.dir;
    int numEdits = 0;
    
    long recentOpcodeOffsets[] = new long[2];
    Arrays.fill(recentOpcodeOffsets, -1);
    EnumMap<FSEditLogOpCodes, Holder<Integer>> opCounts =
        new EnumMap<FSEditLogOpCodes, Holder<Integer>>(FSEditLogOpCodes.class);
    
    boolean error = false;
    boolean reopen = false;    
    boolean quitAfterScan = false;
    
    long sharedLogTxId = FSEditLogLoader.TXID_IGNORE;
    long localLogTxId = FSEditLogLoader.TXID_IGNORE;

    FSEditLogOp op = null;
    FSEditLog localEditLog = fsDir.fsImage.getEditLog();

    while (running && !quitAfterScan) {      
      // if the application requested that we make a final pass over 
      // the transaction log, then we remember it here. We close and
      // reopen the file to ensure that we can see all the data in the
      // file, one reason being that NFS has open-to-close cache
      // coherancy and the edit log could be stored in NFS.
      //
      if (reopen || lastScan) {
        inputEditStream.close();
        inputEditStream = standby.setupIngestStreamWithRetries(startTxId);
        if (lastScan) {
          // QUIESCE requested by Standby thread
          LOG.info("Ingest: Starting last scan of transaction log: " + this.toString());
          quitAfterScan = true;
        }

        // discard older buffers and start a fresh one.
        inputEditStream.refresh(currentPosition, localEditLog.getLastWrittenTxId());       
        setCatchingUp();
        reopen = false;
      }

      //
      // Process all existing transactions till end of file
      //
      while (running) {
        
        if (lastScan && !quitAfterScan) {
          // Standby thread informed the ingest to quiesce
          // we should refresh the input stream as soon as possible
          // then quitAfterScan will be true
          break;
        }
        
        // record the current file offset.
        currentPosition = inputEditStream.getPosition(); 
        InjectionHandler.processEvent(InjectionEvent.INGEST_BEFORE_LOAD_EDIT);

        fsNamesys.writeLock();
        try {
          error = false;
          op = ingestFSEdit(inputEditStream);
          
          /*
           * In the case of segments recovered on primary namenode startup, we
           * have segments that are finalized (by name), but not containing the
           * ending transaction. Without this check, we will keep looping until
           * the next checkpoint to discover this situation.
           */
          if (!inputEditStream.isInProgress()
              && standby.getLastCorrectTxId() == inputEditStream.getLastTxId()) {
            // this is a correct segment with no end segment transaction
            LOG.info("Ingest: Reached finalized log segment end with no end marker. "
                + this.toString());
            tearDown(localEditLog, false, true);
            break;
          }

          if (op == null) {
            FSNamesystem.LOG.debug("Ingest: Invalid opcode, reached end of log " +
                                   "Number of transactions found " + 
                                   numEdits);
            break; // No more transactions.
          }
          
          sharedLogTxId = op.txid;
          // Verify transaction ids match.
          localLogTxId = localEditLog.getLastWrittenTxId() + 1;
          // Fatal error only when the log contains transactions from the future
          // we allow to process a transaction with smaller txid than local
          // we will simply skip it later after reading from the ingest edits
          if (localLogTxId < sharedLogTxId
              || InjectionHandler
                  .falseCondition(InjectionEvent.INGEST_TXID_CHECK)) {
            String message = "The transaction id in the edit log : "
                + sharedLogTxId
                + " does not match the transaction id inferred"
                + " from FSIMAGE : " + localLogTxId;
            LOG.fatal(message);
            throw new RuntimeException(message);
          }

          // skip previously loaded transactions
          if (!canApplyTransaction(sharedLogTxId, localLogTxId, op))
            continue;
          
          // for recovery, we do not want to re-load transactions,
          // but we want to populate local log with them
          if (shouldLoad(sharedLogTxId)) {
            FSEditLogLoader.loadEditRecord(
                logVersion, 
                inputEditStream,
                recentOpcodeOffsets, 
                opCounts, 
                fsNamesys, 
                fsDir, 
                numEdits, 
                op);
          }     
          
          LOG.info("Ingest: " + this.toString() 
                            + ", size: " + inputEditStream.length()
                            + ", processing transaction at offset: " + currentPosition 
                            + ", txid: " + op.txid
                            + ", opcode: " + op.opCode);
          
          if (op.opCode == FSEditLogOpCodes.OP_START_LOG_SEGMENT) {
            LOG.info("Ingest: Opening log segment: " + this.toString());
            localEditLog.open();
          } else if (op.opCode == FSEditLogOpCodes.OP_END_LOG_SEGMENT) {
            InjectionHandler
                .processEventIO(InjectionEvent.INGEST_CLEAR_STANDBY_STATE);
            LOG.info("Ingest: Closing log segment: " + this.toString());
            tearDown(localEditLog, true, true);
            numEdits++;
            LOG.info("Ingest: Reached log segment end. " + this.toString());
            break;
          } else {
            localEditLog.logEdit(op);
            if (inputEditStream.getReadChecksum() != FSEditLog
                .getChecksumForWrite().getValue()) {
              throw new IOException(
                  "Ingest: mismatched r/w checksums for transaction #"
                      + numEdits);
            }
          }
          
          numEdits++;
          standby.setLastCorrectTxId(op.txid);
        }
        catch (ChecksumException cex) {
          LOG.info("Checksum error reading the transaction #" + numEdits +
                   " reopening the file");
          reopen = true;
          break;
        }
        catch (IOException e) {
          LOG.info("Encountered error reading transaction", e);
          error = true; // if we haven't reached eof, then error.
          break;
        } finally {
          if (localEditLog.isOpen()) {
            localEditLog.logSyncIfNeeded();
          }
          fsNamesys.writeUnlock();
        }
      } // end inner while(running) -- all breaks come here
   
      // if we failed to read the entire transaction from disk, 
      // then roll back to the offset where there was a last good 
      // read, sleep for sometime for new transaction to
      // appear in the file and then continue;
      if (error || running) {
        // discard older buffers and start a fresh one.
      	inputEditStream.refresh(currentPosition, localEditLog.getLastWrittenTxId());
        setCatchingUp();

        if (error) {
          LOG.info("Ingest: Incomplete transaction record at offset " + 
              inputEditStream.getPosition() +
                   " but the file is of size " + inputEditStream.length() + 
                   ". Continuing....");
        }

        if (running && !lastScan) {
          try {
            Thread.sleep(100); // sleep for a second
          } catch (InterruptedException e) {
            // break out of waiting if we receive an interrupt.
          }
        }
      }
    } //end outer while(running)
    
    ///////////////////// FINAL ACTIONS /////////////////////
    
    // This was the last scan of the file but we could not read a full
    // transaction from disk. If we proceed this will corrupt the image
    if (error) {
      String errorMessage = FSEditLogLoader.getErrorMessage(recentOpcodeOffsets, currentPosition);
      LOG.error(errorMessage);
      throw new IOException("Failed to read the edits log. " + 
          "Incomplete transaction at " + currentPosition);
    }
    
    // If the last Scan was completed, then stop the Ingest thread.
    if (lastScan && quitAfterScan) {
      LOG.info("Ingest: lastScan completed. " + this.toString());
      running = false;
      if(localEditLog.isOpen()) {
        // quiesced non-finalized segment
        LOG.info("Ingest: Reached non-finalized log segment end. "+ this.toString());
        tearDown(localEditLog, false, localLogTxId != startTxId);
      }
    }
    FSEditLogLoader.dumpOpCounts(opCounts);
    return numEdits; // total transactions consumed
  }
512755
def _update_descriptor(self, state, allocation):
        ''''''
        if state.descriptor.shard is None or state.descriptor.shard == 'lobby':
            own_shard = state.agent.get_shard_id()
            state.descriptor.shard = own_shard
        state.descriptor.resources = allocation.alloc
        f = fiber.succeed(state.descriptor)
        f.add_callback(state.agent.save_document)
        f.add_callback(self._store_descriptor)
        return f
301236
private void quiesceIngestWithReprocess() throws IOException {
    if (ingest != null) {
      LOG.info("Standby: Quiescing - quiescing ongoing ingest");
      quiesceIngest();
      reprocessCurrentSegmentIfNeeded(ingest.getIngestStatus());
    }
  }
396892
def write_file_to_descriptor(input_queue, descriptor):
    
    try:
        file_name = input_queue.get(timeout=2)
        descriptor.write("{}\n".format(file_name))
        descriptor.flush()
        input_queue.task_done()
        return True
    except Empty:
        # no more files in queue
        descriptor.close()
        return False
    except IOError:
        return False
859544
public static void badArgs(String msg) {
        System.err.println("Command: fedora-ingest");
        System.err.println();
        System.err
                .println("Summary: Ingests one or more objects into a Fedora repository, from either");
        System.err
                .println("         the local filesystem or another Fedora repository.");
        System.err.println();
        System.err.println("Syntax:");
        System.err
                .println("  fedora-ingest f[ile] path format targetHost:targetPort targetUser targetPassword targetProtocol [log] [context]");
        System.err
                .println("  fedora-ingest d[ir]  path format targetHost:targetPort targetUser targetPassword targetProtocol [log] [context]");
        System.err
                .println("  fedora-ingest r[epos] sourceHost:sourcePort sourceUser sourcePassword pid|* targetHost:targetPort targetUser targetPassword sourceProtocol targetProtocol [log] [context]");
        System.err.println();
        System.err.println("Where:");
        System.err
                .println("  path                           is the local file or directory name that is ingest source.");
        System.err
                .println("  format                         is a string value which indicates the XML format of the ingest file(s)");
        System.err.println("                                 ('" + FOXML1_1.uri
                + "',");
        System.err.println("                                 '" + FOXML1_0.uri
                + "',");
        System.err.println("                                 '"
                + METS_EXT1_1.uri + "',");
        System.err.println("                                 '"
                + METS_EXT1_0.uri + "',");
        System.err.println("                                 '" + ATOM1_1.uri
                + "',");
        System.err.println("                                 or '"
                + ATOM_ZIP1_1.uri + "')");
        System.err
                .println("  pid | *                        is the id of the object to ingest from the source repository OR * in case of all objects from the source repository.");
        System.err
                .println("  sourceHost/targetHost          is the source or target repository's hostname.");
        System.err
                .println("  sourcePort/targetPort          is the source or target repository's port number.");
        System.err
                .println("  sourceUser/targetUser          is the id of the source or target repository user.");
        System.err
                .println("  sourcePassword/targetPassword  is the password of the source or target repository user.");
        System.err
                .println("  sourceProtocol                 is the protocol to communicate with source repository (http or https)");
        System.err
                .println("  targetProtocol                 is the protocol to communicate with target repository (http or https)");
        System.err
                .println("  log                            is the optional log message.  If unspecified, the log message");
        System.err
                .println("                                 will indicate the source filename or repository of the object(s).");
        System.err
                .println("  context                        is the optional parameter for specifying the context name under which ");
        System.err
                .println("                                 the Fedora server is deployed. The default is fedora.");
        System.err.println();
        System.err.println("Examples:");
        System.err.println("fedora-ingest f obj1.xml " + FOXML1_1.uri
                + " myrepo.com:8443 jane jpw https");
        System.err.println();
        System.err
                .println("  Ingests obj1.xml (encoded in FOXML 1.1 format) from the");
        System.err
                .println("  current directory into the repository at myrepo.com:80");
        System.err
                .println("  as user 'jane' with password 'jpw' using the secure https protocol (SSL).");
        System.err
                .println("  The logmessage will be system-generated, indicating");
        System.err.println("  the source path+filename.");
        System.err.println();
        System.err.println("fedora-ingest d c:\\archive " + FOXML1_1.uri
                + " myrepo.com:80 jane janepw http \"\"");
        System.err.println();
        System.err
                .println("  Traverses entire directory structure of c:\\archive, and ingests any file.");
        System.err
                .println("  It assumes all files will be in the FOXML 1.1 format");
        System.err
                .println("  and will fail on ingests of files that are not of this format.");
        System.err.println("  All log messages will be the quoted string.");
        System.err.println();
        System.err.println("fedora-ingest d c:\\archive " + FOXML1_1.uri
                + " myrepo.com:80 jane janepw http \"\" my-fedora");
        System.err
                .println("  Traverses entire directory structure of c:\\archive, and ingests any file.");
        System.err
                .println("  It assumes all files will be in the FOXML 1.1 format");
        System.err
                .println("  and will fail on ingests of files that are not of this format.");
        System.err.println("  All log messages will be the quoted string.");
        System.err
                .println("  Additionally the Fedora server is assumed to be running under the context name ");
        System.err
                .println("  http://myrepo:80/my-fedora instead of http://myrepo:80/fedora ");
        System.err.println();
        System.err
                .println("fedora-ingest r jrepo.com:8081 mike mpw demo:1 myrepo.com:8443 jane jpw http https \"\"");
        System.err.println();
        System.err
                .println("  Ingests the object whose pid is 'demo:1' from the source repository");
        System.err
                .println("  'srcrepo.com:8081' into the target repository 'myrepo.com:80'.");
        System.err
                .println("  The object will be exported from the source repository in the default");
        System.err.println("  export format configured at the source.");
        System.err.println("  All log messages will be empty.");
        System.err.println();
        System.err
                .println("fedora-ingest r jrepo.com:8081 mike mpw O myrepo.com:8443 jane jpw http https \"\"");
        System.err.println();
        System.err
                .println("  Same as above, but ingests all data objects (type O).");
        System.err.println();
        System.err.println("ERROR  : " + msg);
        System.exit(1);
    }