Default processing loop

        :param ifile: Input file object.
        :type ifile: file

        :param process: Bound method to call in processing loop.
        :type process: instancemethod

        :return: :const:`None`.
        :rtype: NoneType
199239
def _execute(self, ifile, process):
        
        self._record_writer.write_records(process(self._records(ifile)))
        self.finish()
542980
function preprocessorcery(infile, info, callback) {
  applicable(infile, info, (err, preprocessors) => {
    if (err) return callback(err);

    const q = queue(1);
    preprocessors.forEach((preprocessor) => {
      const outfile = newfile(infile);
      q.defer((next) => {
        preprocessor(infile, outfile, (err) => {
          if (err) return next(err);
          infile = outfile;
          next();
        });
      });
    });

    q.await((err) => {
      if (err) return callback(err);

      // infile has been changed to the output file by this point
      callback(null, infile);
    });
  });
}
111423
function processFile(file, flow){
  var fconsole = flow.console;
  var scriptSequenceId = 0;
  var jsSequence = false;
  var conditionalBlock;
  var head;
  var body;

  // get ast
  var ast = at.parse(file.content);

  // debug output
  //console.log(require('util').inspect(ast, false, null));

  var handlers = {
    '*': function(node){
      // nothing to do if text node contains whitespace only
      if (node.type == 'text' && !NON_WHITESPACE.test(node.data))
        return;

      // scripts with src continue script sequence
      if (node.type == 'tag' && node.name == 'script')
        return;

      scriptSequenceId += jsSequence;
      jsSequence = false;
    },
    comment: function(node){
      var conditional = node.data.match(/^(\[if\s+[^\]]+\]>)((?:.|[\r\n])*)(<!\[endif\])$/i);
      if (conditional)
      {
        node.prefix = conditional[1];
        node.ast = at.parse(conditional[2]);
        node.postfix = conditional[3];

        scriptSequenceId += jsSequence;
        jsSequence = false;

        conditionalBlock = true;
        at.walk(node.ast, handlers, { file: file });
        conditionalBlock = false;
      }
    },
    tag: function(node){
      var attrs = at.getAttrs(node);

      switch (node.name)
      {
        case 'head':
          if (!head)
          {
            fconsole.log('<head> tag found (store reference)');
            head = node;
          }
          else
          {
            flow.warn({
              file: this.file.relpath,
              message: 'more than one <head> tag is prohibited (ignore)'
            });
          }

          break;

        case 'body':
          if (!body)
          {
            fconsole.log('<body> tag found (store reference)');
            body = node;
          }
          else
          {
            flow.warn({
              file: this.file.relpath,
              message: 'more than one <body> tag is prohibited (ignore)'
            });
          }

          break;

        case 'script':
          // ignore <script> tags with type other than text/javascript
          if (attrs.type && attrs.type != 'text/javascript')
          {
            fconsole.log('<script> with unknown type `' + attrs.type + '` ignored');
            return;
          }

          // external script
          if (attrs.src)
          {
            var uri = resolveUri(attrs.src);

            if (uri.filename)
            {
              fconsole.start('External script found: <script src="' + attrs.src + '">');
              jsSequence = true;
              node.loc = file.location(node.info.start);

              var scriptFile = flow.files.add({
                initiator: {
                  file: file,
                  loc: node.loc,
                  token: at.translate(node)
                },

                type: 'script',
                htmlFile: file,
                htmlNode: node,
                filename: file.resolve(attrs.src),
                package: 'script' + (scriptSequenceId || '')
              });

              if (!scriptFile)
                return;

              file.link(scriptFile, node);

              fconsole.endl();
              return;
            }

            if (uri.mime == 'text/javascript')
            {
              fconsole.start('Inline script with data uri found');
              jsSequence = true;

              var scriptFile = flow.files.add({
                type: 'script',
                inline: true,
                htmlFile: file,
                htmlNode: node,
                baseURI: file.baseURI,
                content: uri.content,
                package: 'script' + (scriptSequenceId || '')
              });

              if (attrs['build-filename'])
                scriptFile.sourceFilename = attrs['build-filename'];

              file.link(scriptFile, node);

              fconsole.endl();
              return;
            }

            if (!uri.url) // external url
            {
              flow.warn({
                file: this.file.relpath,
                message: 'script ignored: ' + at.translate(node)
              });
            }
          }
          else
          {
            fconsole.log('Inline script found\n');

            file.link(flow.files.add({
              type: 'script',
              inline: true,
              htmlFile: file,
              htmlNode: node,
              baseURI: file.baseURI,
              content: at.getText(node),
              embed: {
                file: file.filename,
                start: node.info.startContent,
                end: node.info.endContent
              }
            }), node);
          }

          break;

        //
        // style
        //
        case 'link':

          if (at.rel(node, 'stylesheet') && attrs.href)
          {
            // <link rel="stylesheet">');
            fconsole.log('External style found: ' + at.translate(node));
            node.loc = file.location(node.info.start);

            var styleLinkFile = flow.files.add({
              initiator: {
                file: file,
                loc: node.loc,
                token: at.translate(node)
              },

              type: 'style',
              htmlFile: file,
              htmlNode: node,
              htmlId: attrs.id,
              filename: file.resolve(attrs.href),
              media: attrs.media || 'all',
              conditional: conditionalBlock
            });

            if (styleLinkFile)
            {
              if (styleLinkFile.htmlNode !== node)
                flow.warn({
                  file: this.file.relpath,
                  fatal: true,
                  message: 'Duplicate <link> found: ' + at.translate(node)
                });

              file.link(styleLinkFile, node);
            }
          }

          break;

        case 'style':
          var attrs = at.getAttrs(node);

          // ignore <style> with type other than text/css
          if (attrs.type && attrs.type != 'text/css')
          {
            flow.warn({
              file: this.file.relpath,
              message: '<style> with unknown type ' + attrs.type + ' ignored'
            });
            return;
          }

          // <style> or <style type="text/css">
          fconsole.log('Inline style found');

          var styleFile = flow.files.add({
            type: 'style',
            inline: true,
            htmlFile: file,
            htmlNode: node,
            htmlId: attrs.id,
            baseURI: file.baseURI,
            media: attrs.media || 'all',
            conditional: conditionalBlock,
            content: at.getText(node),
            embed: {
              file: file.filename,
              start: node.info.startContent,
              end: node.info.endContent
            }
          });

          if (attrs['build-filename'])
            styleFile.sourceFilename = attrs['build-filename'];

          file.link(styleFile, node);

          break;

        default:
          var attrs = at.getAttrs(node);
          if (attrs.style)
          {
            var styleFile = flow.files.add({
              type: 'style-block',
              inline: true,
              htmlFile: file,
              htmlNode: node,
              baseURI: file.baseURI,
              rule: true,
              content: attrs.style
            });

            if (flow.options.verbose)
            {
              var nodeClone = {};
              for (var key in node)
                if (key != 'children')
                  nodeClone[key] = node[key];
                else
                  nodeClone[key] = [];

              fconsole.log('Style attribute found: ' + at.translate(nodeClone));
            }

            file.link(styleFile, node);
          }
      }
    }
  };

  at.walk(ast, handlers, { file: file });

  // save result in file
  file.ast = ast;
}
418239
def select_files_by_ifo_combination(ifocomb, insps):
    
    inspcomb = FileList()
    for ifo, ifile in zip(*insps.categorize_by_attr('ifo')):
        if ifo in ifocomb:
            inspcomb += ifile

    return inspcomb
1294737
def process(processor_or_config = nil)
      if processor_or_config.respond_to?(:process)
        processor = processor_or_config
      elsif processor_or_config.nil?
        processor = Processor.new
      else
        processor = Processor.new(processor_or_config)
      end

      processor.process(self)
    end
356552
def get_pid(rundir, process_type=PROCESS_TYPE, name=None):
    
    pidPath = get_pidpath(rundir, process_type, name)
    log.log('run', 'pidfile for %s %s is %s' % (process_type, name, pidPath))

    if not os.path.exists(pidPath):
        return

    pidFile = open(pidPath, 'r')
    pid = pidFile.readline()
    pidFile.close()
    if not pid or int(pid) == 0:
        return

    return int(pid)
79131
def start(self):
        
        # If pidfile already exists, we should read pid from there; to overwrite it, if locking
        # will fail, because locking attempt somehow purges the file contents.
        if os.path.isfile(self.pid):
            with open(self.pid, "r") as old_pidfile:
                old_pid = old_pidfile.read()
        # Create a lockfile so that only one instance of this daemon is running at any time.
        try:
            lockfile = open(self.pid, "w")
        except IOError:
            print("Unable to create the pidfile.")
            sys.exit(1)
        try:
            # Try to get an exclusive lock on the file. This will fail if another process has the file
            # locked.
            fcntl.flock(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
        except IOError:
            print("Unable to lock on the pidfile.")
            # We need to overwrite the pidfile if we got here.
            with open(self.pid, "w") as pidfile:
                pidfile.write(old_pid)
            sys.exit(1)

        # skip fork if foreground is specified
        if not self.foreground:
            # Fork, creating a new process for the child.
            try:
                process_id = os.fork()
            except OSError as e:
                self.logger.error("Unable to fork, errno: {0}".format(e.errno))
                sys.exit(1)
            if process_id != 0:
                if self.keep_fds:
                    # This is the parent process. Exit without cleanup,
                    # see https://github.com/thesharp/daemonize/issues/46
                    os._exit(0)
                else:
                    sys.exit(0)
            # This is the child process. Continue.

            # Stop listening for signals that the parent process receives.
            # This is done by getting a new process id.
            # setpgrp() is an alternative to setsid().
            # setsid puts the process in a new parent group and detaches its controlling terminal.
            process_id = os.setsid()
            if process_id == -1:
                # Uh oh, there was a problem.
                sys.exit(1)

            # Add lockfile to self.keep_fds.
            self.keep_fds.append(lockfile.fileno())

            # Close all file descriptors, except the ones mentioned in self.keep_fds.
            devnull = "/dev/null"
            if hasattr(os, "devnull"):
                # Python has set os.devnull on this system, use it instead as it might be different
                # than /dev/null.
                devnull = os.devnull

            if self.auto_close_fds:
                for fd in range(3, resource.getrlimit(resource.RLIMIT_NOFILE)[0]):
                    if fd not in self.keep_fds:
                        try:
                            os.close(fd)
                        except OSError:
                            pass

            devnull_fd = os.open(devnull, os.O_RDWR)
            os.dup2(devnull_fd, 0)
            os.dup2(devnull_fd, 1)
            os.dup2(devnull_fd, 2)
            os.close(devnull_fd)

        if self.logger is None:
            # Initialize logging.
            self.logger = logging.getLogger(self.app)
            self.logger.setLevel(logging.DEBUG)
            # Display log messages only on defined handlers.
            self.logger.propagate = False

            # Initialize syslog.
            # It will correctly work on OS X, Linux and FreeBSD.
            if sys.platform == "darwin":
                syslog_address = "/var/run/syslog"
            else:
                syslog_address = "/dev/log"

            # We will continue with syslog initialization only if actually have such capabilities
            # on the machine we are running this.
            if os.path.exists(syslog_address):
                syslog = handlers.SysLogHandler(syslog_address)
                if self.verbose:
                    syslog.setLevel(logging.DEBUG)
                else:
                    syslog.setLevel(logging.INFO)
                # Try to mimic to normal syslog messages.
                formatter = logging.Formatter("%(asctime)s %(name)s: %(message)s",
                                              "%b %e %H:%M:%S")
                syslog.setFormatter(formatter)

                self.logger.addHandler(syslog)

        # Set umask to default to safe file permissions when running as a root daemon. 027 is an
        # octal number which we are typing as 0o27 for Python3 compatibility.
        os.umask(0o27)

        # Change to a known directory. If this isn't done, starting a daemon in a subdirectory that
        # needs to be deleted results in "directory busy" errors.
        os.chdir(self.chdir)

        # Execute privileged action
        privileged_action_result = self.privileged_action()
        if not privileged_action_result:
            privileged_action_result = []

        # Change owner of pid file, it's required because pid file will be removed at exit.
        uid, gid = -1, -1

        if self.group:
            try:
                gid = grp.getgrnam(self.group).gr_gid
            except KeyError:
                self.logger.error("Group {0} not found".format(self.group))
                sys.exit(1)

        if self.user:
            try:
                uid = pwd.getpwnam(self.user).pw_uid
            except KeyError:
                self.logger.error("User {0} not found.".format(self.user))
                sys.exit(1)

        if uid != -1 or gid != -1:
            os.chown(self.pid, uid, gid)

        # Change gid
        if self.group:
            try:
                os.setgid(gid)
            except OSError:
                self.logger.error("Unable to change gid.")
                sys.exit(1)

        # Change uid
        if self.user:
            try:
                uid = pwd.getpwnam(self.user).pw_uid
            except KeyError:
                self.logger.error("User {0} not found.".format(self.user))
                sys.exit(1)
            try:
                os.setuid(uid)
            except OSError:
                self.logger.error("Unable to change uid.")
                sys.exit(1)

        try:
            lockfile.write("%s" % (os.getpid()))
            lockfile.flush()
        except IOError:
            self.logger.error("Unable to write pid to the pidfile.")
            print("Unable to write pid to the pidfile.")
            sys.exit(1)

        # Set custom action on SIGTERM.
        signal.signal(signal.SIGTERM, self.sigterm)
        atexit.register(self.exit)

        self.logger.warning("Starting daemon.")

        try:
            self.action(*privileged_action_result)
        except Exception:
            for line in traceback.format_exc().split("\n"):
                self.logger.error(line)
126319
def process_inputs(self):
        
        ret = []
        files = self.args.files
        if files is None:
            return ret

        for fname in files:
            if fname == '-':
                if self.args.base64stdin:
                    for line in sys.stdin:
                        data = base64.b64decode(line)
                        ret.append(self.process_file(data, fname))

                    continue
                else:
                    fh = sys.stdin

            elif fname.endswith('.tar') or fname.endswith('.tar.gz'):
                sub = self.process_tar(fname)
                ret.append(sub)
                continue

            elif not os.path.isfile(fname):
                sub = self.process_dir(fname)
                ret.append(sub)
                continue

            else:
                fh = open(fname, 'rb')

            with fh:
                data = fh.read()
                sub = self.process_file(data, fname)
                ret.append(sub)

        return ret
1207751
def _set_process_async(process):
    """"""
    for fd in [process.stderr.fileno(), process.stdout.fileno()]:
        fl = fcntl.fcntl(fd, fcntl.F_GETFL)
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
391130
def process_file(self, path, dryrun):
        
        # special case - skip output file so we won't include it in result
        if path == self._output_path:
            return None

        # if dryrun skip and return file
        if dryrun:
            return path

        # concat file with output file
        with open(path, "rb") as infile:
            data = infile.read()
            self._output_file.write(data)

        # return processed file path
        return path
737802
def _write_file_iface(iface, data, folder, pattern):
    '''
    
    '''
    filename = os.path.join(folder, pattern.format(iface))
    if not os.path.exists(folder):
        msg = '{0} cannot be written. {1} does not exist'
        msg = msg.format(filename, folder)
        log.error(msg)
        raise AttributeError(msg)
    with salt.utils.files.fopen(filename, 'w') as fp_:
        fp_.write(salt.utils.stringutils.to_str(data))
774929
function processFile() {
  const options = {};
  return (file, callback) => {
    const data = csso.minify(file.data, options).css;
    async.nextTick(callback, null, Object.assign({}, file, { data }));
  };
}
681964
def _iter_process():
    
    # TODO: Process32{First,Next} does not return full executable path, only
    # the name. To get the full path, Module32{First,Next} is needed, but that
    # does not contain parent process information. We probably need to call
    # BOTH to build the correct process tree.
    h_process = windll.kernel32.CreateToolhelp32Snapshot(
        2,  # dwFlags=TH32CS_SNAPPROCESS (include all processes).
        0,  # th32ProcessID=0 (the current process).
    )
    if h_process == INVALID_HANDLE_VALUE:
        raise WinError()
    pe = PROCESSENTRY32()
    pe.dwSize = sizeof(PROCESSENTRY32)
    success = windll.kernel32.Process32First(h_process, byref(pe))
    while True:
        if not success:
            errcode = windll.kernel32.GetLastError()
            if errcode == ERROR_NO_MORE_FILES:
                # No more processes to iterate through, we're done here.
                return
            elif errcode == ERROR_INSUFFICIENT_BUFFER:
                # This is likely because the file path is longer than the
                # Windows limit. Just ignore it, it's likely not what we're
                # looking for. We can fix this when it actually matters. (#8)
                continue
            raise WinError()

        # The executable name would be encoded with the current code page if
        # we're in ANSI mode (usually). Try to decode it into str/unicode,
        # replacing invalid characters to be safe (not thoeratically necessary,
        # I think). Note that we need to use 'mbcs' instead of encoding
        # settings from sys because this is from the Windows API, not Python
        # internals (which those settings reflect). (pypa/pipenv#3382)
        executable = pe.szExeFile
        if isinstance(executable, bytes):
            executable = executable.decode('mbcs', 'replace')

        info = {'executable': executable}
        if pe.th32ParentProcessID:
            info['parent_pid'] = pe.th32ParentProcessID
        yield pe.th32ProcessID, info
        success = windll.kernel32.Process32Next(h_process, byref(pe))
340801
def main():
    
    global args
    args = parse_args()

    if not args:
        return 1

    state = MyState(args)

    for path in args.paths:
        if os.path.isdir(path):
            walk_dir(path, args, state)
        else:
            safe_process_files(os.path.dirname(path), [os.path.basename(path)], args, state)
        if state.should_quit():
            break
    if state.failed_files:
        sys.stderr.write("error: %i/%i AEADs failed\n" % (len(state.failed_files), state.file_count))
        return 1
    if args.debug:
        sys.stderr.write("Successfully processed %i AEADs\n" % (state.file_count))
1345036
def load_file(self, path=None, env=None, silent=True, key=None):
        
        env = (env or self.current_env).upper()
        files = ensure_a_list(path)
        if files:
            self.logger.debug("Got %s files to process", files)
            already_loaded = set()
            for _filename in files:
                self.logger.debug("Processing file %s", _filename)
                filepath = os.path.join(self._root_path, _filename)
                self.logger.debug("File path is %s", filepath)
                # Handle possible *.globs sorted alphanumeric
                for path in sorted(glob.glob(filepath)):
                    self.logger.debug("Loading %s", path)
                    if path in already_loaded:  # pragma: no cover
                        self.logger.debug("Skipping %s, already loaded", path)
                        continue
                    settings_loader(
                        obj=self,
                        env=env,
                        silent=silent,
                        key=key,
                        filename=path,
                    )
                    already_loaded.add(path)
            if not already_loaded:
                self.logger.warning(
                    "Not able to locate the files %s to load", files
                )
672468
function (file) {
        var self = this,
            ext = path.extname(file.name).toLowerCase();
        return this.isZip(ext) ? self.processZip(file) : self.processFile(file, ext);
    }
204772
def process(self):
        

        for locale in self._fields.keys():
            self._client._put(
                "{0}/files/{1}/process".format(
                    self.__class__.base_url(
                        self.space.id,
                        self.id,
                        environment_id=self._environment_id
                    ),
                    locale
                ),
                {},
                headers=self._update_headers()
            )
        return self.reload()