Preprocess the given data, ready for parsing.
437594
def Preprocess(self, data):
    """"""
    # Add whitespace to line continuations.
    data = data.replace(":\\", ": \\")

    # Strip comments manually because sudoers has multiple meanings for '#'.
    data = SudoersFieldParser.COMMENTS_RE.sub("", data)
    return data
1186655
def preprocess_tree(self, tree):
        """"""
        visitor = RinohTreePreprocessor(tree, self)
        tree.walkabout(visitor)
1162682
def preprocess_cell(self, cell, resources, cell_index):
        """"""
        from nbformat.notebooknode import NotebookNode

        attach_names = []

        # Just move the attachment into an output

        for k, attach in cell.get('attachments', {}).items():
            for mime_type in self.extract_output_types:
                if mime_type in attach:

                    if not 'outputs' in cell:
                        cell['outputs'] = []

                    o = NotebookNode({
                        'data': NotebookNode({mime_type: attach[mime_type]}),
                        'metadata': NotebookNode({
                            'filenames': {mime_type: k}  # Will get re-written
                        }),
                        'output_type': 'display_data'
                    })

                    cell['outputs'].append(o)

                    attach_names.append((mime_type, k))

        nb, resources = super().preprocess_cell(cell, resources, cell_index)

        output_names = list(resources.get('outputs', {}).keys())

        if attach_names:
            # We're going to assume that attachments are only on Markdown cells, and Markdown cells
            # can't generate output, so all of the outputs wee added.

            # reverse + zip matches the last len(attach_names) elements from output_names

            for output_name, (mimetype, an) in zip(reversed(output_names), reversed(attach_names)):
                # We'll post process to set the final output directory
                cell.source = re.sub('\(attachment:{}\)'.format(an),
                                     '(__IMGDIR__/{})'.format(output_name), cell.source)

        return nb, resources
186528
protected function prepareImportData()
    {
        if (empty($this->arImportData)) {
            return;
        }

        foreach ($this->arImportData as $sKey => $sValue) {
            if (!is_string($sValue)) {
                continue;
            }

            $this->arImportData[$sKey] = trim($sValue);
        }
    }
451125
protected function preprocess() {
		if ( $this->preprocessed || $this->skipPreprocess ) {
			return;
		}

		// Normalize line endings
		if ( strcspn( $this->text, "\r" ) !== strlen( $this->text ) ) {
			$this->text = preg_replace( '/\r\n?/', "\n", $this->text );
			$this->length = strlen( $this->text );
		}

		// Raise parse errors for any control characters
		static $re;
		if ( $re === null ) {
			// Note that we deliberately do not use the 'u' flag on the
			// regexp below, as that bypasses the PCRE JIT.  Instead we
			// rewrite character classes containing codepoints which
			// require more than one UTF-8 byte as alternations.
			$re = '/[' .
				// "C0 controls" (u0000 - u001F) but not
				// "ASCII whitespace" (u0009, u000A, u000C, u000D, u0020) or
				// NULL (u0000)
				// https://infra.spec.whatwg.org/#c0-control
				// https://infra.spec.whatwg.org/#ascii-whitespace
				'\x{0001}-\x{0008}' .
				'\x{000B}' .
				'\x{000E}-\x{001F}' .
				// "Controls" other than C0 controls (u007F - u009F)
				// https://infra.spec.whatwg.org/#control
				'\x{007F}]|' .
				// (We can't use character classes above u007F)
				"\u{0080}|\u{0081}|\u{0082}|\u{0083}|" .
				"\u{0084}|\u{0085}|\u{0086}|\u{0087}|" .
				"\u{0088}|\u{0089}|\u{008A}|\u{008B}|" .
				"\u{008C}|\u{008D}|\u{008E}|\u{008F}|" .
				"\u{0090}|\u{0091}|\u{0092}|\u{0093}|" .
				"\u{0094}|\u{0095}|\u{0096}|\u{0097}|" .
				"\u{0098}|\u{0099}|\u{009A}|\u{009B}|" .
				"\u{009C}|\u{009D}|\u{009E}|\u{009F}|" .
				// HTML spec calls these "noncharacters"
				// https://infra.spec.whatwg.org/#noncharacter
				"\u{FDD0}|\u{FDD1}|\u{FDD2}|\u{FDD3}|" .
				"\u{FDD4}|\u{FDD5}|\u{FDD6}|\u{FDD7}|" .
				"\u{FDD8}|\u{FDD9}|\u{FDDA}|\u{FDDB}|" .
				"\u{FDDC}|\u{FDDD}|\u{FDDE}|\u{FDDF}|" .
				"\u{FDE0}|\u{FDE1}|\u{FDE2}|\u{FDE3}|" .
				"\u{FDE4}|\u{FDE5}|\u{FDE6}|\u{FDE7}|" .
				"\u{FDE8}|\u{FDE9}|\u{FDEA}|\u{FDEB}|" .
				"\u{FDEC}|\u{FDED}|\u{FDEE}|\u{FDEF}|" .
				"\u{FFFE}|\u{FFFF}|" .
				"\u{1FFFE}|\u{1FFFF}|" .
				"\u{2FFFE}|\u{2FFFF}|" .
				"\u{3FFFE}|\u{3FFFF}|" .
				"\u{4FFFE}|\u{4FFFF}|" .
				"\u{5FFFE}|\u{5FFFF}|" .
				"\u{6FFFE}|\u{6FFFF}|" .
				"\u{7FFFE}|\u{7FFFF}|" .
				"\u{8FFFE}|\u{8FFFF}|" .
				"\u{9FFFE}|\u{9FFFF}|" .
				"\u{AFFFE}|\u{AFFFF}|" .
				"\u{BFFFE}|\u{BFFFF}|" .
				"\u{CFFFE}|\u{CFFFF}|" .
				"\u{DFFFE}|\u{DFFFF}|" .
				"\u{EFFFE}|\u{EFFFF}|" .
				"\u{FFFFE}|\u{FFFFF}|" .
				"\u{10FFFE}|\u{10FFFF}/S";
		}
		if ( !$this->ignoreErrors ) {
			$pos = 0;
			while ( $pos < $this->length ) {
				$count = preg_match( $re, $this->text, $m, PREG_OFFSET_CAPTURE, $pos );
				if ( $count === false ) {
					$this->throwPregError();
				} elseif ( !$count ) {
					break;
				}
				$pos = $m[0][1];
				$this->error( "disallowed control character", $pos );
				$pos += strlen( $m[0][0] );
			}
		}
	}
823414
def prepare_static_data(self, data):
        
        d = data.copy()
        for f in self.get_fields():
            if f['static'] and f['name'] in d:
                d[f['name']] = make_view_field(f, None, self.types_convert_map, self.fields_convert_map, d[f['name']])['display']
        return d
289677
private void preprocess(IoBuffer pat) {
    this.b = new int[256];
    for (int i = this.patternPos; i < this.patternLimit; i++) {
      int p = ByteBufferUtils.uByte(pat.get(i));
      this.b[p] = this.b[p] | (1 << i);
    }
  }
186042
function processData (data) {
    if (_cb.beforeReturnData) { _cb.beforeReturnData(data); }
    if (_cb.returnData) { _cb.returnData(data, response); }
    if (_cb.afterReturnData) { _cb.afterReturnData(data); }
  }
468405
def preprocess(ops, nlp, rows, get_ids):
    """"""
    Xs = []
    ys = []
    for (text1, text2), label in rows:
        Xs.append((get_ids([nlp(text1)])[0], get_ids([nlp(text2)])[0]))
        ys.append(label)
    return Xs, ops.asarray(ys, dtype="float32")
1305577
def standardize(self):
        

        if self.preprocessed_data.empty:
            data = self.original_data
        else:
            data = self.preprocessed_data
        
        scaler = preprocessing.StandardScaler()
        data = pd.DataFrame(scaler.fit_transform(data), columns=data.columns, index=data.index)
        self.preprocessed_data = data
606066
def main(folder):
    """"""
    raw_datapath, outputpath, p_queue = get_parameters(folder)
    create_preprocessed_dataset(raw_datapath, outputpath, p_queue)
    utils.create_run_logfile(folder)
283169
def preprocess(train_dataset, output_dir, eval_dataset, checkpoint, pipeline_option):
    """"""

    import apache_beam as beam
    import google.datalab.utils
    from . import _preprocess

    if checkpoint is None:
      checkpoint = _util._DEFAULT_CHECKPOINT_GSURL

    job_name = ('preprocess-image-classification-' +
                datetime.datetime.now().strftime('%y%m%d-%H%M%S'))

    staging_package_url = _util.repackage_to_staging(output_dir)
    tmpdir = tempfile.mkdtemp()
    # suppress DataFlow warnings about wheel package as extra package.
    original_level = logging.getLogger().getEffectiveLevel()
    logging.getLogger().setLevel(logging.ERROR)
    try:
      # Workaround for DataFlow 2.0, which doesn't work well with extra packages in GCS.
      # Remove when the issue is fixed and new version of DataFlow is included in Datalab.
      extra_packages = [staging_package_url, _TF_GS_URL, _PROTOBUF_GS_URL]
      local_packages = [os.path.join(tmpdir, os.path.basename(p))
                        for p in extra_packages]
      for source, dest in zip(extra_packages, local_packages):
        file_io.copy(source, dest, overwrite=True)

      options = {
          'staging_location': os.path.join(output_dir, 'tmp', 'staging'),
          'temp_location': os.path.join(output_dir, 'tmp'),
          'job_name': job_name,
          'project': _util.default_project(),
          'extra_packages': local_packages,
          'teardown_policy': 'TEARDOWN_ALWAYS',
          'no_save_main_session': True
      }
      if pipeline_option is not None:
        options.update(pipeline_option)

      opts = beam.pipeline.PipelineOptions(flags=[], **options)
      p = beam.Pipeline('DataflowRunner', options=opts)
      _preprocess.configure_pipeline(p, train_dataset, eval_dataset,
                                     checkpoint, output_dir, job_name)
      job_results = p.run()
    finally:
      shutil.rmtree(tmpdir)
      logging.getLogger().setLevel(original_level)

    if (_util.is_in_IPython()):
      import IPython
      dataflow_url = 'https://console.developers.google.com/dataflow?project=%s' % \
                     _util.default_project()
      html = 'Job "%s" submitted.' % job_name
      html += '<p>Click <a href="%s" target="_blank">here</a> to track preprocessing job. <br/>' \
          % dataflow_url
      IPython.display.display_html(html, raw=True)
    return google.datalab.utils.DataflowJob(job_results)
153863
def _preprocess_inputs(durations, event_observed, timeline, entry, weights):
    

    n = len(durations)
    durations = np.asarray(pass_for_numeric_dtypes_or_raise_array(durations)).reshape((n,))

    # set to all observed if event_observed is none
    if event_observed is None:
        event_observed = np.ones(n, dtype=int)
    else:
        event_observed = np.asarray(event_observed).reshape((n,)).copy().astype(int)

    if entry is not None:
        entry = np.asarray(entry).reshape((n,))

    event_table = survival_table_from_events(durations, event_observed, entry, weights=weights)
    if timeline is None:
        timeline = event_table.index.values
    else:
        timeline = np.asarray(timeline)

    return (durations, event_observed, timeline.astype(float), entry, event_table)
562514
@Override
    public void preProcess(DataSet toPreProcess) {
        for (DataSetPreProcessor preProcessor : preProcessors) {
            preProcessor.preProcess(toPreProcess);
        }
    }
257816
def preprocess(ops, nlp, rows, get_ids):
    """"""
    Xs = []
    ys = []
    for (text1, text2), label in rows:
        Xs.append((get_ids([nlp(text1)])[0], get_ids([nlp(text2)])[0]))
        ys.append(label)
    return Xs, to_categorical(ys, nb_classes=2)
753312
function _prepUpdate(data) {
    var vals = [];
    var sorted = Object.keys(data).sort();
    var i = -1;
    while (++i < sorted.length) {
      vals.push(this.formatter.wrap(sorted[i]) + ' = ' + this.formatter.parameter(data[sorted[i]]));
    }
    return vals;
  }
600320
def _preprocess_input(self):
    """"""
    # Compute the sets of MTF dimensions used in operations/tensors.

    # a {string: frozenset(string)}, mapping operation name to MTF dimension
    # names.
    self._operation_name_to_mtf_dimension_set = {}
    # a {string: frozenset(string)}, mapping tensor name to MTF dimension names.
    self._tensor_name_to_mtf_dimension_set = {}

    for operation_name in self._graph.get_all_operation_names():
      self._operation_name_to_mtf_dimension_set[operation_name] = frozenset(
          set(self._graph.get_operation_mtf_dimension_names(
              operation_name)).intersection(
                  self._layout_validator.splittable_mtf_dimension_names))
    for tensor_name in self._graph.get_all_tensor_names():
      self._tensor_name_to_mtf_dimension_set[tensor_name] = frozenset(
          set(self._graph.get_tensor_mtf_dimension_names(tensor_name))
          .intersection(self._layout_validator.splittable_mtf_dimension_names))

    self._operation_mtf_dimension_sets = set(
        self._operation_name_to_mtf_dimension_set.values())
    self._mtf_dimension_sets = self._operation_mtf_dimension_sets | set(
        self._tensor_name_to_mtf_dimension_set.values())

    # Compute possible assignments for each set of MTF dimensions.
    self._assignments = {}  # indexed by MTF dimension set
    for mtf_dimension_set in self._mtf_dimension_sets:
      self._assignments[mtf_dimension_set] = _generate_assignments(
          mtf_dimension_set, self._layout_validator.mesh_dimension_name_to_size)