Transforms the rows for the source system into (partial) dimensional data.
494212
def transform_source_rows(self):
        
        # Start timer for overall progress.
        self._time0 = time.perf_counter()

        self.pre_transform_source_rows()

        # Transform all source rows.
        with self._source_reader:
            with self._transformed_writer:
                with self._parked_writer:
                    with self._ignored_writer:
                        self._transform_rows()

        # Time end of transformation.
        self._time1 = time.perf_counter()

        # Load transformed rows into the fact table.
        self._load_transformed_rows()

        # Time end of loading transformed rows.
        self._time2 = time.perf_counter()

        # Load parked and ignored rows into the parked and ignored rows.
        self._load_ignored_rows()
        self._load_parked_rows()

        # Time end of loading parked and ignored rows.
        self._time3 = time.perf_counter()

        # Show statistics about number of rows and performance.
        self._log_statistics()
660056
def wrap(self, row: Union[Mapping[str, Any], Sequence[Any]]):
        """"""
        return (
            self.dataclass(
                **{
                    ident: row[column_name]
                    for ident, column_name in self.ids_and_column_names.items()
                }
            )
            if isinstance(row, Mapping)
            else self.dataclass(
                **{ident: val for ident, val in zip(self.ids_and_column_names.keys(), row)}
            )
        )
328122
def convert_descriptor_and_rows(self, descriptor, rows):
        

        # Prepare
        primary_key = None
        schema = tableschema.Schema(descriptor)
        if len(schema.primary_key) == 1:
            primary_key = schema.primary_key[0]
        elif len(schema.primary_key) > 1:
            message = 'Multi-column primary keys are not supported'
            raise tableschema.exceptions.StorageError(message)

        # Get data/index
        data_rows = []
        index_rows = []
        jtstypes_map = {}
        for row in rows:
            values = []
            index = None
            for field, value in zip(schema.fields, row):
                try:
                    if isinstance(value, float) and np.isnan(value):
                        value = None
                    if value and field.type == 'integer':
                        value = int(value)
                    value = field.cast_value(value)
                except tableschema.exceptions.CastError:
                    value = json.loads(value)
                # http://pandas.pydata.org/pandas-docs/stable/gotchas.html#support-for-integer-na
                if value is None and field.type in ('number', 'integer'):
                    jtstypes_map[field.name] = 'number'
                    value = np.NaN
                if field.name == primary_key:
                    index = value
                else:
                    values.append(value)
            data_rows.append(tuple(values))
            index_rows.append(index)

        # Get dtypes
        dtypes = []
        for field in schema.fields:
            if field.name != primary_key:
                field_name = field.name
                if six.PY2:
                    field_name = field.name.encode('utf-8')
                dtype = self.convert_type(jtstypes_map.get(field.name, field.type))
                dtypes.append((field_name, dtype))

        # Create dataframe
        index = None
        columns = schema.headers
        array = np.array(data_rows, dtype=dtypes)
        if primary_key:
            index_field = schema.get_field(primary_key)
            index_dtype = self.convert_type(index_field.type)
            index_class = pd.Index
            if index_field.type in ['datetime', 'date']:
                index_class = pd.DatetimeIndex
            index = index_class(index_rows, name=primary_key, dtype=index_dtype)
            columns = filter(lambda column: column != primary_key, schema.headers)
        dataframe = pd.DataFrame(array, index=index, columns=columns)

        return dataframe
1292491
def _reduce(self):
        """"""
        ctx = context.cpu()
        if self._stype == 'default':
            block = self.list_data()
            data = ndarray.add_n(*(w.copyto(ctx) for w in block)) / len(block)
        else:
            # fetch all rows for 'row_sparse' param
            all_row_ids = ndarray.arange(0, self.shape[0], dtype='int64', ctx=ctx)
            data = ndarray.zeros(self.shape, stype='row_sparse', ctx=ctx)
            self._trainer._row_sparse_pull(self, data, all_row_ids, full_idx=True)
        return data
312153
def build_row(data = self.data)
      values = data.to_a
      keys = self.data.column_names.to_a
      hash = {}
      values.each_with_index do |val, i|
        key = (keys[i] || i).to_s
        hash[key] = val
      end
      line = hash.to_json.to_s
      output << "  #{line}"
    end
205501
def normalize_rows(features):
    
    # Normalize each row of term frequencies to 1
    features = features.tocsr()
    features = normalize(features, norm="l2")
    # for i in range(features.shape[0]):
    #     term_frequency = features.getrow(i).data
    #     if term_frequency.size > 0:
    #         features.data[features.indptr[i]: features.indptr[i + 1]] =\
    #             features.data[features.indptr[i]: features.indptr[i + 1]]/np.sqrt(np.sum(np.power(term_frequency, 2)))

    return features
477161
def prepare_data(self, rows):
        
        self._rows = dict()
        for row in copy.copy(rows) if self.copy else rows:
            pseudo_key = self._get_pseudo_key(row)
            if pseudo_key not in self._rows:
                self._rows[pseudo_key] = list()
            self._rows[pseudo_key].append(row)

        # Convert begin and end dates to integers.
        self._date_type = None
        for pseudo_key, rows in self._rows.items():
            self._rows_date2int(rows)
696005
def apply_mapping(raw_row, mapping):
    '''
    
    '''
    row = {target: mapping_func(raw_row[source_key])
           for target, (mapping_func, source_key)
           in mapping.fget().items()}
    return row
422352
def <<(row)
      if row == Border::SEPARATOR
        separators << columns_size - (header ? 0 : 2)
      else
        rows_copy = rows.dup
        assert_row_sizes rows_copy << row
        rows << to_row(row)
      end
      self
    end
536244
def transpose(self, rows):
        
        res = OrderedDict()
        for row, cols in rows.items():
            for col, cell in cols.items():
                if col not in res:
                    res[col] = OrderedDict()
                res[col][row] = cell
        return res
495964
def expand_multirow_data(data):
    

    num_cols = len(data[0]) # number of columns

    # calculates row heights
    row_heights = []
    for mlrow in data:
        row_height = 0
        for j, cell in enumerate(mlrow):
            row_height = max(row_height, 1 if not isinstance(cell, (list, tuple)) else len(cell))
        row_heights.append(row_height)
    num_lines = sum(row_heights) # line != row (rows are multiline)

    # rebuilds table data
    new_data = [[""]*num_cols for i in range(num_lines)]
    i0 = 0
    for row_height, mlrow in zip(row_heights, data):
        for j, cell in enumerate(mlrow):
            if not isinstance(cell, (list, tuple)):
                cell = [cell]

            for incr, x in enumerate(cell):
                new_data[i0+incr][j] = x

        i0 += row_height

    return new_data, row_heights
566592
def _transform_columns(component, inner)
      col_count = component.parent.elements.size

      small_val = component.attr('small')
      large_val = component.attr('large')

      small_size = small_val || column_count
      large_size = large_val || small_val || (column_count / col_count).to_i

      classes = _combine_classes(component, "small-#{small_size} large-#{large_size} columns")

      classes << ' first' unless component.previous_element
      classes << ' last' unless component.next_element

      subrows = component.elements.css(".row").to_a.concat(component.elements.css("row").to_a)
      expander = %{<th class="expander"></th>} if large_size.to_i == column_count && subrows.empty?

      %{<#{INTERIM_TH_TAG} class="#{classes}" #{_pass_through_attributes(component)}><table><tr><th>#{inner}</th>#{expander}</tr></table></#{INTERIM_TH_TAG}>}
    end
87897
def _transform_row(self, row):
        
        tmp_row = []

        for i, column in enumerate(row.value):
            if column.has_array_value:
                raise NotImplementedError('array types are not supported')
            elif column.scalar_value.null:
                tmp_row.append(None)
            else:
                field_name, rep, mutate_to, cast_from = self._column_data_types[i]

                # get the value from the field_name
                value = getattr(column.scalar_value, field_name)

                # cast the value
                if cast_from is not None:
                    value = cast_from(value)

                tmp_row.append(value)
        return tmp_row
834497
def row_to_obj(self, row, cur):
        """"""
        obj = tornado.util.ObjectDict()
        for val, desc in zip(row, cur.description):
            obj[desc.name] = val
        return obj
910838
def format_dimension(dimension):
        """"""
        result = ""
        if "type" in dimension.attrib:
            result += "[R" if dimension.attrib["type"] == "row" else "[C"
        else:
            result += "[C"
        if "index" in dimension.attrib:
            result += ":" + dimension.attrib["index"]

        result += "] " + re.sub("\s+", " ", dimension.text.replace("\n", " "))
        return result
1039472
def iterboxed(self, rows):
        

        def asvalues(raw):
            """Convert a row of raw bytes into a flat row.  Result will
            be a freshly allocated object, not shared with
            argument.
            """

            if self.bitdepth == 8:
                return array('B', raw)
            if self.bitdepth == 16:
                raw = tostring(raw)
                return array('H', struct.unpack('!%dH' % (len(raw)//2), raw))
            assert self.bitdepth < 8
            width = self.width
            # Samples per byte
            spb = 8//self.bitdepth
            out = array('B')
            mask = 2**self.bitdepth - 1
            shifts = map(self.bitdepth.__mul__, reversed(range(spb)))
            for o in raw:
                out.extend(map(lambda i: mask&(o>>i), shifts))
            return out[:width]

        return imap(asvalues, rows)
1273573
def to_multidimensional(tpm):
    
    # Cast to np.array.
    tpm = np.array(tpm)
    # Get the number of nodes.
    N = tpm.shape[-1]
    # Reshape. We use Fortran ordering here so that the rows use the
    # little-endian convention (least-significant bits correspond to low-index
    # nodes). Note that this does not change the actual memory layout (C- or
    # Fortran-contiguous), so there is no performance loss.
    return tpm.reshape([2] * N + [N], order="F").astype(float)