helper for infer_columns. removes paths that are direct children of the n-1 or n-2 path
192976
def eliminate_sequential_children(paths):
  ""
  return [p for i,p in enumerate(paths) if not ((i>0 and paths[i-1]==p[:-1]) or (i>1 and paths[i-2]==p[:-1]))]
828684
def filter_exclude(df, s):
    
    keep = ~np.array( [s in c for c in df.columns.values] )
    return df.iloc[:, keep]
112422
def _get_gtf_column(column_name, gtf_path, df):
    
    if column_name in df.columns:
        return list(df[column_name])

    else:
        raise ValueError(
            "Missing '%s' in columns of %s, available: %s" % (
                column_name,
                gtf_path,
                list(df.columns)))
929147
def none_missing(df, columns=None):
    
    if columns is None:
        columns = df.columns
    assert not df[columns].isnull().any().any()
    return df
940705
function removeChildInternals (withoutInternal) {
  return _.chain(withoutInternal)
    .map((item, key) => {
      const withoutInternal = removeInternalValues(item)
      if (withoutInternal !== undefined && withoutInternal !== item) {
        return [key, withoutInternal]
      }
    })
    .filter()
    .fromPairs()
    .value()
}
1163116
def _inferSchema(self, rdd, samplingRatio=None, names=None):
        
        first = rdd.first()
        if not first:
            raise ValueError("The first row in RDD is empty, "
                             "can not infer schema")
        if type(first) is dict:
            warnings.warn("Using RDD of dict to inferSchema is deprecated. "
                          "Use pyspark.sql.Row instead")

        if samplingRatio is None:
            schema = _infer_schema(first, names=names)
            if _has_nulltype(schema):
                for row in rdd.take(100)[1:]:
                    schema = _merge_type(schema, _infer_schema(row, names=names))
                    if not _has_nulltype(schema):
                        break
                else:
                    raise ValueError("Some of types cannot be determined by the "
                                     "first 100 rows, please try again with sampling")
        else:
            if samplingRatio < 0.99:
                rdd = rdd.sample(False, float(samplingRatio))
            schema = rdd.map(lambda row: _infer_schema(row, names)).reduce(_merge_type)
        return schema
681690
def select_paths(cls, dataset, selection):
        
        return [s[0] for s in np.array([{0: p} for p in dataset.data])[selection]]
1137092
def normalize_paths(value, parent=os.curdir):
    
    if not value:
        return []
    if isinstance(value, list):
        return value
    paths = []
    for path in value.split(','):
        path = path.strip()
        if '/' in path:
            path = os.path.abspath(os.path.join(parent, path))
        paths.append(path.rstrip('/'))
    return paths
521633
private void removeDominatedColumns(DoubleMatrix1D c,
			DoubleMatrix2D A, DoubleMatrix1D b, 
			DoubleMatrix1D lb, DoubleMatrix1D ub, 
			DoubleMatrix1D ylb, DoubleMatrix1D yub,
			DoubleMatrix1D zlb, DoubleMatrix1D zub) {
		for(short col=0; col<this.vColPositions.length; col++){
			short[] vColPositionsCol = vColPositions[col];
			if(vColPositionsCol==null || vColPositionsCol.length==0){
				continue;
			}
			double e = 0.;
			double d = 0.;
			//it will be e <= d
			for(int i=0; i<vColPositionsCol.length; i++){
				short row = vColPositionsCol[i]; 
				double AIJ = A.getQuick(row, col);
				if(AIJ > 0){
					e += AIJ * ylb.getQuick(row); 
					d += AIJ * yub.getQuick(row);
				}else{
					e += AIJ * yub.getQuick(row); 
					d += AIJ * ylb.getQuick(row);
				}
			}
			
			double cmd = c.getQuick(col) - d;
			double cme = c.getQuick(col) - e;
			//logger.debug("cmd: " +cmd);
			//logger.debug("cme: " +cme);
			boolean isCmdPositive = cmd > 0 && !isZero(cmd);//strictly > 0
			boolean isCmeNegative = cme < 0 && !isZero(cme);//strictly < 0
			boolean isLBColUnbounded = isLBUnbounded(lb.getQuick(col));
			boolean isUBColUnbounded = isUBUnbounded(ub.getQuick(col));
			
			if(isCmdPositive || isCmeNegative){
			  //dominated column
				log.debug("found dominated column: " + col);
				if(isCmdPositive){
					zlb.setQuick(col, 0);
					if(isLBColUnbounded){
						log.debug("unbounded problem");
						throw new RuntimeException("unbounded problem");
					}
					ub.setQuick(col, lb.getQuick(col));
					log.debug("x[" + col + "]="+lb.getQuick(col));
					addToPresolvingStack(new LinearDependency(col, null, null, lb.getQuick(col)));
					pruneFixedVariable(col, c, A, b, lb, ub, ylb, yub, zlb, zub);
				}else if(isCmeNegative){
					zub.setQuick(col, 0);
					if(isUBColUnbounded){
						log.debug("unbounded problem");
						throw new RuntimeException("unbounded problem");
					}
					lb.setQuick(col, ub.getQuick(col));
					log.debug("x[" + col + "]="+ub.getQuick(col));
					addToPresolvingStack(new LinearDependency(col, null, null, ub.getQuick(col)));
					pruneFixedVariable(col, c, A, b, lb, ub, ylb, yub, zlb, zub);
				}
				continue;
			}
			
			//here we have cmd<=0 and cme>=0 (can even be unbounded)
			
			if(vColPositionsCol.length>1){//the column singletons are used to generate the bounds d and e and therefore they cannot be dropped with this test
				if(!isLBColUnbounded && isZero(cmd)){	
					//weakly dominated column, see A. & A. (27), (28)
					log.debug("found weakly dominated column: " + col);
					ub.setQuick(col, lb.getQuick(col));
					log.debug("x[" + col + "]="+lb.getQuick(col));
					addToPresolvingStack(new LinearDependency(col, null, null, lb.getQuick(col)));
					pruneFixedVariable(col, c, A, b, lb, ub, ylb, yub, zlb, zub);
					continue;
				}
				if(!isUBColUnbounded && isZero(cme)){	
					//weakly dominated column, see A. & A. (27), (28)
					log.debug("found weakly dominated column: " + col);
					lb.setQuick(col, ub.getQuick(col));
					log.debug("x[" + col + "]="+ub.getQuick(col));
					addToPresolvingStack(new LinearDependency(col, null, null, ub.getQuick(col)));
					pruneFixedVariable(col, c, A, b, lb, ub, ylb, yub, zlb, zub);
					continue;
				}
			}
			
			if(!isLBColUnbounded && isUBColUnbounded){
				//new bounds on the optimal Lagrange multipliers y
				for(int i=0; i<vColPositionsCol.length; i++){
					short row = vColPositionsCol[i]; 
					double AIJ = A.getQuick(row, col);
					if(AIJ > 0){
						if(!isUBUnbounded(cme/AIJ + ylb.getQuick(row))){
							log.debug("set new bounds on the optimal Lagrange multipliers: " + row);
							yub.setQuick(row, Math.min(yub.getQuick(row), cme/AIJ + ylb.getQuick(row)));
						}
					}else{
						if(!isLBUnbounded(cme/AIJ + yub.getQuick(row))){
							log.debug("set new bounds on the optimal Lagrange multipliers: " + row);
							ylb.setQuick(row, Math.max(ylb.getQuick(row), cme/AIJ + yub.getQuick(row)));
						}
					}
				}
			}
			if(isLBColUnbounded && !isUBColUnbounded){
			  //new bounds on the optimal Lagrange multipliers y
				for(int i=0; i<vColPositionsCol.length; i++){
					short row = vColPositionsCol[i]; 
					double AIJ = A.getQuick(row, col);
					if(AIJ > 0){
						if(!isLBUnbounded(cmd/AIJ + yub.getQuick(row))){
							log.debug("set new bounds on the optimal Lagrange multipliers: " + row);
							ylb.setQuick(row, Math.max(ylb.getQuick(row), cmd/AIJ + yub.getQuick(row)));
						}
					}else{
						if(!isUBUnbounded(cmd/AIJ + ylb.getQuick(row))){
							log.debug("set new bounds on the optimal Lagrange multipliers: " + row);
							yub.setQuick(row, Math.min(yub.getQuick(row), cmd/AIJ + ylb.getQuick(row)));
						}
					}
				}
			}
		}
	}
202209
def _get_inferred_data_res(column, age):
    
    try:
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            # Get the values for this column
            values = column["values"]
            # Make sure that age and values are numpy arrays
            _values = np.array(copy.copy(values), dtype=float)
            # _values = _values[np.where(~np.isnan(_values))[0]]
            _age = np.array(age, dtype=float)
            # _age = _age[np.where(~np.isnan(_age))[0]]

            # If we have values, keep going
            if len(_values) != 0:
                # Get the resolution for this age and column values data
                res = _get_resolution(_age, _values)
                # If we have successful resolution data, keep going
                if len(res) != 0:
                    column["hasResolution"] = __get_inferred_data_res_2(res)

                # Remove the NaNs from the values list.
                _values = _values[np.where(~np.isnan(_values))[0]]
                # Calculate column non-resolution data, update the column with the results.
                column.update(__get_inferred_data_res_2(_values))

    except KeyError as e:
        logger_inferred_data.debug("get_inferred_data_column: KeyError: {}".format(e))
    except Exception as e:
        logger_inferred_data.debug("get_inferred_data_column: Exception: {}".format(e))

    return column
870090
def p_attr(self,t):
    
    # careful: sqex.infer_columns relies on AttrX not containing anything but a name
    t[0] = AttrX(NameX(t[1]), AsterX() if t[3]=='*' else NameX(t[3]))
919649
def remove_length_outliers(df, columnname):
    """"""
    return df[df[columnname] < (np.median(df[columnname]) + 3 * np.std(df[columnname]))]
196122
def infer_list(values: List[GenericAny]) -> Array:
    """"""
    if not values:
        return Array(null)
    return Array(highest_precedence(map(infer, values)))
1329786
def _set_pattern_columns(self, path_column):
        
        try:
            # CategoricalDtype allows specifying known categories when
            # creating objects. It was added in pandas 0.21.0.
            from pandas.api.types import CategoricalDtype
            _HAS_CDT = True
        except ImportError:
            _HAS_CDT = False

        col = self._dataframe[path_column]
        paths = col.cat.categories

        column_by_field = {field:
            col.cat.codes.map(dict(enumerate(values))).astype(
                "category" if not _HAS_CDT else CategoricalDtype(set(values))
            ) for field, values in reverse_formats(self.pattern, paths).items()
        }
        self._dataframe = self._dataframe.assign(**column_by_field)
430439
private static LocationPath compressAnywhere(LocationPath path){
        int noOfNulls = 0;
        Step steps[] = path.steps;

        Step prevStep = steps[0];
        for(int i=1, len=steps.length; i<len; i++){
            Step curStep = steps[i];

            if(!curStep.predicateSet.hasPosition && prevStep.predicateSet.getPredicate()==null){
                if(prevStep.axis==Axis.DESCENDANT_OR_SELF && prevStep.constraint.id==Constraint.ID_NODE){
                    if(curStep.axis==Axis.CHILD){
                        steps[i-1] = null;
                        Step newStep = new Step(Axis.DESCENDANT, curStep.constraint);
                        newStep.setPredicate(curStep);
                        steps[i] = curStep = newStep;
                        noOfNulls++;
                    }
                }
            }
            prevStep = curStep;
        }
        return noOfNulls>0 ? removeNullSteps(path, noOfNulls) : path;
    }
889724
def split_dataframe(path_df):
    
    splits = np.where(path_df.iloc[:, 0].isnull())[0]+1
    return [df for df in np.split(path_df, splits) if len(df) > 1]
475677
function filter(p, rules, root, isDirectory, included) {
  p = path.relative(root, p);
  for (var i in rules) {
    var rule = rules[i];
    if (isDirectory) {
      if (rule.match(p) || rule.match("/" + p) ||
          rule.match(p + "/") || rule.match("/" + p + "/")
         ) {
        return rule.negate;
      }
    } else {
      if (rule.match(p) || rule.match("/" + p)) {
        return rule.negate;
      }
    }
  }
  return included;
}