Link Node Dataset File Read from File Method
979713
def _read(self, directory, filename, session, path, name, extension, spatial, spatialReferenceID, replaceParamFile):
        
        # Set file extension property
        self.fileExtension = extension

        # Dictionary of keywords/cards and parse function names
        KEYWORDS = ('NUM_LINKS',
                    'TIME_STEP',
                    'NUM_TS',
                    'START_TIME',
                    'TS')

        # Parse file into chunks associated with keywords/cards
        with open(path, 'r') as f:
            self.name = f.readline().strip()
            chunks = pt.chunk(KEYWORDS, f)

        # Parse chunks associated with each key
        for card, chunkList in iteritems(chunks):
            # Parse each chunk in the chunk list
            for chunk in chunkList:
                schunk = chunk[0].strip().split()

                # Cases
                if card == 'NUM_LINKS':
                    # NUM_LINKS handler
                    self.numLinks = schunk[1]

                elif card == 'TIME_STEP':
                    # TIME_STEP handler
                    self.timeStepInterval = schunk[1]

                elif card == 'NUM_TS':
                    # NUM_TS handler
                    self.numTimeSteps = schunk[1]

                elif card == 'START_TIME':
                    # START_TIME handler
                    self.startTime = '%s  %s    %s  %s  %s  %s' % (
                        schunk[1],
                        schunk[2],
                        schunk[3],
                        schunk[4],
                        schunk[5],
                        schunk[6])

                elif card == 'TS':
                    # TS handler
                    for line in chunk:
                        sline = line.strip().split()
                        token = sline[0]

                        # Cases
                        if token == 'TS':
                            # Time Step line handler
                            timeStep = LinkNodeTimeStep(timeStep=sline[1])
                            timeStep.linkNodeDataset = self

                        else:
                            # Split the line
                            spLinkLine = line.strip().split()

                            # Create LinkDataset GSSHAPY object
                            linkDataset = LinkDataset()
                            linkDataset.numNodeDatasets = int(spLinkLine[0])
                            linkDataset.timeStep = timeStep
                            linkDataset.linkNodeDatasetFile = self

                            # Parse line into NodeDatasets
                            NODE_VALUE_INCREMENT = 2
                            statusIndex = 1
                            valueIndex = statusIndex + 1

                            # Parse line into node datasets
                            if linkDataset.numNodeDatasets > 0:
                                for i in range(0, linkDataset.numNodeDatasets):
                                    # Create NodeDataset GSSHAPY object
                                    nodeDataset = NodeDataset()
                                    nodeDataset.status = int(spLinkLine[statusIndex])
                                    nodeDataset.value = float(spLinkLine[valueIndex])
                                    nodeDataset.linkDataset = linkDataset
                                    nodeDataset.linkNodeDatasetFile = self

                                    # Increment to next status/value pair
                                    statusIndex += NODE_VALUE_INCREMENT
                                    valueIndex += NODE_VALUE_INCREMENT
                            else:
                                nodeDataset = NodeDataset()
                                nodeDataset.value = float(spLinkLine[1])
                                nodeDataset.linkDataset = linkDataset
                                nodeDataset.linkNodeDatasetFile = self
174402
public void readTrnData(String dataFile) {
	if (cpStr2Int != null) {
	    cpStr2Int.clear();
	} else {
	    cpStr2Int = new HashMap();
	}
	
	if (cpInt2Str != null) {
	    cpInt2Str.clear();	    
	} else {
	    cpInt2Str = new HashMap();
	}
	
	if (lbStr2Int != null) {
	    lbStr2Int.clear();
	} else {
	    lbStr2Int = new HashMap();
	}
	
	if (lbInt2Str != null) {
	    lbInt2Str.clear();
	} else {
	    lbInt2Str = new HashMap();
	}
	
	if (trnData != null) {
	    trnData.clear();
	} else {
	    trnData = new ArrayList();
	}

	// open data file	
	BufferedReader fin = null;
	
	try {
	    fin = new BufferedReader(new InputStreamReader(new FileInputStream(dataFile), "UTF-8"));
	//    BufferedWriter flog = new BufferedWriter(new OutputStreamWriter(
//	/	new FileOutputStream((new File(dataFile)).getParent() + File.separator + "log.txt"), "UTF-8"));
	    
	    System.out.println("Reading training data ...");
	    
	    String line;
	    while ((line = fin.readLine()) != null) {
		StringTokenizer strTok = new StringTokenizer(line, " \t\r\n");
		int len = strTok.countTokens();
				
		if (len <= 1) {
		    // skip this invalid line
		    continue;
		}
		
		List strCps = new ArrayList();
		for (int i = 0; i < len - 1; i++) {
		    strCps.add(strTok.nextToken());
		}
		
		String labelStr = strTok.nextToken();
//		
//		String [] tags = {"N", "Np", "Nc", "Nu", "V", "A", "P", "L", "M", 
//		"R", "E", "C", "I", "T", "U", "Y", "X", "LBKT", "RBKT"};
//
//
//		//System.out.println("--" + labelStr);
//		//if (!StringUtils.isSign(labelStr)){
//			boolean flag = false;
//			for (String tag : tags){
//				if (labelStr.equalsIgnoreCase(tag)){
//					flag = true;
//				}
//			}
//			
//			if (!flag){
//			//	flog.write(line + "\n");
//				//System.out.println("--" + labelStr);
//			}
//		//}
		
		List intCps = new ArrayList();
		
		for (int i = 0; i < strCps.size(); i++) {	
		    String cpStr = (String)strCps.get(i);		    
		    Integer cpInt = (Integer)cpStr2Int.get(cpStr);		    
		    if (cpInt != null) {
			intCps.add(cpInt);
		    } else {
			intCps.add(new Integer(cpStr2Int.size()));
			cpStr2Int.put(cpStr, new Integer(cpStr2Int.size()));
			cpInt2Str.put(new Integer(cpInt2Str.size()), cpStr);
		    }
		}
		
		Integer labelInt = (Integer)lbStr2Int.get(labelStr);
		if (labelInt == null) {
		    labelInt = new Integer(lbStr2Int.size());
		    
//		    System.out.println("hey:" + labelStr);
//		    flog.write(labelStr + "\t" + line + "\n");
		    lbStr2Int.put(labelStr, labelInt);
		    lbInt2Str.put(labelInt, labelStr);
		}
		
		int[] cps = new int[intCps.size()];
		for (int i = 0; i < cps.length; i++) {
		    cps[i] = ((Integer)intCps.get(i)).intValue();
		}
		
		Observation obsr = new Observation(labelInt.intValue(), cps);
		
		// add this observation to the data
		trnData.add(obsr);
	    }
	    
	    System.out.println("Reading " + Integer.toString(trnData.size()) +
			" training data examples completed!");
	  // flog.close();
	
	} catch (IOException e) {
	    System.out.println(e.toString());
	    return;
	}

	option.numCps = cpStr2Int.size();
	option.numLabels = lbStr2Int.size();	
	option.numTrainExps = trnData.size();	
    }
1130509
def push(description, datasets, wait, verbose):
    """"""
    # Loading config
    config = _get_config()
    file_patterns = config["contents"]
    if not isinstance(file_patterns, type([])):
        # put it into an array so we can iterate it, if it isn't already an array
        file_patterns = [file_patterns]
    if datasets:
        datasets_string = datasets.split(',')
    else:
        datasets_string = config.get("datasets", '')


    # Getting all file names/globs -- making sure we get CONFIG_FILE
    files = {CONFIG_FILE}  # use a set so we don't get duplicates
    for pattern in file_patterns:
        files.update(glob.glob(pattern))
    if not files:
        print "No files could be found? Check your contents section in `CONFIG_FILE`!"
        exit(-1)

    if verbose:
        print ""
        print "gatherered files:"
        for path in files:
            print "\t", path
        print ""

    # Zipping
    _print("zipping...")

    if not os.path.exists("brains_history"):
        os.mkdir("brains_history")

    zip_path = 'brains_history/%s.zip' % str(datetime.datetime.now())
    with ZipFile(zip_path, 'w') as zip_file:
        for path in files:
            zip_file.write(path)
    cprint("done", 'green')

    # Sending to server
    with open(zip_path, 'rb') as zip_file:
        _print("sending to server...")
        try:
            response = requests.post(
                URL_SUBMIT,
                files={
                    "zip_file": zip_file,
                },
                data={
                    "name": config["name"],
                    "description": description or '',
                    "languages": config["languages"],
                    "datasets": datasets_string,
                    "wait": wait,
                },
                stream=wait  # if we're waiting for response then stream
            )

            if response.status_code == 200:
                cprint("done", 'green')

                if wait:
                    _print("\nOutput: ")
                    cprint(" " * 72, 'green', attrs=('underline',))

                    chunk_buffer = ""
                    # read in 1 chunk at a time for carriage return detection
                    for chunk in response.iter_content(chunk_size=1):
                        chunk_buffer += chunk

                        if chunk == '\r':
                            # We hit the end of a message!
                            try:
                                data = json.loads(chunk_buffer)
                                if "stdout" not in data or "stderr" not in data:
                                    print "dis one"
                                    continue
                            except (ValueError,):
                                continue

                            if data["stdout"]:
                                # Get rid of termination string, if it's there
                                data["stdout"] = data["stdout"].replace("-%-%-%-%-END BRAIN SEQUENCE-%-%-%-%-", "")
                                _print(data["stdout"])
                            if data["stderr"]:
                                _print(colored(data["stderr"], 'red'))

                            # Clear buffer after reading message
                            chunk_buffer = ""
            else:
                cprint(response.json()["error"], 'red')
        except requests.exceptions.ConnectionError:
            cprint("failed to connect to server!", 'red')
            exit(-2)
1324753
def read(filename):
    
    import h5py

    f = h5py.File(filename, "r")
    dset = f["tstt"]

    points = dset["nodes"]["coordinates"][()]
    # read point data
    point_data = {}
    if "tags" in dset["nodes"]:
        for name, dataset in dset["nodes"]["tags"].items():
            point_data[name] = dataset[()]

    # # Assert that the GLOBAL_IDs are contiguous.
    # point_gids = dset['nodes']['tags']['GLOBAL_ID'][()]
    # point_start_gid = dset['nodes']['coordinates'].attrs['start_id']
    # point_end_gid = point_start_gid + len(point_gids) - 1
    # assert all(point_gids == range(point_start_gid, point_end_gid + 1))

    h5m_to_meshio_type = {"Edge2": "line", "Tri3": "triangle", "Tet4": "tetra"}
    cells = {}
    cell_data = {}
    for h5m_type, data in dset["elements"].items():
        meshio_type = h5m_to_meshio_type[h5m_type]
        conn = data["connectivity"]
        # Note that the indices are off by 1 in h5m.
        cells[meshio_type] = conn[()] - 1

        # TODO bring cell data back
        # if 'tags' in data:
        #     for name, dataset in data['tags'].items():
        #         cell_data[name] = dataset[()]

    # The `sets` in H5M are special in that they represent a segration of data
    # in the current file, particularly by a load balancer (Metis, Zoltan,
    # etc.). This segregation has no equivalent in other data types, but is
    # certainly worthwhile visualizing.
    # Hence, we will translate the sets into cell data with the prefix "set::"
    # here.
    field_data = {}
    # TODO deal with sets
    # if 'sets' in dset and 'contents' in dset['sets']:
    #     # read sets
    #     sets_contents = dset['sets']['contents'][()]
    #     sets_list = dset['sets']['list'][()]
    #     sets_tags = dset['sets']['tags']

    #     cell_start_gid = conn.attrs['start_id']
    #     cell_gids = cell_start_gid + elems['tags']['GLOBAL_ID'][()]
    #     cell_end_gid = cell_start_gid + len(cell_gids) - 1
    #     assert all(cell_gids == range(cell_start_gid, cell_end_gid + 1))

    #     # create the sets
    #     for key, value in sets_tags.items():
    #         mod_key = 'set::' + key
    #         cell_data[mod_key] = numpy.empty(len(cells), dtype=int)
    #         end = 0
    #         for k, row in enumerate(sets_list):
    #             bits = _int_to_bool_list(row[3])
    #             # is_owner = bits[0]
    #             # is_unique = bits[1]
    #             # is_ordered = bits[2]
    #             is_range_compressed = bits[3]
    #             if is_range_compressed:
    #                 start_gids = sets_contents[end:row[0]+1:2]
    #                 lengths = sets_contents[end+1:row[0]+1:2]
    #                 for start_gid, length in zip(start_gids, lengths):
    #                     end_gid = start_gid + length - 1
    #                     if start_gid >= cell_start_gid and \
    #                             end_gid <= cell_end_gid:
    #                         i0 = start_gid - cell_start_gid
    #                         i1 = end_gid - cell_start_gid + 1
    #                         cell_data[mod_key][i0:i1] = value[k]
    #                     else:
    #                         # TODO deal with point data
    #                         raise RuntimeError('')
    #             else:
    #                 gids = sets_contents[end:row[0]+1]
    #                 cell_data[mod_key][gids - cell_start_gid] = value[k]

    #             end = row[0] + 1

    return Mesh(
        points, cells, point_data=point_data, cell_data=cell_data, field_data=field_data
    )
1109652
def choose_raw_dataset(currently=""):
    """"""
    folder = os.path.join(get_project_root(), "raw-datasets")
    files = [os.path.join(folder, name) for name in os.listdir(folder)
             if name.endswith(".pickle")]
    default = -1
    for i, filename in enumerate(files):
        if os.path.basename(currently) == os.path.basename(filename):
            default = i
        if i != default:
            print("[%i]\t%s" % (i, os.path.basename(filename)))
        else:
            print("\033[1m[%i]\033[0m\t%s" % (i, os.path.basename(filename)))
    i = input_int_default("Choose a dataset by number: ", default)
    return files[i]
1301411
def create_dataset_file(root_dir, data):
  """"""
  file_path = os.path.join(root_dir, '{dataset_type}', '{dataset_name}.py')
  context = (
      _HEADER + _DATASET_DEFAULT_IMPORTS + _CITATION
      + _DESCRIPTION + _DATASET_DEFAULTS
  )

  with gfile.GFile(file_path.format(**data), 'w') as f:
    f.write(context.format(**data))
1319747
def _read(self, directory, filename, session, path, name, extension, spatial, spatialReferenceID, replaceParamFile):
        
        # Set file extension property
        self.fileExtension = extension

        # Dictionary of keywords/cards and parse function names
        KEYWORDS = {'ALPHA': cic.cardChunk,
                    'BETA': cic.cardChunk,
                    'THETA': cic.cardChunk,
                    'LINKS': cic.cardChunk,
                    'MAXNODES': cic.cardChunk,
                    'CONNECT': cic.connectChunk,
                    'LINK': cic.linkChunk}

        links = []
        connectivity = []

        # Parse file into chunks associated with keywords/cards
        with open(path, 'r') as f:
            chunks = pt.chunk(KEYWORDS, f)

        # Parse chunks associated with each key
        for key, chunkList in iteritems(chunks):
            # Parse each chunk in the chunk list
            for chunk in chunkList:
                # Call chunk specific parsers for each chunk
                result = KEYWORDS[key](key, chunk)

                # Cases
                if key == 'LINK':
                    # Link handler
                    links.append(self._createLink(result, replaceParamFile))

                elif key == 'CONNECT':
                    # Connectivity handler
                    connectivity.append(result)

                else:
                    # Global variable handler
                    card = result['card']
                    value = result['values'][0]
                    # Cases
                    if card == 'LINKS':
                        self.links = int(value)
                    elif card == 'MAXNODES':
                        self.maxNodes = int(value)
                    elif card == 'ALPHA':
                        self.alpha = float(vrp(value, replaceParamFile))
                    elif card == 'BETA':
                        self.beta = float(vrp(value, replaceParamFile))
                    elif card == 'THETA':
                        self.theta = float(vrp(value, replaceParamFile))

        self._createConnectivity(linkList=links, connectList=connectivity)

        if spatial:
            self._createGeometry(session, spatialReferenceID)
1123350
def _read(self, directory, filename, session, path, name, extension, spatial, spatialReferenceID, maskMap):
        
        # Assign file extension attribute to file object
        self.fileExtension = extension

        if isinstance(maskMap, RasterMapFile) and maskMap.fileExtension == 'msk':
            # Vars from mask map
            columns = maskMap.columns
            rows = maskMap.rows
            upperLeftX = maskMap.west
            upperLeftY = maskMap.north

            # Derive the cell size (GSSHA cells are square, so it is the same in both directions)
            cellSizeX = int(abs(maskMap.west - maskMap.east) / columns)
            cellSizeY = -1 * cellSizeX

            # Dictionary of keywords/cards and parse function names
            KEYWORDS = {'DATASET': wdc.datasetHeaderChunk,
                        'TS': wdc.datasetScalarTimeStepChunk}

            # Open file and read plain text into text field
            with open(path, 'r') as f:
                chunks = pt.chunk(KEYWORDS, f)

            # Parse header chunk first
            header = wdc.datasetHeaderChunk('DATASET', chunks['DATASET'][0])

            # Parse each time step chunk and aggregate
            timeStepRasters = []

            for chunk in chunks['TS']:
                timeStepRasters.append(wdc.datasetScalarTimeStepChunk(chunk, columns, header['numberCells']))

            # Set WMS dataset file properties
            self.name = header['name']
            self.numberCells = header['numberCells']
            self.numberData = header['numberData']
            self.objectID = header['objectID']

            if header['type'] == 'BEGSCL':
                self.objectType = header['objectType']
                self.type = self.SCALAR_TYPE

            elif header['type'] == 'BEGVEC':
                self.vectorType = header['objectType']
                self.type = self.VECTOR_TYPE

            # Create WMS raster dataset files for each raster
            for timeStep, timeStepRaster in enumerate(timeStepRasters):
                # Create new WMS raster dataset file object
                wmsRasterDatasetFile = WMSDatasetRaster()

                # Set the wms dataset for this WMS raster dataset file
                wmsRasterDatasetFile.wmsDataset = self

                # Set the time step and timestamp and other properties
                wmsRasterDatasetFile.iStatus = timeStepRaster['iStatus']
                wmsRasterDatasetFile.timestamp = timeStepRaster['timestamp']
                wmsRasterDatasetFile.timeStep = timeStep + 1

                # If spatial is enabled create PostGIS rasters
                if spatial:
                    # Process the values/cell array
                    wmsRasterDatasetFile.raster = RasterLoader.makeSingleBandWKBRaster(session,
                                                                                       columns, rows,
                                                                                       upperLeftX, upperLeftY,
                                                                                       cellSizeX, cellSizeY,
                                                                                       0, 0,
                                                                                       spatialReferenceID,
                                                                                       timeStepRaster['cellArray'])

                # Otherwise, set the raster text properties
                else:
                    wmsRasterDatasetFile.rasterText = timeStepRaster['rasterText']

            # Add current file object to the session
            session.add(self)

        else:
            log.warning("Could not read {0}. Mask Map must be supplied "
                     "to read WMS Datasets.".format(filename))
487899
def link_node(abs_src: str, abs_dest: str, force: bool=False):
    """"""
    dest_parent_dir = split(abs_dest)[0]
    if not isdir(dest_parent_dir):
        # exist_ok=True in case of concurrent creation of the same
        # parent dir
        os.makedirs(dest_parent_dir, exist_ok=True)
    if isfile(abs_src):
        # sync file by linking it to dest
        link_func(abs_src, abs_dest, force)
    elif isdir(abs_src):
        # sync dir by recursively linking files under it to dest
        shutil.copytree(abs_src, abs_dest,
                        copy_function=functools.partial(link_func,
                                                        force=force),
                        ignore=shutil.ignore_patterns('.git'))
    else:
        raise FileNotFoundError(abs_src)
920645
def _read_node_data(subnw, current_node, node_type, matcher, formatcode):
    

    if node_type == "leaf" or node_type == "single":
        if node_type == "leaf":
            node = current_node.add_child()
        else:
            node = current_node
    else:
        node = current_node

    subnw = subnw.strip()
    
    if not subnw and node_type == 'leaf' and formatcode != 100:
        raise NewickError('Empty leaf node found')
    elif not subnw:
        return

    container1, container2, converterFn1, converterFn2, compiled_matcher = matcher[node_type]
    data = re.match(compiled_matcher, subnw)
    if data:
        data = data.groups()
        # This prevents ignoring errors even in flexible nodes:
        if subnw and data[0] is None and data[1] is None and data[2] is None:
            raise NewickError("Unexpected newick format '%s'" %subnw)

        if data[0] is not None and data[0] != '':
            node.add_feature(container1, converterFn1(data[0].strip()))

        if data[1] is not None and data[1] != '':
            node.add_feature(container2, converterFn2(data[1][1:].strip()))

        if data[2] is not None \
                and data[2].startswith("[&&NHX"):
            _parse_extra_features(node, data[2])
    else:
        raise NewickError("Unexpected newick format '%s' " %subnw[0:50])
    return
290024
private ExampleSet readDataSet(File in) throws IOException
	{
		logger.info("read the example set");

		//
		ExampleSet inputSet = new SentenceSetCopy();
		inputSet.read(new BufferedReader(new FileReader(in)));

		String trainFrac = parameter.getProperty("train-frac");
		if (trainFrac != null)
		{
			double f = Double.parseDouble(trainFrac);
			logger.info("training original size: " + inputSet.size());
			logger.info("training fraction: " + (100 * f) + "%");
			return inputSet.subSet(0, (int) (inputSet.size() * f));
		}

		return inputSet;
	}
390041
def _read_data(prefix, batch_size, augmentation=None):
    
    prefix = prefix.replace("\\", "/")
    folder = "/".join(prefix.split("/")[:-1])
    phase = prefix.split("/")[-1]
    config = json.load(open(prefix + '_config.json'))
    num_threads = config["num_threads"]

    filenames = [folder + "/" + f for f in listdir(folder) if isfile(join(folder, f)) and phase in f and not "config.json" in f]

    dataset = tf.data.TFRecordDataset(filenames=filenames, num_parallel_reads=num_threads)
    dataset = dataset.shuffle(buffer_size=10 * batch_size)
    dataset = dataset.repeat()
    dataset = dataset.map(map_func=_create_parser_fn(config, phase), num_parallel_calls=num_threads)
    if augmentation is not None:
        dataset = dataset.map(map_func=augmentation, num_parallel_calls=num_threads)
    dataset = dataset.batch(batch_size=batch_size)
    dataset = dataset.prefetch(buffer_size=1)

    return dataset
78754
def build_dataset(filename, sequence_length, alphabet_size, max_lines=-1):
    """"""
    with open(filename) as f:
        content = f.readlines()
    content = content[:max_lines]
    content = [line for line in content if len(line) > 2]   # Remove blank lines
    seqs = np.zeros((sequence_length, len(content), alphabet_size))
    for ix, line in enumerate(content):
        padded_line = (line + " " * sequence_length)[:sequence_length]
        seqs[:, ix, :] = string_to_one_hot(padded_line, alphabet_size)
    return seqs
274387
private ExampleSet readDataSet(File in) throws IOException {
        logger.info("read the example set");

        //
        ExampleSet inputSet1 = new SentenceSetCopy();
        inputSet1.read(new BufferedReader(new FileReader(in)));

        return inputSet1;
    }
999902
def _load(self, dataset='train'):
        
        data, labels = None, None
        if dataset is 'train':
            files = [os.path.join(self.cifar10_dir, 'data_batch_%d' % i) for i in range(1, 6)]
        else:
            files = [os.path.join(self.cifar10_dir, 'test_batch')]

        for file in files:
            if not os.path.exists(file):
                raise FileNotFoundError('Failed to find file: ' + file)

        # Load the data from the batch files
        for file in files:
            with open(file, 'rb') as f:
                cifar10 = pickle.load(f, encoding='latin1')

            if labels is None:
                labels = np.array(cifar10['labels'])
            else:
                labels = np.concatenate((labels, cifar10['labels']), axis=0)

            if data is None:
                data = cifar10['data']
            else:
                data = np.concatenate((data, cifar10['data']), axis=0)

        # Adapt the format of the data to our convnet
        data = np.array(data, dtype=float) / 255.0
        data = data.reshape([-1, self.num_channels, self.img_size, self.img_size])
        data = data.transpose([0, 2, 3, 1])

        # One-hot encode labels (see https://stackoverflow.com/a/42874726)
        labels = np.eye(self.num_classes)[np.array(labels).reshape(-1)]

        if dataset is 'train':
            self._train_data, self._train_labels = data, labels
        else:
            self._test_data, self._test_labels = data, labels
893891
def copy_from_server_to_local(dataset_remote_dir, dataset_local_dir,
                              remote_fname, local_fname):
    
    log.debug("Copying file `{}` to a local directory `{}`."
              .format(remote_fname, dataset_local_dir))

    head, tail = os.path.split(local_fname)
    head += os.path.sep
    if not os.path.exists(head):
        os.makedirs(os.path.dirname(head))

    shutil.copyfile(remote_fname, local_fname)

    # Copy the original group id and file permission
    st = os.stat(remote_fname)
    os.chmod(local_fname, st.st_mode)
    # If the user have read access to the data, but not a member
    # of the group, he can't set the group. So we must catch the
    # exception. But we still want to do this, for directory where
    # only member of the group can read that data.
    try:
        os.chown(local_fname, -1, st.st_gid)
    except OSError:
        pass

    # Need to give group write permission to the folders
    # For the locking mechanism
    # Try to set the original group as above
    dirs = os.path.dirname(local_fname).replace(dataset_local_dir, '')
    sep = dirs.split(os.path.sep)
    if sep[0] == "":
        sep = sep[1:]
    for i in range(len(sep)):
        orig_p = os.path.join(dataset_remote_dir, *sep[:i + 1])
        new_p = os.path.join(dataset_local_dir, *sep[:i + 1])
        orig_st = os.stat(orig_p)
        new_st = os.stat(new_p)
        if not new_st.st_mode & stat.S_IWGRP:
            os.chmod(new_p, new_st.st_mode | stat.S_IWGRP)
        if orig_st.st_gid != new_st.st_gid:
            try:
                os.chown(new_p, -1, orig_st.st_gid)
            except OSError:
                pass
465789
def _load_dataset(dsid, ds_info, file_handlers, dim='y'):
        """"""
        slice_list = []
        failure = True
        for fh in file_handlers:
            try:
                projectable = fh.get_dataset(dsid, ds_info)
                if projectable is not None:
                    slice_list.append(projectable)
                    failure = False
            except KeyError:
                logger.warning("Failed to load {} from {}".format(dsid, fh),
                               exc_info=True)

        if failure:
            raise KeyError(
                "Could not load {} from any provided files".format(dsid))

        if dim not in slice_list[0].dims:
            return slice_list[0]
        res = xr.concat(slice_list, dim=dim)

        combined_info = file_handlers[0].combine_info(
            [p.attrs for p in slice_list])

        res.attrs = combined_info
        return res