Convert nested set of directories to
324971
def convert_dirs(base_dir, hdf_name, complib=None, complevel=0):
    
    print('Converting directories in {}'.format(base_dir))

    dirs = glob.glob(os.path.join(base_dir, '*'))
    dirs = {d for d in dirs if os.path.basename(d) in DIRECTORIES}
    if not dirs:
        raise RuntimeError('No direcotries found matching known data.')

    store = pd.HDFStore(
        hdf_name, mode='w', complevel=complevel, complib=complib)

    for dirpath in dirs:
        dirname = os.path.basename(dirpath)

        print(dirname)
        df = cache_to_df(dirpath)

        if dirname == 'travel_data':
            keys = ['from_zone_id', 'to_zone_id']
        elif dirname == 'annual_employment_control_totals':
            keys = ['sector_id', 'year', 'home_based_status']
        elif dirname == 'annual_job_relocation_rates':
            keys = ['sector_id']
        elif dirname == 'annual_household_control_totals':
            keys = ['year']
        elif dirname == 'annual_household_relocation_rates':
            keys = ['age_of_head_max', 'age_of_head_min',
                    'income_min', 'income_max']
        elif dirname == 'building_sqft_per_job':
            keys = ['zone_id', 'building_type_id']
        elif dirname == 'counties':
            keys = ['county_id']
        elif dirname == 'development_event_history':
            keys = ['building_id']
        elif dirname == 'target_vacancies':
            keys = ['building_type_id', 'year']
        else:
            keys = [dirname[:-1] + '_id']

        if dirname != 'annual_household_relocation_rates':
            df = df.set_index(keys)

        for colname in df.columns:
            if df[colname].dtype == np.float64:
                df[colname] = df[colname].astype(np.float32)
            elif df[colname].dtype == np.int64:
                df[colname] = df[colname].astype(np.int32)
            else:
                df[colname] = df[colname]

        df.info()
        print(os.linesep)
        store.put(dirname, df)

    store.close()
1017950
def glob_in_parents(dir, patterns, upper_limit=None):
    """"""

    assert(isinstance(dir, str))
    assert(isinstance(patterns, list))

    result = []

    absolute_dir = os.path.join(os.getcwd(), dir)
    absolute_dir = os.path.normpath(absolute_dir)
    while absolute_dir:
        new_dir = os.path.split(absolute_dir)[0]
        if new_dir == absolute_dir:
            break
        result = glob([new_dir], patterns)
        if result:
            break
        absolute_dir = new_dir

    return result
1166877
def subdir_path(directory, relative):
    """"""
    item_bits = directory.split(os.sep)
    relative_bits = relative.split(os.sep)
    for i, _item in enumerate(item_bits):
        if i == len(relative_bits) - 1:
            return os.sep.join(item_bits[i:])
        else:
            if item_bits[i] != relative_bits[i]:
                return None

    return None
178628
def _normalize_dir(string_):
    '''
    
    '''
    return os.path.normpath(salt.utils.stringutils.to_unicode(string_))
965307
def split_path(path):
    '''
    
    '''
    def recurse_path(path,retlist):
        if len(retlist) > 100:
            fullpath = os.path.join(*([ path, ] + retlist))
            print("Directory '{}' contains too many levels".format(fullpath))
            exit(1)
        head, tail = os.path.split(path)
        if len(tail) > 0:
            retlist.insert(0,tail)
            recurse_path(head,retlist)
        elif len(head) > 1:
            recurse_path(head,retlist)
        else:
            return

    retlist = []
    path = os.path.realpath(os.path.normpath(path))
    drive, path = os.path.splitdrive(path)
    if len(drive) > 0: retlist.append(drive)
    recurse_path(path,retlist)
    return retlist
1166673
def recursive_path(pack, path):
    """"""
    matches = []
    for root, _, filenames in os.walk(os.path.join(pack, path)):
        for filename in filenames:
            matches.append(os.path.join(root, filename)[len(pack) + 1:])
    return matches
656979
def itersubdirs(self, pattern=None, abspath=False):
        
        if pattern is not None:
            globster = Globster([pattern])
        for root, dirs, files in self.walk():
            for d in dirs:
                if pattern is None or (pattern is not None and globster.match(d)):
                    if abspath:
                        yield os.path.join(root, d)
                    else:
                        yield self.relpath(os.path.join(root, d))
447773
function mkdir(dirname, options = {}) {
  if (fs.existsSync(dirname)) return;
  assert.equal(typeof dirname, 'string', 'expected dirname to be a string');
  let opts = Object.assign({ cwd: process.cwd(), fs }, options);
  let mode = opts.mode || 0o777 & ~process.umask();
  let segs = path.relative(opts.cwd, dirname).split(path.sep);
  let make = dir => !exists(dir, opts) && fs.mkdirSync(dir, mode);
  for (let i = 0; i <= segs.length; i++) {
    try {
      make((dirname = path.join(opts.cwd, ...segs.slice(0, i))));
    } catch (err) {
      handleError(dirname, opts)(err);
    }
  }
  return dirname;
}
218163
def flatten_dir_tree(self, tree):
        

        result = {}

        def helper(tree, leading_path = ''):
            dirs  = tree['dirs']; files = tree['files']
            for name, file_info in files.iteritems():
                file_info['path'] = leading_path + '/'  + name
                result[file_info['path']] = file_info

            for name, contents in dirs.iteritems():
                helper(contents, leading_path +'/'+ name)
        helper(tree); return result
180436
def rmdir(path, recursive=True, **kwargs):
    """"""
    if recursive:
        import shutil
        return shutil.rmtree(path, **kwargs)
    else:
        import os
        return os.remdir(path, **kwargs)
49884
def path_dirs(urls):
    '''
    
    '''
    path_dirs = StringCounter()
    for url in urls:
        for path_dir in filter(None, urlparse(url).path.split('/')):
            path_dirs[path_dir] += urls[url]
    return path_dirs
1192839
function traverse (root, dirs, collect, next) {
  const rootPath = path.join (root, ...dirs);
  fs.lstat (rootPath, (err0) => {
    if (err0) {
      next (err0);
    } else {
      fs.readdir (rootPath, (_, files) => {
        let pending = files.length;
        for (let file of files) {
          const filePath = path.join (root, ...dirs, file);
          fs.lstat (filePath, (err1, stats) => {
            if (err1) {
              next (err1);
            } else {
              if (stats.isDirectory ()) {
                traverse (root, [...dirs, file], collect, err2 => {
                  if (err2) {
                    next (err2);
                  } else {
                    if (--pending === 0) {
                      next ();
                    }
                  }
                });
              } else {
                if (stats.isFile ()) {
                  collect (dirs, file, files);
                  if (--pending === 0) {
                    next ();
                  }
                }
              }
            }
          });
        }
      });
    }
  });
}
721433
def setdirs(outfiles):
    """"""
    for k in outfiles:
        fname=outfiles[k]
        dname= os.path.dirname(fname)
        if not os.path.isdir(dname):
            os.mkdir(dname)
569916
def walk_dir(dir_path, walk_after, recurse=None, archive_mtime=None):
    """"""
    if recurse is None:
        recurse = Settings.recurse

    result_set = set()
    if not recurse:
        return result_set

    for root, _, filenames in os.walk(dir_path):
        for filename in filenames:
            filename_full = os.path.join(root, filename)
            try:
                results = walk_file(filename_full, walk_after, recurse,
                                    archive_mtime)
                result_set = result_set.union(results)
            except Exception:
                print("Error with file: {}".format(filename_full))
                raise

    return result_set
163756
private Set<String> convertAbsToRelative(Collection<File> files) {
        Set<String> result = new HashSet<String>();
        for (File f : files) {
            String fAbsPath = PathUtils.fixPathString(f); //assumption here that f is a dir, and path will end in File.seperator
            if (fAbsPath.startsWith(rootAbsolutePath)) {
                String absPath = fAbsPath; // canonicalPath(f);
                absPath = absPath.substring(rootAbsolutePath.length());
                if ("\\".equals(File.separator)) {
                    absPath = absPath.replace('\\', '/');
                }
                //handle / case. (path will have been the same, so the substring created emptystring)
                if (absPath.length() == 0) {
                    absPath = "/";
                }
                result.add(absPath);
            } else
                throw new IllegalStateException(fAbsPath + " " + rootAbsolutePath);
        }
        return result;
    }
1207120
def objects_to_root(objects: List) -> Root:
    

    def _to_tree(objs: Iterable) -> Dict:
        """
        Build a tree structure from a flat list of objects.

        :param objs: The raw iterable of S3 `ObjectSummary`s, as returned by a
                     bucket listing.
        :return: The listing as a nested dictionary where keys are directory
                 and file names. The values of directories will in turn be a
                 dict. The values of keys representing files will be the
                 `ObjectSummary` instance.
        """
        path_tree = {}
        for obj in objs:
            is_dir = obj.key.endswith('/')
            chunks = [chunk for chunk in obj.key.split('/') if chunk]
            chunk_count = len(chunks)
            tmp = path_tree
            for i, chunk in enumerate(chunks):
                is_last_chunk = i == chunk_count - 1

                if is_last_chunk and not is_dir:
                    tmp[chunk] = obj
                else:
                    # must be a directory
                    if chunk not in tmp:
                        # it doesn't exist - create it
                        tmp[chunk] = {}
                    tmp = tmp[chunk]
        return path_tree

    def _to_entity(key: str, value: Union[Dict, Any]) -> Entity:
        """
        Turn a nested dictionary representing an S3 bucket into the correct
        `Entity` object.

        :param key: The name of the entity.
        :param value: If the entity is a directory, the nested dict
                      representing its contents. Otherwise, the `ObjectSummary`
                      instance representing the file.
        :return: The entity representing the entity name and value pair.
        """
        if isinstance(value, dict):
            return Directory(
                key,
                {key_: _to_entity(key_, value_)
                 for key_, value_ in value.items()})

        return File(pathlib.PurePath(value.key).name, value.size,
                    value.e_tag.strip('"'))

    tree = _to_tree(objects)
    return Root({pathlib.PurePath(key).name: _to_entity(key, value)
                 for key, value in tree.items()})
495715
def _get_sub_dirs(parent):
    """"""
    return [child for child in os.listdir(parent) if os.path.isdir(os.path.join(parent, child))]