Command line utility for serving datasets in a directory over HTTP.
427678
def cli():
    """"""
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "dataset_directory",
        help="Directory with datasets to be served"
    )
    parser.add_argument(
        "-p",
        "--port",
        type=int,
        default=8081,
        help="Port to serve datasets on (default 8081)"
    )
    args = parser.parse_args()
    if not os.path.isdir(args.dataset_directory):
        parser.error("Not a directory: {}".format(args.dataset_directory))

    serve_dtool_directory(args.dataset_directory, args.port)
779649
def command_line_runner():
    """"""
    parser = get_parser()
    args = vars(parser.parse_args())
    if args['version']:
        print(__version__)
        return
    if args['clear_cache']:
        utils.clear_cache()
        print('Cleared {0}.'.format(utils.CACHE_DIR))
        return
    if not args['query']:
        parser.print_help()
        return

    # Enable cache unless user sets environ variable SCRAPE_DISABLE_CACHE
    if not os.getenv('SCRAPE_DISABLE_CACHE'):
        utils.enable_cache()

    # Save images unless user sets environ variable SCRAPE_DISABLE_IMGS
    if os.getenv('SCRAPE_DISABLE_IMGS'):
        args['no_images'] = True

    # Prompt user for filetype if none specified
    prompt_filetype(args)

    # Prompt user to save images when crawling (for pdf and HTML formats)
    prompt_save_images(args)

    # Scrape webpage content
    scrape(args)
416224
def resource_to_rdf(resource, dataset=None, graph=None):
    '''
    
    '''
    graph = graph or Graph(namespace_manager=namespace_manager)
    if dataset and dataset.id:
        id = URIRef(url_for('datasets.show_redirect', dataset=dataset.id,
                            _external=True,
                            _anchor='resource-{0}'.format(resource.id)))
    else:
        id = BNode(resource.id)
    permalink = url_for('datasets.resource', id=resource.id, _external=True)
    r = graph.resource(id)
    r.set(RDF.type, DCAT.Distribution)
    r.set(DCT.identifier, Literal(resource.id))
    r.add(DCT.title, Literal(resource.title))
    r.add(DCT.description, Literal(resource.description))
    r.add(DCAT.downloadURL, URIRef(resource.url))
    r.add(DCAT.accessURL, URIRef(permalink))
    r.add(DCT.issued, Literal(resource.published))
    r.add(DCT.modified, Literal(resource.modified))
    if dataset and dataset.license:
        r.add(DCT.rights, Literal(dataset.license.title))
        if dataset.license.url:
            r.add(DCT.license, URIRef(dataset.license.url))
    if resource.filesize is not None:
        r.add(DCAT.bytesSize, Literal(resource.filesize))
    if resource.mime:
        r.add(DCAT.mediaType, Literal(resource.mime))
    if resource.format:
        r.add(DCT.term('format'), Literal(resource.format))
    if resource.checksum:
        checksum = graph.resource(BNode())
        checksum.set(RDF.type, SPDX.Checksum)
        algorithm = 'checksumAlgorithm_{0}'.format(resource.checksum.type)
        checksum.add(SPDX.algorithm, getattr(SPDX, algorithm))
        checksum.add(SPDX.checksumValue, Literal(resource.checksum.value))
        r.add(SPDX.checksum, checksum)
    return r
105738
def main():
    """"""
    # Dynamically determine all the providers available and gather command line arguments.
    parsed_args = generate_cli_main_parser().parse_args()

    log_level = logging.getLevelName(parsed_args.log_level)
    logging.basicConfig(stream=sys.stdout, level=log_level,
                        format='%(message)s')
    logger.debug('Arguments: %s', parsed_args)

    # In the CLI context, will get configuration interactively:
    #   * from the command line
    #   * from the environment variables
    #   * from lexicon configuration files found in given --config-dir (default is current dir)
    config = ConfigResolver()
    config.with_args(parsed_args).with_env().with_config_dir(parsed_args.config_dir)

    client = Client(config)

    results = client.execute()

    handle_output(results, parsed_args.output, config.resolve('lexicon:action'))
14448
def prepare_data_maybe_download(directory):
    
    filename = 'ubuntu_dialogs.tgz'
    url = 'http://cs.mcgill.ca/~jpineau/datasets/ubuntu-corpus-1.0/ubuntu_dialogs.tgz'
    dialogs_path = os.path.join(directory, 'dialogs')

    # test it there are some dialogs in the path
    if not os.path.exists(os.path.join(directory, "10", "1.tst")):
        # dialogs are missing
        archive_path = os.path.join(directory, filename)
        if not os.path.exists(archive_path):
            # archive missing, download it
            print("Downloading %s to %s" % (url, archive_path))
            filepath, _ = urllib.request.urlretrieve(url, archive_path)
            print "Successfully downloaded " + filepath

        # unpack data
        if not os.path.exists(dialogs_path):
            print("Unpacking dialogs ...")
            with tarfile.open(archive_path) as tar:
                tar.extractall(path=directory)
            print("Archive unpacked.")

        return
1008278
def cli():
    """"""
    parser = argparse.ArgumentParser()
    parser.add_argument('api', help="""
        API to call.
        One one of 'article', 'frontpage', 'product', 'image', 'analyze', or
        'discussion'.
    """)
    parser.add_argument('url', help="""
        URL to pass as the 'url' parameter.
    """)
    parser.add_argument('token', help="""
        API key (token).
        Get one at https://www.diffbot.com/.
    """)
    parser.add_argument('-a', '--all', help="""
        Request all fields.
    """, action='store_true')
    parser.add_argument('-f', '--file', help="""
        File to read data from.
        Use '-' to read from STDIN.
    """)
    fields = text = html = None
    _args = parser.parse_args()
    if _args.all:
        fields = '*'
    if _args.file == '-':
        text = sys.stdin.read()
    elif _args.file:
        with open(_args.file, 'rb') as src:
            if os.path.splitext(_args.file)[1] in ('.html', '.htm'):
                html = src.read().decode(ENCODING)
            else:
                text = src.read().decode(ENCODING)
    print(json.dumps((api(_args.api, _args.url, _args.token,
                          html=html or None,
                          text=text or None,
                          fields=fields)),
                     sort_keys=True,
                     indent=2))
1281453
def exporter(directory, method, datasets):
    """"""
    if method.lower() == 'json':
        # Convert json_dict to a JSON styled string
        json_string = json.dumps(datasets, indent=4)
        savefile = open('{}/exported.json'.format(directory), 'w+')
        savefile.write(json_string)
        savefile.close()

    if method.lower() == 'csv':
        with open('{}/exported.csv'.format(directory), 'w+') as csvfile:
            csv_writer = csv.writer(
                csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
            for key, values in datasets.items():
                if values is None:
                    csv_writer.writerow([key])
                else:
                    csv_writer.writerow([key] + values)
        csvfile.close()
1270723
def _create_handler_directory(self, handler_name, api_version, server_type):
        """"""

        directory = "{}/api/v{}/{}/{}".format(self._directory, api_version, server_type, handler_name)
        os.makedirs(directory, exist_ok=True)

        with open("{}/api/v{}/{}/{}.rst".format(self._directory, api_version, server_type, handler_name), "w+") as f:
            f.write(handler_name.replace("api.", "").replace("_", " ", ).capitalize())
            f.write("\n-----------------------------\n\n")
            f.write(".. toctree::\n   :glob:\n   :maxdepth: 2\n\n   {}/*\n".format(handler_name))
103838
def list_datasets():
    """"""
    lines = []
    for name, resource in itertools.chain(LOCAL_DATASETS.items(), REMOTE_DATASETS.items()):

        if isinstance(resource, LocalFileMetadata):
            location = "local: {}".format(resource.filename)
        elif isinstance(resource, RemoteFileMetadata):
            location = "remote: {}".format(resource.url)
        else:
            location = "unknown"
        lines.append("{}\n{}\n{}\n{}".format(name, "=" * len(name), resource.description, location))

    return "\n\n{}\n\n".format(10 * "-").join(lines)
1007812
def main():
    
    parser = argparse.ArgumentParser()
    parser.add_argument('--batch_size', type=int, default=64)
    parser.add_argument('--image_path', type=str, default='./data/datasets/')
    parser.add_argument('--align_path', type=str, default='./data/align/')
    parser.add_argument('--num_gpus', type=int, default=1)
    parser.add_argument('--num_workers', type=int, default=0)
    parser.add_argument('--data_type', type=str, default='valid')
    parser.add_argument('--model_path', type=str, default=None)
    config = parser.parse_args()
    trainer = Train(config)
    trainer.build_model(path=config.model_path)
    trainer.load_dataloader()

    if config.data_type == 'train':
        data_loader = trainer.train_dataloader
    elif config.data_type == 'valid':
        data_loader = trainer.valid_dataloader

    trainer.infer_batch(data_loader)
1335724
def glob_files_remotely(cmd, client, share_name, pattern):
    """"""
    from collections import deque
    t_dir, t_file = cmd.get_models('file.models#Directory', 'file.models#File')

    queue = deque([""])
    while queue:
        current_dir = queue.pop()
        for f in client.list_directories_and_files(share_name, current_dir):
            if isinstance(f, t_file):
                if not pattern or _match_path(os.path.join(current_dir, f.name), pattern):
                    yield current_dir, f.name
            elif isinstance(f, t_dir):
                queue.appendleft(os.path.join(current_dir, f.name))
176764
def _get_vqa_v2_image_raw_dataset(directory, image_root_url, image_urls):
  """"""
  for url in image_urls:
    filename = os.path.basename(url)
    download_url = os.path.join(image_root_url, url)
    path = generator_utils.maybe_download(directory, filename, download_url)
    unzip_dir = os.path.join(directory, filename.strip(".zip"))
    if not tf.gfile.Exists(unzip_dir):
      zipfile.ZipFile(path, "r").extractall(directory)
1131731
def manage(settingspath, root_dir, argv):
    
    # add settings.json to environment variables
    os.environ[ENV_VAR_SETTINGS] = settingspath
    # add root_dir
    os.environ[ENV_VAR_ROOT_DIR] = root_dir
    # get datasets list
    with open(settingspath) as settings_file:
        settings = json.load(settings_file)
    # manage args
    datasets_list = generate_datasets_list(settings, argv)
    if "make-data-file" == argv[1]:
        make_data_file(datasets_list, argv)
    elif "parse-data" == argv[1]:
        parse_data(datasets_list, argv)
    elif "do-operations" == argv[1]:
        do_operations(datasets_list, argv)
    else:
        print_help()
61851
def list_dataset_uris(cls, base_uri, config_path):
        """"""
        parsed_uri = generous_parse_uri(base_uri)
        irods_path = parsed_uri.path

        uri_list = []

        logger.info("irods_path: '{}'".format(irods_path))

        for dir_path in _ls_abspaths(irods_path):

            logger.info("dir path: '{}'".format(dir_path))

            base, uuid = os.path.split(dir_path)
            base_uri = "irods:{}".format(base)
            uri = cls.generate_uri(
                name=None,
                uuid=uuid,
                base_uri=base_uri
            )

            storage_broker = cls(uri, config_path)

            if storage_broker.has_admin_metadata():
                uri_list.append(uri)

        return uri_list
721028
def main(reactor):
    """"""
    parser = argparse.ArgumentParser(
        description='Fetch a URI or series of URIs and print a title '
                    'or summary for each.',
        epilog='If no URIs are passed on the command line, they are '
               'read from standard input, one per line.')
    parser.add_argument(
        'uris', metavar='URI', nargs='*', help='URI to fetch')
    parser.add_argument(
        '-H', '--hostname-tag', action='store_true',
        help='prefix titles with a hostname tag')
    args = parser.parse_args()
    uris = args.uris or imap(lambda x: x.strip(), sys.stdin)
    finished = DeferredList([
        fetch_title(uri.decode(locale.getpreferredencoding()),
                    hostname_tag=args.hostname_tag, friendly_errors=True)
        for uri in uris])
    finished.addCallback(print_and_exit)
    return finished
178823
def dataset(directory, images_file, labels_file):
  """"""

  images_file = download(directory, images_file)
  labels_file = download(directory, labels_file)

  check_image_file_header(images_file)
  check_labels_file_header(labels_file)

  def decode_image(image):
    # Normalize from [0, 255] to [0.0, 1.0]
    image = tf.decode_raw(image, tf.uint8)
    image = tf.cast(image, tf.float32)
    image = tf.reshape(image, [784])
    return image / 255.0

  def decode_label(label):
    label = tf.decode_raw(label, tf.uint8)  # tf.string -> [tf.uint8]
    label = tf.reshape(label, [])  # label is a scalar
    return tf.to_int32(label)

  images = tf.data.FixedLengthRecordDataset(
      images_file, 28 * 28, header_bytes=16).map(decode_image)
  labels = tf.data.FixedLengthRecordDataset(
      labels_file, 1, header_bytes=8).map(decode_label)
  return tf.data.Dataset.zip((images, labels))
184156
def Main():
  
  argument_parser = argparse.ArgumentParser(description=(
      'Calculates a message digest hash for every file in a directory or '
      'storage media image.'))

  argument_parser.add_argument(
      'source', nargs='?', action='store', metavar='image.raw', default=None,
      help=('path of the directory or filename of a storage media image '
            'containing the file.'))

  argument_parser.add_argument(
      '--no-auto-recurse', '--no_auto_recurse', dest='no_auto_recurse',
      action='store_true', default=False, help=(
          'Indicate that the source scanner should not auto-recurse.'))

  options = argument_parser.parse_args()

  if not options.source:
    print('Source value is missing.')
    print('')
    argument_parser.print_help()
    print('')
    return False

  logging.basicConfig(
      level=logging.INFO, format='[%(levelname)s] %(message)s')

  output_writer = StdoutWriter()

  if not output_writer.Open():
    print('Unable to open output writer.')
    print('')
    return False

  return_value = True
  source_analyzer = SourceAnalyzer(auto_recurse=not options.no_auto_recurse)

  try:
    source_analyzer.Analyze(options.source, output_writer)

    print('Completed.')

  except KeyboardInterrupt:
    return_value = False

    print('Aborted by user.')

  output_writer.Close()

  return return_value