Upload all new files to the bucket
324487
def upload_files
      files_to_upload = local_compiled_assets.reject { |f| exists_on_remote?(f) }
      files_to_upload.each do |asset|
        upload_file(asset)
      end
    end
102380
def upload():
    """"""
    try:
        o = check_output(['twine', 'upload'] + glob('dist/*'))
    except CalledProcessError:
        call(['twine', 'upload'] + glob('dist/*'))
        raise
    print(o.decode('utf-8'))
272685
def upload_index_file(self, test_address, timestamp):
        
        global already_uploaded_files
        already_uploaded_files = list(set(already_uploaded_files))
        already_uploaded_files.sort()
        file_name = "%s/%s/index.html" % (test_address, timestamp)
        index = self.get_key(file_name)
        index_str = []
        for completed_file in already_uploaded_files:
            index_str.append("<a href='" + self.bucket_url + ""
                             "%s'>%s</a>" % (completed_file, completed_file))
        index.set_contents_from_string(
            "<br>".join(index_str),
            headers={"Content-Type": "text/html"})
        index.make_public()
        return "%s%s" % (self.bucket_url, file_name)
335054
protected final void addAllFilesToSuccessfullyUploadedFiles() {
        for (Iterator<UploadFile> iterator = params.files.iterator(); iterator.hasNext(); ) {
            UploadFile file = iterator.next();

            if (!successfullyUploadedFiles.contains(file.path)) {
                successfullyUploadedFiles.add(file.path);
            }
            iterator.remove();
        }
    }
1066872
def upload(
    ctx, files, max_threads, prompt, forward, reverse, tags, metadata, project_id, coerce_ascii
):
    """"""

    appendables = {}
    if tags:
        appendables["tags"] = []
        for tag in tags:
            appendables["tags"].append(tag)

    if metadata:
        appendables["metadata"] = {}
        for metadata_kv in metadata:
            split_metadata = metadata_kv.split("=", 1)
            if len(split_metadata) > 1:
                metadata_value = split_metadata[1]
                appendables["metadata"][split_metadata[0]] = metadata_value

    appendables = validate_appendables(appendables, ctx.obj["API"])

    if (forward or reverse) and not (forward and reverse):
        click.echo("You must specify both forward and reverse files", err=True)
        ctx.exit(1)

    if forward and reverse:
        if len(files) > 0:
            click.echo(
                "You may not pass a FILES argument when using the "
                " --forward and --reverse options.",
                err=True,
            )
            ctx.exit(1)
        files = [(forward, reverse)]
    elif len(files) == 0:
        click.echo(ctx.get_help())
        return
    else:
        files = list(files)

        # "intelligently" find paired files and tuple them
        paired_files = []
        single_files = set(files)

        for filename in files:
            # convert "read 1" filenames into "read 2" and check that they exist; if they do
            # upload the files as a pair, autointerleaving them
            pair = re.sub("[._][Rr]1[._]", lambda x: x.group().replace("1", "2"), filename)

            # we don't necessary need the R2 to have been passed in; we infer it anyways
            if pair != filename and os.path.exists(pair):
                if not prompt and pair not in single_files:
                    # if we're not prompting, don't automatically pull in files
                    # not in the list the user passed in
                    continue

                paired_files.append((filename, pair))

                if pair in single_files:
                    single_files.remove(pair)

                single_files.remove(filename)

        auto_pair = True

        if prompt and len(paired_files) > 0:
            pair_list = ""
            for p in paired_files:
                pair_list += "\n  {}  &  {}".format(os.path.basename(p[0]), os.path.basename(p[1]))

            answer = click.confirm(
                "It appears there are paired files:{}\nInterleave them after upload?".format(
                    pair_list
                ),
                default="Y",
            )

            if not answer:
                auto_pair = False

        if auto_pair:
            files = paired_files + list(single_files)

    total_size = sum(
        [
            (_file_size(x[0], uncompressed=True) + _file_size(x[1], uncompressed=True))
            if isinstance(x, tuple)
            else _file_size(x, uncompressed=False)
            for x in files
        ]
    )

    upload_kwargs = {
        "metadata": appendables["valid_metadata"],
        "tags": appendables["valid_tags"],
        "project": project_id,
        "coerce_ascii": coerce_ascii,
        "progressbar": progressbar(length=total_size, label="Uploading..."),
    }

    run_via_threadpool(
        ctx.obj["API"].Samples.upload,
        files,
        upload_kwargs,
        max_threads=max_threads,
        graceful_exit=False,
    )
1154653
def upload(s3_conn, filepath, s3_path):
    
    bucket_name, prefix = split_s3_path(s3_path)
    bucket = s3_conn.get_bucket(bucket_name)
    filename = os.path.basename(filepath)

    key = boto.s3.key.Key(
        bucket=bucket,
        name='{}/{}'.format(prefix, filename)
    )
    logging.info('uploading from %s to %s', filepath, key)
    key.set_contents_from_filename(filepath)
1319011
def upload_to_s3(file_path, config):
    
    logging.info("Uploading file to S3 bucket: %s", config['s3_bucket_name'])
    s3 = boto3.resource('s3')
    s3_filename = config['s3_bucket_path'] + config['rendered_filename']
    s3.Bucket(config['s3_bucket_name']).upload_file(
        file_path, s3_filename, ExtraArgs={
            'ContentType': 'text/html', 'ACL': 'public-read'})
1085290
def put_files(self, files, content_type=None, compress=None, cache_control=None, block=True):
    
    def base_uploadfn(path, content, interface):
      interface.put_file(path, content, content_type, compress, cache_control=cache_control)

    for path, content in files:
      content = compression.compress(content, method=compress)
      uploadfn = partial(base_uploadfn, path, content)

      if len(self._threads):
        self.put(uploadfn)
      else:
        uploadfn(self._interface)

    if block:
      desc = 'Uploading' if self.progress else None
      self.wait(desc)

    return self
571360
function split({url, bucket, key, arn='', maxFiles=0, linesPerFile=500, maxLambdas=20, reverse=false, cb=null}) {

  let fileCounter = 0
  let lineCounter = 0
  const lineBuffer = new Buffer(4096)
  const gunzip = zlib.createGunzip()
  let newStream
  let currentFile
  let lineLength = 0
  let stopSplitting = false
  let header

  switch (url.substr(url.lastIndexOf('.') + 1)) {
    case 'csv':
      newStream = got.stream(url)
      break
    case 'gz':
      newStream = got.stream(url).pipe(gunzip)
      break
  }

  const build = function buildFile(line) {
    // get the csv header
    if (fileCounter === 0 && lineCounter === 0) header = line.toString()

    // create a new file or add to existing
    if (lineCounter === 0) {
      currentFile = new stream.PassThrough()
      currentFile.push(header)
    } else {
      currentFile.push(line.toString())
    }
    lineCounter += 1 // increment the filename
    lineLength = 0 // reset the buffer

    if (lineCounter > linesPerFile) {
      fileCounter += 1
      const fileName = `${key}${fileCounter}.csv`;
      const params = {
        Body: currentFile,
        Bucket: bucket,
        Key: fileName
      }
      currentFile.end()
      s3.upload(params, (e, d) => { if (e) console.log(e) })
      lineCounter = 0 // start counting the lines again
      if ((fileCounter) % 250 === 0 && fileCounter != 0) console.log(`uploaded ${fileCounter} files`)
      // sentinel csv is ordered from old to new so always have to go all the way back
      if ((fileCounter >= maxFiles) && maxFiles != 0 && !reverse) {
        stopSplitting = true
      }
    }
  }

  newStream.on('data', (data) => {
    if (!stopSplitting) {
      const dataLen = data.length
      for (let i = 0; i < dataLen; i++) {
        lineBuffer[lineLength] = data[i] // Buffer new line data.
        lineLength++
        if (data[i] === 10) { // Newline char was found.
          build(lineBuffer.slice(0, lineLength))
        }
      }
    }
  })

  newStream.on('end', () => {
    // write the last records
    if (lineCounter > 0) {
        fileCounter += 1
        const params = {
          Body: currentFile,
          Bucket: bucket,
          Key: `${key}${fileCounter}.csv`
        }
        s3.upload(params, (e, d) => { if (e) console.log(e) })
        currentFile.end()
    }
    console.log(`${fileCounter-1} total files`)
    // determine batches and run lambdas
    if (arn != '') {
      maxFiles = (maxFiles === 0) ? fileCounter : Math.min(maxFiles, fileCounter)
      var numLambdas = Math.min(maxFiles, maxLambdas)
      var batchSize = Math.floor(maxFiles / numLambdas)
      var extra = maxFiles % numLambdas
      var maxEndFile = reverse ? fileCounter : maxFiles
      
      var startFile = reverse ? fileCounter - maxFiles + 1: 1
      var endFile
      console.log(`Invoking ${numLambdas} batches of Lambdas of ${batchSize} files each with ${extra} extra (Files ${startFile}-${maxEndFile})`)
      for (var i = 0; i < numLambdas; i++) {
        endFile = (i < extra) ? startFile + batchSize: startFile + batchSize - 1
        invokeLambda(bucket, key, startFile, Math.min(endFile, maxEndFile), arn)
        startFile = endFile + 1
      }
    }
    cb()
  })
  newStream.on('error', e => cb(e))
}
465093
def copy_file(aws_access_key_id, aws_secret_access_key, bucket_name, file, s3_folder):
    

    #  Connect to the bucket

    bucket = s3_bucket(aws_access_key_id, aws_secret_access_key, bucket_name)
    key = boto.s3.key.Key(bucket)

    if s3_folder:
        target_name = '%s/%s' % (s3_folder, os.path.basename(file))
    else:
        target_name = os.path.basename(file)

    key.key = target_name
    print('Uploading %s to %s' % (file, target_name))
    key.set_contents_from_filename(file)
    print('Upload %s FINISHED: %s' % (file, dt.now()))
655286
function (req, res) {
                var s3 = new AWS.S3(),
                    bucket = req.params.name,
                    key = getFullKeyParam(req.params, req.route),
                    contentType = mime.lookup(key),
                    params = {
                        Bucket: bucket,
                        Key: key,
                        // Pipe the file through the request body
                        Body: req
                    },
                    trackUploadProgress = function (evt) {
                        // This function is called each time a chunk of the file is uploaded
                        //console.log(evt);
                    },
                    handleUploadComplete = function (err, data) {
                        if (err) {
                            res.status(500).send(err);
                        } else {
                            // Add s3g link to reference the resource                            
                            data.links = [{
                                rel: 'self',
                                href: getOriginalink(req)
                            }];
                            
                            res.json(data);
                        }
                    };

				// Set content type for object if we have one
                if (contentType) {
                    params.ContentType = contentType;
                }

                s3.upload(params).on('httpUploadProgress', trackUploadProgress).send(handleUploadComplete);
            }
1153524
def get_files(client, bucket, prefix=''):
    """"""
    bucket = client.get_bucket(bucket)
    files = list(bucket.list_blobs(prefix=prefix))    
    return files
197316
def upload_file_handle(
            self,
            bucket: str,
            key: str,
            src_file_handle: typing.BinaryIO,
            content_type: str=None,
            metadata: dict=None):
        
        raise NotImplementedError()
516379
def upload_scripts(client, script_dir, overwrite=True):
    """"""
    local_dir = os.path.join(genometools._root, 'data', 'gcloud', 'scripts')
    match = _BUCKET_PAT.match(script_dir)
    script_bucket = match.group(1)
    script_prefix = match.group(2)

    depth = len(local_dir.split(os.sep))

    for root, dirs, files in os.walk(local_dir):
        rel_path = '/'.join(root.split(os.sep)[depth:])
        for f in files:
            local_path = os.path.join(root, f)

            if rel_path:
                remote_path = '/'.join([script_prefix, rel_path, f])
            else:
                remote_path = '/'.join([script_prefix, f])
            _LOGGER.info('Uploading "%s"...', remote_path)
            storage.upload_file(client, script_bucket, local_path, remote_path,
                                overwrite=overwrite)
1338709
def upload_files(self, container, src_dst_map, content_type=None):
        """"""
        if not content_type:
            content_type = "application/octet.stream"
        url = self.make_url(container, None, None)
        headers = self._base_headers
        multi_files = []
        try:
            for src_path in src_dst_map:
                dst_name = src_dst_map[src_path]
                if not dst_name:
                    dst_name = os.path.basename(src_path)
                multi_files.append(
                    ('files', (dst_name, open(src_path, 'rb'), content_type)))

            rsp = requests.post(url, headers=headers, files=multi_files,
                                timeout=self._timeout)
        except requests.exceptions.ConnectionError as e:
            RestHttp._raise_conn_error(e)
        finally:
            for n, info in multi_files:
                dst, f, ctype = info
                f.close()

        return self._handle_response(rsp)
457696
def upload():
    

    def twine(*args):
        """ run a twine command """
        process = run(sys.executable, '-m', 'twine', *args)

        return process.wait() != 0

    if run(sys.executable, 'setup.py', 'sdist', 'bdist_wheel').wait() != 0:
        error('failed building packages')

    if twine('register', glob.glob('dist/*')[0]):
        error('register failed')

    if twine('upload', '-s', '-i', 'CB164668', '--skip-existing', 'dist/*'):
        error('upload failed')
104706
def upload_file(token, channel_name, file_name):
    

    slack = Slacker(token)

    slack.files.upload(file_name, channels=channel_name)