Uploads the compressed SQL file to S3
197379
def upload_output_to_s3
      update_status "Uploading output file #{::File.basename @sql_file}"
      dump_path = "#{@s3_path}/#{::File.basename @sql_file}"
      s3.put_object(@bucket, dump_path, File.read(@sql_file))
      upload = s3.directories.get(@bucket).files.get dump_path
      @files = [ {
        name: ::File.basename(@sql_file),
        size: upload.content_length,
        url:  upload.url(Time.now + (3600 * 24 * 30))  # 30 days from now
      } ]
      @log.info "Deleting tmp directory #{File.dirname @sql_file}"
      FileUtils.rm_rf(File.dirname @sql_file)
    end
1162856
def save_sql_to_files(overwrite=False):
    
    ext_size = len(SQL_EXTENSION)
    path = DATA_PATH / 'scripts'
    save_dir = DATA_PATH / "raw"

    for file in os.listdir(path):
        if file.endswith(SQL_EXTENSION):
            file_path = os.path.join(save_dir,
                                     file[:-ext_size] + '.' + FILE_EXTENSION)
            if not os.path.isfile(file_path) or overwrite:
                query_result = make_query(path / file)
                save_dataframe_as_pickle(query_result, file_path)
            else:
                print(("file {} already exists, if you would like to update"
                       " it, use -f flag\n").format(file_path))
249120
def upload(self, fileobj, bucket, key, transfer_config=None, subscribers=None):
        '''  '''
        check_io_access(fileobj, os.R_OK, True)
        return self._queue_task(bucket, [FilePair(key, fileobj)], transfer_config,
                                subscribers, enumAsperaDirection.SEND)
805976
def upload_file_boto(fname, remote_fname, mditems=None):
    
    r_fname = objectstore.parse_remote(remote_fname)
    conn = objectstore.connect(remote_fname)
    bucket = conn.lookup(r_fname.bucket)
    if not bucket:
        bucket = conn.create_bucket(r_fname.bucket, location=objectstore.get_region(remote_fname))
    key = bucket.get_key(r_fname.key, validate=False)
    if mditems is None:
        mditems = {}
    if "x-amz-server-side-encryption" not in mditems:
        mditems["x-amz-server-side-encryption"] = "AES256"
    for name, val in mditems.items():
        key.set_metadata(name, val)
    key.set_contents_from_filename(fname, encrypt_key=True)
129802
def uncompress_file(s3_object)
      result = ''

      if s3_object.exists?
        io = StringIO.new
        io.write(s3_object.read)
        io.rewind

        gz = Zlib::GzipReader.new(io)
        result = gz.read

        gz.close
      end

      result
    end
789959
def _decompress(self, cmd, s3_snap):
        
        compressor = COMPRESSORS.get(s3_snap.compressor)
        if compressor is None:
            return cmd
        decompress_cmd = compressor['decompress']
        return "{} | {}".format(decompress_cmd, cmd)
1332838
def backup(id):
    """"""
    filename = dump_database(id)
    key = "{}.dump".format(id)

    bucket = user_s3_bucket()
    bucket.upload_file(filename, key)
    return _generate_s3_url(bucket, key)
559785
async function uploadFiles( store, session, caslib, datadir ) {
    let files = readdir(datadir);
    let executeCmd = session.links('execute');

    for (let i = 0 ; i < files.length ;i++) {
    
        let parms = { caslib: caslib, name: files[i],  quiet : true};
        await store.runAction(session, {action: 'table.dropTable', data: parms})

        parms = { caslib: caslib, source: files[i],quiet : true};
        await store.runAction(session, {action: 'table.deleteSource', data: parms})

        parms = {
            rstore: { name: files[i], caslib: caslib, replace: true},
            store : readFile(files[i], datadir)
        };
        payload = { action: 'aStore.upload', data: parms };
        console.log( `uploading  ${files[i]}`);
        await store.runAction(session, payload);
        
        parms = { rstore: { name: files[i], caslib: caslib } };
        payload = { action: 'aStore.describe', data: parms };
        await store.runAction(session, payload);
       
        parms = {
            caslib : caslib,
            name   : files[i],
            replace: true,
            table: {
                caslib: caslib,
                name  : files[i]
            }
        };
        payload ={action: 'table.save', data: parms};
        await store.runAction(session, payload);

    }
}
35773
function () {
    // There is a possibility that the incoming stream was empty, therefore the MPU never started
    // and cannot be finalized.
    if (multipartUploadID) {
      cachedClient.completeMultipartUpload(
        {
          Bucket: destinationDetails.Bucket,
          Key: destinationDetails.Key,
          UploadId: multipartUploadID,
          MultipartUpload: {
            Parts: partIds
          }
        },
        function (err, result) {
          if (err)
            abortUpload('Failed to complete the multipart upload on S3: ' + JSON.stringify(err));
          else {
            // Emit both events for backwards compatibility, and to follow the spec.
            ws.emit('uploaded', result);
            ws.emit('finish', result);
            started = false;
          }
        }
      );
    }
  }
758428
def upload_csv(files)
      files.each do |file|
        basename = File.basename(file)
        next if basename == File.basename(exporter.replication.key)
        RailsRedshiftReplicator.logger.info I18n.t(:uploading_notice,
                                                   file: file,
                                                   key: self.class.s3_file_key(exporter.source_table, basename),
                                                   scope: :rails_redshift_replicator)
        s3_client.put_object(
          key: self.class.s3_file_key(exporter.source_table, basename),
          body: File.open(file),
          bucket: bucket
        )
      end
      files.each { |f| FileUtils.rm f }
    end
744687
def do_lzop_put(creds, url, local_path, gpg_key):
    
    assert url.endswith('.lzo')
    blobstore = get_blobstore(storage.StorageLayout(url))

    with tempfile.NamedTemporaryFile(
            mode='r+b', buffering=pipebuf.PIPE_BUF_BYTES) as tf:
        with pipeline.get_upload_pipeline(
                open(local_path, 'rb'), tf, gpg_key=gpg_key):
            pass

        tf.flush()

        clock_start = time.time()
        tf.seek(0)
        k = blobstore.uri_put_file(creds, url, tf)
        clock_finish = time.time()

        kib_per_second = format_kib_per_second(
            clock_start, clock_finish, k.size)

        return kib_per_second
1071705
private static function uploadFile($bucket, $srcPath, $dstPath, $bizAttr = null, $insertOnly = null) {
        $srcPath = realpath($srcPath);
	    $dstPath = self::cosUrlEncode($dstPath);

	    if (filesize($srcPath) >= self::MAX_UNSLICE_FILE_SIZE ) {
		    return array(
                'code' => COSAPI_PARAMS_ERROR,
                'message' => 'file '.$srcPath.' larger then 20M, please use uploadBySlicing interface',
                'data' => array()
            );
	    }

        $expired = time() + self::EXPIRED_SECONDS;
        $url = self::generateResUrl($bucket, $dstPath);
        $signature = Auth::createReusableSignature($expired, $bucket);
        $fileSha = hash_file('sha1', $srcPath);

        $data = array(
            'op' => 'upload',
            'sha' => $fileSha,
            'biz_attr' => (isset($bizAttr) ? $bizAttr : ''),
        );

        $data['filecontent'] = file_get_contents($srcPath);

        if (isset($insertOnly) && strlen($insertOnly) > 0) {
            $data['insertOnly'] = (($insertOnly == 0 || $insertOnly == '0' ) ? 0 : 1);
        }

        $req = array(
            'url' => $url,
            'method' => 'post',
            'timeout' => self::$timeout,
            'data' => $data,
            'header' => array(
                'Authorization: ' . $signature,
            ),
        );

        return self::sendRequest($req);
    }
763964
function deployS3(config) {
  if (!config.credentials.accessKeyId || !config.credentials.secretAccessKey) {
    gutil.log(
      gutil.colors.red('AWS S3 credentials missing, skipping upload...'),
    );
    return;
  }

  // create a new publisher
  const publisher = aws.create(config.credentials);

  // define custom headers
  const headers = {
    'Cache-Control': 'max-age=315360000, no-transform, public',
  };
  return (
    gulp
      .src(config.assetsPath)
      .pipe(
        rename(function(filepath) {
          filepath.dirname = path.join(config.dirname, filepath.dirname);
        }),
      )
      // gzip, Set Content-Encoding headers and add .gz extension
      .pipe(gulpIf(file => !file.path.match(/\.(mp4|ogg|webm|avi|mov|mkv)$/), aws.gzip()))
      // publisher will add Content-Length, Content-Type and headers specified above
      // If not specified it will set x-amz-acl to public-read by default
      .pipe(publisher.publish(headers))
      // create a cache file to speed up consecutive uploads
      .pipe(publisher.cache())
      // print upload updates to console
      .pipe(aws.reporter())
  );
}
596521
def upload_bam_to_s3(job, job_vars):
    
    input_args, ids = job_vars
    work_dir = job.fileStore.getLocalTempDir()
    uuid = input_args['uuid']
    # I/O
    job.fileStore.readGlobalFile(ids['alignments.bam'], os.path.join(work_dir, 'alignments.bam'))
    bam_path = os.path.join(work_dir, 'alignments.bam')
    sample_name = uuid + '.bam'
    # Parse s3_dir to get bucket and s3 path
    s3_dir = input_args['s3_dir']
    bucket_name = s3_dir.split('/')[0]
    bucket_dir = os.path.join('/'.join(s3_dir.split('/')[1:]), 'bam_files')
    # Upload to S3 via S3AM
    s3am_command = ['s3am',
                    'upload',
                    'file://{}'.format(bam_path),
                    os.path.join('s3://', bucket_name, bucket_dir, sample_name)]
    subprocess.check_call(s3am_command)
92953
function uploadNewBackup(s3, backupTmpFilePath, bucket, key) {
  return new Promise((resolve, reject) => {
    debug(`Uploading from ${backupTmpFilePath} to ${bucket}/${key}`);

    function uploadFromStream(s3, bucket, key) {
      const pass = new stream.PassThrough();

      const params = {
        Bucket: bucket,
        Key: key,
        Body: pass
      };
      s3.upload(params, function(err, data) {
        debug('S3 upload done');
        if (err) {
          debug(err);
          reject(new VError(err, 'Upload failed'));
          return;
        }
        debug('Upload succeeded');
        debug(data);
        resolve();
      }).httpUploadProgress = (progress) => {
        debug(`S3 upload progress: ${progress}`);
      };

      return pass;
    }

    const inputStream = fs.createReadStream(backupTmpFilePath);
    const s3Stream = uploadFromStream(s3, bucket, key);
    inputStream.pipe(s3Stream);
  });
}
655286
function (req, res) {
                var s3 = new AWS.S3(),
                    bucket = req.params.name,
                    key = getFullKeyParam(req.params, req.route),
                    contentType = mime.lookup(key),
                    params = {
                        Bucket: bucket,
                        Key: key,
                        // Pipe the file through the request body
                        Body: req
                    },
                    trackUploadProgress = function (evt) {
                        // This function is called each time a chunk of the file is uploaded
                        //console.log(evt);
                    },
                    handleUploadComplete = function (err, data) {
                        if (err) {
                            res.status(500).send(err);
                        } else {
                            // Add s3g link to reference the resource                            
                            data.links = [{
                                rel: 'self',
                                href: getOriginalink(req)
                            }];
                            
                            res.json(data);
                        }
                    };

				// Set content type for object if we have one
                if (contentType) {
                    params.ContentType = contentType;
                }

                s3.upload(params).on('httpUploadProgress', trackUploadProgress).send(handleUploadComplete);
            }
505036
function Chunk(options)
{
  // make sure the options object contains all the necessary information
  Errors.assertInternal(Util.isObject(options));
  Errors.assertInternal(Util.isObject(options.statement));
  Errors.assertInternal(Util.isObject(options.services));
  Errors.assertInternal(Util.isNumber(options.startIndex));
  Errors.assertInternal(Util.isArray(options.columns));
  Errors.assertInternal(Util.isObject(options.mapColumnNameToIndices));
  Errors.assertInternal(Util.isObject(options.statementParameters));
  Errors.assertInternal(Util.isString(options.resultVersion));
  Errors.assertInternal(Util.isNumber(options.rowCount));

  // if the result is small (i.e. not persisted on S3/Blob), there's no
  // compressed and uncompressed size, so default to -1
  this._compressedSize   = options.compressedSize   || -1;
  this._uncompressedSize = options.uncompressedSize || -1;

  // copy out other information from the options object and save it
  this._statement              = options.statement;
  this._services               = options.services;
  this._startIndex             = options.startIndex;
  this._url                    = options.url;
  this._columns                = options.columns;
  this._mapColumnNameToIndices = options.mapColumnNameToIndices;
  this._chunkHeaders           = options.chunkHeaders;
  this._rowset                 = options.rowset;

  // use the start index and row count to compute the end index
  this._endIndex = this._startIndex + options.rowCount - 1;

  // use the start and end index to build an id for this chunk
  this._id = buildId(this._startIndex, this._endIndex);
}