[VARIABLE "gs://my_bucket/filename.csv"]
496414
public Job loadSingle(String sourceUri) {
    // [START bigquery_load_table_gcs_csv]
    Job job = table.load(FormatOptions.csv(), sourceUri);
    // Wait for the job to complete
    try {
      Job completedJob =
          job.waitFor(
              RetryOption.initialRetryDelay(Duration.ofSeconds(1)),
              RetryOption.totalTimeout(Duration.ofMinutes(3)));
      if (completedJob != null && completedJob.getStatus().getError() == null) {
        // Job completed successfully
      } else {
        // Handle error case
      }
    } catch (InterruptedException e) {
      // Handle interrupted wait
    }
    // [END bigquery_load_table_gcs_csv]
    return job;
  }
235615
def get_full_filename(self, dataType, subjectArea):
        
        return dataPath + os.sep + 'core' + os.sep + dataType + '_' + subjectArea + '.CSV'
595375
public static String createReadableString(String bucketName, String objectName) {
    if (bucketName == null && objectName != null) {
      throw new IllegalArgumentException(
          String.format("Invalid bucketName/objectName pair: gs://%s/%s", bucketName, objectName));
    }
    // TODO(user): Unify this method with other methods that convert bucketName/objectName
    // to a URI; maybe use the single slash for compatibility.
    StringBuilder result = new StringBuilder("gs://");
    if (bucketName != null) {
      result.append(bucketName);
    }
    if (objectName != null) {
      result.append('/').append(objectName);
    }
    return result.toString();
  }
571360
function split({url, bucket, key, arn='', maxFiles=0, linesPerFile=500, maxLambdas=20, reverse=false, cb=null}) {

  let fileCounter = 0
  let lineCounter = 0
  const lineBuffer = new Buffer(4096)
  const gunzip = zlib.createGunzip()
  let newStream
  let currentFile
  let lineLength = 0
  let stopSplitting = false
  let header

  switch (url.substr(url.lastIndexOf('.') + 1)) {
    case 'csv':
      newStream = got.stream(url)
      break
    case 'gz':
      newStream = got.stream(url).pipe(gunzip)
      break
  }

  const build = function buildFile(line) {
    // get the csv header
    if (fileCounter === 0 && lineCounter === 0) header = line.toString()

    // create a new file or add to existing
    if (lineCounter === 0) {
      currentFile = new stream.PassThrough()
      currentFile.push(header)
    } else {
      currentFile.push(line.toString())
    }
    lineCounter += 1 // increment the filename
    lineLength = 0 // reset the buffer

    if (lineCounter > linesPerFile) {
      fileCounter += 1
      const fileName = `${key}${fileCounter}.csv`;
      const params = {
        Body: currentFile,
        Bucket: bucket,
        Key: fileName
      }
      currentFile.end()
      s3.upload(params, (e, d) => { if (e) console.log(e) })
      lineCounter = 0 // start counting the lines again
      if ((fileCounter) % 250 === 0 && fileCounter != 0) console.log(`uploaded ${fileCounter} files`)
      // sentinel csv is ordered from old to new so always have to go all the way back
      if ((fileCounter >= maxFiles) && maxFiles != 0 && !reverse) {
        stopSplitting = true
      }
    }
  }

  newStream.on('data', (data) => {
    if (!stopSplitting) {
      const dataLen = data.length
      for (let i = 0; i < dataLen; i++) {
        lineBuffer[lineLength] = data[i] // Buffer new line data.
        lineLength++
        if (data[i] === 10) { // Newline char was found.
          build(lineBuffer.slice(0, lineLength))
        }
      }
    }
  })

  newStream.on('end', () => {
    // write the last records
    if (lineCounter > 0) {
        fileCounter += 1
        const params = {
          Body: currentFile,
          Bucket: bucket,
          Key: `${key}${fileCounter}.csv`
        }
        s3.upload(params, (e, d) => { if (e) console.log(e) })
        currentFile.end()
    }
    console.log(`${fileCounter-1} total files`)
    // determine batches and run lambdas
    if (arn != '') {
      maxFiles = (maxFiles === 0) ? fileCounter : Math.min(maxFiles, fileCounter)
      var numLambdas = Math.min(maxFiles, maxLambdas)
      var batchSize = Math.floor(maxFiles / numLambdas)
      var extra = maxFiles % numLambdas
      var maxEndFile = reverse ? fileCounter : maxFiles
      
      var startFile = reverse ? fileCounter - maxFiles + 1: 1
      var endFile
      console.log(`Invoking ${numLambdas} batches of Lambdas of ${batchSize} files each with ${extra} extra (Files ${startFile}-${maxEndFile})`)
      for (var i = 0; i < numLambdas; i++) {
        endFile = (i < extra) ? startFile + batchSize: startFile + batchSize - 1
        invokeLambda(bucket, key, startFile, Math.min(endFile, maxEndFile), arn)
        startFile = endFile + 1
      }
    }
    cb()
  })
  newStream.on('error', e => cb(e))
}
86498
@Override
  public URI getGcsPath(Path hadoopPath) {
    logger.atFine().log("GHFS.getGcsPath: %s", hadoopPath);

    // Convert to fully qualified absolute path; the Path object will callback to get our current
    // workingDirectory as part of fully resolving the path.
    Path resolvedPath = makeQualified(hadoopPath);

    String objectName = resolvedPath.toUri().getPath();
    if (objectName != null && resolvedPath.isAbsolute()) {
      // Strip off leading '/' because GoogleCloudStorageFileSystem.getPath appends it explicitly
      // between bucket and objectName.
      objectName = objectName.substring(1);
    }

    // Construct GCS path uri.
    URI gcsPath = pathCodec.getPath(rootBucket, objectName, true);
    logger.atFine().log("GHFS.getGcsPath: %s -> %s", hadoopPath, gcsPath);
    return gcsPath;
  }
256573
def get_csv(filename):
    
    check_if_this_file_exist(filename)

    #Process this function
    filename = os.path.abspath(filename)
    s = command_line(['exiftool', '-G', '-csv', '-sort', filename])
    if s:
        #convert bytes to string
        s = s.decode('utf-8')
        return s
    else:
        return 0
1009583
def _gcs_list(args, _):
  
  target = args['objects']
  project = args['project']
  if target is None:
    return _gcs_list_buckets(project, '*')  # List all buckets.

  bucket_name, key = google.datalab.storage._bucket.parse_name(target)
  if bucket_name is None:
    raise Exception('Cannot list %s; not a valid bucket name' % target)

  # If a target was specified, list keys inside it
  if target:
    if not re.search('\?|\*|\[', target):
      # If no wild characters are present in the key string, append a '/*' suffix to show all keys
      key = key.strip('/') + '/*' if key else '*'

    if project:
      # Only list if the bucket is in the project
      for bucket in google.datalab.storage.Buckets(_make_context(project)):
        if bucket.name == bucket_name:
          break
      else:
        raise Exception('%s does not exist in project %s' % (target, project))
    else:
      bucket = google.datalab.storage.Bucket(bucket_name)

    if bucket.exists():
      return _gcs_list_keys(bucket, key)
    else:
      raise Exception('Bucket %s does not exist' % target)

  else:
    # Treat the bucket name as a pattern and show matches. We don't use bucket_name as that
    # can strip off wildchars and so we need to strip off gs:// here.
    return _gcs_list_buckets(project, target.strip('/')[5:])
854211
def get_csv_import_info(self, path):
        

        csvfilename = os.path.split(path)[1]

        try:
            filterdlg = CsvImportDialog(self.main_window, csvfilepath=path)

        except csv.Error, err:
            # Display modal warning dialog

            msg = _("'{filepath}' does not seem to be a valid CSV file.\n \n"
                    "Opening it yielded the error:\n{error}")
            msg = msg.format(filepath=csvfilename, error=err)
            short_msg = _('Error reading CSV file')

            self.display_warning(msg, short_msg)

            return

        if filterdlg.ShowModal() == wx.ID_OK:
            dialect, has_header = filterdlg.csvwidgets.get_dialect()
            digest_types = filterdlg.grid.dtypes
            encoding = filterdlg.csvwidgets.encoding

        else:
            filterdlg.Destroy()

            return

        filterdlg.Destroy()

        return dialect, has_header, digest_types, encoding
1331464
public Bucket getBucket(String bucket) {
        if (bucket.contains("..") || bucket.contains("/") || bucket.contains("\\")) {
            throw Exceptions.createHandled()
                            .withSystemErrorMessage(
                                    "Invalid bucket name: %s. A bucket name must not contain '..' '/' or '\\'",
                                    bucket)
                            .handle();
        }
        return new Bucket(new File(getBaseDir(), bucket));
    }
956161
private static void statFile(String fname) {
    try {
      Path path = Paths.get(new URI(fname));
      long size = Files.size(path);
      System.out.println(fname + ": " + size + " bytes.");
    } catch (Exception ex) {
      System.out.println(fname + ": " + ex.toString());
    }
  }
810145
def _gcs_create(args, _):
  
  errs = []
  for name in args['bucket']:
    try:
      bucket, key = google.datalab.storage._bucket.parse_name(name)
      if bucket and not key:
        google.datalab.storage.Bucket(bucket).create(_make_context(args['project']))
      else:
        raise Exception("Invalid bucket name %s" % name)
    except Exception as e:
      errs.append("Couldn't create %s: %s" %
                  (name, _extract_gcs_api_response_error(str(e))))
  if errs:
    raise Exception('\n'.join(errs))
233559
function processFiles(bucket, key, transform, cb, currentFileNum=0, lastFileNum=0, arn=null, retries=0) {
  const maxRetries = 5

  var nextFileNum = (currentFileNum < lastFileNum) ? currentFileNum + 1 : null
  //invokeLambda(bucket, key, currentFileNum, lastFileNum, arn)

  processFile(
    bucket, `${key}${currentFileNum}.csv`, transform
  ).then((n_scenes) => {
    invokeLambda(bucket, key, nextFileNum, lastFileNum, arn, 0)
    cb()
  }).catch((e) => {
    // if CSV failed, try it again
    if (retries < maxRetries) {
      invokeLambda(bucket, key, currentFileNum, lastFileNum, arn, retries + 1)
    } else {
      // log and move onto the next one
      console.log(`error: maxRetries hit in file ${currentFileNum}`)
      invokeLambda(bucket, key, nextFileNum, lastFileNum, arn, 0)
    }
    cb()
  })
}
59661
private void checkBucketName(String name) throws InvalidBucketNameException {
    if (name == null) {
      throw new InvalidBucketNameException(NULL_STRING, "null bucket name");
    }

    // Bucket names cannot be no less than 3 and no more than 63 characters long.
    if (name.length() < 3 || name.length() > 63) {
      String msg = "bucket name must be at least 3 and no more than 63 characters long";
      throw new InvalidBucketNameException(name, msg);
    }
    // Successive periods in bucket names are not allowed.
    if (name.matches("\\.\\.")) {
      String msg = "bucket name cannot contain successive periods. For more information refer "
          + "http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html";
      throw new InvalidBucketNameException(name, msg);
    }
    // Bucket names should be dns compatible.
    if (!name.matches("^[a-z0-9][a-z0-9\\.\\-]+[a-z0-9]$")) {
      String msg = "bucket name does not follow Amazon S3 standards. For more information refer "
          + "http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html";
      throw new InvalidBucketNameException(name, msg);
    }
  }
1019690
public static FileNotFoundException getFileNotFoundException(
      String bucketName, String objectName) {
    checkArgument(!isNullOrEmpty(bucketName), "bucketName must not be null or empty");
    return new FileNotFoundException(
        String.format(
            "Item not found: '%s'."
                + " If you enabled STRICT generation consistency, it is possible that"
                + " the live version is still available but the intended generation is deleted.",
            StorageResourceId.createReadableString(bucketName, nullToEmpty(objectName))));
  }
419095
private static String getDownloadUrl(String bucket, String objectName) {
    return String.format(GCS_URL_FORMAT, bucket, objectName);
  }
1006209
def _read(self, directory, filename, session, path, name, extension, spatial, spatialReferenceID, replaceParamFile):
        
        # Set file extension property
        self.fileExtension = extension

        # Open file and parse into a data structure
        with open(path, 'r') as f:
            for line in f:
                sline = line.strip().split()
                if len(sline) == 1:
                    self.numParameters = sline[0]
                else:
                    # Create GSSHAPY TargetParameter object
                    target = TargetParameter(targetVariable=sline[0],
                                             varFormat=sline[1])

                    # Associate TargetParameter with ReplaceParamFile
                    target.replaceParamFile = self
1039341
def error_in_upload(rows)
      csv_file_path = "#{@file_path}/errors_#{@csv_file_name}"
      begin
        CSV.open(csv_file_path, "wb") do |csv|
          rows.each do |row|
            csv << row
          end
        end
      rescue Exception => e
        Rails.logger.error e.formatted_exception("Building error file for #{@file_name} failed!")
      end
    end