Returns a BigQuery Extract Job configuration for the given source table and destination URI.
490292
public static ExtractJobConfiguration of(TableId sourceTable, String destinationUri) {
    return newBuilder(sourceTable, destinationUri).build();
  }
557733
def get_task_db(self, source_path):
        '''
        '''
        sql = 'SELECT * FROM upload WHERE source_path=?'
        req = self.cursor.execute(sql, [source_path, ])
        if req:
            return req.fetchone()
        else:
            None
1122393
public Job copyTableId(String dataset, String tableName) throws BigQueryException {
    // [START bigquery_copy_table]
    TableId destinationId = TableId.of(dataset, tableName);
    JobOption options = JobOption.fields(JobField.STATUS, JobField.USER_EMAIL);
    Job job = table.copy(destinationId, options);
    // Wait for the job to complete.
    try {
      Job completedJob =
          job.waitFor(
              RetryOption.initialRetryDelay(Duration.ofSeconds(1)),
              RetryOption.totalTimeout(Duration.ofMinutes(3)));
      if (completedJob != null && completedJob.getStatus().getError() == null) {
        // Job completed successfully.
      } else {
        // Handle error case.
      }
    } catch (InterruptedException e) {
      // Handle interrupted wait
    }
    // [END bigquery_copy_table]
    return job;
  }
936609
def get_eager_datasource(cls, session, datasource_type, datasource_id):
        """"""
        datasource_class = ConnectorRegistry.sources[datasource_type]
        return (
            session.query(datasource_class)
            .options(
                subqueryload(datasource_class.columns),
                subqueryload(datasource_class.metrics),
            )
            .filter_by(id=datasource_id)
            .one()
        )
1215859
@Override
  public void checkOutputSpecs(JobContext job) throws FileAlreadyExistsException, IOException {
    Configuration conf = job.getConfiguration();

    // Validate the output configuration.
    BigQueryOutputConfiguration.validateConfiguration(conf);

    // Get the output path.
    Path outputPath = BigQueryOutputConfiguration.getGcsOutputPath(conf);
    logger.atInfo().log("Using output path '%s'.", outputPath);

    // Error if the output path already exists.
    FileSystem outputFileSystem = outputPath.getFileSystem(conf);
    if (outputFileSystem.exists(outputPath)) {
      throw new IOException("The output path '" + outputPath + "' already exists.");
    }

    // Error if compression is set as there's mixed support in BigQuery.
    if (FileOutputFormat.getCompressOutput(job)) {
      throw new IOException("Compression isn't supported for this OutputFormat.");
    }

    // Error if unable to create a BigQuery helper.
    try {
      new BigQueryFactory().getBigQueryHelper(conf);
    } catch (GeneralSecurityException gse) {
      throw new IOException("Failed to create BigQuery client", gse);
    }

    // Let delegate process its checks.
    getDelegate(conf).checkOutputSpecs(job);
  }
1172389
def extract(request, calc_id, what):
    
    job = logs.dbcmd('get_job', int(calc_id))
    if job is None:
        return HttpResponseNotFound()
    if not utils.user_has_permission(request, job.user_name):
        return HttpResponseForbidden()

    try:
        # read the data and save them on a temporary .npz file
        with datastore.read(job.ds_calc_dir + '.hdf5') as ds:
            fd, fname = tempfile.mkstemp(
                prefix=what.replace('/', '-'), suffix='.npz')
            os.close(fd)
            n = len(request.path_info)
            query_string = unquote_plus(request.get_full_path()[n:])
            aw = _extract(ds, what + query_string)
            a = {}
            for key, val in vars(aw).items():
                key = str(key)  # can be a numpy.bytes_
                if isinstance(val, str):
                    # without this oq extract would fail
                    a[key] = numpy.array(val.encode('utf-8'))
                elif isinstance(val, dict):
                    # this is hack: we are losing the values
                    a[key] = list(val)
                else:
                    a[key] = val
            numpy.savez_compressed(fname, **a)
    except Exception as exc:
        tb = ''.join(traceback.format_tb(exc.__traceback__))
        return HttpResponse(
            content='%s: %s\n%s' % (exc.__class__.__name__, exc, tb),
            content_type='text/plain', status=500)

    # stream the data back
    stream = FileWrapper(open(fname, 'rb'))
    stream.close = lambda: (FileWrapper.close(stream), os.remove(fname))
    response = FileResponse(stream, content_type='application/octet-stream')
    response['Content-Disposition'] = (
        'attachment; filename=%s' % os.path.basename(fname))
    response['Content-Length'] = str(os.path.getsize(fname))
    return response
4646
public static CopyJobConfiguration of(TableId destinationTable, List<TableId> sourceTables) {
    return newBuilder(destinationTable, sourceTables).build();
  }
1133960
def all_known(cls, sources=None, prefixes=None):
        

        sources = sources or ("config", "jobs", "raw_subqueues")

        queues = set()

        if "config" in sources and not prefixes:
            # Some queues are explicitly declared in the config (including all root raw queues)
            cfg = context.get_current_config()

            queues_from_config = [
                t.get("queue")
                for t in (cfg.get("tasks") or {}).values()
                if t.get("queue")
            ]

            queues_from_config += Queue.get_queues_config().keys()

            queues_from_config += [
                t.get("retry_queue")
                for t in Queue.get_queues_config().values()
                if t.get("retry_queue")
            ]

            queues |= set(queues_from_config)

        if "jobs" in sources:

            # This will get all queues from mongodb, including those where we have only non-queued jobs
            for q in context.connections.mongodb_jobs.mrq_jobs.distinct("queue"):
                if prefixes and not any(q.startswith(p) for p in prefixes):
                    continue
                queues.add(q)

        if "raw_subqueues" in sources:
            for q in Queue.get_queues_config():
                if prefixes and not any(q + "/" == p for p in prefixes):
                    continue
                queue_obj = Queue(q)
                if queue_obj.is_raw and queue_obj.has_subqueues:
                    # TODO: optimize this with a single SUNION on all keys
                    queues |= queue_obj.get_known_subqueues()

        return queues
233537
def get_remoteci_configuration(topic_id, remoteci_id, db_conn=None):
    """"""

    db_conn = db_conn or flask.g.db_conn
    last_rconfiguration_id = get_last_rconfiguration_id(
        topic_id, remoteci_id, db_conn=db_conn)
    _RCONFIGURATIONS = models.REMOTECIS_RCONFIGURATIONS
    _J_RCONFIGURATIONS = models.JOIN_REMOTECIS_RCONFIGURATIONS
    query = sql.select([_RCONFIGURATIONS]). \
        select_from(_J_RCONFIGURATIONS.
                    join(_RCONFIGURATIONS)). \
        where(_J_RCONFIGURATIONS.c.remoteci_id == remoteci_id)
    query = query.where(sql.and_(_RCONFIGURATIONS.c.state != 'archived',
                                 _RCONFIGURATIONS.c.topic_id == topic_id))
    query = query.order_by(sql.desc(_RCONFIGURATIONS.c.created_at))
    query = query.order_by(sql.asc(_RCONFIGURATIONS.c.name))
    all_rconfigurations = db_conn.execute(query).fetchall()

    if len(all_rconfigurations) > 0:
        for i in range(len(all_rconfigurations)):
            if str(all_rconfigurations[i]['id']) == last_rconfiguration_id:
                # if i==0, then indice -1 is the last element
                return all_rconfigurations[i - 1]
        return all_rconfigurations[0]
    else:
        return None
869644
@Override
  public void sourceConfInit(FlowProcess<? extends Configuration> flowProcess,
      Tap<Configuration, RecordReader, OutputCollector> tap, Configuration conf) {
    conf.setBoolean("mapred.mapper.new-api", false);
    conf.setClass("mapred.input.format.class", CorcInputFormat.class, InputFormat.class);
    // ORC cannot be combined.
    conf.setBoolean("cascading.hadoop.hfs.combine.files", false);
    CorcInputFormat.setSchemaTypeInfo(conf, schemaTypeInfo);
    CorcInputFormat.setTypeInfo(conf, typeInfo);
    CorcInputFormat.setSearchArgumentKryo(conf, searchArgumentKryo);
    CorcInputFormat.setConverterFactoryClass(conf, converterFactory.getClass().asSubclass(ConverterFactory.class));
  }
279649
public MigrationJob[] retrieveJobs(int batchSize, int startIndex, JobType jobType)
            throws IOException, LightblueException {
        LOGGER.debug("Retrieving jobs: batchSize={}, startIndex={}", batchSize, startIndex);

        DataFindRequest findRequest = new DataFindRequest("migrationJob", null);

        List<Query> conditions = new ArrayList<>(Arrays.asList(new Query[] {
                // get jobs for this configuration
                Query.withValue("configurationName", Query.eq, migrationConfiguration.getConfigurationName()),
                // get jobs whose state ara available
                Query.withValue("status", Query.eq, "available"),
                // only get jobs that are
                Query.withValue("scheduledDate", Query.lte, new Date())
        }));

        if (jobType == JobType.GENERATED) {
            LOGGER.debug("Looking for generated job");
            conditions.add(Query.withValue("generated", Query.eq, true));
        } else if (jobType == JobType.NONGENERATED) {
            LOGGER.debug("Looking for non generated job");
            conditions.add(Query.withValue("generated", Query.eq, false));
        }

        findRequest.where(Query.and(conditions));

        findRequest.select(Projection.includeField("*"));

        findRequest.range(startIndex, startIndex + batchSize - 1);

        LOGGER.debug("Finding Jobs to execute: {}", findRequest.getBody());

        return lbClient.data(findRequest, MigrationJob[].class);
    }
452576
def get_job_logs_from_workflow(workflow_id):
    """"""
    query_result = (
        db.session.query(
            models.CrawlerJob.logs,
        )
        .join(
            models.CrawlerWorkflowObject,
            models.CrawlerJob.job_id == models.CrawlerWorkflowObject.job_id,
        )
        .filter(models.CrawlerWorkflowObject.object_id == workflow_id)
        .one_or_none()
    )

    if query_result is None:
        click.secho(
            (
                "Workflow %s was not found, maybe it's not a crawl workflow?" %
                workflow_id
            ),
            fg='yellow',
        )
        sys.exit(1)

    _show_file(
        file_path=query_result[0],
        header_name='Log',
    )
1302448
def _get_metadata(job_id):
    """"""
    # Avoid SQLAlchemy "Unicode type received non-unicode bind param value"
    # warnings.
    job_id = unicode(job_id)

    results = ENGINE.execute(
        METADATA_TABLE.select().where(
            METADATA_TABLE.c.job_id == job_id)).fetchall()
    metadata = {}
    for row in results:
        value = row['value']
        if row['type'] == 'json':
            value = json.loads(value)
        metadata[row['key']] = value
    return metadata
1296918
def worker(work_unit):
    '''

    '''
    if 'config' not in work_unit.spec:
        raise coordinate.exceptions.ProgrammerError(
            'could not run extraction without global config')

    web_conf = Config()
    unitconf = work_unit.spec['config']
    #logger.info(unitconf)
    with yakonfig.defaulted_config([coordinate, kvlayer, dblogger, web_conf],
                                   config=unitconf):
        traverse_extract_fetch(web_conf, work_unit.key)
300009
private List<JobExecutionInfo> executeQuery(JobExecutionQuery query) throws RemoteInvocationException {
    JobExecutionQueryResult result = this.client.get(query);

    if (result != null && result.hasJobExecutions()) {
      return result.getJobExecutions();
    }
    return Collections.emptyList();
  }
247840
def _get_logs(job_id):
    """"""
    # Avoid SQLAlchemy "Unicode type received non-unicode bind param value"
    # warnings.
    job_id = unicode(job_id)

    results = ENGINE.execute(
        LOGS_TABLE.select().where(LOGS_TABLE.c.job_id == job_id)).fetchall()

    results = [dict(result) for result in results]

    for result in results:
        result.pop("job_id")

    return results
585566
public static Job createSubmittableJob(Configuration conf, String[] args)
  throws IOException {
    conf.setIfUnset("hbase.client.connection.impl",
      BigtableConfiguration.getConnectionClass().getName());
    conf.setIfUnset(BigtableOptionsFactory.BIGTABLE_RPC_TIMEOUT_MS_KEY, "60000");
    conf.setBoolean(TableInputFormat.SHUFFLE_MAPS, true);

    String tableName = args[0];
    Path outputDir = new Path(args[1]);
    Job job = Job.getInstance(conf, NAME + "_" + tableName);
    job.setJobName(NAME + "_" + tableName);
    job.setJarByClass(Export.class);
    // Set optional scan parameters
    Scan s = getConfiguredScanForJob(conf, args);
    TableMapReduceUtil.initTableMapperJob(tableName, s, IdentityTableMapper.class, ImmutableBytesWritable.class, Result.class, job, false);
    // No reducers.  Just write straight to output files.
    job.setNumReduceTasks(0);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    job.setOutputKeyClass(ImmutableBytesWritable.class);
    job.setOutputValueClass(Result.class);
    FileOutputFormat.setOutputPath(job, outputDir); // job conf doesn't contain the conf so doesn't have a default fs.
    return job;
  }