:param data_in: path to submission.yaml
        :param args:
        :param kwargs:
        :raise ValueError:
969348
def parse(self, data_in, *args, **kwargs):
        
        if not os.path.exists(data_in):
            raise ValueError("File / Directory does not exist: %s" % data_in)

        if os.path.isdir(data_in):
            submission_filepath = os.path.join(data_in, 'submission.yaml')
            if not os.path.exists(submission_filepath):
                submission_filepath = os.path.join(data_in, 'submission.yml')
                if not os.path.exists(submission_filepath):
                    raise ValueError("No submission file in %s" % data_in)
            data_in = submission_filepath

        # first validate submission file:
        with open(data_in, 'r') as submission_file:
            submission_data = list(yaml.load_all(submission_file, Loader=Loader))

            if len(submission_data) == 0:
                raise RuntimeError("Submission file (%s) is empty" % data_in)

            submission_file_validator = SubmissionFileValidator()
            if not submission_file_validator.validate(file_path=data_in,
                                                      data=submission_data):
                raise RuntimeError(
                    "Submission file (%s) did not pass validation: %s" %
                    (data_in, self._pretty_print_errors(
                        submission_file_validator.get_messages())))

        metadata = {}
        tables = []

        # validator for table data
        data_file_validator = DataFileValidator()

        index = 0
        for i in range(0, len(submission_data)):
            if not submission_data[i]: # empty YAML document
                continue
            if 'data_file' not in submission_data[i]:
                metadata = submission_data[i] # information about whole submission
                continue
            table_filepath = os.path.join(os.path.dirname(data_in),
                                          submission_data[i]['data_file'])
            with open(table_filepath, 'r') as table_file:
                if not os.path.exists(table_filepath):
                    raise ValueError(
                        "table file: %s does not exist" % table.data_file)

                table_data = yaml.load(table_file, Loader=Loader)

                if not data_file_validator.validate(data=table_data,
                                                    file_path=table_filepath):
                    raise RuntimeError(
                        "Data file (%s) did not pass validation: %s" %
                        (table_filepath, self._pretty_print_errors(
                            data_file_validator.get_messages())))

                index = index + 1
                table = Table(index=index, metadata=submission_data[i],
                              data=table_data)
                tables.append(table)

        return ParsedData(metadata, tables)
334622
def handle_data(self, context, data, dt):
        
        if self.rule.should_trigger(dt):
            self.callback(context, data)
152793
def main(args):
  
  print_in_box('Validating submission ' + args.submission_filename)
  random.seed()
  temp_dir = args.temp_dir
  delete_temp_dir = False
  if not temp_dir:
    temp_dir = tempfile.mkdtemp()
    logging.info('Created temporary directory: %s', temp_dir)
    delete_temp_dir = True
  validator = submission_validator_lib.SubmissionValidator(temp_dir,
                                                           args.use_gpu)
  if validator.validate_submission(args.submission_filename,
                                   args.submission_type):
    print_in_box('Submission is VALID!')
  else:
    print_in_box('Submission is INVALID, see log messages for details')
  if delete_temp_dir:
    logging.info('Deleting temporary directory: %s', temp_dir)
    subprocess.call(['rm', '-rf', temp_dir])
1233225
def _match_data_to_parameter(cls, data):
    
    in_value = data["in"]
    for cls in [QueryParameter, HeaderParameter, FormDataParameter,
                PathParameter, BodyParameter]:
        if in_value == cls.IN:
            return cls
    return None
630077
def update_data_request(data_request_id, update_submission_data_request_data, opts = {})
      data, _status_code, _headers = update_data_request_with_http_info(data_request_id, update_submission_data_request_data, opts)
      data
    end
1014444
def run(self, target, payload, instance=None, hook_id=None, **kwargs):
        
        self.post_data(target, payload, hook_id)
586547
def submit(self, func, *args, **kwargs):
        
        self.op_sequence += 1
        self.sqs.send_message(
            QueueUrl=self.map_queue,
            MessageBody=utils.dumps({'args': args, 'kwargs': kwargs}),
            MessageAttributes={
                'sequence_id': {
                    'StringValue': str(self.op_sequence),
                    'DataType': 'Number'},
                'op': {
                    'StringValue': named(func),
                    'DataType': 'String',
                },
                'ser': {
                    'StringValue': 'json',
                    'DataType': 'String'}}
        )

        self.futures[self.op_sequence] = f = SQSFuture(
            self.op_sequence)
        return f
645721
def write_yaml (self, data, encoding=None, errors=None, newline=None, **kwargs):
        
        import yaml

        with self.open (mode='wt', encoding=encoding, errors=errors, newline=newline) as f:
            return yaml.dump (data, stream=f, **kwargs)
1132089
function(data, param, value) {
    if(_definition[param]) {
      var paramInstance = Param(param, value, _definition[param]);
      _definition[param].param.forEach(function(p) {
        data[p] = paramInstance;
      });

    } else {
      // Unsupported if operating in strict mode
      if(_useStrict) {
        throw(new Error('Invalid argument syntax ('+param+')'));

      } else {
        data[param] = Param(param, value);
      }
    }
  }
1129608
def get_data_for_model(self, args, kwargs):
        
        kwargs_from_invocation = self.get_raw_data()
        args_from_invocation = deque(self.path_args)

        defaults = kwargs
        values = args + list(kwargs.keys())

        output = {}

        raw = False
        for i, field in enumerate(values):
            ## going through each bit of data that the model needs
            ## `field` here is the name of each needed var.

            # the 'default' value that may be defined in the model.
            # this variable might be a string or int or might even be a primitive object.
            # NotImplemented here is used as to preserve if a default value is None.
            # it is used here as a sort of MetaNone.
            default_defined_in_model = defaults.get(field, NotImplemented)
            
            # the value in kwarg arguments such as --values and GET params
            from_data_kwargs = kwargs_from_invocation.get(field, None)

            # The value that will end up being used.
            value_to_use = None

            if default_defined_in_model == RAW_INVOCATION_ARGS:
                # flag that the RAW_INVOCATION_ARGS primitive has been invoked
                # used later to suppress errors for unused program args
                # when this primitive is invoked, all positional args are invalid. 
                raw = True

            if type(default_defined_in_model) == GiottoPrimitive:
                value_to_use = self.get_primitive(default_defined_in_model.name)
            elif from_data_kwargs:
                value_to_use = from_data_kwargs
            elif not raw and args_from_invocation:
                value_to_use = args_from_invocation.popleft()
            elif default_defined_in_model is not NotImplemented:
                value_to_use = default_defined_in_model
            else:
                raise InvalidInvocation("Data Missing For Program. Missing: %s" % field)
            
            output[field] = value_to_use

        if args_from_invocation and not raw:
            msg = "Too many arguments. Program `%s` takes %s arguments, %s given" % (
                self.program.name, len(args) + len(kwargs), len(args_from_invocation)
            )
            raise InvalidInvocation(msg)

        return output
1015806
def init_from_datastore(self):
    
    self._attacks = {}
    self._targeted_attacks = {}
    self._defenses = {}
    for entity in self._datastore_client.query_fetch(kind=KIND_SUBMISSION):
      submission_id = entity.key.flat_path[-1]
      submission_path = entity['submission_path']
      participant_id = {k: entity[k]
                        for k in ['team_id', 'baseline_id']
                        if k in entity}
      submission_descr = SubmissionDescriptor(path=submission_path,
                                              participant_id=participant_id)
      if list(entity.key.flat_path[0:2]) == ATTACKS_ENTITY_KEY:
        self._attacks[submission_id] = submission_descr
      elif list(entity.key.flat_path[0:2]) == TARGET_ATTACKS_ENTITY_KEY:
        self._targeted_attacks[submission_id] = submission_descr
      elif list(entity.key.flat_path[0:2]) == DEFENSES_ENTITY_KEY:
        self._defenses[submission_id] = submission_descr
580893
def write_yaml(data, out_path, Dumper=yaml.Dumper, **kwds):
    ''''''
    import errno
    import os

    class OrderedDumper(Dumper):
        pass
    def _dict_representer(dumper, data):
        return dumper.represent_mapping(
            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
            data.items())

    OrderedDumper.add_representer(OrderedDict, _dict_representer)

    # Make directory for calibration file if does not exist
    base_path, file_name = os.path.split(out_path)
    try:
        os.makedirs(base_path)
    except OSError as exc: # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(base_path):
            pass
        else: raise

    # Write dictionary to YAML
    with open(out_path, 'w') as f:
        return yaml.dump(data, f, OrderedDumper, default_flow_style=False, **kwds)
1215060
def runGetDataset(self, id_):
        
        dataset = self.getDataRepository().getDataset(id_)
        return self.runGetRequest(dataset)
2010
def _package_and_submit(args):
    
    args = _parse_and_validate_args(args)

    logging.debug(args)
    dist = __make_tmp_dir()
    try:
        __package_dependencies(dist_dir=dist, additional_reqs=args['requirements'],
                               silent=args['silent'])
        __package_app(tasks_pkg=args['package'],
                      dist_dir=dist,
                      custom_main=args['main'],
                      extra_data=args['extra_data'])
        __run_spark_submit(lane_yaml=args['yaml'],
                           dist_dir=dist,
                           spark_home=args['spark_home'],
                           spark_args=args['spark_args'],
                           silent=args['silent'])

    except Exception as exc:
        __clean_up(dist)
        raise exc
    __clean_up(dist)
749917
def submit(self, options=[]):
        
        self.complete_node_set()
        self._write_job_file()

        args = ['condor_submit_dag']
        args.extend(options)
        args.append(self.dag_file)

        log.info('Submitting workflow %s with options: %s', self.name, args)
        return super(Workflow, self).submit(args)
542064
def invoke(proc, args=None, *, _instance = None, **kwargs):
        
        if args is None:
            args=[]
        for kwargname in kwargs:
            args.append('--'+kwargname)
            args.append('{}'.format(kwargs[kwargname]))
        parser = proc.invoke_parser(noexit=(_instance is not None))
        opts = parser.parse_args(args)
        kwargs0 = {}


        def handle_set(opts, dataset, kwargs0, canMulti = False):
            for elem in dataset:
                elemname = elem.name
                # ml-run-process passes values for not provided inputs, outputs and params as empty strings ('')
                if hasattr(opts, elemname) and getattr(opts, elemname) not in [None, '']:
                    # value for element was given in the invocation
                    elemvalue = getattr(opts, elemname)
                    if canMulti and isinstance(elemvalue, list):
                        elemlist = elemvalue
                    else:
                        elemlist = [ elemvalue ]
                    for elemelem in elemlist:
                        for validator in elem.validators: validator(elemelem)
                    if hasattr(opts, elem.name):
                        prepared = elem.prepare(elemvalue) or elemvalue
                        kwargs0[elem.name] = prepared
                elif elem.optional:
                    # value was not set but is optional so ignore it
                    kwargs0[elem.name] = None
                else:
                    # value was not set and is mandatory -- error
                    raise AttributeError('Missing value for {} '.format(elemname))

        try:
            handle_set(opts, proc.INPUTS, kwargs0, True)
            handle_set(opts, proc.OUTPUTS, kwargs0, True)

            for param in proc.PARAMETERS:
                if hasattr(opts, param.name) and getattr(opts, param.name) is not None and getattr(opts, param.name) is not '':
                    value = getattr(opts, param.name)
                    # validate if needed
                    for validator in param.validators:
                        validator(value)
                    # if param is a tuple of choices, each choice is a tuple itself
                    # with first element of the input value and second element
                    # containing the value to be passed to the processor
                    if param.choices and isinstance(param.choices, tuple):
                        for choice in param.choices:
                            if choice[0] == value:
                                kwargs0[param.name] = choice[1]
                                break
                    else:
                        kwargs0[param.name] = value
                elif param.optional:
                    kwargs0[param.name] = param.default
                else:
                    raise AttributeError('Missing value for {} parameter'.format(param.name))
            if not _instance:
                _instance = proc(**kwargs0)
            else:
                _instance.apply(_instance, **kwargs0)
            return _instance.run()
            # todo: cleanup
        except Exception as e:
            print("Error:", e)
#            traceback.print_exc()
            raise
906089
def _get_data(self, kwargs):
        """"""
        if "ds" not in kwargs:
            raise ValueError("Keyword argument 'ds' missing.")

        ds = kwargs["ds"]

        if self.req_feature:
            if "feature" not in kwargs:
                raise ValueError("Keyword argument 'feature' missing.")
            return self.get_feature(ds, kwargs["feature"])
        else:
            return ds