Default validation for Process Steps data structure
489815
def validate_process_steps(prop, value):
    

    if value is not None:
        validate_type(prop, value, (dict, list))

        procstep_keys = set(_complex_definitions[prop])

        for idx, procstep in enumerate(wrap_value(value)):
            ps_idx = prop + '[' + str(idx) + ']'
            validate_type(ps_idx, procstep, dict)

            for ps_prop, ps_val in iteritems(procstep):
                ps_key = '.'.join((ps_idx, ps_prop))

                if ps_prop not in procstep_keys:
                    _validation_error(prop, None, value, ('keys: {0}'.format(','.join(procstep_keys))))

                if ps_prop != 'sources':
                    validate_type(ps_key, ps_val, string_types)
                else:
                    validate_type(ps_key, ps_val, (string_types, list))

                    for src_idx, src_val in enumerate(wrap_value(ps_val)):
                        src_key = ps_key + '[' + str(src_idx) + ']'
                        validate_type(src_key, src_val, string_types)
354942
def __quick_validate(r, check=('ResultSet', 'Result', 'totalRecords')):
    '''
    
    '''
    if 'ResultSet' in check and 'ResultSet' not in r:
        raise ResultSetError('no ResultSet in server response')
    if 'Result' in check and 'Result' not in r['ResultSet']:
        raise ResultSetError('no Result in server response')
    if 'totalRecords' in check and 'totalRecords' not in r['ResultSet']:
        raise ResultSetError('no totalRecords in server response')
    return True
969359
def validate(cls, data):
        """"""
        try:
            jsonschema.validate(
                data, cls.SCHEMA,
                # See: https://github.com/Julian/jsonschema/issues/148
                types={'array': (list, tuple)})
        except jsonschema.ValidationError as e:
            raise InvalidFormat("Failure data not of the"
                                " expected format: %s" % (e.message))
        else:
            # Ensure that all 'exc_type_names' originate from one of
            # base exceptions, because those are the root exceptions that
            # python mandates/provides and anything else is invalid...
            causes = collections.deque([data])
            while causes:
                cause = causes.popleft()
                try:
                    generated_on = cause['generated_on']
                    ok_bases = cls.BASE_EXCEPTIONS[generated_on[0]]
                except (KeyError, IndexError):
                    ok_bases = []
                root_exc_type = cause['exc_type_names'][-1]
                if root_exc_type not in ok_bases:
                    raise InvalidFormat(
                        "Failure data 'exc_type_names' must"
                        " have an initial exception type that is one"
                        " of %s types: '%s' is not one of those"
                        " types" % (ok_bases, root_exc_type))
                sub_cause = cause.get('cause')
                if sub_cause is not None:
                    causes.append(sub_cause)
1349109
def validate_nesting(self):
        '''
        
        '''
        error_dict = {}
        story_root = self.story_tree_root
        story_nodes = story_root.get_descendants().prefetch_related('arcelementnode_set').filter(depth=2)

        def parse_children(node_object, seq=0):
            '''
            Recursive method to check all generations of a node for arc
            elements.
            '''
            returnable_ae_dict = {}
            if node_object.get_children_count():
                for node in node_object.get_children().prefetch_related('arcelementnode_set'):
                    if node.get_children_count():
                        local_ae_dict, newseq = parse_children(node, seq)
                        for arc in local_ae_dict.keys():
                            if arc not in returnable_ae_dict.keys():
                                returnable_ae_dict[arc] = []
                            returnable_ae_dict[arc] = returnable_ae_dict[arc] + (local_ae_dict[arc])
                        seq = newseq
                    else:
                        local_arc_elements = node.arcelementnode_set.all()
                        for ae in local_arc_elements:
                            if ae.is_milestone:
                                logger.debug('Found a milestone to sequence.')
                                if ae.arc not in returnable_ae_dict.keys():
                                    returnable_ae_dict[ae.arc] = []
                                returnable_ae_dict[ae.arc].append({
                                    ae.arc_element_type: seq,
                                })
                                logger.debug('Appended an element of type %s  from arc %s at sequence %d' % (
                                    ae.arc_element_type,
                                    ae.arc.name,
                                    seq))
                        seq += 1
            return returnable_ae_dict, seq

        arcs_out_of_sequence = []
        arc_milestone_elements = {}
        x = 0
        arcs_with_nest_conflicts = []
        for node in story_nodes:
            arc_elements_found, x = parse_children(node, x)
            for arc in arc_elements_found.keys():
                if arc not in arc_milestone_elements.keys():
                    arc_milestone_elements[arc] = []
                arc_milestone_elements[arc] = arc_milestone_elements[arc] + arc_elements_found[arc]
        arc_entry_exit = {}
        for arc, elements in arc_milestone_elements.items():
            z = {}
            last_local_seq = 0
            logger.debug("There are %d elements to parse" % len(elements))
            for ae_elements in elements:
                logger.debug('Parsing %s' % ae_elements)
                for k, v in ae_elements.items():
                    z[k] = v
            logger.debug('%s' % OrderedDict(sorted(z.items(), key=lambda t: t[1])))
            for key, value in OrderedDict(sorted(z.items(), key=lambda t: t[1])).items():
                if last_local_seq > ARC_NODE_ELEMENT_DEFINITIONS[key]['milestone_seq']:
                    arcs_out_of_sequence.append(arc)
                    break
                last_local_seq = ARC_NODE_ELEMENT_DEFINITIONS[key]['milestone_seq']
                if 'mile_hook' in z.keys() and 'mile_reso' in z.keys():
                    arc_entry_exit[arc] = {
                        'entry': z['mile_hook'],
                        'exit': z['mile_reso'],
                    }
        if arc_entry_exit:
            temp_ent = {}
            temp_ex = {}
            for en_arc, enter in arc_entry_exit.items():
                temp_ent[en_arc] = enter['entry']
            for ex_arc, leave in arc_entry_exit.items():
                temp_ex[ex_arc] = leave['exit']
            arc_entries = OrderedDict(sorted(temp_ent.items(), key=lambda t: t[1]))
            arc_exits = OrderedDict(sorted(temp_ex.items(), key=lambda t: t[1], reverse=True))
            logger.debug("Evaluating entries vs exits: %s vs %s" % (arc_entries, arc_exits))
            index_num = 0
            for entrance in arc_entries.keys():
                test_exit = list(arc_exits.keys())[index_num]
                logger.debug('Checking %s vs %s arcs' % (entrance, test_exit))
                if entrance != test_exit:
                    logger.debug("Different arcs.. making sure they don't share an entry/exist point")
                    if (arc_entry_exit[entrance]['entry'] != arc_entry_exit[test_exit]['entry'] and
                        arc_entry_exit[entrance]['exit'] != arc_entry_exit[test_exit]['exit']):
                        if ((arc_entry_exit[entrance]['entry'] < arc_entry_exit[test_exit]['entry'] and
                                arc_entry_exit[entrance]['exit'] > arc_entry_exit[test_exit]['exit']) or
                            (arc_entry_exit[entrance]['entry'] > arc_entry_exit[test_exit]['entry'] and
                             arc_entry_exit[entrance]['exit'] < arc_entry_exit[test_exit]['exit'])):
                            logger.debug("No same entry point, but the nest is still valid.")
                        else:
                            logger.debug("nesting error found: %s should resole before %s" % (entrance, test_exit))
                            if entrance not in arcs_with_nest_conflicts:
                                arcs_with_nest_conflicts.append(entrance)
                            if test_exit not in arcs_with_nest_conflicts:
                                arcs_with_nest_conflicts.append(test_exit)
                    else:
                        logger.debug('Arcs share an entry/exit point so nesting error is ignored.')
                index_num += 1
        unique_nest_errors = list(set(arcs_with_nest_conflicts))
        logger.debug("%d arcs with nesting errors found" % len(unique_nest_errors))
        logger.debug('Arcs with nesting errors: %s' % unique_nest_errors)
        story_tree = story_root.get_descendants().prefetch_related('arcelementnode_set')
        if arcs_out_of_sequence:
            error_dict['nest_arc_seq'] = {
                'error_message': "Arc element milestones are out of sequence",
                'offending_arcs': []
            }
            logger.debug('There are %d arcs out of internal sequence' % len(arcs_out_of_sequence))
            arc_sequence_error_dict = {}
            story_root.refresh_from_db()
            for arc in arcs_out_of_sequence:
                arc_sequence_error_dict['offending_nodes'] = story_tree.filter(arcelementnode__arc=arc)
            error_dict['nest_arc_seq']['offending_arcs'].append(arc_sequence_error_dict)
        if unique_nest_errors:
            error_dict['nest_reso_error'] = {
                'error_message': "Arcs should resolve in the opposite order that they were introduced",
                'offending_nodes': story_tree.filter(
                    arcelementnode__arc__in=unique_nest_errors,
                    arcelementnode__arc_element_type='mile_reso'
                )
            }
        return error_dict
609734
def _is_process_filtered(self, process, key=None):
        """"""
        if key is None:
            key = self.filter_key
        try:
            # If the item process[key] is a list, convert it to a string
            # in order to match it with the current regular expression
            if isinstance(process[key], list):
                value = ' '.join(process[key])
            else:
                value = process[key]
        except KeyError:
            # If the key did not exist
            return False
        try:
            return self._filter_re.match(value) is None
        except (AttributeError, TypeError):
            # AttributeError
            # Filter processes crashs with a bad regular expression pattern (issue #665)
            # TypeError
            # Filter processes crashs if value is None (issue #1105)
            return False
906767
private String isNotNullStep(Step<?, ?> currentStep) {
        if (currentStep instanceof TraversalFilterStep<?>) {
            TraversalFilterStep<?> tfs = (TraversalFilterStep<?>) currentStep;
            List<?> c = tfs.getLocalChildren();
            if (c != null && c.size() == 1) {
                Traversal.Admin<?, ?> a = (Traversal.Admin<?, ?>) c.iterator().next();
                Step<?, ?> s = a.getEndStep();
                if (a.getSteps().size() == 1 && s instanceof PropertiesStep<?>) {
                    PropertiesStep<?> ps = (PropertiesStep<?>) s;
                    String[] keys = ps.getPropertyKeys();
                    if (keys != null && keys.length == 1) {
                        return keys[0];
                    }
                }
            }
        }
        return null;
    }
48990
private void validateWorkflow(WorkflowDef workflowDef, Map<String, Object> workflowInput, String externalStoragePath) {
        try {
            //Check if the input to the workflow is not null
            if (workflowInput == null && StringUtils.isBlank(externalStoragePath)) {
                LOGGER.error("The input for the workflow '{}' cannot be NULL", workflowDef.getName());
                throw new ApplicationException(INVALID_INPUT, "NULL input passed when starting workflow");
            }
        } catch (Exception e) {
            Monitors.recordWorkflowStartError(workflowDef.getName(), WorkflowContext.get().getClientApp());
            throw e;
        }
    }
34929
def validate_unit_process_ids(self, expected, actual):
        """"""
        self.log.debug('Checking units for running processes...')
        self.log.debug('Expected PIDs: {}'.format(expected))
        self.log.debug('Actual PIDs: {}'.format(actual))

        if len(actual) != len(expected):
            return ('Unit count mismatch.  expected, actual: {}, '
                    '{} '.format(len(expected), len(actual)))

        for (e_sentry, e_proc_names) in six.iteritems(expected):
            e_sentry_name = e_sentry.info['unit_name']
            if e_sentry in actual.keys():
                a_proc_names = actual[e_sentry]
            else:
                return ('Expected sentry ({}) not found in actual dict data.'
                        '{}'.format(e_sentry_name, e_sentry))

            if len(e_proc_names.keys()) != len(a_proc_names.keys()):
                return ('Process name count mismatch.  expected, actual: {}, '
                        '{}'.format(len(expected), len(actual)))

            for (e_proc_name, e_pids), (a_proc_name, a_pids) in \
                    zip(e_proc_names.items(), a_proc_names.items()):
                if e_proc_name != a_proc_name:
                    return ('Process name mismatch.  expected, actual: {}, '
                            '{}'.format(e_proc_name, a_proc_name))

                a_pids_length = len(a_pids)
                fail_msg = ('PID count mismatch. {} ({}) expected, actual: '
                            '{}, {} ({})'.format(e_sentry_name, e_proc_name,
                                                 e_pids, a_pids_length,
                                                 a_pids))

                # If expected is a list, ensure at least one PID quantity match
                if isinstance(e_pids, list) and \
                        a_pids_length not in e_pids:
                    return fail_msg
                # If expected is not bool and not list,
                # ensure PID quantities match
                elif not isinstance(e_pids, bool) and \
                        not isinstance(e_pids, list) and \
                        a_pids_length != e_pids:
                    return fail_msg
                # If expected is bool True, ensure 1 or more PIDs exist
                elif isinstance(e_pids, bool) and \
                        e_pids is True and a_pids_length < 1:
                    return fail_msg
                # If expected is bool False, ensure 0 PIDs exist
                elif isinstance(e_pids, bool) and \
                        e_pids is False and a_pids_length != 0:
                    return fail_msg
                else:
                    self.log.debug('PID check OK: {} {} {}: '
                                   '{}'.format(e_sentry_name, e_proc_name,
                                               e_pids, a_pids))
        return None
211465
def validate_next(stepper, metrics, val_iter):
    """"""
    stepper.reset(False)
    with no_grad_context():
        (*x,y) = val_iter.next()
        preds,l = stepper.evaluate(VV(x), VV(y))
        res = [delistify(to_np(l))]
        res += [f(datafy(preds), datafy(y)) for f in metrics]
    stepper.reset(True)
    return res
703624
@Override
    public void validateProcessOutput(FaxClientSpi faxClientSpi,ProcessOutput processOutput,FaxActionType faxActionType)
    {
        //call exit code validation
        super.validateProcessOutput(faxClientSpi,processOutput,faxActionType);

        //get output
        String output=processOutput.getOutputText();

        if((output!=null)&&(output.length()>0))
        {
            //if no valid done output was found
            if(output.indexOf(Fax4jExeConstants.OPERATION_OUTPUT_DONE.toString())==-1)
            {
                //get error message
                String prefix=Fax4jExeConstants.FAX_ERROR_OUTPUT_PREFIX.toString();
                int index=output.indexOf(prefix);
                int prefixLenght=prefix.length();
                String errorMessage=null;
                if(index!=-1)
                {
                    //get index
                    index=index+prefixLenght;

                    if(output.length()>index)
                    {
                        //get output
                        errorMessage=output.substring(index);
                        
                        errorMessage=errorMessage.trim();
                        index=errorMessage.indexOf("\n");
                        if(index!=-1)
                        {
                            errorMessage=errorMessage.substring(0,index);
                            errorMessage=errorMessage.trim();
                        }
                        
                        //ignore empty message
                        if(errorMessage.length()==0)
                        {
                            errorMessage=null;
                        }
                    }
                }
                
                if(errorMessage==null)
                {
                    throw new FaxException("Error while invoking fax4j.exe, no error message detected.\nNative Output:\n"+output);
                }
                throw new FaxException("Error while invoking fax4j.exe, error: "+errorMessage+"\nNative Output:\n"+output);
            }
        }
        else
        {
            throw new FaxException("Error while invoking fax4j.exe, no output detected.");
        }
    }
801376
def _process_dependlist(self, dependlist, anexec, isSubroutine, mode="insert"):
        """"""
        for i in range(len(dependlist)):
            #Since we are looping over all the elements and some will
            #be lists of parameters, we need to skip any items that are lists.
            if isinstance(dependlist[i], list):
                continue

            key = dependlist[i].lower()
            if len(dependlist) > i + 1:
                has_params = isinstance(dependlist[i + 1], list)
            else:
                has_params = False

            #Clean the dependency key to make sure we are only considering valid
            #executable symbols.
            cleanmatch = self.RE_DEPCLEAN.match(key + "(")
            if not cleanmatch:
                continue
            else:
                key = cleanmatch.group("key")

            #We need to handle if and do constructs etc. separately
            if key not in ["then", ",", "", "elseif"] and has_params \
               and not "=" in key and not ">" in key:
                if key in ["if", "do"]:
                    self._process_dependlist(dependlist[i + 1], anexec, False, mode)
                else:
                    #This must be valid call to an executable, add it to the list
                    #with its parameters and then process its parameters list
                    #to see if there are more nested executables
                    if mode == "insert":
                        self._add_dependency(key, dependlist, i, isSubroutine, anexec)
                    elif mode == "delete":
                        #Try and find a dependency already in the executable that 
                        #has the same call signature; then remove it.
                        self._remove_dependency(dependlist, i, isSubroutine, anexec)

                    self._process_dependlist(dependlist[i + 1], anexec, False, mode)
1073990
def steps_for_spec(builder, spec, processed_modules=None):
    '''
    
    '''
    if processed_modules is None:
        processed_modules = set()
    steps = []
    for module in spec.get('depends_on', []):
        if module not in processed_modules:
            processed_modules.add(module)
            steps.extend(steps_for_spec(
                builder,
                module.fbcode_builder_spec(builder),
                processed_modules
            ))
    steps.extend(spec.get('steps', []))
    return steps
22409
def validate(self):
        """"""
        if not self.recipe:
            raise workflows.Error("Invalid recipe: No recipe defined")

        # Without a 'start' node nothing would happen
        if "start" not in self.recipe:
            raise workflows.Error('Invalid recipe: "start" node missing')
        if not self.recipe["start"]:
            raise workflows.Error('Invalid recipe: "start" node empty')
        if not all(
            isinstance(x, (list, tuple)) and len(x) == 2 for x in self.recipe["start"]
        ):
            raise workflows.Error('Invalid recipe: "start" node invalid')
        if any(x[0] == "start" for x in self.recipe["start"]):
            raise workflows.Error('Invalid recipe: "start" node points to itself')

        # Check that 'error' node points to regular nodes only
        if "error" in self.recipe and isinstance(
            self.recipe["error"], (list, tuple, basestring)
        ):
            if "start" in self.recipe["error"]:
                raise workflows.Error(
                    'Invalid recipe: "error" node points to "start" node'
                )
            if "error" in self.recipe["error"]:
                raise workflows.Error('Invalid recipe: "error" node points to itself')

        # All other nodes must be numeric
        nodes = list(
            filter(
                lambda x: not isinstance(x, int) and x not in ("start", "error"),
                self.recipe,
            )
        )
        if nodes:
            raise workflows.Error('Invalid recipe: Node "%s" is not numeric' % nodes[0])

        # Detect cycles
        touched_nodes = set(["start", "error"])

        def flatten_links(struct):
            """Take an output/error link object, list or dictionary and return flat list of linked nodes."""
            if struct is None:
                return []
            if isinstance(struct, int):
                return [struct]
            if isinstance(struct, list):
                if not all(isinstance(x, int) for x in struct):
                    raise workflows.Error(
                        "Invalid recipe: Invalid link in recipe (%s)" % str(struct)
                    )
                return struct
            if isinstance(struct, dict):
                joined_list = []
                for sub_list in struct.values():
                    joined_list += flatten_links(sub_list)
                return joined_list
            raise workflows.Error(
                "Invalid recipe: Invalid link in recipe (%s)" % str(struct)
            )

        def find_cycles(path):
            """Depth-First-Search helper function to identify cycles."""
            if path[-1] not in self.recipe:
                raise workflows.Error(
                    'Invalid recipe: Node "%s" is referenced via "%s" but missing'
                    % (str(path[-1]), str(path[:-1]))
                )
            touched_nodes.add(path[-1])
            node = self.recipe[path[-1]]
            for outgoing in ("output", "error"):
                if outgoing in node:
                    references = flatten_links(node[outgoing])
                    for n in references:
                        if n in path:
                            raise workflows.Error(
                                "Invalid recipe: Recipe contains cycle (%s -> %s)"
                                % (str(path), str(n))
                            )
                        find_cycles(path + [n])

        for link in self.recipe["start"]:
            find_cycles(["start", link[0]])
        if "error" in self.recipe:
            if isinstance(self.recipe["error"], (list, tuple)):
                for link in self.recipe["error"]:
                    find_cycles(["error", link])
            else:
                find_cycles(["error", self.recipe["error"]])

        # Test recipe for unreferenced nodes
        for node in self.recipe:
            if node not in touched_nodes:
                raise workflows.Error(
                    'Invalid recipe: Recipe contains unreferenced node "%s"' % str(node)
                )
597104
def fill_with_defaults(process_input, input_schema):
    """"""
    for field_schema, fields, path in iterate_schema(process_input, input_schema):
        if 'default' in field_schema and field_schema['name'] not in fields:
            dict_dot(process_input, path, field_schema['default'])
51364
def validate_process_subtype(supertype_name, supertype, subtype_name, subtype):
    
    errors = []
    for item in supertype:
        # Ensure that the item exists in subtype and has the same schema.
        for subitem in subtype:
            if item['name'] != subitem['name']:
                continue

            for key in set(item.keys()) | set(subitem.keys()):
                if key in ('label', 'description'):
                    # Label and description can differ.
                    continue
                elif key == 'required':
                    # A non-required item can be made required in subtype, but not the
                    # other way around.
                    item_required = item.get('required', True)
                    subitem_required = subitem.get('required', False)

                    if item_required and not subitem_required:
                        errors.append("Field '{}' is marked as required in '{}' and optional in '{}'.".format(
                            item['name'],
                            supertype_name,
                            subtype_name,
                        ))
                elif item.get(key, None) != subitem.get(key, None):
                    errors.append("Schema for field '{}' in type '{}' does not match supertype '{}'.".format(
                        item['name'],
                        subtype_name,
                        supertype_name
                    ))

            break
        else:
            errors.append("Schema for type '{}' is missing supertype '{}' field '{}'.".format(
                subtype_name,
                supertype_name,
                item['name']
            ))

    return errors
206244
def _validated(self, data):
        """"""
        errors = []
        for sub in self.schemas:
            try:
                return sub(data)
            except NotValid as ex:
                errors.extend(ex.args)

        raise NotValid(' and '.join(errors))
1114482
def nf_step_to_process(step, out_handle):
    
    pprint.pprint(step)

    directives = []
    for req in step["task_definition"]["requirements"]:
        if req["requirement_type"] == "docker":
            directives.append("container '%s'" % req["value"])
        elif req["requirement_type"] == "cpu":
            directives.append("cpus %s" % req["value"])
        elif req["requirement_type"] == "memory":
            directives.append("memory '%s'" % req["value"])

    task_id = step["task_id"]
    directives = "\n  ".join(directives)
    inputs = "\n    ".join(nf_io_to_process(step["inputs"], step["task_definition"]["inputs"],
                                        step["scatter"]))
    outputs = "\n    ".join(nf_io_to_process(step["outputs"], step["task_definition"]["outputs"]))
    commandline = (step["task_definition"]["baseCommand"] + " " +
                   " ".join([nf_input_to_cl(i) for i in step["task_definition"]["inputs"]]))
    out_handle.write(_nf_process_tmpl.format(**locals()))