Reads flow results of a given flow using given query options.
194607
def ReadFlowResults(self,
                      client_id,
                      flow_id,
                      offset,
                      count,
                      with_tag=None,
                      with_type=None,
                      with_substring=None,
                      cursor=None):
    """"""

    query = ("SELECT payload, type, UNIX_TIMESTAMP(timestamp), tag "
             "FROM flow_results "
             "FORCE INDEX (flow_results_by_client_id_flow_id_timestamp) "
             "WHERE client_id = %s AND flow_id = %s ")
    args = [db_utils.ClientIDToInt(client_id), db_utils.FlowIDToInt(flow_id)]

    if with_tag is not None:
      query += "AND tag = %s "
      args.append(with_tag)

    if with_type is not None:
      query += "AND type = %s "
      args.append(with_type)

    if with_substring is not None:
      query += "AND payload LIKE %s "
      args.append("%{}%".format(with_substring))

    query += "ORDER BY timestamp ASC LIMIT %s OFFSET %s"
    args.append(count)
    args.append(offset)

    cursor.execute(query, args)

    ret = []
    for serialized_payload, payload_type, ts, tag in cursor.fetchall():
      if payload_type in rdfvalue.RDFValue.classes:
        payload = rdfvalue.RDFValue.classes[payload_type]()
        payload.ParseFromString(serialized_payload)
      else:
        payload = rdf_objects.SerializedValueOfUnrecognizedType(
            type_name=payload_type, value=serialized_payload)

      timestamp = mysql_utils.TimestampToRDFDatetime(ts)
      result = rdf_flow_objects.FlowResult(payload=payload, timestamp=timestamp)
      if tag:
        result.tag = tag

      ret.append(result)

    return ret
210470
def ReadHuntFlowsStatesAndTimestamps(self, hunt_id, cursor=None):
    """"""

    query = """
      SELECT
        flow_state, UNIX_TIMESTAMP(timestamp), UNIX_TIMESTAMP(last_update)
      FROM flows
      FORCE INDEX(flows_by_hunt)
      WHERE parent_hunt_id = %s AND parent_flow_id IS NULL
    """

    cursor.execute(query, [db_utils.HuntIDToInt(hunt_id)])

    result = []
    for fs, ct, lup in cursor.fetchall():
      result.append(
          db.FlowStateAndTimestamps(
              flow_state=rdf_flow_objects.Flow.FlowState.FromInt(fs),
              create_time=mysql_utils.TimestampToRDFDatetime(ct),
              last_update_time=mysql_utils.TimestampToRDFDatetime(lup)))

    return result
812191
def ReadFlowObject(self, client_id, flow_id, cursor=None):
    """"""
    query = ("SELECT " + self.FLOW_DB_FIELDS +
             "FROM flows WHERE client_id=%s AND flow_id=%s")
    cursor.execute(
        query,
        [db_utils.ClientIDToInt(client_id),
         db_utils.FlowIDToInt(flow_id)])
    result = cursor.fetchall()
    if not result:
      raise db.UnknownFlowError(client_id, flow_id)
    row, = result
    return self._FlowObjectFromRow(row)
1179523
def perfom_query(iterator, &block)
      if example
        connection.by_example(example, options).each(&iterator)
      else
        connection.all(options).each(&iterator)
      end
    end
364343
def CountFlowResultsByType(self, client_id, flow_id):
    """"""
    result = collections.Counter()
    for hr in self.ReadFlowResults(client_id, flow_id, 0, sys.maxsize):
      key = compatibility.GetName(hr.payload.__class__)
      result[key] += 1

    return result
354554
def _ReadAndLockNextRequestsToProcess(self, flow_keys, cursor):
    """"""

    query = """
      SELECT client_id, flow_id, next_request_to_process
      FROM flows
      WHERE {conditions}
      FOR UPDATE
    """
    condition_template = "(client_id = %s AND flow_id = %s)"
    conditions = [condition_template] * len(flow_keys)
    query = query.format(conditions=" OR ".join(conditions))
    args = []
    for client_id, flow_id in flow_keys:
      args.append(db_utils.ClientIDToInt(client_id))
      args.append(db_utils.FlowIDToInt(flow_id))

    cursor.execute(query, args)
    next_requests = {}
    for client_id_int, flow_id_int, next_request in cursor.fetchall():
      flow_key = (db_utils.IntToClientID(client_id_int),
                  db_utils.IntToFlowID(flow_id_int))
      next_requests[flow_key] = next_request
    return next_requests
79550
def _ReadFlowResponseCounts(self, request_keys, cursor=None):
    """"""

    query = """
      SELECT
        flow_requests.client_id, flow_requests.flow_id,
        flow_requests.request_id, COUNT(*)
      FROM flow_responses, flow_requests
      WHERE ({conditions}) AND
        flow_requests.client_id = flow_responses.client_id AND
        flow_requests.flow_id = flow_responses.flow_id AND
        flow_requests.request_id = flow_responses.request_id AND
        flow_requests.needs_processing = FALSE
      GROUP BY
        flow_requests.client_id,
        flow_requests.flow_id,
        flow_requests.request_id
    """

    condition_template = """
      (flow_requests.client_id=%s AND
       flow_requests.flow_id=%s AND
       flow_requests.request_id=%s)"""
    conditions = [condition_template] * len(request_keys)
    args = []
    for client_id, flow_id, request_id in request_keys:
      args.append(db_utils.ClientIDToInt(client_id))
      args.append(db_utils.FlowIDToInt(flow_id))
      args.append(request_id)

    query = query.format(conditions=" OR ".join(conditions))
    cursor.execute(query, args)
    response_counts = {}
    for (client_id_int, flow_id_int, request_id, count) in cursor.fetchall():
      request_key = (db_utils.IntToClientID(client_id_int),
                     db_utils.IntToFlowID(flow_id_int), request_id)
      response_counts[request_key] = count
    return response_counts
1277081
def _LoadFlows(self, client_id, min_create_time, token):
    
    if data_store.RelationalDBEnabled():
      if isinstance(client_id, rdfvalue.RDFURN):
        client_id = client_id.Basename()

      flow_list = data_store.REL_DB.ReadAllFlowObjects(
          client_id=client_id,
          min_create_time=min_create_time,
          include_child_flows=False)
      for flow_obj in flow_list:
        yield flow_obj
    else:
      now = rdfvalue.RDFDatetime.Now()
      client_id_urn = rdf_client.ClientURN(client_id)
      flows_dir = aff4.FACTORY.Open(client_id_urn.Add("flows"), token=token)
      # Save DB roundtrips by checking both conditions at once.
      flow_list = flows_dir.ListChildren(
          age=(min_create_time.AsMicrosecondsSinceEpoch(),
               now.AsMicrosecondsSinceEpoch()))
      for flow_obj in aff4.FACTORY.MultiOpen(flow_list, token=token):
        yield rdf_flow_objects.Flow(
            args=flow_obj.args,
            flow_class_name=flow_obj.runner_args.flow_name,
            flow_id=flow_obj.urn.Basename(),
            create_time=flow_obj.context.create_time,
            creator=flow_obj.creator,
        )
962623
protected void flow(Env<AttrContext> env, Queue<Env<AttrContext>> results) {
        if (compileStates.isDone(env, CompileState.FLOW)) {
            results.add(env);
            return;
        }

        try {
            if (shouldStop(CompileState.FLOW))
                return;

            if (relax) {
                results.add(env);
                return;
            }

            if (verboseCompilePolicy)
                printNote("[flow " + env.enclClass.sym + "]");
            JavaFileObject prev = log.useSource(
                                                env.enclClass.sym.sourcefile != null ?
                                                env.enclClass.sym.sourcefile :
                                                env.toplevel.sourcefile);
            try {
                make.at(Position.FIRSTPOS);
                TreeMaker localMake = make.forToplevel(env.toplevel);
                flow.analyzeTree(env, localMake);
                compileStates.put(env, CompileState.FLOW);

                if (shouldStop(CompileState.FLOW))
                    return;

                results.add(env);
            }
            finally {
                log.useSource(prev);
            }
        }
        finally {
            if (!taskListener.isEmpty()) {
                TaskEvent e = new TaskEvent(TaskEvent.Kind.ANALYZE, env.toplevel, env.enclClass.sym);
                taskListener.finished(e);
            }
        }
    }
397502
@ReadOperation
    public Map<?, ?> getReport(@Nullable final String flowId) {
        val jsonMap = new LinkedHashMap<String, Object>();
        val map = this.applicationContext.getBeansOfType(FlowDefinitionRegistry.class, false, true);

        map.forEach((k, v) -> Arrays.stream(v.getFlowDefinitionIds())
            .filter(currentId -> {
                if (StringUtils.isNotBlank(flowId)) {
                    return flowId.equalsIgnoreCase(currentId);
                }
                return true;
            })
            .forEach(id -> {
                val flowDetails = new LinkedHashMap<String, Object>();
                val def = Flow.class.cast(v.getFlowDefinition(id));
                flowDetails.put("startState", def.getStartState().getId());

                val states = new LinkedHashMap<String, Map>();
                Arrays.stream(def.getStateIds()).forEach(st -> {

                    val state = (State) def.getState(st);
                    val stateMap = new LinkedHashMap<String, Object>();

                    if (!state.getAttributes().asMap().isEmpty()) {
                        stateMap.put("attributes", CollectionUtils.wrap(state.getAttributes()));
                    }
                    if (StringUtils.isNotBlank(state.getCaption())) {
                        stateMap.put("caption", state.getCaption());
                    }

                    var acts = StreamSupport.stream(state.getEntryActionList().spliterator(), false)
                        .map(Object::toString)
                        .collect(Collectors.toList());

                    if (!acts.isEmpty()) {
                        stateMap.put("entryActions", acts);
                    }

                    if (state instanceof ActionState) {
                        acts = StreamSupport.stream(ActionState.class.cast(state).getActionList().spliterator(), false)
                            .map(Object::toString)
                            .collect(Collectors.toList());
                        if (!acts.isEmpty()) {
                            stateMap.put("actionList", acts);
                        }
                    }

                    if (state instanceof EndState) {
                        stateMap.put("isEndState", Boolean.TRUE);
                    }
                    if (state.isViewState()) {
                        stateMap.put("isViewState", state.isViewState());
                        stateMap.put("isRedirect", ((ViewState) state).getRedirect());

                        acts = StreamSupport.stream(state.getEntryActionList().spliterator(), false)
                            .map(Object::toString)
                            .collect(Collectors.toList());

                        if (!acts.isEmpty()) {
                            stateMap.put("renderActions", ((ViewState) state).getRenderActionList());
                        }

                        acts = Arrays.stream(((ViewState) state).getVariables())
                            .map(value -> value.getName() + " -> " + value.getValueFactory().toString())
                            .collect(Collectors.toList());

                        if (!acts.isEmpty()) {
                            stateMap.put("viewVariables", acts);
                        }

                        val field = ReflectionUtils.findField(((ViewState) state).getViewFactory().getClass(), "viewId");
                        if (field != null) {
                            ReflectionUtils.makeAccessible(field);
                            val exp = (Expression) ReflectionUtils.getField(field, ((ViewState) state).getViewFactory());
                            stateMap.put("viewId", StringUtils.defaultIfBlank(exp.getExpressionString(), exp.getValue(null).toString()));
                        } else {
                            LOGGER.warn("Field viewId cannot be located on view state [{}]", state);
                        }
                    }

                    if (state instanceof TransitionableState) {
                        val stDef = TransitionableState.class.cast(state);

                        acts = StreamSupport.stream(stDef.getExitActionList().spliterator(), false)
                            .map(Object::toString)
                            .collect(Collectors.toList());

                        if (!acts.isEmpty()) {
                            stateMap.put("exitActions", acts);
                        }

                        acts = Arrays.stream(stDef.getTransitions())
                            .map(tr -> tr.getId() + " -> " + tr.getTargetStateId())
                            .collect(Collectors.toList());

                        if (!acts.isEmpty()) {
                            stateMap.put("transitions", acts);
                        }
                    }
                    states.put(st, stateMap);
                });
                flowDetails.put("states", states);
                flowDetails.put("possibleOutcomes", def.getPossibleOutcomes());
                flowDetails.put("stateCount", def.getStateCount());

                var acts = StreamSupport.stream(def.getEndActionList().spliterator(), false)
                    .map(Object::toString)
                    .collect(Collectors.toList());
                if (!acts.isEmpty()) {
                    flowDetails.put("endActions", acts);
                }

                acts = StreamSupport.stream(def.getGlobalTransitionSet().spliterator(), false)
                    .map(tr -> tr.getId() + " -> " + tr.getTargetStateId() + " @ " + tr.getExecutionCriteria().toString())
                    .collect(Collectors.toList());
                if (!acts.isEmpty()) {
                    flowDetails.put("globalTransitions", acts);
                }

                acts = Arrays.stream(def.getExceptionHandlerSet().toArray())
                    .map(Object::toString)
                    .collect(Collectors.toList());
                if (!acts.isEmpty()) {
                    flowDetails.put("exceptionHandlers", acts);
                }

                val vars = Arrays.stream(def.getVariables())
                    .map(FlowVariable::getName)
                    .collect(Collectors.joining(","));

                if (StringUtils.isNotBlank(vars)) {
                    flowDetails.put("variables", vars);
                }

                jsonMap.put(id, flowDetails);
            }));

        return jsonMap;
    }
808089
def FromResponses(cls, request=None, responses=None):
    """"""
    res = cls()
    res.request = request
    if request:
      res.request_data = request.request_data

    for r in responses or []:
      if isinstance(r, rdf_flow_objects.FlowResponse):
        res.responses.append(r.payload)
      elif isinstance(r, rdf_flow_objects.FlowStatus):
        res.status = r
        res.success = r.status == "OK"
      elif isinstance(r, rdf_flow_objects.FlowIterator):
        pass
      else:
        raise TypeError("Got unexpected response type: %s" % type(r))
    return res
765311
def _query(options, collection_name, num_to_skip,
           num_to_return, query, field_selector, opts, check_keys):
    """"""
    encoded = _dict_to_bson(query, check_keys, opts)
    if field_selector:
        efs = _dict_to_bson(field_selector, False, opts)
    else:
        efs = b""
    max_bson_size = max(len(encoded), len(efs))
    return b"".join([
        _pack_int(options),
        _make_c_string(collection_name),
        _pack_int(num_to_skip),
        _pack_int(num_to_return),
        encoded,
        efs]), max_bson_size
29914
def ProcessFlow(self, flow_processing_request):
    """"""

    client_id = flow_processing_request.client_id
    flow_id = flow_processing_request.flow_id

    data_store.REL_DB.AckFlowProcessingRequests([flow_processing_request])

    try:
      rdf_flow = data_store.REL_DB.LeaseFlowForProcessing(
          client_id, flow_id, processing_time=rdfvalue.Duration("6h"))
    except db.ParentHuntIsNotRunningError:
      flow_base.TerminateFlow(client_id, flow_id, "Parent hunt stopped.")
      return

    logging.info("Processing Flow %s/%s (%s).", client_id, flow_id,
                 rdf_flow.flow_class_name)

    flow_cls = registry.FlowRegistry.FlowClassByName(rdf_flow.flow_class_name)
    flow_obj = flow_cls(rdf_flow)

    if not flow_obj.IsRunning():
      raise ValueError(
          "Received a request to process flow %s on client %s that is not "
          "running." % (flow_id, client_id))

    processed = flow_obj.ProcessAllReadyRequests()
    if processed == 0:
      raise ValueError(
          "Unable to process any requests for flow %s on client %s." %
          (flow_id, client_id))

    while not self._ReleaseProcessedFlow(flow_obj):
      processed = flow_obj.ProcessAllReadyRequests()
      if processed == 0:
        raise ValueError(
            "%s/%s: ReleaseProcessedFlow returned false but no "
            "request could be processed (next req: %d)." %
            (client_id, flow_id, flow_obj.rdf_flow.next_request_to_process))
379705
def read_results(self):
        """"""
        
        if self.submited:
            # The job has been submitted. Check the state.
            if not self.job_ready() :
                if self.block :
                    while not self.job_ready() :
                        time.sleep(self.job_check_time)
                else :
                    raise CalcNotReadyError
                    

            # Assume the calc finished. Copy the files back.
            subprocess.call(self.copy_in_cmd % {
                'ldir': self.wdir,
                'rdir': os.path.join(self.parameters['rdir'],os.path.split(self.directory)[-1]),
                'user': self.parameters['user'],
                'host': self.parameters['host']
            }, shell=True)
                
        
        fn=os.path.join(self.directory,'pw.out')
        # Read the pan-ultimate line of the output file
        try: 
            ln=open(fn).readlines()[-2]
            if ln.find('JOB DONE.')>-1 :
                # Job is done we can read the output
                r=read_quantumespresso_textoutput(fn)
                self.submited=False
                self.jobid=None
            else :
                # Job not ready.
                raise CalcNotReadyError
        except (IOError, IndexError) :
            # Job not ready.
            raise CalcNotReadyError
        
        # All is fine - really read the results
        self.calc.read_results(self)
862490
def CountFlowResults(self,
                       client_id,
                       flow_id,
                       with_tag=None,
                       with_type=None,
                       cursor=None):
    """"""
    query = ("SELECT COUNT(*) "
             "FROM flow_results "
             "FORCE INDEX (flow_results_by_client_id_flow_id_timestamp) "
             "WHERE client_id = %s AND flow_id = %s ")
    args = [db_utils.ClientIDToInt(client_id), db_utils.FlowIDToInt(flow_id)]

    if with_tag is not None:
      query += "AND tag = %s "
      args.append(with_tag)

    if with_type is not None:
      query += "AND type = %s "
      args.append(with_type)

    cursor.execute(query, args)
    return cursor.fetchone()[0]
1108950
def InitFromFlowResult(self, flow_result):
    """"""

    self.payload_type = compatibility.GetName(flow_result.payload.__class__)
    self.payload = flow_result.payload
    self.client_id = flow_result.client_id
    self.timestamp = flow_result.timestamp

    return self
295669
private Map<Optional<Executor>, List<ExecutableFlow>> getFlowToExecutorMap() {
    final HashMap<Optional<Executor>, List<ExecutableFlow>> exFlowMap = new HashMap<>();
    try {
      for (final Pair<ExecutionReference, ExecutableFlow> runningFlow : this
          .executorLoader.fetchActiveFlows().values()) {
        final Optional<Executor> executor = runningFlow.getFirst().getExecutor();
        List<ExecutableFlow> flows = exFlowMap.get(executor);
        if (flows == null) {
          flows = new ArrayList<>();
          exFlowMap.put(executor, flows);
        }
        flows.add(runningFlow.getSecond());
      }
    } catch (final ExecutorManagerException e) {
      logger.error("Failed to get flow to executor map");
    }
    return exFlowMap;
  }