Manager a topology
1. Instantiate necessary resources
2. Valid whether the runtime management is legal
3. Complete the runtime management for a specific command
499524
public void manageTopology()
      throws TopologyRuntimeManagementException, TMasterException, PackingException {
    String topologyName = Context.topologyName(config);
    // 1. Do prepare work
    // create an instance of state manager
    String statemgrClass = Context.stateManagerClass(config);
    IStateManager statemgr;
    try {
      statemgr = ReflectionUtils.newInstance(statemgrClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
      throw new TopologyRuntimeManagementException(String.format(
          "Failed to instantiate state manager class '%s'",
          statemgrClass), e);
    }

    // Put it in a try block so that we can always clean resources
    try {
      // initialize the statemgr
      statemgr.initialize(config);

      // TODO(mfu): timeout should read from config
      SchedulerStateManagerAdaptor adaptor = new SchedulerStateManagerAdaptor(statemgr, 5000);

      boolean hasExecutionData = validateRuntimeManage(adaptor, topologyName);

      // 2. Try to manage topology if valid
      // invoke the appropriate command to manage the topology
      LOG.log(Level.FINE, "Topology: {0} to be {1}ed", new Object[]{topologyName, command});

      // build the runtime config
      Config runtime = Config.newBuilder()
          .put(Key.TOPOLOGY_NAME, Context.topologyName(config))
          .put(Key.SCHEDULER_STATE_MANAGER_ADAPTOR, adaptor)
          .build();

      // Create a ISchedulerClient basing on the config
      ISchedulerClient schedulerClient = getSchedulerClient(runtime);

      callRuntimeManagerRunner(runtime, schedulerClient, !hasExecutionData);
    } finally {
      // 3. Do post work basing on the result
      // Currently nothing to do here

      // 4. Close the resources
      SysUtils.closeIgnoringExceptions(statemgr);
    }
  }
506916
@Override
    protected void executeImpl(CommandLine cmd) {
        try {
            List<PropertyDefinition> defintions =
                new PropertyDefinitionListBuilder().addMillDb()
                                                   .build();
            PropertyVerifier verifier = new PropertyVerifier(defintions);
            verifier.verify(System.getProperties());
            ApplicationContext context =
                new AnnotationConfigApplicationContext("org.duracloud.mill.db",
                                                       "org.duracloud.mill.manifest.builder");
            log.info("context initialized");

            boolean dryRun = Boolean.valueOf(cmd.hasOption("d"));
            boolean clean = Boolean.valueOf(cmd.hasOption("C"));
            String host = cmd.getOptionValue("host");
            String port = cmd.getOptionValue("port", "443");
            String username = cmd.getOptionValue("u");
            String password = cmd.getOptionValue("p");
            List<String> spaceList = new LinkedList<>();
            if (cmd.hasOption("s")) {
                String spaces = cmd.getOptionValue("s", "");
                if (!StringUtils.isEmpty(spaces)) {
                    spaceList = Arrays.asList(spaces.split("[,]"));
                }
            }

            List<String> storeList = new LinkedList<>();

            if (cmd.hasOption("t")) {
                String stores = cmd.getOptionValue("t", "");
                if (!StringUtils.isEmpty(stores)) {
                    storeList = Arrays.asList(stores.split("[,]"));
                }
            }

            int threads = 0;
            if (cmd.hasOption("T")) {
                threads = Integer.parseInt(cmd.getOptionValue("T", "10"));
            }

            String account = host.split("[.]")[0];
            ContentStoreManager storeManager = new ContentStoreManagerImpl(host, port);
            storeManager.login(new Credential(username, password));
            Map<String, ContentStore> contentStoreMap = storeManager.getContentStores();
            List<ContentStore> contentStores = new LinkedList<>();
            for (String storeId : contentStoreMap.keySet()) {
                if (storeList.isEmpty() || storeList.contains(storeId)) {
                    ContentStore contentStore = contentStoreMap.get(storeId);
                    contentStores.add(contentStore);
                }
            }

            ManifestBuilder builder = (ManifestBuilder) context.getBean(ManifestBuilder.class);
            builder.init(account, contentStores, spaceList, clean, dryRun, threads);
            builder.execute();
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        } finally {
            log.info("exiting...");
        }
    }
1079563
private static TopologyAPI.TopologyState getRuntimeTopologyState(
      String topologyName,
      SchedulerStateManagerAdaptor statemgr) throws TMasterException {
    PhysicalPlans.PhysicalPlan plan = statemgr.getPhysicalPlan(topologyName);

    if (plan == null) {
      throw new TMasterException(String.format(
        "Failed to get physical plan for topology '%s'", topologyName));
    }

    return plan.getTopology().getState();
  }
1096055
public Config createPrimaryRuntime(TopologyAPI.Topology topology) {
    return Config.newBuilder()
        .put(Key.TOPOLOGY_ID, topology.getId())
        .put(Key.TOPOLOGY_NAME, topology.getName())
        .put(Key.TOPOLOGY_DEFINITION, topology)
        .put(Key.NUM_CONTAINERS, 1 + TopologyUtils.getNumContainers(topology))
        .build();
  }
956716
def create_manager(arguments):
  """"""

  if arguments.statefile is None:
    arguments.statefile = os.path.join(os.path.dirname(arguments.logdir), 'submitted.db')

  arguments.statefile = os.path.realpath(arguments.statefile)

  return manager.JobManager(statefile=arguments.statefile)
842121
def _initialize_manager(self, runtime):
        """"""
        if self._runtime is not None:
            raise errors.IllegalState('this manager has already been initialized.')
        self._runtime = runtime
        self._config = runtime.get_configuration()
        set_json_client(runtime)
492174
def main(argv=None):
    """"""
   
    program_name = __programm_name__
    program_version = "v%s" % __version__
    program_descrption = __programm_description__

    try:
        # Setup argument parser
        parser = ArgumentParser(prog=program_name, description=program_descrption)
        parser.add_argument("-v", "--verbose", dest="verbose", action="count", help="run in verbose mode (-vvv for more, -vvvv to enable connection debugging)")
        parser.add_argument("-s", "--sudo", action="store_true", help="run supervisorctl actions with sudo (nopasswd))")
        parser.add_argument("-V", "--version", action="version", version=program_version)
        parser.add_argument("host-pattern", help="A host-pattern usually refers to a group of hosts. For more details, see Ansible documentation about Patterns.")
        #parser.add_argument("supervisorctl-action", help="A supervisorctl action (and optional argument). For more details, see Supervisor documentation about the available supervisorctl actions.")
        
        subparsers = parser.add_subparsers(help="One of the available supervisorctl actions.", dest="supervisorctl-action")
        subparsers.add_parser("status", help="Get status info of all processes.")
        subparsers.add_parser("reread", help="Reread the configuration files of supervisord")
        subparsers.add_parser("reload", help="Restart remote supervisord")
        subparsers.add_parser("update", help="Reload the configuration files of supervisord and add/remove processes as necessary")
        start_subparser = subparsers.add_parser("start", help="Start a process by name")
        start_subparser.add_argument("process-name", help="Name of the process")
        stop_subparser = subparsers.add_parser("stop", help="Stop a process by name")
        stop_subparser.add_argument("process-name", help="Name of the process")
        restart_subparser = subparsers.add_parser("restart", help="Restart a process by name")
        restart_subparser.add_argument("process-name", help="Name of the process")
        remove_subparser = subparsers.add_parser("remove", help="Remove a process by name")
        remove_subparser.add_argument("process-name", help="Name of the process")
        
        # Process arguments
        args = parser.parse_args(argv)
        verbose = args.verbose
        host_pattern = getattr(args, "host-pattern")
        supervisorctl_action = getattr(args, "supervisorctl-action")        
        
        sudo = args.sudo
        ansible_executable = "ansible"
        supervisorctl_executable = "supervisorctl"
        ansible_action_option = "-a"
        
        if sudo:
            supervisorctl_command = "sudo " + supervisorctl_executable + " " + supervisorctl_action
        else:
            supervisorctl_command = supervisorctl_executable + " " + supervisorctl_action
          
        if supervisorctl_action not in ['status', 'reread', 'update', 'reload']:
            supervisorctl_argument = getattr(args, "process-name")
            supervisorctl_command = supervisorctl_command + ' ' + supervisorctl_argument
            
        if verbose >0:
            verbose_level = "-"+ "v"*verbose
            print("Verbose mode on: " + verbose_level)
            print "Parsed arguments:"
            print args   
            retcode = call([ansible_executable, host_pattern, verbose_level, ansible_action_option, supervisorctl_command,])
        else:
            retcode = call([ansible_executable, host_pattern, ansible_action_option, supervisorctl_command])
        
        return retcode
    except KeyboardInterrupt:
        ### handle keyboard interrupt ###
        return 0
1271021
def create_system(self, **system_options):
        
        if self.master is None:
            raise ValueError('Handler {} is not able to create systems.'.format(self))

        if isinstance(self.master, ForceField):
            system = self.master.createSystem(self.topology, **system_options)
        elif isinstance(self.master, (AmberPrmtopFile, GromacsTopFile, DesmondDMSFile)):
            system = self.master.createSystem(**system_options)
        elif isinstance(self.master, CharmmPsfFile):
            if not hasattr(self.master, 'parmset'):
                raise ValueError('PSF topology files require Charmm parameters.')
            system = self.master.createSystem(self.master.parmset, **system_options)
        else:
            raise NotImplementedError('Handler {} is not able to create systems.'.format(self))

        if self.has_box:
            system.setDefaultPeriodicBoxVectors(*self.box)
        return system
782598
def _submit_topology(cmd_args, app):
    
    cfg = app.cfg
    if cmd_args.create_bundle:
        ctxtype = ctx.ContextTypes.BUNDLE
    elif cmd_args.service_name:
        cfg[ctx.ConfigParams.FORCE_REMOTE_BUILD] = True
        cfg[ctx.ConfigParams.SERVICE_NAME] = cmd_args.service_name
        ctxtype = ctx.ContextTypes.STREAMING_ANALYTICS_SERVICE
    sr = ctx.submit(ctxtype, app.app, cfg)
    return sr
269486
def _configure_manager(self):
        
        self._manager = ContainerManager(self, resource_class=Container,
                response_key="", uri_base="")
942624
private static Options constructOptions() {
    Options options = new Options();

    Option topName = Option.builder("t")
        .desc("Name of the topology")
        .longOpt("topologyname")
        .hasArgs()
        .argName("topologyname")
        .required()
        .build();

    Option topId = Option.builder("i")
        .desc("Id of the topology")
        .longOpt("topologyid")
        .hasArgs()
        .argName("topologyid")
        .required()
        .build();

    Option ckptMgrId = Option.builder("c")
        .desc("Id of the checkpoint manager")
        .longOpt("ckptmgrid")
        .hasArgs()
        .argName("ckptmgrid")
        .required()
        .build();

    Option ckptMgrPort = Option.builder("p")
        .desc("Port of the checkpoint manager")
        .longOpt("ckptmgrport")
        .hasArgs()
        .argName("ckptmgrport")
        .required()
        .build();

    Option ckptMgrConfig = Option.builder("f")
        .desc("Config name of the checkpoint manager")
        .longOpt("ckptmgrconfig")
        .hasArgs()
        .argName("ckptmgrconfig")
        .required()
        .build();

    Option ckptMgrOverridenConfig = Option.builder("o")
        .desc("Config name of the checkpoint manager")
        .longOpt("ckptMgrOverridenConfig")
        .hasArgs()
        .argName("ckptMgrOverridenConfig")
        .build();

    Option heronInternalConfig = Option.builder("g")
        .desc("Heron internal config filename")
        .longOpt("heroninternalconfig")
        .hasArgs()
        .argName("heroninternalconfig")
        .required()
        .build();

    options.addOption(topName);
    options.addOption(topId);
    options.addOption(ckptMgrId);
    options.addOption(ckptMgrPort);
    options.addOption(ckptMgrConfig);
    options.addOption(ckptMgrOverridenConfig);
    options.addOption(heronInternalConfig);

    return options;
  }
801046
def get_a_manager(threadPool_settings=None):
    
    global __singleton_manager
    if __singleton_manager is None:
        def _thread_entry():
            if prctl:
                prctl.set_name('mirte manager')
            m.run()
            l.info('manager.run() returned')
        l = logging.getLogger('mirte.get_a_manager')
        l.info("Creating new instance")
        m = Manager(logging.getLogger('mirte'))
        if threadPool_settings:
            m.update_instance('threadPool', threadPool_settings)
        threading.Thread(target=_thread_entry).start()
        m.running_event.wait()
        __singleton_manager = m
    return __singleton_manager
1059427
@Override
  public boolean onSchedule(PackingPlan packing) {
    LOG.info("Starting to deploy topology: " + LocalContext.topologyName(config));

    synchronized (processToContainer) {
      LOG.info("Starting executor for TMaster");
      startExecutor(0, null);

      // for each container, run its own executor
      for (PackingPlan.ContainerPlan container : packing.getContainers()) {
        startExecutor(container.getId(), container.getInstances());
      }
    }

    LOG.info("Executor for each container have been started.");

    return true;
  }
873848
def _manage_commands():
    
    for name, (command, *aliases) in COMMANDS.items():
        COMMAND_MAPPER[name] = command
        for alias in aliases:
            COMMAND_MAPPER[alias] = command
        COMMAND_ALIASES[name] = aliases
27
def get_manager_cmd(self):
        """"""
        cmd = os.path.abspath(os.path.join(os.path.dirname(__file__), "server", "notebook_daemon.py"))
        assert os.path.exists(cmd)
        return cmd
298724
def main():
    
    arg_parse = setup_argparse()
    args = arg_parse.parse_args()

    if not args.quiet:
        print('GNS3 Topology Converter')

    if args.debug:
        logging_level = logging.DEBUG
    else:
        logging_level = logging.WARNING

    logging.basicConfig(level=logging_level,
                        format=LOG_MSG_FMT, datefmt=LOG_DATE_FMT)

    logging.getLogger(__name__)

    # Add the main topology to the list of files to convert
    if args.topology == 'topology.net':
        args.topology = os.path.join(os.getcwd(), 'topology.net')

    topology_files = [{'file': topology_abspath(args.topology),
                       'snapshot': False}]

    # Add any snapshot topologies to be converted
    topology_files.extend(get_snapshots(args.topology))

    topology_name = name(args.topology, args.name)

    # Do the conversion
    for topology in topology_files:
        do_conversion(topology, topology_name, args.output, args.debug)
1305334
def _build_cmdline(self, args, env={}):
        
        if self._user is None:
            return super(RunExecutor, self)._build_cmdline(args, env)
        result = _SUDO_ARGS + [self._user]
        for var, value in env.items():
            result.append(var + '=' + value)
        return result + ['--'] + args