Displays a confirmation message showing the configuration of the
 Elastic MapReduce job flow and cluster.
480159
def confirm_message
      cluster = Cluster.new

      case self.type
      when 'Elasticrawl::ParseJob'
        message = segment_list
      else
        message = []
      end

      message.push('Job configuration')
      message.push(self.job_desc)
      message.push('')
      message.push(cluster.cluster_desc)

      message.join("\n")
    end
1328067
def main():
    

    template = Template()

    # Description
    template.add_description(
        'AWS CloudFormation Sample Template ElastiCache_Redis:'
        'Sample template showing how to create an Amazon'
        'ElastiCache Redis Cluster. **WARNING** This template'
        'creates an Amazon EC2 Instance and an Amazon ElastiCache'
        'Cluster. You will be billed for the AWS resources used'
        'if you create a stack from this template.')

    # Mappings
    template.add_mapping('AWSInstanceType2Arch', {
        't1.micro':     {'Arch': 'PV64'},
        't2.micro':     {'Arch': 'HVM64'},
        't2.small':     {'Arch': 'HVM64'},
        't2.medium':    {'Arch': 'HVM64'},
        'm1.small':     {'Arch': 'PV64'},
        'm1.medium':    {'Arch': 'PV64'},
        'm1.large':     {'Arch': 'PV64'},
        'm1.xlarge':    {'Arch': 'PV64'},
        'm2.xlarge':    {'Arch': 'PV64'},
        'm2.2xlarge':   {'Arch': 'PV64'},
        'm2.4xlarge':   {'Arch': 'PV64'},
        'm3.medium':    {'Arch': 'HVM64'},
        'm3.large':     {'Arch': 'HVM64'},
        'm3.xlarge':    {'Arch': 'HVM64'},
        'm3.2xlarge':   {'Arch': 'HVM64'},
        'c1.medium':    {'Arch': 'PV64'},
        'c1.xlarge':    {'Arch': 'PV64'},
        'c3.large':     {'Arch': 'HVM64'},
        'c3.xlarge':    {'Arch': 'HVM64'},
        'c3.2xlarge':   {'Arch': 'HVM64'},
        'c3.4xlarge':   {'Arch': 'HVM64'},
        'c3.8xlarge':   {'Arch': 'HVM64'},
        'c4.large':     {'Arch': 'HVM64'},
        'c4.xlarge':    {'Arch': 'HVM64'},
        'c4.2xlarge':   {'Arch': 'HVM64'},
        'c4.4xlarge':   {'Arch': 'HVM64'},
        'c4.8xlarge':   {'Arch': 'HVM64'},
        'g2.2xlarge':   {'Arch': 'HVMG2'},
        'r3.large':     {'Arch': 'HVM64'},
        'r3.xlarge':    {'Arch': 'HVM64'},
        'r3.2xlarge':   {'Arch': 'HVM64'},
        'r3.4xlarge':   {'Arch': 'HVM64'},
        'r3.8xlarge':   {'Arch': 'HVM64'},
        'i2.xlarge':    {'Arch': 'HVM64'},
        'i2.2xlarge':   {'Arch': 'HVM64'},
        'i2.4xlarge':   {'Arch': 'HVM64'},
        'i2.8xlarge':   {'Arch': 'HVM64'},
        'd2.xlarge':    {'Arch': 'HVM64'},
        'd2.2xlarge':   {'Arch': 'HVM64'},
        'd2.4xlarge':   {'Arch': 'HVM64'},
        'd2.8xlarge':   {'Arch': 'HVM64'},
        'hi1.4xlarge':  {'Arch': 'HVM64'},
        'hs1.8xlarge':  {'Arch': 'HVM64'},
        'cr1.8xlarge':  {'Arch': 'HVM64'},
        'cc2.8xlarge':  {'Arch': 'HVM64'}
        })

    template.add_mapping('AWSRegionArch2AMI', {
        'us-east-1': {'PV64': 'ami-0f4cfd64',
                      'HVM64': 'ami-0d4cfd66',
                      'HVMG2': 'ami-5b05ba30'},
        'us-west-2': {'PV64': 'ami-d3c5d1e3',
                      'HVM64': 'ami-d5c5d1e5',
                      'HVMG2': 'ami-a9d6c099'},
        'us-west-1': {'PV64': 'ami-85ea13c1',
                      'HVM64': 'ami-87ea13c3',
                      'HVMG2': 'ami-37827a73'},
        'eu-west-1': {'PV64': 'ami-d6d18ea1',
                      'HVM64': 'ami-e4d18e93',
                      'HVMG2': 'ami-72a9f105'},
        'eu-central-1': {'PV64': 'ami-a4b0b7b9',
                         'HVM64': 'ami-a6b0b7bb',
                         'HVMG2': 'ami-a6c9cfbb'},
        'ap-northeast-1': {'PV64': 'ami-1a1b9f1a',
                           'HVM64': 'ami-1c1b9f1c',
                           'HVMG2': 'ami-f644c4f6'},
        'ap-southeast-1': {'PV64': 'ami-d24b4280',
                           'HVM64': 'ami-d44b4286',
                           'HVMG2': 'ami-12b5bc40'},
        'ap-southeast-2': {'PV64': 'ami-ef7b39d5',
                           'HVM64': 'ami-db7b39e1',
                           'HVMG2': 'ami-b3337e89'},
        'sa-east-1': {'PV64': 'ami-5b098146',
                      'HVM64': 'ami-55098148',
                      'HVMG2': 'NOT_SUPPORTED'},
        'cn-north-1': {'PV64': 'ami-bec45887',
                       'HVM64': 'ami-bcc45885',
                       'HVMG2': 'NOT_SUPPORTED'}
        })

    template.add_mapping('Region2Principal', {
        'us-east-1': {'EC2Principal': 'ec2.amazonaws.com',
                      'OpsWorksPrincipal': 'opsworks.amazonaws.com'},
        'us-west-2': {'EC2Principal': 'ec2.amazonaws.com',
                      'OpsWorksPrincipal': 'opsworks.amazonaws.com'},
        'us-west-1': {'EC2Principal': 'ec2.amazonaws.com',
                      'OpsWorksPrincipal': 'opsworks.amazonaws.com'},
        'eu-west-1': {'EC2Principal': 'ec2.amazonaws.com',
                      'OpsWorksPrincipal': 'opsworks.amazonaws.com'},
        'ap-southeast-1': {'EC2Principal': 'ec2.amazonaws.com',
                           'OpsWorksPrincipal': 'opsworks.amazonaws.com'},
        'ap-northeast-1': {'EC2Principal': 'ec2.amazonaws.com',
                           'OpsWorksPrincipal': 'opsworks.amazonaws.com'},
        'ap-southeast-2': {'EC2Principal': 'ec2.amazonaws.com',
                           'OpsWorksPrincipal': 'opsworks.amazonaws.com'},
        'sa-east-1': {'EC2Principal': 'ec2.amazonaws.com',
                      'OpsWorksPrincipal': 'opsworks.amazonaws.com'},
        'cn-north-1': {'EC2Principal': 'ec2.amazonaws.com.cn',
                       'OpsWorksPrincipal': 'opsworks.amazonaws.com.cn'},
        'eu-central-1': {'EC2Principal': 'ec2.amazonaws.com',
                         'OpsWorksPrincipal': 'opsworks.amazonaws.com'}
        })

    # Parameters
    cachenodetype = template.add_parameter(Parameter(
        'ClusterNodeType',
        Description='The compute and memory capacity of the nodes in the Redis'
                    ' Cluster',
        Type='String',
        Default='cache.m1.small',
        AllowedValues=['cache.m1.small',
                       'cache.m1.large',
                       'cache.m1.xlarge',
                       'cache.m2.xlarge',
                       'cache.m2.2xlarge',
                       'cache.m2.4xlarge',
                       'cache.c1.xlarge'],
        ConstraintDescription='must select a valid Cache Node type.',
        ))

    instancetype = template.add_parameter(Parameter(
        'InstanceType',
        Description='WebServer EC2 instance type',
        Type='String',
        Default='t2.micro',
        AllowedValues=['t1.micro',
                       't2.micro',
                       't2.small',
                       't2.medium',
                       'm1.small',
                       'm1.medium',
                       'm1.large',
                       'm1.xlarge',
                       'm2.xlarge',
                       'm2.2xlarge',
                       'm2.4xlarge',
                       'm3.medium',
                       'm3.large',
                       'm3.xlarge',
                       'm3.2xlarge',
                       'c1.medium',
                       'c1.xlarge',
                       'c3.large',
                       'c3.xlarge',
                       'c3.2xlarge',
                       'c3.4xlarge',
                       'c3.8xlarge',
                       'c4.large',
                       'c4.xlarge',
                       'c4.2xlarge',
                       'c4.4xlarge',
                       'c4.8xlarge',
                       'g2.2xlarge',
                       'r3.large',
                       'r3.xlarge',
                       'r3.2xlarge',
                       'r3.4xlarge',
                       'r3.8xlarge',
                       'i2.xlarge',
                       'i2.2xlarge',
                       'i2.4xlarge',
                       'i2.8xlarge',
                       'd2.xlarge',
                       'd2.2xlarge',
                       'd2.4xlarge',
                       'd2.8xlarge',
                       'hi1.4xlarge',
                       'hs1.8xlarge',
                       'cr1.8xlarge',
                       'cc2.8xlarge',
                       'cg1.4xlarge'],
        ConstraintDescription='must be a valid EC2 instance type.',
        ))

    keyname = template.add_parameter(Parameter(
        'KeyName',
        Description='Name of an existing EC2 KeyPair to enable SSH access'
                    ' to the instance',
        Type='AWS::EC2::KeyPair::KeyName',
        ConstraintDescription='must be the name of an existing EC2 KeyPair.',
        ))

    sshlocation = template.add_parameter(Parameter(
        'SSHLocation',
        Description='The IP address range that can be used to SSH to'
                    ' the EC2 instances',
        Type='String',
        MinLength='9',
        MaxLength='18',
        Default='0.0.0.0/0',
        AllowedPattern='(\\d{1,3})\\.(\\d{1,3})\\.'
                       '(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})',
        ConstraintDescription='must be a valid IP CIDR range of the'
                              ' form x.x.x.x/x.'
        ))

    # Resources
    webserverrole = template.add_resource(iam.Role(
        'WebServerRole',
        AssumeRolePolicyDocument=Policy(
            Statement=[
                Statement(
                    Effect=Allow,
                    Action=[AssumeRole],
                    Principal=Principal('Service',
                                        [FindInMap('Region2Principal',
                                                   Ref('AWS::Region'),
                                                   'EC2Principal')]),
                    )
                ]
            ),
        Path='/',
    ))

    template.add_resource(iam.PolicyType(
        'WebServerRolePolicy',
        PolicyName='WebServerRole',
        PolicyDocument=awacs.aws.Policy(
            Statement=[awacs.aws.Statement(
                Action=[awacs.aws.Action("elasticache",
                        "DescribeCacheClusters")],
                Resource=["*"],
                Effect=awacs.aws.Allow
            )]
        ),
        Roles=[Ref(webserverrole)],
    ))

    webserverinstanceprofile = template.add_resource(iam.InstanceProfile(
        'WebServerInstanceProfile',
        Path='/',
        Roles=[Ref(webserverrole)],
    ))

    webserversg = template.add_resource(ec2.SecurityGroup(
        'WebServerSecurityGroup',
        GroupDescription='Enable HTTP and SSH access',
        SecurityGroupIngress=[
            ec2.SecurityGroupRule(
                IpProtocol='tcp',
                FromPort='22',
                ToPort='22',
                CidrIp=Ref(sshlocation),
                ),
            ec2.SecurityGroupRule(
                IpProtocol='tcp',
                FromPort='80',
                ToPort='80',
                CidrIp='0.0.0.0/0',
                )
            ]
        ))

    webserverinstance = template.add_resource(ec2.Instance(
        'WebServerInstance',
        Metadata=cloudformation.Metadata(
            cloudformation.Init({
                'config': cloudformation.InitConfig(
                    packages={
                        'yum': {
                            'httpd':     [],
                            'php':       [],
                            'php-devel': [],
                            'gcc':       [],
                            'make':      []
                            }
                        },

                    files=cloudformation.InitFiles({
                        '/var/www/html/index.php': cloudformation.InitFile(
                            content=Join('', [
                                '<?php\n',
                                'echo \"<h1>AWS CloudFormation sample'
                                ' application for Amazon ElastiCache'
                                ' Redis Cluster</h1>\";\n',
                                '\n',
                                '$cluster_config = json_decode('
                                'file_get_contents(\'/tmp/cacheclusterconfig\''
                                '), true);\n',
                                '$endpoint = $cluster_config[\'CacheClusters'
                                '\'][0][\'CacheNodes\'][0][\'Endpoint\'][\'Add'
                                'ress\'];\n',
                                '$port = $cluster_config[\'CacheClusters\'][0]'
                                '[\'CacheNodes\'][0][\'Endpoint\'][\'Port\'];'
                                '\n',
                                '\n',
                                'echo \"<p>Connecting to Redis Cache Cluster '
                                'node \'{$endpoint}\' on port {$port}</p>\";'
                                '\n',
                                '\n',
                                '$redis=new Redis();\n',
                                '$redis->connect($endpoint, $port);\n',
                                '$redis->set(\'testkey\', \'Hello World!\');'
                                '\n',
                                '$return = $redis->get(\'testkey\');\n',
                                '\n',
                                'echo \"<p>Retrieved value: $return</p>\";'
                                '\n',
                                '?>\n'
                                ]),
                            mode='000644',
                            owner='apache',
                            group='apache'
                            ),
                        '/etc/cron.d/get_cluster_config':
                            cloudformation.InitFile(
                                content='*/5 * * * * root'
                                        ' /usr/local/bin/get_cluster_config',
                                mode='000644',
                                owner='root',
                                group='root'
                                ),
                        '/usr/local/bin/get_cluster_config':
                            cloudformation.InitFile(
                                content=Join('', [
                                    '#! /bin/bash\n',
                                    'aws elasticache describe-cache-clusters ',
                                    '         --cache-cluster-id ',
                                    Ref('RedisCluster'),
                                    '         --show-cache-node-info'
                                    ' --region ', Ref('AWS::Region'),
                                    ' > /tmp/cacheclusterconfig\n'
                                    ]),
                                mode='000755',
                                owner='root',
                                group='root'
                                ),
                        '/usr/local/bin/install_phpredis':
                            cloudformation.InitFile(
                                content=Join('', [
                                    '#! /bin/bash\n',
                                    'cd /tmp\n',
                                    'wget https://github.com/nicolasff/'
                                    'phpredis/zipball/master -O phpredis.zip'
                                    '\n',
                                    'unzip phpredis.zip\n',
                                    'cd nicolasff-phpredis-*\n',
                                    'phpize\n',
                                    './configure\n',
                                    'make && make install\n',
                                    'touch /etc/php.d/redis.ini\n',
                                    'echo extension=redis.so > /etc/php.d/'
                                    'redis.ini\n'
                                    ]),
                                mode='000755',
                                owner='root',
                                group='root'
                                ),
                        '/etc/cfn/cfn-hup.conf': cloudformation.InitFile(
                            content=Join('', [
                                '[main]\n',
                                'stack=', Ref('AWS::StackId'), '\n',
                                'region=', Ref('AWS::Region'), '\n'
                                ]),
                            mode='000400',
                            owner='root',
                            group='root'
                            ),
                        '/etc/cfn/hooks.d/cfn-auto-reloader.conf':
                            cloudformation.InitFile(
                                content=Join('', [
                                    '[cfn-auto-reloader-hook]\n',
                                    'triggers=post.update\n',
                                    'path=Resources.WebServerInstance.Metadata'
                                    '.AWS::CloudFormation::Init\n',
                                    'action=/opt/aws/bin/cfn-init -v ',
                                    '         --stack ', Ref('AWS::StackName'),
                                    '         --resource WebServerInstance ',
                                    '         --region ', Ref('AWS::Region'),
                                    '\n',
                                    'runas=root\n'
                                    ]),
                                # Why doesn't the Amazon template have this?
                                # mode='000400',
                                # owner='root',
                                # group='root'
                                ),
                        }),

                    commands={
                        '01-install_phpredis': {
                            'command': '/usr/local/bin/install_phpredis'
                            },
                        '02-get-cluster-config': {
                            'command': '/usr/local/bin/get_cluster_config'
                            }
                        },

                    services={
                        "sysvinit": cloudformation.InitServices({
                            "httpd": cloudformation.InitService(
                                enabled=True,
                                ensureRunning=True,
                                ),
                            "cfn-hup": cloudformation.InitService(
                                enabled=True,
                                ensureRunning=True,
                                files=['/etc/cfn/cfn-hup.conf',
                                       '/etc/cfn/hooks.d/'
                                       'cfn-auto-reloader.conf']
                                ),
                            }),
                        },
                    )
                })
            ),
        ImageId=FindInMap('AWSRegionArch2AMI', Ref('AWS::Region'),
                          FindInMap('AWSInstanceType2Arch',
                                    Ref(instancetype), 'Arch')),
        InstanceType=Ref(instancetype),
        SecurityGroups=[Ref(webserversg)],
        KeyName=Ref(keyname),
        IamInstanceProfile=Ref(webserverinstanceprofile),
        UserData=Base64(Join('', [
            '#!/bin/bash -xe\n',
            'yum update -y aws-cfn-bootstrap\n',

            '# Setup the PHP sample application\n',
            '/opt/aws/bin/cfn-init -v ',
            '         --stack ', Ref('AWS::StackName'),
            '         --resource WebServerInstance ',
            '         --region ', Ref('AWS::Region'), '\n',

            '# Signal the status of cfn-init\n',
            '/opt/aws/bin/cfn-signal -e $? ',
            '         --stack ', Ref('AWS::StackName'),
            '         --resource WebServerInstance ',
            '         --region ', Ref('AWS::Region'), '\n'
            ])),
        CreationPolicy=CreationPolicy(
            ResourceSignal=ResourceSignal(Timeout='PT15M')
            ),
        Tags=Tags(Application=Ref('AWS::StackId'),
                  Details='Created using Troposhpere')
        ))

    redisclustersg = template.add_resource(elasticache.SecurityGroup(
        'RedisClusterSecurityGroup',
        Description='Lock the cluster down',
        ))

    template.add_resource(elasticache.SecurityGroupIngress(
        'RedisClusterSecurityGroupIngress',
        CacheSecurityGroupName=Ref(redisclustersg),
        EC2SecurityGroupName=Ref(webserversg),
        ))

    template.add_resource(elasticache.CacheCluster(
        'RedisCluster',
        Engine='redis',
        CacheNodeType=Ref(cachenodetype),
        NumCacheNodes='1',
        CacheSecurityGroupNames=[Ref(redisclustersg)],
        ))

    # Outputs
    template.add_output([
        Output(
            'WebsiteURL',
            Description='Application URL',
            Value=Join('', [
                'http://',
                GetAtt(webserverinstance, 'PublicDnsName'),

                ])
            )
        ])

    # Print CloudFormation Template
    print(template.to_json())
962344
def build_caching_info_message(job_spec,
                               job_id,
                               workflow_workspace,
                               workflow_json,
                               result_path):
    """"""
    caching_info_message = {
        "job_spec": job_spec,
        "job_id": job_id,
        "workflow_workspace": workflow_workspace,
        "workflow_json": workflow_json,
        "result_path": result_path
    }
    return caching_info_message
1146534
void showFilesConfiguration() {

        // Message to communicate the resolved configuration:
        String message;
        if (filesUrl != null) {
            String reload = filesReloadable ? "reloadable" : "non-reloadable";
            message = "Files will be served from: " + filesUrl + " (" + reload + ")";
        } else {
            message = "No static files will be served.";
        }
        log.info("Files: {}", message);
    }
374820
public String buildConfirmation() {

        StringBuffer result = new StringBuffer(512);

        result.append("<p><div id='conf-msg'>\n");
        if (!isDirectPublish()) {
            result.append(key(Messages.GUI_PUBLISH_PROJECT_CONFIRMATION_1, new Object[] {getProjectname()}));
        } else {
            boolean isFolder = false;
            if (!isMultiOperation()) {
                try {
                    isFolder = getCms().readResource(getParamResource(), CmsResourceFilter.ALL).isFolder();
                } catch (CmsException e) {
                    // ignore
                }
            }
            if (isMultiOperation() || isFolder || (hasSiblings() && hasCorrectLockstate())) {
                result.append(key(Messages.GUI_PUBLISH_MULTI_CONFIRMATION_0));
            } else {
                result.append(key(Messages.GUI_PUBLISH_CONFIRMATION_0));
            }
        }
        result.append("\n</div></p>\n");
        return result.toString();
    }
316471
def print_brokers(cluster_config, brokers):
    
    print("Will restart the following brokers in {0}:".format(cluster_config.name))
    for id, host in brokers:
        print("  {0}: {1}".format(id, host))
4280
public static org.apache.hadoop.mapred.JobConf asJobConf(Configuration cfg) {
        return cfg instanceof org.apache.hadoop.mapred.JobConf
            ? (org.apache.hadoop.mapred.JobConf) cfg
            : new org.apache.hadoop.mapred.JobConf(cfg);
    }
1365177
public void configure(JobConf job) {
    iconf = new IndexUpdateConfiguration(job);
    analyzer =
        (Analyzer) ReflectionUtils.newInstance(
            iconf.getDocumentAnalyzerClass(), job);

    localAnalysis =
        (ILocalAnalysis) ReflectionUtils.newInstance(
            iconf.getLocalAnalysisClass(), job);
    localAnalysis.configure(job);

    shards = Shard.getIndexShards(iconf);

    distributionPolicy =
        (IDistributionPolicy) ReflectionUtils.newInstance(
            iconf.getDistributionPolicyClass(), job);
    distributionPolicy.init(shards);

    LOG.info("sea.document.analyzer = " + analyzer.getClass().getName());
    LOG.info("sea.local.analysis = " + localAnalysis.getClass().getName());
    LOG.info(shards.length + " shards = " + iconf.getIndexShards());
    LOG.info("sea.distribution.policy = "
        + distributionPolicy.getClass().getName());
  }
626566
def _update_status(self, sub_job_num=None):
        
        job_id = '%s.%s' % (self.cluster_id, sub_job_num) if sub_job_num else str(self.cluster_id)
        format = ['-format', '"%d"', 'JobStatus']
        cmd = 'condor_q {0} {1} && condor_history {0} {1}'.format(job_id, ' '.join(format))
        args = [cmd]
        out, err = self._execute(args, shell=True, run_in_job_dir=False)
        if err:
            log.error('Error while updating status for job %s: %s', job_id, err)
            raise HTCondorError(err)
        if not out:
            log.error('Error while updating status for job %s: Job not found.', job_id)
            raise HTCondorError('Job not found.')

        out = out.replace('\"', '').split('\n')


        status_code = 0
        for status_code_str in out:
            try:
                status_code = int(status_code_str.strip())
            except:
                pass

        log.info('Job %s status: %d', job_id, status_code)

        key = CONDOR_JOB_STATUSES[status_code]

        return key
1004209
def __create_and_save_state(cls, job_config, mapreduce_spec):
    
    state = model.MapreduceState.create_new(job_config.job_id)
    state.mapreduce_spec = mapreduce_spec
    state.active = True
    state.active_shards = 0
    state.app_id = job_config._app
    config = datastore_rpc.Configuration(force_writes=job_config._force_writes)
    state.put(config=config)
    return state
148983
def service_confirmation(self, bslpdu):
        """"""
        if _debug: ProxyServiceNetworkAdapter._debug("service_confirmation %r", bslpdu)

        # build a PDU
        pdu = NPDU(bslpdu.pduData)

        # the source is from the original source, not the proxy itself
        pdu.pduSource = bslpdu.bslciAddress

        # if the proxy received a broadcast, send it upstream as a broadcast
        if isinstance(bslpdu, ProxyToServerBroadcastNPDU):
            pdu.pduDestination = LocalBroadcast()
        if _debug: ProxyServiceNetworkAdapter._debug("    - pdu: %r", pdu)

        # decode it, the nework layer needs NPDUs
        npdu = NPDU()
        npdu.decode(pdu)
        if _debug: ProxyServiceNetworkAdapter._debug("    - npdu: %r", npdu)

        # send it to the service access point for processing
        self.adapterSAP.process_npdu(self, npdu)
362393
@Override
  protected void configure() {

    // Bind and expose LogWriter (a bit hacky, but needed by MapReduce for now)
    bind(LogWriter.class).to(LocalLogWriter.class);
    expose(LogWriter.class);

    // Bind ServiceAnnouncer for procedure.
    bind(ServiceAnnouncer.class).to(DiscoveryServiceAnnouncer.class);

    // For Binding queue stuff
    bind(QueueReaderFactory.class).in(Scopes.SINGLETON);

    // Bind ProgramRunner
    MapBinder<ProgramRunnerFactory.Type, ProgramRunner> runnerFactoryBinder =
      MapBinder.newMapBinder(binder(), ProgramRunnerFactory.Type.class, ProgramRunner.class);
    runnerFactoryBinder.addBinding(ProgramRunnerFactory.Type.FLOW).to(FlowProgramRunner.class);
    runnerFactoryBinder.addBinding(ProgramRunnerFactory.Type.FLOWLET).to(FlowletProgramRunner.class);

    bind(ProgramRunnerFactory.class).to(InMemoryFlowProgramRunnerFactory.class).in(Scopes.SINGLETON);
    // Note: Expose for test cases. Need to refactor test cases.
    expose(ProgramRunnerFactory.class);

    // For binding DataSet transaction stuff
    install(new DataFabricFacadeModule());
  }
939754
def _on_cluster_discovery(self, future):
        
        LOGGER.debug('_on_cluster_discovery(%r)', future)
        common.maybe_raise_exception(future)
        nodes = future.result()
        for node in nodes:
            name = '{}:{}'.format(node.ip, node.port)
            if name in self._cluster:
                LOGGER.debug('Updating cluster connection info for %s:%s',
                             node.ip, node.port)
                self._cluster[name].set_slots(node.slots)
                self._cluster[name].set_read_only('slave' in node.flags)
            else:
                self._create_cluster_connection(node)
        self._discovery = True
509904
def clusterflow_pipelines_printout(self):
        
        data = dict()
        html = ''
        for f,d in self.clusterflow_runfiles.items():
            pid = d.get('pipeline_id', 'unknown')
            data[pid] = [
                d.get('pipeline_name'),
                "\n".join(d.get('pipeline_steps', []))
            ]
        for pid, d in data.items():
            html += '''
                <div class="panel panel-default">
                    <div class="panel-heading"><h3 class="panel-title">Pipeline Steps: {} (<code>{}</code>)</h3></div>
                    <pre class="panel-body" style="border:0; background-color:transparent; padding:0 15px; margin:0; color:#666; font-size:90%;">{}</pre>
                </div>
                '''.format(pid, d[0], d[1])
        return html
269642
def notify(self, new_jobs_count):
        

        if not self.use_notify():
            return

        # Not really useful to send more than 100 notifs (to be configured)
        count = min(new_jobs_count, 100)

        notify_key = redis_key("notify", self)

        context.connections.redis.lpush(notify_key, *([1] * count))
        context.connections.redis.expire(notify_key, max(1, int(context.get_current_config()["max_latency"] * 2)))
54420
@Override
  public void sinkConfInit(FlowProcess<? extends Configuration> flowProcess,
      Tap<Configuration, RecordReader, OutputCollector> tap, Configuration conf) {
    conf.setBoolean("mapred.mapper.new-api", false);
    conf.setClass("mapred.output.format.class", CorcOutputFormat.class, OutputFormat.class);
    conf.setClass("mapreduce.job.output.key.class", NullWritable.class, Writable.class);
    conf.setClass("mapreduce.job.output.value.class", Corc.class, Writable.class);
  }
643578
def _mapreduce_job_counters_metrics(self, running_jobs, auth, ssl_verify, addl_tags):
        
        for job_metrics in itervalues(running_jobs):
            job_name = job_metrics['job_name']

            # Check if the job_name exist in the custom metrics
            if self.general_counters or (job_name in self.job_specific_counters):
                job_specific_metrics = self.job_specific_counters.get(job_name)

                metrics_json = self._rest_request_to_json(
                    job_metrics['tracking_url'],
                    auth,
                    ssl_verify,
                    'counters',
                    self.MAPREDUCE_SERVICE_CHECK,
                    tags=addl_tags,
                )

                if metrics_json.get('jobCounters'):
                    if metrics_json['jobCounters'].get('counterGroup'):

                        # Cycle through all the counter groups for this job
                        for counter_group in metrics_json['jobCounters']['counterGroup']:
                            group_name = counter_group.get('counterGroupName')

                            if group_name:
                                counter_metrics = set([])

                                # Add any counters in the job specific metrics
                                if job_specific_metrics and group_name in job_specific_metrics:
                                    counter_metrics = counter_metrics.union(job_specific_metrics[group_name])

                                # Add any counters in the general metrics
                                if group_name in self.general_counters:
                                    counter_metrics = counter_metrics.union(self.general_counters[group_name])

                                if counter_metrics:
                                    # Cycle through all the counters in this counter group
                                    if counter_group.get('counter'):
                                        for counter in counter_group['counter']:
                                            counter_name = counter.get('name')

                                            # Check if the counter name is in the custom metrics for this group name
                                            if counter_name and counter_name in counter_metrics:
                                                tags = [
                                                    'app_name:' + job_metrics.get('app_name'),
                                                    'user_name:' + job_metrics.get('user_name'),
                                                    'job_name:' + job_name,
                                                    'counter_name:' + str(counter_name).lower(),
                                                ]

                                                tags.extend(addl_tags)

                                                self._set_metrics_from_json(
                                                    counter, self.MAPREDUCE_JOB_COUNTER_METRICS, tags
                                                )