There is a bug in SDC' Docker implementation and intolerable prometheus_client code, due to which
        its process_collector will fail.

        See https://github.com/prometheus/client_python/issues/80
438165
def _disable_prometheus_process_collector(self) -> None:
        
        logger.info("Removing prometheus process collector")
        try:
            core.REGISTRY.unregister(PROCESS_COLLECTOR)
        except KeyError:
            logger.debug("PROCESS_COLLECTOR already removed from prometheus")
714030
private void prepareDockerCli() {
        clientConfig = createDefaultConfigBuilder()
                .build();

        dockerCmdExecFactory = new JerseyDockerCmdExecFactory()
                .withConnectTimeout(10 * 1000)
                .withReadTimeout(100 * 1000);

        dockerClient = DockerClientBuilder.getInstance(clientConfig)
                .withDockerCmdExecFactory(dockerCmdExecFactory)
                .build();
    }
1262017
private void updatePoolConf() throws DevFailed {
        if (!polledDevices.contains(deviceName)) {
            polledDevices.add(deviceName);
            final Map<String, String[]> properties = new HashMap<String, String[]>();
            properties.put(POLLING_THREADS_POOL_CONF, polledDevices.toArray(new String[0]));
            DatabaseFactory.getDatabase().setDeviceProperties(ServerManager.getInstance().getAdminDeviceName(),
                    properties);
        }
    }
261703
def from_sdp(cls, sdp):
        
        bits = sdp.split()
        if len(bits) < 8:
            raise ValueError('SDP does not have enough properties')

        kwargs = {
            'foundation': bits[0],
            'component': int(bits[1]),
            'transport': bits[2],
            'priority': int(bits[3]),
            'host': bits[4],
            'port': int(bits[5]),
            'type': bits[7],
        }

        for i in range(8, len(bits) - 1, 2):
            if bits[i] == 'raddr':
                kwargs['related_address'] = bits[i + 1]
            elif bits[i] == 'rport':
                kwargs['related_port'] = int(bits[i + 1])
            elif bits[i] == 'tcptype':
                kwargs['tcptype'] = bits[i + 1]
            elif bits[i] == 'generation':
                kwargs['generation'] = int(bits[i + 1])

        return Candidate(**kwargs)
829679
async def send_container_update_to_client(self, client_addrs):
        
        self._logger.debug("Sending containers updates...")
        available_containers = tuple(self._containers.keys())
        msg = BackendUpdateContainers(available_containers)
        for client in client_addrs:
            await ZMQUtils.send_with_addr(self._client_socket, client, msg)
832165
def create_dependencies task
      project_name = "minke#{SecureRandom.urlsafe_base64(12)}".downcase.gsub(/[^0-9a-z ]/i, '')
      network_name = ENV['DOCKER_NETWORK'] ||= "#{project_name}_default"
      ENV['DOCKER_PROJECT'] = project_name
      ENV['DOCKER_NETWORK'] = network_name

      logger = Minke::Logging.create_logger(STDOUT, self.verbose)
      shell = Minke::Helpers::Shell.new(logger)

      task_runner = Minke::Tasks::TaskRunner.new ({
        :ruby_helper       => Minke::Helpers::Ruby.new,
        :copy_helper       => Minke::Helpers::Copy.new,
        :service_discovery => Minke::Docker::ServiceDiscovery.new(project_name, Minke::Docker::DockerRunner.new(logger), network_name),
        :logger_helper     => logger
      })

      consul = Minke::Docker::Consul.new(
        {
          :health_check => Minke::Docker::HealthCheck.new(logger),
          :service_discovery => Minke::Docker::ServiceDiscovery.new( project_name, Minke::Docker::DockerRunner.new(logger, network_name), network_name),
          :consul_loader => ConsulLoader::Loader.new(ConsulLoader::ConfigParser.new),
          :docker_runner => Minke::Docker::DockerRunner.new(logger, network_name),
          :network => network_name,
          :project_name => project_name,
          :logger_helper => logger
        }
      )

      network = Minke::Docker::Network.new(
        network_name,
        shell
      )

      return {
        :config                 => @config,
        :task_name              => task,
        :docker_runner          => Minke::Docker::DockerRunner.new(logger, network_name, project_name),
        :task_runner            => task_runner,
        :shell_helper           => shell,
        :logger_helper          => logger,
        :generator_config       => generator_config,
        :docker_compose_factory => Minke::Docker::DockerComposeFactory.new(shell, project_name, network_name),
        :consul                 => consul,
        :docker_network         => network,
        :health_check           => Minke::Docker::HealthCheck.new(logger),
        :service_discovery      => Minke::Docker::ServiceDiscovery.new(project_name, Minke::Docker::DockerRunner.new(logger), network_name)
      } 
    end
1082589
def create(self):
        
        data = self._node_data()
        data["node_id"] = self._id
        if self._node_type == "docker":
            timeout = None
        else:
            timeout = 1200
        trial = 0
        while trial != 6:
            try:
                response = yield from self._compute.post("/projects/{}/{}/nodes".format(self._project.id, self._node_type), data=data, timeout=timeout)
            except ComputeConflict as e:
                if e.response.get("exception") == "ImageMissingError":
                    res = yield from self._upload_missing_image(self._node_type, e.response["image"])
                    if not res:
                        raise e
                else:
                    raise e
            else:
                yield from self.parse_node_response(response.json)
                return True
            trial += 1
907726
def _create_cadvisor_prometheus_instance(self, instance):
        
        cadvisor_instance = deepcopy(instance)
        cadvisor_instance.update(
            {
                'namespace': self.NAMESPACE,
                # We need to specify a prometheus_url so the base class can use it as the key for our config_map,
                # we specify a dummy url that will be replaced in the `check()` function. We append it with "cadvisor"
                # so the key is different than the kubelet scraper.
                'prometheus_url': instance.get('cadvisor_metrics_endpoint', 'dummy_url/cadvisor'),
                'ignore_metrics': [
                    'container_cpu_cfs_periods_total',
                    'container_fs_inodes_free',
                    'container_fs_inodes_total',
                    'container_fs_io_current',
                    'container_fs_io_time_seconds_total',
                    'container_fs_io_time_weighted_seconds_total',
                    'container_fs_read_seconds_total',
                    'container_fs_reads_merged_total',
                    'container_fs_reads_total',
                    'container_fs_sector_reads_total',
                    'container_fs_sector_writes_total',
                    'container_fs_write_seconds_total',
                    'container_fs_writes_merged_total',
                    'container_fs_writes_total',
                    'container_last_seen',
                    'container_start_time_seconds',
                    'container_spec_memory_swap_limit_bytes',
                    'container_scrape_error',
                ],
                # Defaults that were set when CadvisorPrometheusScraper was based on PrometheusScraper
                'send_monotonic_counter': instance.get('send_monotonic_counter', False),
                'health_service_check': instance.get('health_service_check', False),
            }
        )
        return cadvisor_instance
160474
private DockerSlave provisionWithWait(DockerSlaveTemplate template, ProvisioningActivity.Id id)
            throws Exception {
        final DockerContainerLifecycle dockerContainerLifecycle = template.getDockerContainerLifecycle();
        final String imageId = dockerContainerLifecycle.getImage();
        final DockerComputerLauncher computerLauncher = template.getLauncher();

        //pull image
        dockerContainerLifecycle.getPullImage().exec(getClient(), imageId, new StreamTaskListener(new NullStream()));

        // set the operating system if it's not already cached
        if (template.getOsType() == null) {
            template.setOsType(determineOsType(imageId));
        }

        LOG.debug("Trying to run container for {}, operating system {}", imageId, template.getOsType());
        final String containerId = runContainer(template);

        InspectContainerResponse ir;
        try {
            ir = getClient().inspectContainerCmd(containerId).exec();
        } catch (ProcessingException ex) {
            LOG.error("Failed to run container for {}, clean-up container", imageId);
            dockerContainerLifecycle.getRemoveContainer().exec(getClient(), containerId);
            throw ex;
        }

        // Build a description up:
        String nodeDescription = "Docker Node [" + imageId + " on ";
        try {
            nodeDescription += getDisplayName();
        } catch (Exception ex) {
            nodeDescription += "???";
        }
        nodeDescription += "]";

        String slaveName = String.format("%s-%s", getDisplayName(), containerId.substring(0, 12));

        if (computerLauncher.waitUp(getDisplayName(), template, ir)) {
            LOG.debug("Container {} is ready for slave connection", containerId);
        } else {
            LOG.error("Container {} is not ready for slave connection.", containerId);
        }

        final DockerComputerLauncher launcher = computerLauncher.getPreparedLauncher(getDisplayName(), template, ir);
        return new DockerSlave(slaveName, nodeDescription, launcher, containerId, template, getDisplayName(), id);
    }
1030742
function(appName) {

    // Query docker for list of containers.
    return dockerInstance()
    .then(function(dockerInstance) {
      // Make sure to retry call to list containers.
      return Promise.fromNode(function(cb) {
        dockerInstance.listContainers({all: 1}, cb);
      });
    })
    // Make sure we have a timeout.
    .timeout(30 * 1000)
    // Wrap errors.
    .catch(function(err) {
      throw new VError(err, 'Error querying docker for list of containers.');
    })
    // Filter out containers with invalid status
    .filter(function(data) {
      return data.Status !== 'Removal In Progress';
    })
    // Map docker containers to generic containers.
    .map(toGenericContainer)
    // Filter out nulls and undefineds.
    .filter(_.identity)
    // Filter out all non-kalabox containers
    .filter(function(data) {
      return data.kalabox === true;
    })
    // Filter by app name if an app name was given.
    .then(function(containers) {
      if (appName) {
        return Promise.filter(containers, function(container) {
          return container.app === appName;
        });
      } else {
        return containers;
      }
    });

  }
1236357
def _create_kubelet_prometheus_instance(self, instance):
        
        kubelet_instance = deepcopy(instance)
        kubelet_instance.update(
            {
                'namespace': self.NAMESPACE,
                # We need to specify a prometheus_url so the base class can use it as the key for our config_map,
                # we specify a dummy url that will be replaced in the `check()` function. We append it with "kubelet"
                # so the key is different than the cadvisor scraper.
                'prometheus_url': instance.get('kubelet_metrics_endpoint', 'dummy_url/kubelet'),
                'metrics': [
                    {
                        'apiserver_client_certificate_expiration_seconds': 'apiserver.certificate.expiration',
                        'rest_client_requests_total': 'rest.client.requests',
                        'rest_client_request_latency_seconds': 'rest.client.latency',
                        'kubelet_runtime_operations': 'kubelet.runtime.operations',
                        'kubelet_runtime_operations_errors': 'kubelet.runtime.errors',
                        'kubelet_network_plugin_operations_latency_microseconds': 'kubelet.network_plugin.latency',
                        'kubelet_volume_stats_available_bytes': 'kubelet.volume.stats.available_bytes',
                        'kubelet_volume_stats_capacity_bytes': 'kubelet.volume.stats.capacity_bytes',
                        'kubelet_volume_stats_used_bytes': 'kubelet.volume.stats.used_bytes',
                        'kubelet_volume_stats_inodes': 'kubelet.volume.stats.inodes',
                        'kubelet_volume_stats_inodes_free': 'kubelet.volume.stats.inodes_free',
                        'kubelet_volume_stats_inodes_used': 'kubelet.volume.stats.inodes_used',
                    }
                ],
                # Defaults that were set when the Kubelet scraper was based on PrometheusScraper
                'send_monotonic_counter': instance.get('send_monotonic_counter', False),
                'health_service_check': instance.get('health_service_check', False),
            }
        )
        return kubelet_instance
8080
def pull(self):
        
        for json_e in self.d.pull(repository=self.name, tag=self.tag, stream=True, decode=True):
            logger.debug(json_e)
            status = graceful_get(json_e, "status")
            if status:
                logger.info(status)
            else:
                error = graceful_get(json_e, "error")
                logger.error(status)
                raise ConuException("There was an error while pulling the image %s: %s",
                                    self.name, error)
        self.using_transport(SkopeoTransport.DOCKER_DAEMON)
899701
def docker_client():
    
    cert_path = os.environ.get('DOCKER_CERT_PATH', '')
    if cert_path == '':
        cert_path = os.path.join(os.environ.get('HOME', ''), '.docker')

    base_url = os.environ.get('DOCKER_HOST')
    tls_config = None

    if os.environ.get('DOCKER_TLS_VERIFY', '') != '':
        parts = base_url.split('://', 1)
        base_url = '%s://%s' % ('https', parts[1])

        client_cert = (os.path.join(cert_path, 'cert.pem'),
                       os.path.join(cert_path, 'key.pem'))
        ca_cert = os.path.join(cert_path, 'ca.pem')

        tls_config = tls.TLSConfig(
            ssl_version=ssl.PROTOCOL_TLSv1,
            verify=True,
            assert_hostname=False,
            client_cert=client_cert,
            ca_cert=ca_cert,
        )

    timeout = int(os.environ.get('DOCKER_CLIENT_TIMEOUT', 60))
    return Client(
        base_url=base_url, tls=tls_config, version='1.15', timeout=timeout
    )
862864
def _handle_stream_line(line):
    
    # XXX This necessary processing is probably a bug from docker-py,
    # hence, might break if the bug is fixed, i.e. we should get decoded JSON
    # directly from the API.
    line = json.loads(line)
    if 'error' in line:
        content = "ERROR: " + line['error']
    else:
        content = line.get('stream', '')
    for streamline in content.split('\n'):
        if streamline:
            yield streamline
1180747
def register_endpoint(self, path, app=None):
        

        if is_running_from_reloader() and not os.environ.get('DEBUG_METRICS'):
            return

        if app is None:
            app = self.app or current_app

        @app.route(path)
        @self.do_not_track()
        def prometheus_metrics():
            # import these here so they don't clash with our own multiprocess module
            from prometheus_client import multiprocess, CollectorRegistry

            if 'prometheus_multiproc_dir' in os.environ:
                registry = CollectorRegistry()
            else:
                registry = self.registry

            if 'name[]' in request.args:
                registry = registry.restricted_registry(request.args.getlist('name[]'))

            if 'prometheus_multiproc_dir' in os.environ:
                multiprocess.MultiProcessCollector(registry)

            headers = {'Content-Type': CONTENT_TYPE_LATEST}
            return generate_latest(registry), 200, headers
1013823
function sdkReady(type, callback) {
        callback = callback || function () {};
        if (!Embedo.defaults.SOURCES[type]) {
          return callback(new Error('unsupported_sdk_type'));
        }
        var counter = 0;
        (function check() {
          counter++;
          if (counter > 15) {
            return callback(new Error(type + ':sdk_not_available'));
          }
          if (window[Embedo.defaults.SOURCES[type].GLOBAL]) {
            return callback(null, window[Embedo.defaults.SOURCES[type].GLOBAL]);
          }
          setTimeout(check, 10 * counter);
        })(type);
      }
728331
def xmlrpc_run(self, port=23333, bind='127.0.0.1', logRequests=False):
        ''''''
        from pyspider.libs.wsgi_xmlrpc import WSGIXMLRPCApplication

        application = WSGIXMLRPCApplication()

        application.register_function(self.quit, '_quit')
        application.register_function(self.__len__, 'size')

        def dump_counter(_time, _type):
            try:
                return self._cnt[_time].to_dict(_type)
            except:
                logger.exception('')
        application.register_function(dump_counter, 'counter')

        def new_task(task):
            if self.task_verify(task):
                self.newtask_queue.put(task)
                return True
            return False
        application.register_function(new_task, 'newtask')

        def send_task(task):
            '''dispatch task to fetcher'''
            self.send_task(task)
            return True
        application.register_function(send_task, 'send_task')

        def update_project():
            self._force_update_project = True
        application.register_function(update_project, 'update_project')

        def get_active_tasks(project=None, limit=100):
            allowed_keys = set((
                'type',
                'taskid',
                'project',
                'status',
                'url',
                'lastcrawltime',
                'updatetime',
                'track',
            ))
            track_allowed_keys = set((
                'ok',
                'time',
                'follows',
                'status_code',
            ))

            iters = [iter(x.active_tasks) for k, x in iteritems(self.projects)
                     if x and (k == project if project else True)]
            tasks = [next(x, None) for x in iters]
            result = []

            while len(result) < limit and tasks and not all(x is None for x in tasks):
                updatetime, task = t = max(t for t in tasks if t)
                i = tasks.index(t)
                tasks[i] = next(iters[i], None)
                for key in list(task):
                    if key == 'track':
                        for k in list(task[key].get('fetch', [])):
                            if k not in track_allowed_keys:
                                del task[key]['fetch'][k]
                        for k in list(task[key].get('process', [])):
                            if k not in track_allowed_keys:
                                del task[key]['process'][k]
                    if key in allowed_keys:
                        continue
                    del task[key]
                result.append(t)
            # fix for "<type 'exceptions.TypeError'>:dictionary key must be string"
            # have no idea why
            return json.loads(json.dumps(result))
        application.register_function(get_active_tasks, 'get_active_tasks')

        def get_projects_pause_status():
            result = {}
            for project_name, project in iteritems(self.projects):
                result[project_name] = project.paused
            return result
        application.register_function(get_projects_pause_status, 'get_projects_pause_status')

        def webui_update():
            return {
                'pause_status': get_projects_pause_status(),
                'counter': {
                    '5m_time': dump_counter('5m_time', 'avg'),
                    '5m': dump_counter('5m', 'sum'),
                    '1h': dump_counter('1h', 'sum'),
                    '1d': dump_counter('1d', 'sum'),
                    'all': dump_counter('all', 'sum'),
                },
            }
        application.register_function(webui_update, 'webui_update')

        import tornado.wsgi
        import tornado.ioloop
        import tornado.httpserver

        container = tornado.wsgi.WSGIContainer(application)
        self.xmlrpc_ioloop = tornado.ioloop.IOLoop()
        self.xmlrpc_server = tornado.httpserver.HTTPServer(container, io_loop=self.xmlrpc_ioloop)
        self.xmlrpc_server.listen(port=port, address=bind)
        logger.info('scheduler.xmlrpc listening on %s:%s', bind, port)
        self.xmlrpc_ioloop.start()