Override content download URL generator to generate download links
to files with Netgen Site download route.
423167
public function process(ContainerBuilder $container): void
    {
        if ($container->has('ezpublish.fieldType.ease.download_url_generator')) {
            $container
                ->findDefinition('ezpublish.fieldType.ezbinarybase.download_url_generator')
                ->setClass(ContentDownloadUrlGenerator::class);
        }
    }
74206
private void generateBackLinks(final Metadata m, final Element e) {
        final Element backLinksElements = new Element("backLinks", NS);
        for (final URL backLink : m.getBackLinks()) {
            addNotNullElement(backLinksElements, "backLink", backLink);
        }
        if (!backLinksElements.getChildren().isEmpty()) {
            e.addContent(backLinksElements);
        }
    }
245343
function getPage(url, cb) {

    var page = generator.page$[u.urlPath(url)];
    var getFn = page && page.get && generator['get'+page.get];

    if (getFn) {
      getFn(page, function() { cb(null, page); });
    }
    else {
      process.nextTick(function() { cb(null, page); });
    }
  }
974529
def gen_tasks(self, urls):
        
        tasks = []
        if isinstance(urls, list):
            for url in urls:
                dat = {
                    'url': url,
                    'name': url.split('/')[-1]
                }
                if not self.overwrite and helper.is_file_ok(dat['name']):
                    self.result['skip'] += 1
                else:
                    tasks.append(dat)
        elif isinstance(urls, dict):
            # {k: v} => k is name, v is url value
            for k, v in urls.items():
                dat = {
                    'url': v,
                    'name': k,
                }
                if not self.overwrite and helper.is_file_ok(dat['name']):
                    self.result['skip'] += 1
                else:
                    tasks.append(dat)
        else:
            raise CrawlerParamsError('urls should be list/dict')
        return tasks
693830
def _build_page(self, filepath):
        
        filename = filepath.split("/")[-1]
        # If filename starts with _ (underscore) or . (dot) do not build
        if not filename.startswith(("_", ".")) and (filename.endswith(PAGE_FORMAT)):
            meta = self._get_page_meta(filepath)
            content = self._get_page_content(filepath)

            # The default context for the page
            _default_page = {
                "build_dir": self.build_dir,
                "filepath": meta["filepath"],
                "context": {"page": meta},
                "content": content,
                "markup": meta.get("markup"),
                "template": meta.get("template"),
                "layout": meta.get("layout") or self.default_layout
            }

            # GENERATOR
            # Allows to generate
            _generator = meta.get("_generator")
            if _generator:
                data = self._data.get(_generator.get("data_source"))

                # We want these back in meta in they exists in the data
                special_meta = ["title", "slug", "description"]

                # SINGLE
                if _generator.get("type") == "single":
                    for d in data:
                        dmeta = copy.deepcopy(meta)
                        page = copy.deepcopy(_default_page)
                        for _ in special_meta:
                            if _ in d:
                                dmeta[_] = d.get(_)

                        # If generator has the slug, it will substitute if
                        # Slug in the generator must have token from the data
                        # to generate the slug
                        if "slug" in _generator:
                            dmeta["slug"] = _generator.get("slug").format(**d)

                        # Slug is required
                        if "slug" not in dmeta:
                            print("WARNING: Skipping page because it's missing `slug`")
                            continue
                        slug = dmeta.get("slug")
                        dmeta["url"] = slug
                        dmeta["context"] = d

                        page.update({
                            "filepath": slug,
                            "context": {"page": dmeta}
                        })
                        self.create_page(**page)

                if _generator.get("type") == "pagination":

                    per_page = int(_generator.get("per_page", self.site_config.get("pagination.per_page", 10)))
                    left_edge = int(_generator.get("left_edge", self.site_config.get("pagination.left_edge", 2)))
                    left_current = int(_generator.get("left_edge", self.site_config.get("pagination.left_current", 3)))
                    right_current = int(_generator.get("right_current", self.site_config.get("pagination.right_current", 4)))
                    right_edge = int(_generator.get("right_edge", self.site_config.get("pagination.right_edge", 2)))
                    padding = _generator.get("padding")
                    slug = _generator.get("slug")
                    limit = _generator.get("limit")

                    if "limit" in _generator:
                        data = data[:int(limit)]
                    data_chunks = utils.chunk_list(data, per_page)
                    len_data = len(data)

                    for i, d in enumerate(data_chunks):
                        dmeta = copy.deepcopy(meta)
                        page = copy.deepcopy(_default_page)

                        page_num = i + 1
                        _paginator = Paginator([],
                                               total=len_data,
                                               page=page_num,
                                               per_page=per_page,
                                               padding=padding,
                                               left_edge=left_edge,
                                               right_edge=right_edge,
                                               left_current=left_current,
                                               right_current=right_current)
                        _paginator.slug = slug
                        _paginator.index_slug = _generator.get("index_slug")

                        _slug = slug.format(**{"page_num": page_num})
                        dmeta["url"] = _slug
                        dmeta["context"] = d
                        dmeta["paginator"] = _paginator
                        page.update({
                            "filepath": _slug,
                            "context": {"page": dmeta}
                        })
                        self.create_page(**page)

                        # First page need to generate the index
                        if i == 0 and _generator.get("index_slug"):
                            page["filepath"] = _generator.get("index_slug")
                            self.create_page(**page)

            # NORMAL PAGE
            else:
                self.create_page(**_default_page)
1012411
def generate_static_site(self, output_root=None, extra_context=None):
        
        self.app.config['BUILD_PATH'] = output_root

        # use this hook for registering URLs to freeze
        self.call_hook("generate", self, output_root, extra_context)

        if output_root is not None:
            # realpath or this gets generated relative to the tarbell package
            self.app.config['FREEZER_DESTINATION'] = os.path.realpath(output_root)

        self.freezer.freeze()
253391
def _page_gen(self):
        
        track = ""
        for page in self.__pages__:
            track += "/{page}".format(page=page)
        return track
1169068
def gen_url(url, text)
      if url =~ /([A-Za-z]+):(.*)/
        type = $1
        path = $2
      else
        type = "http"
        path = url
        url  = "http://#{url}"
      end

      if type == "link"
        if path[0,1] == '#'     # is this meaningful?
          url = path
        else
          url = HTMLGenerator.gen_url(@from_path, path)
        end
      end

      if (type == "http" || type == "link") && 
          url =~ /\.(gif|png|jpg|jpeg|bmp)$/

        "<img src=\"#{url}\" />"
      else
        "<a href=\"#{url}\">#{text.sub(%r{^#{type}:/*}, '')}</a>"
      end
    end
1108388
def download(self, sub_url):
        """"""
        response = requests.get(sub_url, headers=self.headers).text
        soup = BS(response, 'lxml')
        downlink = self.base_url+soup.select('.download a')[0]['href']
        data = requests.get(downlink, headers=self.headers)
        z = zipfile.ZipFile(cStringIO.StringIO(data.content))
        srt_files = [f.filename for f in z.filelist
                     if f.filename.rsplit('.')[-1].lower() in ['srt', 'ass']]
        z.extract(srt_files[0], '/tmp/')
        return srt_files[0]
787304
def gen_files(path, prefix="_"):
    "  "

    if op.isdir(path):
        for name in listdir(path):
            fpath = op.join(path, name)
            if is_parsed_file(fpath):
                yield op.abspath(fpath)

    elif is_parsed_file(path):
        yield op.abspath(path)
1325899
def download(self, uuid, url, forced_file_name=None,
                 progress_bar=True, chunk_size=256,
                 directory=None, overwrite=False):
        
        self.last_req_time = None
        url = self.get_full_url(url)
        self.log.debug("download url : " + url)
        # Building request
        request = urllib2.Request(url)
        # request.add_header('Content-Type', 'application/json; charset=UTF-8')
        request.add_header('Accept', 'application/json,*/*;charset=UTF-8')
        # request start
        starttime = datetime.datetime.now()
        # doRequest
        resultq = urllib2.urlopen(request)
        code = resultq.getcode()
        file_name = uuid
        self.log.debug("ret code : '" + str(code) + "'")
        if code == 200:
            content_lenth = resultq.info().getheader('Content-Length')
            if not content_lenth:
                msg = "No content lengh header found !"
                self.log.debug(msg)
                progress_bar = False
            else:
                file_size = int(content_lenth.strip())
            if forced_file_name:
                file_name = forced_file_name
            else:
                content_dispo = resultq.info().getheader('Content-disposition')
                if content_dispo:
                    content_dispo = content_dispo.strip()
                    file_name = extract_file_name(content_dispo)
            if directory:
                if os.path.isdir(directory):
                    file_name = directory + "/" + file_name
            if os.path.isfile(file_name):
                if not overwrite:
                    cpt = 1
                    while 1:
                        if not os.path.isfile(file_name + "." + str(cpt)):
                            file_name += "." + str(cpt)
                            break
                        cpt += 1
                else:
                    self.log.warn("'%s' already exists. It was overwriten.",
                                  file_name)
            stream = None
            pbar = None
            if progress_bar:
                widgets = [FileTransferSpeed(), ' <<<', Bar(), '>>> ',
                           Percentage(), ' ', ETA()]
                pbar = ProgressBar(widgets=widgets, maxval=file_size)
                stream = FileWithCallback(file_name, 'w', pbar.update,
                                          file_size, file_name)
                pbar.start()
            else:
                stream = file(file_name, 'w')
            while 1:
                chunk = resultq.read(chunk_size)
                if not chunk:
                    break
                stream.write(chunk)
            stream.flush()
            stream.close()
            if pbar:
                pbar.finish()
        # request end
        endtime = datetime.datetime.now()
        self.last_req_time = str(endtime - starttime)
        self.log.debug("download url : %(url)s : request time : %(time)s",
                       {"url": url,
                        "time": self.last_req_time})
        return (file_name, self.last_req_time)
515636
def replace_git_url(generator):
    """"""
    template = g_jinja2.get_template(GIT_TEMPLATE)

    should_cache = generator.context.get('GIT_CACHE_ENABLED')
    cache_location = generator.context.get('GIT_CACHE_LOCATION')

    for article in generator.articles:
        for match in git_regex.findall(article._content):
            params = collections.defaultdict(str)
            repo = match[1]
            filename = match[3]
            branch = match[5]
            hash = match[7]

            params['repo'] = match[1]
            params['filename'] = match[3]
            if match[5]:
                params['branch'] = match[5]
            if match[7]:
                params['hash'] = match[7]

            logger.info('[git]: Found repo {}, filename {}, branch {} and hash {}'.format(repo, filename, branch, hash))
            logger.info('[git]: {}'.format(params))

            body = None if not should_cache else get_cache(cache_location, **params)

            # Fetch the git
            if not body:
                logger.info('[git]: Git did not exist in cache, fetching...')
                response = fetch_git(**params)
                body = get_body(response)

                if should_cache:
                    logger.info('[git]: Saving git to cache...')
                    cache_params = copy.copy(params)
                    cache_params['body'] = body
                    set_cache(cache_location, **cache_params)
            else:
                logger.info('[git]: Found git in cache.')

            # Create a context to render with
            context = generator.context.copy()
            context.update({
                'code': body,
                'footer': 'full',
                'base': __url__,
                'filename': filename,
                'url': git_url(**params)
            })
            replacement = template.render(context)
            article._content = article._content.replace(match[0], replacement)
43714
def generate_contents(startpath, outfilename=DEFAULT_BUILDFILE):
    
    def _ignored_name(name):
        return (
            name.startswith('.') or
            name == PACKAGE_DIR_NAME or
            name.endswith('~') or
            name == outfilename
        )

    def _generate_contents(dir_path):
        safename_duplicates = defaultdict(list)
        for name in os.listdir(dir_path):
            if _ignored_name(name):
                continue

            path = os.path.join(dir_path, name)

            if os.path.isdir(path):
                nodename = name
                ext = None
            elif os.path.isfile(path):
                nodename, ext = splitext_no_dot(name)
            else:
                continue

            safename = to_identifier(nodename)
            safename_duplicates[safename].append((name, nodename, ext))

        safename_to_name = {}
        for safename, duplicates in iteritems(safename_duplicates):
            for name, nodename, ext in duplicates:
                if len(duplicates) > 1 and ext:
                    new_safename = to_identifier(name)  # Name with ext
                else:
                    new_safename = safename
                existing_name = safename_to_name.get(new_safename)
                if existing_name is not None:
                    message = "Duplicate node names in directory %r."
                    message += " %r was renamed to %r, which overlaps with %r"
                    raise BuildException(
                        message % (dir_path, name, new_safename, existing_name)
                    )
                safename_to_name[new_safename] = name

        contents = {}
        for safename, name in iteritems(safename_to_name):
            path = os.path.join(dir_path, name)

            if os.path.isdir(path):
                data = _generate_contents(path)
            else:
                rel_path = os.path.relpath(path, startpath)
                data = dict(file=rel_path)

            contents[safename] = data

        return contents

    return dict(
        contents=_generate_contents(startpath)
    )
402607
function generateSite(config, useServer) {
  var fileMap = new FileMap(config)
    , siteBuilder = new SiteBuilder(config)
    , server = require(__dirname + '/server')(siteBuilder);

  fileMap.walk();
  fileMap.on('ready', function() {
    siteBuilder.fileMap = fileMap;
    siteBuilder.build();
  });

  siteBuilder.once('ready', function() {
    if (useServer) {
      server.run();
      server.watch();
    }
  });

  return siteBuilder;
}
1049890
def generate_overlay_urls(self):
        """"""
        overlays = {}
        for o in self.dataset.list_overlay_names():
            url = self.generate_url(".dtool/overlays/{}.json".format(o))
            overlays[o] = url
        return overlays
425220
def interlink_translated_content(generator):
    '''
    '''
    inspector = GeneratorInspector(generator)
    for content in inspector.all_contents():
        interlink_translations(content)
1053661
function downloadNode(next) {
        if (fs.existsSync(nodeFilePath)) return next();

        var uri = framework;

        if (framework === 'node') {
          uri = 'nodejs'; // if node, use nodejs uri
        } else if (framework === 'nodejs') {
          framework = 'node'; // support nodejs, and node, as framework.
        }

        var type = global.type;
        var url,
          prefix = 'https://' + uri + '.org/dist';

        if (version === 'latest') {
          url = prefix + '/' + framework + '-' + version + '.tar.gz';
        } else {
          url =
            prefix +
            '/v' +
            version +
            '/' +
            framework +
            '-v' +
            version +
            '.tar.gz';
        }

        _log('downloading %s', url);

        var output = fs.createWriteStream(nodeFilePath, {
          flags: 'w+',
        });

        // need to set user-agent to bypass some corporate firewalls
        var requestOptions = {
          url: url,
          headers: {
            'User-Agent': 'Node.js',
          },
        };

        _logProgress(request(requestOptions)).pipe(output);

        output.on('close', function() {
          next();
        });
      }